Skip to content

/ go

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

x/build: farmer hang #10698

Closed
crawshaw opened this issue May 5, 2015 · 1 comment
Closed

x/build: farmer hang #10698

crawshaw opened this issue May 5, 2015 · 1 comment
Labels
FrozenDueToAge

Comments

@crawshaw
Copy link
Contributor

@crawshaw crawshaw commented May 5, 2015

The mangled stack trace from journald, with log prefix removed to fit on github:

2015/05/05 12:37:46 http: Accept error: accept tcp [::]:443: too many open files; retrying in 1s
2015/05/05 12:37:46 http: Accept error: accept tcp [::]:80: too many open files; retrying in 1s
2015/05/05 12:37:46 failed to find new work: Get https://build.golang.org/?mode=json: dial tcp: lookup build.golang.org on 10.240.0.1:53: dial udp 10.240.0.1:53: too many open files
2015/05/05 12:37:47 http: Accept error: accept tcp [::]:443: too many open files; retrying in 1s
2015/05/05 12:37:47 http: Accept error: accept tcp [::]:80: too many open files; retrying in 1s
SIGQUIT: quit
PC=0x70abe8 m=0
goroutine 57 [syscall, 5502 minutes]:
syscall.Syscall6(0x3d, 0x60f1, 0xc208123d24, 0x0, 0xc208467440, 0x0, 0x0, 0x60, 0x0, 0x7f5900000001)
/Users/crawshaw/go/src/syscall/asm_linux_amd64.s:44 +0x5 fp=0xc208123c90 sp=0xc208123c88
syscall.wait4(0x60f1, 0xc208123d24, 0x0, 0xc208467440, 0x90, 0x0, 0x0)
/Users/crawshaw/go/src/syscall/zsyscall_linux_amd64.go:170 +0x75 fp=0xc208123ce8 sp=0xc208123c90
syscall.Wait4(0x60f1, 0xc208123d6c, 0x0, 0xc208467440, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/syscall/syscall_linux.go:256 +0x5c fp=0xc208123d30 sp=0xc208123ce8
os.(*Process).wait(0xc20843a9c0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/os/exec_unix.go:22 +0x108 fp=0xc208123de0 sp=0xc208123d30
os.(*Process).Wait(0xc20843a9c0, 0xc20843a958, 0x0, 0x0)
/Users/crawshaw/go/src/os/doc.go:45 +0x34 fp=0xc208123e08 sp=0xc208123de0
os/exec.(*Cmd).Wait(0xc20843f680, 0x0, 0x0)
/Users/crawshaw/go/src/os/exec/exec.go:364 +0x214 fp=0xc208123ed8 sp=0xc208123e08
os/exec.(*Cmd).Run(0xc20843f680, 0x0, 0x0)
/Users/crawshaw/go/src/os/exec/exec.go:246 +0x6b fp=0xc208123f08 sp=0xc208123ed8
main.startWatching.func2(0xc20800dcc0, 0x40, 0xa3af50, 0x26, 0xa1bd50, 0x1f, 0x2540be400)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/watcher.go:173 +0xda fp=0xc208123fa8 sp=0xc208123f08
runtime.goexit()
/Users/crawshaw/go/src/runtime/asm_amd64.s:1670 +0x1 fp=0xc208123fb0 sp=0xc208123fa8
created by main.startWatching
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/watcher.go:177 +0x7c5
goroutine 1 [select]:
main.main()
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:300 +0xc7e
goroutine 30 [sleep]:
time.Sleep(0xdf8475800)
/Users/crawshaw/go/src/runtime/time.go:59 +0xfc
main.(*gceBuildletPool).cleanUpOldVMs(0xc46bb0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/gce.go:254 +0x34a
created by main.main
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:285 +0xd0d
goroutine 547283 [semacquire, 318 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2091a6420, 0xc2088d3040)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2091a6420, 0xc2088d3040)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2091a6420, 0xc2088d3040)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2091a6420, 0xc2088d3040)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc209898a00)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 9 [chan receive, 493 minutes]:
main.(*reverseBuildletPool).reverseHealthCheck(0xc470a0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:139 +0x3e5
main.init.4.func1()
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:54 +0x3d
created by main.init.4
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:56 +0x33
goroutine 29 [sleep]:
time.Sleep(0x3b9aca00)
/Users/crawshaw/go/src/runtime/time.go:59 +0xfc
net/http.(*Server).Serve(0xc2083deba0, 0x7f59fd696418, 0xc20801e598, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/server.go:1792 +0x2d4
net/http.(*Server).ListenAndServe(0xc2083deba0, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/server.go:1770 +0x138
net/http.ListenAndServe(0x987d60, 0x3, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/server.go:1860 +0x92
main.main.func1()
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:272 +0xa4
created by main.main
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:276 +0x3c7
goroutine 15 [select, 5502 minutes]:
net/http.(*persistConn).readLoop(0xc2080a9810)
/Users/crawshaw/go/src/net/http/transport.go:975 +0x9ec
created by net/http.(*Transport).dialConn
/Users/crawshaw/go/src/net/http/transport.go:684 +0xca9
goroutine 541106 [semacquire, 465 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.(*reverseBuildletPool).WriteHTMLStatus(0xc470a0, 0x7f59fd68b6e8, 0xc2089d32d0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:191 +0x16a
main.handleStatus(0x7f59fd69b578, 0xc208570fd0, 0xc2089a1450)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/status.go:60 +0xf61
net/http.HandlerFunc.ServeHTTP(0xabb148, 0x7f59fd69b578, 0xc208570fd0, 0xc2089a1450)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208570fd0, 0xc2089a1450)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2083deba0, 0x7f59fd69b578, 0xc208570fd0, 0xc2089a1450)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208503f40)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 16 [select, 5502 minutes]:
net/http.(*persistConn).writeLoop(0xc2080a9810)
/Users/crawshaw/go/src/net/http/transport.go:1002 +0x40f
created by net/http.(*Transport).dialConn
/Users/crawshaw/go/src/net/http/transport.go:685 +0xccf
goroutine 14 [sleep]:
time.Sleep(0x12a05f200)
/Users/crawshaw/go/src/runtime/time.go:59 +0xfc
net.loadConfig.func1(0x12a05f200, 0x0, 0x9d90d0, 0x10, 0xc20803f260)
/Users/crawshaw/go/src/net/dnsclient_unix.go:241 +0x36
created by net.loadConfig
/Users/crawshaw/go/src/net/dnsclient_unix.go:270 +0x12c
goroutine 31 [sleep]:
time.Sleep(0xdf8475800)
/Users/crawshaw/go/src/runtime/time.go:59 +0xfc
main.cleanUpOldContainers()
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/watcher.go:195 +0x2a5
created by main.startWatchers
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/watcher.go:48 +0x30a
goroutine 546474 [semacquire, 341 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2092614a0, 0xc208cf88f0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2092614a0, 0xc208cf88f0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2092614a0, 0xc208cf88f0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2092614a0, 0xc208cf88f0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2090cdf40)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 547087 [semacquire, 324 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2098ea580, 0xc2098741a0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2098ea580, 0xc2098741a0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2098ea580, 0xc2098741a0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2098ea580, 0xc2098741a0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208563540)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 547444 [semacquire, 312 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208e92160, 0xc208a72b60)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208e92160, 0xc208a72b60)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208e92160, 0xc208a72b60)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208e92160, 0xc208a72b60)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc20906ed20)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 545531 [semacquire, 364 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208945340, 0xc208c03930)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208945340, 0xc208c03930)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208945340, 0xc208c03930)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208945340, 0xc208c03930)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208bacb40)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 547333 [semacquire, 318 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2099f4dc0, 0xc2099df520)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2099f4dc0, 0xc2099df520)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2099f4dc0, 0xc2099df520)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2099f4dc0, 0xc2099df520)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc20995c3c0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 543378 [semacquire, 387 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc209039550, 0xc208a04b60)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc209039550, 0xc208a04b60)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc209039550, 0xc208a04b60)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc209039550, 0xc208a04b60)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208782f00)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 540287 [semacquire, 490 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc20960f290, 0xc2095ea1a0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc20960f290, 0xc2095ea1a0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20960f290, 0xc2095ea1a0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc20960f290, 0xc2095ea1a0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2085625a0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 542215 [semacquire, 426 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc20960f810, 0xc20960dd40)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc20960f810, 0xc20960dd40)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20960f810, 0xc20960dd40)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc20960f810, 0xc20960dd40)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2095d1900)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 58 [chan receive]:
main.findWorkLoop(0xc2083dea20)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:521 +0x132
created by main.main
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:291 +0xd35
goroutine 59 [chan receive]:
main.findTryWorkLoop()
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:599 +0x137
created by main.main
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:292 +0xd4e
goroutine 60 [sleep]:
time.Sleep(0x3b9aca00)
/Users/crawshaw/go/src/runtime/time.go:59 +0xfc
net/http.(*Server).Serve(0xc2084517a0, 0x7f59fd69cd48, 0xc2083a1b20, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/server.go:1792 +0x2d4
main.serveTLS(0x7f59fd6963e0, 0xc20801e3d0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:221 +0x8bc
main.listenAndServeTLS()
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:194 +0x282
created by main.main
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:296 +0x4ea
goroutine 546317 [semacquire, 348 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.(*reverseBuildletPool).WriteHTMLStatus(0xc470a0, 0x7f59fd68b6e8, 0xc209414a80)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:191 +0x16a
main.handleStatus(0x7f59fd69b578, 0xc2091b2e70, 0xc20804a410)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/status.go:60 +0xf61
net/http.HandlerFunc.ServeHTTP(0xabb148, 0x7f59fd69b578, 0xc2091b2e70, 0xc20804a410)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2091b2e70, 0xc20804a410)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2083deba0, 0x7f59fd69b578, 0xc2091b2e70, 0xc20804a410)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208565f40)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 540228 [IO wait, 493 minutes]:
net.runtime_pollWait(0x7f59fd6bad18, 0x72, 0xc208b60c00)
/Users/crawshaw/go/src/runtime/netpoll.go:157 +0x63
net.(*pollDesc).Wait(0xc2084d48b0, 0x72, 0x0, 0x0)
/Users/crawshaw/go/src/net/fd_poll_runtime.go:73 +0x41
net.(*pollDesc).WaitRead(0xc2084d48b0, 0x0, 0x0)
/Users/crawshaw/go/src/net/fd_poll_runtime.go:78 +0x3d
net.(*netFD).Read(0xc2084d4850, 0xc208b60c00, 0x400, 0x400, 0x0, 0x7f59fd687050, 0xc20800a150)
/Users/crawshaw/go/src/net/fd_unix.go:232 +0x22d
net.(*conn).Read(0xc2080c0180, 0xc208b60c00, 0x400, 0x400, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/net.go:124 +0xe7
crypto/tls.(*block).readFromUntil(0xc20957fdd0, 0x7f59fd68cf30, 0xc2080c0180, 0x5, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:455 +0xda
crypto/tls.(*Conn).readRecord(0xc2087e62c0, 0xabb817, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:540 +0x2d2
crypto/tls.(*Conn).Read(0xc2087e62c0, 0xc2084c3000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:901 +0x16b
net/http.(*liveSwitchReader).Read(0xc208502868, 0xc2084c3000, 0x1000, 0x1000, 0x400, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/server.go:212 +0xa7
io.(*LimitedReader).Read(0xc209668f80, 0xc2084c3000, 0x1000, 0x1000, 0xc2080c0180, 0x0, 0x0)
/Users/crawshaw/go/src/io/io.go:427 +0xca
bufio.(*Reader).fill(0xc2088faba0)
/Users/crawshaw/go/src/bufio/bufio.go:97 +0x1d6
bufio.(*Reader).ReadSlice(0xc2088faba0, 0xc20000000a, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/bufio/bufio.go:328 +0x246
bufio.(*Reader).ReadLine(0xc2088faba0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/bufio/bufio.go:357 +0x56
net/textproto.(*Reader).readLineSlice(0xc2092c7050, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/textproto/reader.go:55 +0x84
net/textproto.(*Reader).ReadLine(0xc2092c7050, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/textproto/reader.go:36 +0x43
net/http.ReadResponse(0xc2088faba0, 0xc20821aa90, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/response.go:126 +0xf8
main.(*reverseRoundTripper).RoundTrip(0xc2084a8b20, 0xc20821aa90, 0x0, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:370 +0x152
net/http.send(0xc20821aa90, 0x7f59fd6a1bc8, 0xc2084a8b20, 0x12, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:220 +0x4e1
net/http.(*Client).send(0xc2089b6420, 0xc20821aa90, 0x12, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:143 +0x15d
net/http.(*Client).doFollowingRedirects(0xc2089b6420, 0xc20821aa90, 0xabb3b0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:380 +0xbba
net/http.(*Client).Do(0xc2089b6420, 0xc20821aa90, 0xc20855cd38, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:175 +0x9f
golang.org/x/build/buildlet.(*Client).do(0xc208dd34a0, 0xc20821aa90, 0xc208324c60, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/buildlet/buildletclient.go:112 +0x8b
golang.org/x/build/buildlet.(*Client).Status(0xc208dd34a0, 0x0, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/buildlet/buildletclient.go:364 +0x454
main.(*reverseBuildletPool).reverseHealthCheck.func1(0xc208dd34a0, 0xc2095c45a0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:122 +0x28
created by main.(*reverseBuildletPool).reverseHealthCheck
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:124 +0xa0a
goroutine 543517 [semacquire, 382 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2086efef0, 0xc20813fe10)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2086efef0, 0xc20813fe10)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2086efef0, 0xc20813fe10)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2086efef0, 0xc20813fe10)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208bad4a0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 542304 [semacquire, 424 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc209317ef0, 0xc2089cd520)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc209317ef0, 0xc2089cd520)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc209317ef0, 0xc2089cd520)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc209317ef0, 0xc2089cd520)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208ce5cc0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 546376 [semacquire, 345 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc20946e4d0, 0xc208cf8000)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc20946e4d0, 0xc208cf8000)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20946e4d0, 0xc208cf8000)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc20946e4d0, 0xc208cf8000)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2089c1360)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 546986 [semacquire, 326 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2087982c0, 0xc209355ba0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2087982c0, 0xc209355ba0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2087982c0, 0xc209355ba0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2087982c0, 0xc209355ba0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208490960)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 544875 [semacquire, 369 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc20919c2c0, 0xc208426340)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc20919c2c0, 0xc208426340)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20919c2c0, 0xc208426340)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc20919c2c0, 0xc208426340)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208860000)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 546575 [semacquire, 339 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc20870ebb0, 0xc2088755f0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc20870ebb0, 0xc2088755f0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20870ebb0, 0xc2088755f0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc20870ebb0, 0xc2088755f0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208d19b80)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 542257 [semacquire, 425 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2086ada20, 0xc208c02b60)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2086ada20, 0xc208c02b60)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2086ada20, 0xc208c02b60)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2086ada20, 0xc208c02b60)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208782460)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 545843 [semacquire, 358 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208ea5550, 0xc208c03ee0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208ea5550, 0xc208c03ee0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208ea5550, 0xc208c03ee0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208ea5550, 0xc208c03ee0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208bad5e0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 541769 [semacquire, 442 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208581080, 0xc2089cc820)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208581080, 0xc2089cc820)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208581080, 0xc2089cc820)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208581080, 0xc2089cc820)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208ce4dc0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 546128 [semacquire, 350 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2091b2840, 0xc2091a21a0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2091b2840, 0xc2091a21a0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2091b2840, 0xc2091a21a0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2091b2840, 0xc2091a21a0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208db1680)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 544553 [semacquire, 372 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc20896ab00, 0xc208aefee0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc20896ab00, 0xc208aefee0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20896ab00, 0xc208aefee0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc20896ab00, 0xc208aefee0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208ce55e0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 543028 [semacquire, 399 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc209303c30, 0xc2088b6680)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc209303c30, 0xc2088b6680)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc209303c30, 0xc2088b6680)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc209303c30, 0xc2088b6680)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc20836e960)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 541066 [semacquire, 465 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2084aa630, 0xc20960cf70)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2084aa630, 0xc20960cf70)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2084aa630, 0xc20960cf70)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2084aa630, 0xc20960cf70)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208566460)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 543423 [semacquire, 385 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208798160, 0xc2092f08f0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208798160, 0xc2092f08f0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208798160, 0xc2092f08f0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208798160, 0xc2092f08f0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208861b80)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 542576 [semacquire, 415 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc209590f20, 0xc208937110)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc209590f20, 0xc208937110)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc209590f20, 0xc208937110)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc209590f20, 0xc208937110)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2082583c0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 540384 [semacquire, 487 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2082f9290, 0xc2089a1c70)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2082f9290, 0xc2089a1c70)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2082f9290, 0xc2089a1c70)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2082f9290, 0xc2089a1c70)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc20836e280)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 541429 [semacquire, 453 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208542fd0, 0xc2089a16c0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208542fd0, 0xc2089a16c0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208542fd0, 0xc2089a16c0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208542fd0, 0xc2089a16c0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208503040)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 543780 [semacquire, 374 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc20973d340, 0xc2092f1ee0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc20973d340, 0xc2092f1ee0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20973d340, 0xc2092f1ee0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc20973d340, 0xc2092f1ee0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc20937d900)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 545844 [semacquire, 358 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208ea5970, 0xc2085e2340)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208ea5970, 0xc2085e2340)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208ea5970, 0xc2085e2340)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208ea5970, 0xc2085e2340)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2095d1680)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 545633 [semacquire, 361 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2085da580, 0xc208cd5860)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2085da580, 0xc208cd5860)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2085da580, 0xc208cd5860)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2085da580, 0xc208cd5860)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2093e4000)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 543643 [semacquire, 378 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc20931b4a0, 0xc2089a1a00)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc20931b4a0, 0xc2089a1a00)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20931b4a0, 0xc2089a1a00)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc20931b4a0, 0xc2089a1a00)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc209008000)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 541330 [semacquire, 457 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2081751e0, 0xc208a731e0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2081751e0, 0xc208a731e0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2081751e0, 0xc208a731e0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2081751e0, 0xc208a731e0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2087832c0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 541998 [semacquire, 434 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2080a9ad0, 0xc2095181a0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2080a9ad0, 0xc2095181a0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2080a9ad0, 0xc2095181a0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2080a9ad0, 0xc2095181a0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2085620a0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 543468 [semacquire, 384 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc209678580, 0xc2088bb380)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc209678580, 0xc2088bb380)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc209678580, 0xc2088bb380)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc209678580, 0xc2088bb380)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208ce4be0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 546822 [semacquire, 332 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208c18370, 0xc2093541a0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208c18370, 0xc2093541a0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208c18370, 0xc2093541a0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208c18370, 0xc2093541a0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208491c20)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 543113 [semacquire, 396 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc20878e790, 0xc208c029c0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc20878e790, 0xc208c029c0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20878e790, 0xc208c029c0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc20878e790, 0xc208c029c0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc20906fae0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 540201 [IO wait, 493 minutes]:
net.runtime_pollWait(0x7f59fd638a00, 0x72, 0xc2081fb400)
/Users/crawshaw/go/src/runtime/netpoll.go:157 +0x63
net.(*pollDesc).Wait(0xc2089d31e0, 0x72, 0x0, 0x0)
/Users/crawshaw/go/src/net/fd_poll_runtime.go:73 +0x41
net.(*pollDesc).WaitRead(0xc2089d31e0, 0x0, 0x0)
/Users/crawshaw/go/src/net/fd_poll_runtime.go:78 +0x3d
net.(*netFD).Read(0xc2089d3180, 0xc2081fb400, 0x400, 0x400, 0x0, 0x7f59fd687050, 0xc20800a150)
/Users/crawshaw/go/src/net/fd_unix.go:232 +0x22d
net.(*conn).Read(0xc2080c0328, 0xc2081fb400, 0x400, 0x400, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/net.go:124 +0xe7
crypto/tls.(*block).readFromUntil(0xc2092c6a80, 0x7f59fd68cf30, 0xc2080c0328, 0x5, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:455 +0xda
crypto/tls.(*Conn).readRecord(0xc208126840, 0xabb817, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:540 +0x2d2
crypto/tls.(*Conn).Read(0xc208126840, 0xc208226000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:901 +0x16b
net/http.(*liveSwitchReader).Read(0xc208566f48, 0xc208226000, 0x1000, 0x1000, 0x400, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/server.go:212 +0xa7
io.(*LimitedReader).Read(0xc20837f140, 0xc208226000, 0x1000, 0x1000, 0xc2080c0328, 0x0, 0x0)
/Users/crawshaw/go/src/io/io.go:427 +0xca
bufio.(*Reader).fill(0xc2095c51a0)
/Users/crawshaw/go/src/bufio/bufio.go:97 +0x1d6
bufio.(*Reader).ReadSlice(0xc2095c51a0, 0xc20000000a, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/bufio/bufio.go:328 +0x246
bufio.(*Reader).ReadLine(0xc2095c51a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/bufio/bufio.go:357 +0x56
net/textproto.(*Reader).readLineSlice(0xc2092c6540, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/textproto/reader.go:55 +0x84
net/textproto.(*Reader).ReadLine(0xc2092c6540, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/textproto/reader.go:36 +0x43
net/http.ReadResponse(0xc2095c51a0, 0xc208426f70, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/response.go:126 +0xf8
main.(*reverseRoundTripper).RoundTrip(0xc2084e83d0, 0xc208426f70, 0x0, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:370 +0x152
net/http.send(0xc208426f70, 0x7f59fd6a1bc8, 0xc2084e83d0, 0x12, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:220 +0x4e1
net/http.(*Client).send(0xc208aa0d50, 0xc208426f70, 0x12, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:143 +0x15d
net/http.(*Client).doFollowingRedirects(0xc208aa0d50, 0xc208426f70, 0xabb3b0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:380 +0xbba
net/http.(*Client).Do(0xc208aa0d50, 0xc208426f70, 0xc20873bd38, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:175 +0x9f
golang.org/x/build/buildlet.(*Client).do(0xc2087b4ae0, 0xc208426f70, 0xc208324340, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/buildlet/buildletclient.go:112 +0x8b
golang.org/x/build/buildlet.(*Client).Status(0xc2087b4ae0, 0x0, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/buildlet/buildletclient.go:364 +0x454
main.(*reverseBuildletPool).reverseHealthCheck.func1(0xc2087b4ae0, 0xc208dd39e0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:122 +0x28
created by main.(*reverseBuildletPool).reverseHealthCheck
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:124 +0xa0a
goroutine 546952 [semacquire, 328 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.(*reverseBuildletPool).WriteHTMLStatus(0xc470a0, 0x7f59fd68b6e8, 0xc2095993b0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:191 +0x16a
main.handleStatus(0x7f59fd69b578, 0xc208cfef20, 0xc20982c270)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/status.go:60 +0xf61
net/http.HandlerFunc.ServeHTTP(0xabb148, 0x7f59fd69b578, 0xc208cfef20, 0xc20982c270)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208cfef20, 0xc20982c270)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2083deba0, 0x7f59fd69b578, 0xc208cfef20, 0xc20982c270)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2094bcb40)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 544891 [semacquire, 368 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.(*reverseBuildletPool).WriteHTMLStatus(0xc470a0, 0x7f59fd68b6e8, 0xc2089d28c0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:191 +0x16a
main.handleStatus(0x7f59fd69b578, 0xc2092b5810, 0xc2088b76c0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/status.go:60 +0xf61
net/http.HandlerFunc.ServeHTTP(0xabb148, 0x7f59fd69b578, 0xc2092b5810, 0xc2088b76c0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2092b5810, 0xc2088b76c0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2083deba0, 0x7f59fd69b578, 0xc2092b5810, 0xc2088b76c0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208ce5860)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 542079 [semacquire, 430 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2095c9130, 0xc2088bc000)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2095c9130, 0xc2088bc000)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2095c9130, 0xc2088bc000)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2095c9130, 0xc2088bc000)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2085037c0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 546175 [semacquire, 350 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208b01ce0, 0xc2092f1380)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208b01ce0, 0xc2092f1380)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208b01ce0, 0xc2092f1380)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208b01ce0, 0xc2092f1380)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208badf40)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 541914 [semacquire, 438 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc209038e70, 0xc2092f0270)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc209038e70, 0xc2092f0270)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc209038e70, 0xc2092f0270)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc209038e70, 0xc2092f0270)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2095d1180)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 547724 [semacquire, 304 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208ad1290, 0xc208aafe10)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208ad1290, 0xc208aafe10)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208ad1290, 0xc208aafe10)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208ad1290, 0xc208aafe10)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc209721a40)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 541868 [semacquire, 438 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2086eed10, 0xc209651ee0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2086eed10, 0xc209651ee0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2086eed10, 0xc209651ee0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2086eed10, 0xc209651ee0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2082a6a00)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 547040 [semacquire, 324 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc209678d10, 0xc2088bcea0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc209678d10, 0xc2088bcea0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc209678d10, 0xc2088bcea0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc209678d10, 0xc2088bcea0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc20842edc0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 543603 [semacquire, 379 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc209344dc0, 0xc208c03040)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc209344dc0, 0xc208c03040)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc209344dc0, 0xc208c03040)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc209344dc0, 0xc208c03040)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208bac8c0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 542575 [semacquire, 415 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2086dd760, 0xc20814d6c0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2086dd760, 0xc20814d6c0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2086dd760, 0xc20814d6c0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2086dd760, 0xc20814d6c0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2092bbe00)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 540423 [semacquire, 486 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208173ce0, 0xc208cd4340)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208173ce0, 0xc208cd4340)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208173ce0, 0xc208cd4340)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208173ce0, 0xc208cd4340)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc20898efa0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 546346 [semacquire, 346 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.(*reverseBuildletPool).WriteHTMLStatus(0xc470a0, 0x7f59fd68b6e8, 0xc20866ddc0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:191 +0x16a
main.handleStatus(0x7f59fd69b578, 0xc2089fb600, 0xc208c11ad0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/status.go:60 +0xf61
net/http.HandlerFunc.ServeHTTP(0xabb148, 0x7f59fd69b578, 0xc2089fb600, 0xc208c11ad0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2089fb600, 0xc208c11ad0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2083deba0, 0x7f59fd69b578, 0xc2089fb600, 0xc208c11ad0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc209720e60)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 542403 [semacquire, 420 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208700420, 0xc208a04dd0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208700420, 0xc208a04dd0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208700420, 0xc208a04dd0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208700420, 0xc208a04dd0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2095d0460)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 540211 [IO wait, 493 minutes]:
net.runtime_pollWait(0x7f59fd6a81a0, 0x72, 0xc208b9f800)
/Users/crawshaw/go/src/runtime/netpoll.go:157 +0x63
net.(*pollDesc).Wait(0xc20899c1b0, 0x72, 0x0, 0x0)
/Users/crawshaw/go/src/net/fd_poll_runtime.go:73 +0x41
net.(*pollDesc).WaitRead(0xc20899c1b0, 0x0, 0x0)
/Users/crawshaw/go/src/net/fd_poll_runtime.go:78 +0x3d
net.(*netFD).Read(0xc20899c150, 0xc208b9f800, 0x400, 0x400, 0x0, 0x7f59fd687050, 0xc20800a150)
/Users/crawshaw/go/src/net/fd_unix.go:232 +0x22d
net.(*conn).Read(0xc2080c0058, 0xc208b9f800, 0x400, 0x400, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/net.go:124 +0xe7
crypto/tls.(*block).readFromUntil(0xc2090a6e40, 0x7f59fd68cf30, 0xc2080c0058, 0x5, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:455 +0xda
crypto/tls.(*Conn).readRecord(0xc2088a3b80, 0xabb817, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:540 +0x2d2
crypto/tls.(*Conn).Read(0xc2088a3b80, 0xc208b1a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:901 +0x16b
net/http.(*liveSwitchReader).Read(0xc208ce40e8, 0xc208b1a000, 0x1000, 0x1000, 0x400, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/server.go:212 +0xa7
io.(*LimitedReader).Read(0xc209632060, 0xc208b1a000, 0x1000, 0x1000, 0xc2080c0058, 0x0, 0x0)
/Users/crawshaw/go/src/io/io.go:427 +0xca
bufio.(*Reader).fill(0xc2087b4120)
/Users/crawshaw/go/src/bufio/bufio.go:97 +0x1d6
bufio.(*Reader).ReadSlice(0xc2087b4120, 0xc20000000a, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/bufio/bufio.go:328 +0x246
bufio.(*Reader).ReadLine(0xc2087b4120, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/bufio/bufio.go:357 +0x56
net/textproto.(*Reader).readLineSlice(0xc2092c6900, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/textproto/reader.go:55 +0x84
net/textproto.(*Reader).ReadLine(0xc2092c6900, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/textproto/reader.go:36 +0x43
net/http.ReadResponse(0xc2087b4120, 0xc208427a00, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/response.go:126 +0xf8
main.(*reverseRoundTripper).RoundTrip(0xc2088f4e20, 0xc208427a00, 0x0, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:370 +0x152
net/http.send(0xc208427a00, 0x7f59fd6a1bc8, 0xc2088f4e20, 0x12, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:220 +0x4e1
net/http.(*Client).send(0xc2090a7560, 0xc208427a00, 0x12, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:143 +0x15d
net/http.(*Client).doFollowingRedirects(0xc2090a7560, 0xc208427a00, 0xabb3b0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:380 +0xbba
net/http.(*Client).Do(0xc2090a7560, 0xc208427a00, 0xc20873e538, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:175 +0x9f
golang.org/x/build/buildlet.(*Client).do(0xc2085bee40, 0xc208427a00, 0xc208324700, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/buildlet/buildletclient.go:112 +0x8b
golang.org/x/build/buildlet.(*Client).Status(0xc2085bee40, 0x0, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/buildlet/buildletclient.go:364 +0x454
main.(*reverseBuildletPool).reverseHealthCheck.func1(0xc2085bee40, 0xc208dd3ec0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:122 +0x28
created by main.(*reverseBuildletPool).reverseHealthCheck
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:124 +0xa0a
goroutine 540220 [IO wait, 493 minutes]:
net.runtime_pollWait(0x7f59fd65c948, 0x72, 0xc20849e400)
/Users/crawshaw/go/src/runtime/netpoll.go:157 +0x63
net.(*pollDesc).Wait(0xc2089d3720, 0x72, 0x0, 0x0)
/Users/crawshaw/go/src/net/fd_poll_runtime.go:73 +0x41
net.(*pollDesc).WaitRead(0xc2089d3720, 0x0, 0x0)
/Users/crawshaw/go/src/net/fd_poll_runtime.go:78 +0x3d
net.(*netFD).Read(0xc2089d36c0, 0xc20849e400, 0x400, 0x400, 0x0, 0x7f59fd687050, 0xc20800a150)
/Users/crawshaw/go/src/net/fd_unix.go:232 +0x22d
net.(*conn).Read(0xc2080c0170, 0xc20849e400, 0x400, 0x400, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/net.go:124 +0xe7
crypto/tls.(*block).readFromUntil(0xc20918a7e0, 0x7f59fd68cf30, 0xc2080c0170, 0x5, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:455 +0xda
crypto/tls.(*Conn).readRecord(0xc208c3a2c0, 0xabb817, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:540 +0x2d2
crypto/tls.(*Conn).Read(0xc208c3a2c0, 0xc208745000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:901 +0x16b
net/http.(*liveSwitchReader).Read(0xc20842f628, 0xc208745000, 0x1000, 0x1000, 0x400, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/server.go:212 +0xa7
io.(*LimitedReader).Read(0xc208468d60, 0xc208745000, 0x1000, 0x1000, 0xc2080c0170, 0x0, 0x0)
/Users/crawshaw/go/src/io/io.go:427 +0xca
bufio.(*Reader).fill(0xc208786240)
/Users/crawshaw/go/src/bufio/bufio.go:97 +0x1d6
bufio.(*Reader).ReadSlice(0xc208786240, 0xc20000000a, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/bufio/bufio.go:328 +0x246
bufio.(*Reader).ReadLine(0xc208786240, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/bufio/bufio.go:357 +0x56
net/textproto.(*Reader).readLineSlice(0xc2092c6d50, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/textproto/reader.go:55 +0x84
net/textproto.(*Reader).ReadLine(0xc2092c6d50, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/textproto/reader.go:36 +0x43
net/http.ReadResponse(0xc208786240, 0xc20821a270, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/response.go:126 +0xf8
main.(*reverseRoundTripper).RoundTrip(0xc208bd8e50, 0xc20821a270, 0x0, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:370 +0x152
net/http.send(0xc20821a270, 0x7f59fd6a1bc8, 0xc208bd8e50, 0x12, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:220 +0x4e1
net/http.(*Client).send(0xc20957e750, 0xc20821a270, 0x12, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:143 +0x15d
net/http.(*Client).doFollowingRedirects(0xc20957e750, 0xc20821a270, 0xabb3b0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:380 +0xbba
net/http.(*Client).Do(0xc20957e750, 0xc20821a270, 0xc208018d38, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:175 +0x9f
golang.org/x/build/buildlet.(*Client).do(0xc208a2c060, 0xc20821a270, 0xc208324980, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/buildlet/buildletclient.go:112 +0x8b
golang.org/x/build/buildlet.(*Client).Status(0xc208a2c060, 0x0, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/buildlet/buildletclient.go:364 +0x454
main.(*reverseBuildletPool).reverseHealthCheck.func1(0xc208a2c060, 0xc2095c42a0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:122 +0x28
created by main.(*reverseBuildletPool).reverseHealthCheck
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:124 +0xa0a
goroutine 546271 [semacquire, 348 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208ad2210, 0xc20813e5b0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208ad2210, 0xc20813e5b0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208ad2210, 0xc20813e5b0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208ad2210, 0xc20813e5b0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2090cde00)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 542574 [semacquire, 415 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2085dad10, 0xc2085e32b0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2085dad10, 0xc2085e32b0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2085dad10, 0xc2085e32b0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2085dad10, 0xc2085e32b0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2095d1c20)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 544874 [semacquire, 369 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc209261e40, 0xc208a72dd0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc209261e40, 0xc208a72dd0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc209261e40, 0xc208a72dd0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc209261e40, 0xc208a72dd0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2092ba0a0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 540207 [IO wait, 493 minutes]:
net.runtime_pollWait(0x7f59fd65cd08, 0x72, 0xc208714400)
/Users/crawshaw/go/src/runtime/netpoll.go:157 +0x63
net.(*pollDesc).Wait(0xc208b3eb50, 0x72, 0x0, 0x0)
/Users/crawshaw/go/src/net/fd_poll_runtime.go:73 +0x41
net.(*pollDesc).WaitRead(0xc208b3eb50, 0x0, 0x0)
/Users/crawshaw/go/src/net/fd_poll_runtime.go:78 +0x3d
net.(*netFD).Read(0xc208b3eaf0, 0xc208714400, 0x400, 0x400, 0x0, 0x7f59fd687050, 0xc20800a150)
/Users/crawshaw/go/src/net/fd_unix.go:232 +0x22d
net.(*conn).Read(0xc2080c0260, 0xc208714400, 0x400, 0x400, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/net.go:124 +0xe7
crypto/tls.(*block).readFromUntil(0xc2087e4c60, 0x7f59fd68cf30, 0xc2080c0260, 0x5, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:455 +0xda
crypto/tls.(*Conn).readRecord(0xc2083dab00, 0xabb817, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:540 +0x2d2
crypto/tls.(*Conn).Read(0xc2083dab00, 0xc2080e5000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:901 +0x16b
net/http.(*liveSwitchReader).Read(0xc208ce5588, 0xc2080e5000, 0x1000, 0x1000, 0x400, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/server.go:212 +0xa7
io.(*LimitedReader).Read(0xc2083f0420, 0xc2080e5000, 0x1000, 0x1000, 0xc2080c0260, 0x0, 0x0)
/Users/crawshaw/go/src/io/io.go:427 +0xca
bufio.(*Reader).fill(0xc2085be720)
/Users/crawshaw/go/src/bufio/bufio.go:97 +0x1d6
bufio.(*Reader).ReadSlice(0xc2085be720, 0xc20000000a, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/bufio/bufio.go:328 +0x246
bufio.(*Reader).ReadLine(0xc2085be720, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/bufio/bufio.go:357 +0x56
net/textproto.(*Reader).readLineSlice(0xc2092c6780, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/textproto/reader.go:55 +0x84
net/textproto.(*Reader).ReadLine(0xc2092c6780, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/textproto/reader.go:36 +0x43
net/http.ReadResponse(0xc2085be720, 0xc2084276c0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/response.go:126 +0xf8
main.(*reverseRoundTripper).RoundTrip(0xc208c94230, 0xc2084276c0, 0x0, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:370 +0x152
net/http.send(0xc2084276c0, 0x7f59fd6a1bc8, 0xc208c94230, 0x12, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:220 +0x4e1
net/http.(*Client).send(0xc208bde150, 0xc2084276c0, 0x12, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:143 +0x15d
net/http.(*Client).doFollowingRedirects(0xc208bde150, 0xc2084276c0, 0xabb3b0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:380 +0xbba
net/http.(*Client).Do(0xc208bde150, 0xc2084276c0, 0xc208738538, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:175 +0x9f
golang.org/x/build/buildlet.(*Client).do(0xc2085502a0, 0xc2084276c0, 0xc208324540, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/buildlet/buildletclient.go:112 +0x8b
golang.org/x/build/buildlet.(*Client).Status(0xc2085502a0, 0x0, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/buildlet/buildletclient.go:364 +0x454
main.(*reverseBuildletPool).reverseHealthCheck.func1(0xc2085502a0, 0xc208dd3ce0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:122 +0x28
created by main.(*reverseBuildletPool).reverseHealthCheck
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:124 +0xa0a
goroutine 543779 [semacquire, 374 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc20973cf20, 0xc208937930)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc20973cf20, 0xc208937930)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20973cf20, 0xc208937930)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc20973cf20, 0xc208937930)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc20813be00)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 541206 [semacquire, 461 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208700a50, 0xc208a73ad0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208700a50, 0xc208a73ad0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208700a50, 0xc208a73ad0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208700a50, 0xc208a73ad0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208783680)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 540744 [semacquire, 475 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208a334a0, 0xc2092f0f70)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208a334a0, 0xc2092f0f70)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208a334a0, 0xc2092f0f70)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208a334a0, 0xc2092f0f70)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2082a6500)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 546783 [semacquire, 332 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc20973ca50, 0xc20814cea0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc20973ca50, 0xc20814cea0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20973ca50, 0xc20814cea0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc20973ca50, 0xc20814cea0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc20898f540)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 547628 [semacquire, 306 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc209b12840, 0xc209ac81a0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc209b12840, 0xc209ac81a0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc209b12840, 0xc209ac81a0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc209b12840, 0xc209ac81a0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc209b45ea0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 547942 [semacquire, 298 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.(*reverseBuildletPool).WriteHTMLStatus(0xc470a0, 0x7f59fd68b6e8, 0xc2084b8af0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:191 +0x16a
main.handleStatus(0x7f59fd69b578, 0xc208859970, 0xc208160b60)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/status.go:60 +0xf61
net/http.HandlerFunc.ServeHTTP(0xabb148, 0x7f59fd69b578, 0xc208859970, 0xc208160b60)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208859970, 0xc208160b60)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2083deba0, 0x7f59fd69b578, 0xc208859970, 0xc208160b60)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208db0320)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 542817 [semacquire, 407 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208aa91e0, 0xc2080d7c70)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208aa91e0, 0xc2080d7c70)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208aa91e0, 0xc2080d7c70)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208aa91e0, 0xc2080d7c70)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2092bb4a0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 546070 [semacquire, 354 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc209391a20, 0xc2089a0750)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc209391a20, 0xc2089a0750)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc209391a20, 0xc2089a0750)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc209391a20, 0xc2089a0750)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc20836e640)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 542857 [semacquire, 406 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208608c60, 0xc208c116c0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208608c60, 0xc208c116c0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208608c60, 0xc208c116c0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208608c60, 0xc208c116c0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208b50b40)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 547088 [semacquire, 324 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2098ea630, 0xc209874270)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2098ea630, 0xc209874270)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2098ea630, 0xc209874270)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2098ea630, 0xc209874270)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2094bc6e0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 540224 [IO wait, 493 minutes]:
net.runtime_pollWait(0x7f59fd65ba48, 0x72, 0xc208b26400)
/Users/crawshaw/go/src/runtime/netpoll.go:157 +0x63
net.(*pollDesc).Wait(0xc2080ffaa0, 0x72, 0x0, 0x0)
/Users/crawshaw/go/src/net/fd_poll_runtime.go:73 +0x41
net.(*pollDesc).WaitRead(0xc2080ffaa0, 0x0, 0x0)
/Users/crawshaw/go/src/net/fd_poll_runtime.go:78 +0x3d
net.(*netFD).Read(0xc2080ffa40, 0xc208b26400, 0x400, 0x400, 0x0, 0x7f59fd687050, 0xc20800a150)
/Users/crawshaw/go/src/net/fd_unix.go:232 +0x22d
net.(*conn).Read(0xc2080c0240, 0xc208b26400, 0x400, 0x400, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/net.go:124 +0xe7
crypto/tls.(*block).readFromUntil(0xc2089b6e40, 0x7f59fd68cf30, 0xc2080c0240, 0x5, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:455 +0xda
crypto/tls.(*Conn).readRecord(0xc2084b6580, 0xabb817, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:540 +0x2d2
crypto/tls.(*Conn).Read(0xc2084b6580, 0xc208b28000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:901 +0x16b
net/http.(*liveSwitchReader).Read(0xc2081e3088, 0xc208b28000, 0x1000, 0x1000, 0x400, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/server.go:212 +0xa7
io.(*LimitedReader).Read(0xc208397300, 0xc208b28000, 0x1000, 0x1000, 0xc2080c0240, 0x0, 0x0)
/Users/crawshaw/go/src/io/io.go:427 +0xca
bufio.(*Reader).fill(0xc20863a8a0)
/Users/crawshaw/go/src/bufio/bufio.go:97 +0x1d6
bufio.(*Reader).ReadSlice(0xc20863a8a0, 0xc20000000a, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/bufio/bufio.go:328 +0x246
bufio.(*Reader).ReadLine(0xc20863a8a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/bufio/bufio.go:357 +0x56
net/textproto.(*Reader).readLineSlice(0xc2092c6ed0, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/textproto/reader.go:55 +0x84
net/textproto.(*Reader).ReadLine(0xc2092c6ed0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/textproto/reader.go:36 +0x43
net/http.ReadResponse(0xc20863a8a0, 0xc20821a680, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/response.go:126 +0xf8
main.(*reverseRoundTripper).RoundTrip(0xc2088f4610, 0xc20821a680, 0x0, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:370 +0x152
net/http.send(0xc20821a680, 0x7f59fd6a1bc8, 0xc2088f4610, 0x12, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:220 +0x4e1
net/http.(*Client).send(0xc208aa1620, 0xc20821a680, 0x12, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:143 +0x15d
net/http.(*Client).doFollowingRedirects(0xc208aa1620, 0xc20821a680, 0xabb3b0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:380 +0xbba
net/http.(*Client).Do(0xc208aa1620, 0xc20821a680, 0xc208016d38, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:175 +0x9f
golang.org/x/build/buildlet.(*Client).do(0xc2087b4d20, 0xc20821a680, 0xc208324aa0, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/buildlet/buildletclient.go:112 +0x8b
golang.org/x/build/buildlet.(*Client).Status(0xc2087b4d20, 0x0, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/buildlet/buildletclient.go:364 +0x454
main.(*reverseBuildletPool).reverseHealthCheck.func1(0xc2087b4d20, 0xc2095c4420)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:122 +0x28
created by main.(*reverseBuildletPool).reverseHealthCheck
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:124 +0xa0a
goroutine 541955 [semacquire, 435 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2095c9600, 0xc2088bd1e0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2095c9600, 0xc2088bd1e0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2095c9600, 0xc2088bd1e0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2095c9600, 0xc2088bd1e0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208503900)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 540782 [semacquire, 475 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2084978c0, 0xc208c02c30)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2084978c0, 0xc208c02c30)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2084978c0, 0xc208c02c30)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2084978c0, 0xc208c02c30)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208ce4500)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 541023 [semacquire, 466 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc20854c580, 0xc2088bd860)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc20854c580, 0xc2088bd860)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20854c580, 0xc2088bd860)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc20854c580, 0xc2088bd860)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2085028c0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 542905 [semacquire, 405 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208c18210, 0xc208c11e10)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208c18210, 0xc208c11e10)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208c18210, 0xc208c11e10)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208c18210, 0xc208c11e10)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208b51040)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 546009 [semacquire, 355 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2096dcf20, 0xc208aae9c0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2096dcf20, 0xc208aae9c0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2096dcf20, 0xc208aae9c0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2096dcf20, 0xc208aae9c0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2093e4320)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 540227 [IO wait, 493 minutes]:
net.runtime_pollWait(0x7f59fd6c3f98, 0x72, 0xc208687000)
/Users/crawshaw/go/src/runtime/netpoll.go:157 +0x63
net.(*pollDesc).Wait(0xc2080ff5d0, 0x72, 0x0, 0x0)
/Users/crawshaw/go/src/net/fd_poll_runtime.go:73 +0x41
net.(*pollDesc).WaitRead(0xc2080ff5d0, 0x0, 0x0)
/Users/crawshaw/go/src/net/fd_poll_runtime.go:78 +0x3d
net.(*netFD).Read(0xc2080ff570, 0xc208687000, 0x400, 0x400, 0x0, 0x7f59fd687050, 0xc20800a150)
/Users/crawshaw/go/src/net/fd_unix.go:232 +0x22d
net.(*conn).Read(0xc2080c01a0, 0xc208687000, 0x400, 0x400, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/net.go:124 +0xe7
crypto/tls.(*block).readFromUntil(0xc20957fce0, 0x7f59fd68cf30, 0xc2080c01a0, 0x5, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:455 +0xda
crypto/tls.(*Conn).readRecord(0xc2087e6b00, 0xabb817, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:540 +0x2d2
crypto/tls.(*Conn).Read(0xc2087e6b00, 0xc208357000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:901 +0x16b
net/http.(*liveSwitchReader).Read(0xc20874ccc8, 0xc208357000, 0x1000, 0x1000, 0x400, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/server.go:212 +0xa7
io.(*LimitedReader).Read(0xc2083a82e0, 0xc208357000, 0x1000, 0x1000, 0xc2080c01a0, 0x0, 0x0)
/Users/crawshaw/go/src/io/io.go:427 +0xca
bufio.(*Reader).fill(0xc208a2c720)
/Users/crawshaw/go/src/bufio/bufio.go:97 +0x1d6
bufio.(*Reader).ReadSlice(0xc208a2c720, 0xc20000000a, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/bufio/bufio.go:328 +0x246
bufio.(*Reader).ReadLine(0xc208a2c720, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/bufio/bufio.go:357 +0x56
net/textproto.(*Reader).readLineSlice(0xc2092c6ff0, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/textproto/reader.go:55 +0x84
net/textproto.(*Reader).ReadLine(0xc2092c6ff0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/textproto/reader.go:36 +0x43
net/http.ReadResponse(0xc208a2c720, 0xc20821a9c0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/response.go:126 +0xf8
main.(*reverseRoundTripper).RoundTrip(0xc2088f4c40, 0xc20821a9c0, 0x0, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:370 +0x152
net/http.send(0xc20821a9c0, 0x7f59fd6a1bc8, 0xc2088f4c40, 0x12, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:220 +0x4e1
net/http.(*Client).send(0xc208aa1f20, 0xc20821a9c0, 0x12, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:143 +0x15d
net/http.(*Client).doFollowingRedirects(0xc208aa1f20, 0xc20821a9c0, 0xabb3b0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:380 +0xbba
net/http.(*Client).Do(0xc208aa1f20, 0xc20821a9c0, 0xc20855c538, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:175 +0x9f
golang.org/x/build/buildlet.(*Client).do(0xc2087b59e0, 0xc20821a9c0, 0xc208324c20, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/buildlet/buildletclient.go:112 +0x8b
golang.org/x/build/buildlet.(*Client).Status(0xc2087b59e0, 0x0, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/buildlet/buildletclient.go:364 +0x454
main.(*reverseBuildletPool).reverseHealthCheck.func1(0xc2087b59e0, 0xc2095c4540)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:122 +0x28
created by main.(*reverseBuildletPool).reverseHealthCheck
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:124 +0xa0a
goroutine 544872 [semacquire, 369 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208944dc0, 0xc208699a00)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208944dc0, 0xc208699a00)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208944dc0, 0xc208699a00)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208944dc0, 0xc208699a00)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208491400)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 540425 [semacquire, 486 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208700000, 0xc208cd44e0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208700000, 0xc208cd44e0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208700000, 0xc208cd44e0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208700000, 0xc208cd44e0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc20898ebe0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 543518 [semacquire, 382 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc20952b4a0, 0xc208849ba0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc20952b4a0, 0xc208849ba0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20952b4a0, 0xc208849ba0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc20952b4a0, 0xc208849ba0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2085677c0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 540560 [semacquire, 481 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2095f62c0, 0xc20960c340)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2095f62c0, 0xc20960c340)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2095f62c0, 0xc20960c340)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2095f62c0, 0xc20960c340)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208502000)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 546320 [semacquire, 346 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2094e7600, 0xc2094d20d0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2094e7600, 0xc2094d20d0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2094e7600, 0xc2094d20d0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2094e7600, 0xc2094d20d0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208d197c0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 541956 [semacquire, 435 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2085dbb80, 0xc208699c70)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2085dbb80, 0xc208699c70)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2085dbb80, 0xc208699c70)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2085dbb80, 0xc208699c70)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2087834a0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 540478 [semacquire, 484 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208496e70, 0xc2088bc9c0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208496e70, 0xc2088bc9c0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208496e70, 0xc2088bc9c0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208496e70, 0xc2088bc9c0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208566000)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 545343 [semacquire, 365 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208b00420, 0xc2095195f0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208b00420, 0xc2095195f0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208b00420, 0xc2095195f0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208b00420, 0xc2095195f0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc209008640)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 546126 [semacquire, 354 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208b01290, 0xc2080d6dd0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208b01290, 0xc2080d6dd0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208b01290, 0xc2080d6dd0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208b01290, 0xc2080d6dd0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208567680)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 543582 [semacquire, 381 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.(*reverseBuildletPool).WriteHTMLStatus(0xc470a0, 0x7f59fd68b6e8, 0xc2081025b0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:191 +0x16a
main.handleStatus(0x7f59fd69b578, 0xc2080a80b0, 0xc2080d7860)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/status.go:60 +0xf61
net/http.HandlerFunc.ServeHTTP(0xabb148, 0x7f59fd69b578, 0xc2080a80b0, 0xc2080d7860)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2080a80b0, 0xc2080d7860)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2080a80b0, 0xc2080d7860)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2088614a0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 541304 [semacquire, 457 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc20874a9a0, 0xc2080f2270)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc20874a9a0, 0xc2080f2270)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20874a9a0, 0xc2080f2270)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc20874a9a0, 0xc2080f2270)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208491d60)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 543728 [semacquire, 375 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc20952d550, 0xc20929ef70)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc20952d550, 0xc20929ef70)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20952d550, 0xc20929ef70)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc20952d550, 0xc20929ef70)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208ce5360)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 541025 [semacquire, 466 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2085211e0, 0xc20804b040)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2085211e0, 0xc20804b040)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2085211e0, 0xc20804b040)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2085211e0, 0xc20804b040)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc20842fc20)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 528350 [sleep]:
time.Sleep(0xdf8475800)
/Users/crawshaw/go/src/runtime/time.go:59 +0xfc
main.(*buildStatus).pingDashboard(0xc208583500)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/dash.go:143 +0x865
created by main.(*buildStatus).start
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:908 +0x5d
goroutine 542256 [semacquire, 425 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc209316e70, 0xc2088bde10)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc209316e70, 0xc2088bde10)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc209316e70, 0xc2088bde10)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc209316e70, 0xc2088bde10)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208ce4280)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 546821 [semacquire, 332 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc209316580, 0xc2093540d0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc209316580, 0xc2093540d0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc209316580, 0xc2093540d0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc209316580, 0xc2093540d0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc209898aa0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 542775 [semacquire, 409 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208927600, 0xc208c02dd0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208927600, 0xc208c02dd0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208927600, 0xc208c02dd0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208927600, 0xc208c02dd0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2095d12c0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 546222 [semacquire, 348 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2086ef340, 0xc208698f70)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2086ef340, 0xc208698f70)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2086ef340, 0xc208698f70)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2086ef340, 0xc208698f70)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208ada320)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 547184 [semacquire, 320 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.(*reverseBuildletPool).WriteHTMLStatus(0xc470a0, 0x7f59fd68b6e8, 0xc2093660e0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:191 +0x16a
main.handleStatus(0x7f59fd69b578, 0xc20870fb80, 0xc2095ea820)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/status.go:60 +0xf61
net/http.HandlerFunc.ServeHTTP(0xabb148, 0x7f59fd69b578, 0xc20870fb80, 0xc2095ea820)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20870fb80, 0xc2095ea820)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc20870fb80, 0xc2095ea820)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2094ecb40)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 547485 [semacquire, 310 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.(*reverseBuildletPool).WriteHTMLStatus(0xc470a0, 0x7f59fd68b6e8, 0xc209a7b030)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:191 +0x16a
main.handleStatus(0x7f59fd69b578, 0xc209892370, 0xc2098b5790)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/status.go:60 +0xf61
net/http.HandlerFunc.ServeHTTP(0xabb148, 0x7f59fd69b578, 0xc209892370, 0xc2098b5790)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc209892370, 0xc2098b5790)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2083deba0, 0x7f59fd69b578, 0xc209892370, 0xc2098b5790)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc209ae6aa0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 540659 [semacquire, 478 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208175b80, 0xc20814c9c0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208175b80, 0xc20814c9c0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208175b80, 0xc20814c9c0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208175b80, 0xc20814c9c0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208502140)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 540783 [semacquire, 475 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208497a20, 0xc208c02d00)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208497a20, 0xc208c02d00)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208497a20, 0xc208c02d00)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208497a20, 0xc208c02d00)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc20898fb80)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 546377 [semacquire, 345 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc20946ebb0, 0xc208cf8270)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc20946ebb0, 0xc208cf8270)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20946ebb0, 0xc208cf8270)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc20946ebb0, 0xc208cf8270)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208562d20)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 543288 [semacquire, 390 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc209591550, 0xc208aae8f0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc209591550, 0xc208aae8f0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc209591550, 0xc208aae8f0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc209591550, 0xc208aae8f0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc20874cfa0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 542998 [semacquire, 400 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc20878e580, 0xc2085e2270)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc20878e580, 0xc2085e2270)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20878e580, 0xc2085e2270)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc20878e580, 0xc2085e2270)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2089c0aa0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 540329 [semacquire, 489 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc20932de40, 0xc20814d5f0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc20932de40, 0xc20814d5f0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20932de40, 0xc20814d5f0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc20932de40, 0xc20814d5f0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc20813a000)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 545937 [semacquire, 357 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2086242c0, 0xc208699930)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2086242c0, 0xc208699930)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2086242c0, 0xc208699930)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2086242c0, 0xc208699930)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2095d08c0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 540878 [semacquire, 472 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208701760, 0xc208a72000)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208701760, 0xc208a72000)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208701760, 0xc208a72000)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208701760, 0xc208a72000)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208503a40)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 540198 [IO wait, 493 minutes]:
net.runtime_pollWait(0x7f59fd65afc8, 0x72, 0xc208149c00)
/Users/crawshaw/go/src/runtime/netpoll.go:157 +0x63
net.(*pollDesc).Wait(0xc208b3f8e0, 0x72, 0x0, 0x0)
/Users/crawshaw/go/src/net/fd_poll_runtime.go:73 +0x41
net.(*pollDesc).WaitRead(0xc208b3f8e0, 0x0, 0x0)
/Users/crawshaw/go/src/net/fd_poll_runtime.go:78 +0x3d
net.(*netFD).Read(0xc208b3f880, 0xc208149c00, 0x400, 0x400, 0x0, 0x7f59fd687050, 0xc20800a150)
/Users/crawshaw/go/src/net/fd_unix.go:232 +0x22d
net.(*conn).Read(0xc2080c0250, 0xc208149c00, 0x400, 0x400, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/net.go:124 +0xe7
crypto/tls.(*block).readFromUntil(0xc208bdfa40, 0x7f59fd68cf30, 0xc2080c0250, 0x5, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:455 +0xda
crypto/tls.(*Conn).readRecord(0xc208c3b8c0, 0xabb817, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:540 +0x2d2
crypto/tls.(*Conn).Read(0xc208c3b8c0, 0xc208528000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/crypto/tls/conn.go:901 +0x16b
net/http.(*liveSwitchReader).Read(0xc208566548, 0xc208528000, 0x1000, 0x1000, 0x400, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/server.go:212 +0xa7
io.(*LimitedReader).Read(0xc2083b0c20, 0xc208528000, 0x1000, 0x1000, 0xc2080c0250, 0x0, 0x0)
/Users/crawshaw/go/src/io/io.go:427 +0xca
bufio.(*Reader).fill(0xc208dd2840)
/Users/crawshaw/go/src/bufio/bufio.go:97 +0x1d6
bufio.(*Reader).ReadSlice(0xc208dd2840, 0xc20000000a, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/bufio/bufio.go:328 +0x246
bufio.(*Reader).ReadLine(0xc208dd2840, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/bufio/bufio.go:357 +0x56
net/textproto.(*Reader).readLineSlice(0xc2092c6420, 0x0, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/textproto/reader.go:55 +0x84
net/textproto.(*Reader).ReadLine(0xc2092c6420, 0x0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/textproto/reader.go:36 +0x43
net/http.ReadResponse(0xc208dd2840, 0xc208426d00, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/response.go:126 +0xf8
main.(*reverseRoundTripper).RoundTrip(0xc2088f43e0, 0xc208426d00, 0x0, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:370 +0x152
net/http.send(0xc208426d00, 0x7f59fd6a1bc8, 0xc2088f43e0, 0x12, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:220 +0x4e1
net/http.(*Client).send(0xc208aa0a50, 0xc208426d00, 0x12, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:143 +0x15d
net/http.(*Client).doFollowingRedirects(0xc208aa0a50, 0xc208426d00, 0xabb3b0, 0x0, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:380 +0xbba
net/http.(*Client).Do(0xc208aa0a50, 0xc208426d00, 0xc20873cd38, 0x0, 0x0)
/Users/crawshaw/go/src/net/http/client.go:175 +0x9f
golang.org/x/build/buildlet.(*Client).do(0xc2087b49c0, 0xc208426d00, 0xc208324260, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/buildlet/buildletclient.go:112 +0x8b
golang.org/x/build/buildlet.(*Client).Status(0xc2087b49c0, 0x0, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/buildlet/buildletclient.go:364 +0x454
main.(*reverseBuildletPool).reverseHealthCheck.func1(0xc2087b49c0, 0xc208dd3860)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:122 +0x28
created by main.(*reverseBuildletPool).reverseHealthCheck
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:124 +0xa0a
goroutine 543684 [semacquire, 376 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2092b4a50, 0xc2087c0270)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2092b4a50, 0xc2087c0270)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2092b4a50, 0xc2087c0270)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2092b4a50, 0xc2087c0270)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2082a6dc0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 541245 [semacquire, 460 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208542e70, 0xc20813eea0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208542e70, 0xc20813eea0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208542e70, 0xc20813eea0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208542e70, 0xc20813eea0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc20874cbe0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 543112 [semacquire, 396 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208805a20, 0xc20929f040)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208805a20, 0xc20929f040)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208805a20, 0xc20929f040)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208805a20, 0xc20929f040)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2087821e0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 544543 [semacquire, 372 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2092b5290, 0xc20960cb60)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2092b5290, 0xc20960cb60)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2092b5290, 0xc20960cb60)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2092b5290, 0xc20960cb60)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208f368c0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 546943 [semacquire, 328 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.(*reverseBuildletPool).WriteHTMLStatus(0xc470a0, 0x7f59fd68b6e8, 0xc2094d1dc0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:191 +0x16a
main.handleStatus(0x7f59fd69b578, 0xc208cfe9a0, 0xc20804b790)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/status.go:60 +0xf61
net/http.HandlerFunc.ServeHTTP(0xabb148, 0x7f59fd69b578, 0xc208cfe9a0, 0xc20804b790)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208cfe9a0, 0xc20804b790)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2083deba0, 0x7f59fd69b578, 0xc208cfe9a0, 0xc20804b790)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2094bc460)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 542995 [semacquire, 400 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc209303810, 0xc2088b65b0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc209303810, 0xc2088b65b0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc209303810, 0xc2088b65b0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc209303810, 0xc2088b65b0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2092bb180)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 546623 [semacquire, 337 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc209280b00, 0xc2087c00d0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc209280b00, 0xc2087c00d0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc209280b00, 0xc2087c00d0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc209280b00, 0xc2087c00d0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2084900a0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 545761 [semacquire, 360 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc20957a9a0, 0xc208aae750)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc20957a9a0, 0xc208aae750)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20957a9a0, 0xc208aae750)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc20957a9a0, 0xc208aae750)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2093e40a0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 541916 [semacquire, 436 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208521d90, 0xc2088ba9c0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208521d90, 0xc2088ba9c0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208521d90, 0xc2088ba9c0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208521d90, 0xc2088ba9c0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208503680)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 541116 [semacquire, 465 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2081313f0, 0xc20804b450)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2081313f0, 0xc20804b450)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2081313f0, 0xc20804b450)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2081313f0, 0xc20804b450)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2089c0780)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 544681 [semacquire, 370 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208ad00b0, 0xc2089cd450)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208ad00b0, 0xc2089cd450)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208ad00b0, 0xc2089cd450)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208ad00b0, 0xc2089cd450)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208ce54a0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 546521 [semacquire, 339 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc20946f600, 0xc208918340)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc20946f600, 0xc208918340)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20946f600, 0xc208918340)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc20946f600, 0xc208918340)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2094edcc0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 541152 [semacquire, 462 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208609550, 0xc2088bbe10)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208609550, 0xc2088bbe10)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208609550, 0xc2088bbe10)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208609550, 0xc2088bbe10)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208502be0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 546319 [semacquire, 346 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2094e7340, 0xc2094d2000)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2094e7340, 0xc2094d2000)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2094e7340, 0xc2094d2000)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2094e7340, 0xc2094d2000)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc20842e0a0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 541867 [semacquire, 438 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2085da0b0, 0xc2080d60d0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2085da0b0, 0xc2080d60d0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2085da0b0, 0xc2080d60d0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2085da0b0, 0xc2080d60d0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2092bbc20)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 547403 [semacquire, 312 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2098a2d10, 0xc209ac80d0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2098a2d10, 0xc209ac80d0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2098a2d10, 0xc209ac80d0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2098a2d10, 0xc209ac80d0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc208db10e0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 547875 [semacquire, 300 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc2094d6b00, 0xc209651ad0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc2094d6b00, 0xc209651ad0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc2094d6b00, 0xc209651ad0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc2094d6b00, 0xc209651ad0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2089c0960)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 540693 [semacquire, 477 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208926b00, 0xc2089181a0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208926b00, 0xc2089181a0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208926b00, 0xc2089181a0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208926b00, 0xc2089181a0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc20898fcc0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 542859 [semacquire, 405 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc209302630, 0xc2088b6340)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc209302630, 0xc2088b6340)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc209302630, 0xc2088b6340)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc209302630, 0xc2088b6340)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2092ba820)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 545205 [semacquire, 366 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc208ad11e0, 0xc208919520)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc208ad11e0, 0xc208919520)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc208ad11e0, 0xc208919520)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc208ad11e0, 0xc208919520)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc209720960)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 540695 [semacquire, 477 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/runtime/time.go:59 +0xfc
main.(*trySet).awaitTryBuild(0xc209e47c20, 0xb, 0x9f9390, 0x14, 0xac2050, 0x74, 0x0, 0x0, 0x9c7050, 0xc, ...)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:755 +0x160
created by main.newTrySet
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:701 +0x519
goroutine 631675 [sleep]:
time.Sleep(0xdf8475800)
/Users/crawshaw/go/src/runtime/time.go:59 +0xfc
main.(*buildStatus).pingDashboard(0xc209a0a1c0)
service: main process exited, code=exited, status=2/INVALIDARGUMENT
uild.service entered failed state.
service failed.
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/dash.go:143 +0x865
created by main.(*buildStatus).start
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:908 +0x5d
goroutine 552385 [sleep]:
time.Sleep(0x6fc23ac00)
/Users/crawshaw/go/src/runtime/time.go:59 +0xfc
main.(*trySet).awaitTryBuild(0xc209e47c20, 0xc, 0x9c4a70, 0x8, 0x0, 0x0, 0x0, 0x0, 0x9f6410, 0x16, ...)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:755 +0x160
created by main.newTrySet
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:701 +0x519
goroutine 552387 [sleep]:
time.Sleep(0x6fc23ac00)
/Users/crawshaw/go/src/runtime/time.go:59 +0xfc
main.(*trySet).awaitTryBuild(0xc209e47c20, 0xd, 0x9c4a90, 0xd, 0x0, 0x0, 0x0, 0x0, 0x9f6410, 0x16, ...)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:755 +0x160
created by main.newTrySet
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:701 +0x519
goroutine 631587 [sleep]:
time.Sleep(0xdf8475800)
/Users/crawshaw/go/src/runtime/time.go:59 +0xfc
main.(*buildStatus).pingDashboard(0xc209d96c40)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/dash.go:143 +0x865
created by main.(*buildStatus).start
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:908 +0x5d
goroutine 552391 [semacquire, 165 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.(*reverseBuildletPool).tryToGrab(0xc470a0, 0x9e9710, 0x12, 0x0, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:71 +0x64
main.(*reverseBuildletPool).GetBuildlet(0xc470a0, 0x9e9710, 0x12, 0xc209f56840, 0x28, 0x7f59fd6a6280, 0xc20898a700, 0x5, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:158 +0x67
main.(*buildStatus).build(0xc20898a700, 0x0, 0x0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:925 +0x1f6
main.(*buildStatus).start.func1(0xc20898a700)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:910 +0x33
created by main.(*buildStatus).start
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:916 +0x80
goroutine 631635 [sleep]:
time.Sleep(0xdf8475800)
/Users/crawshaw/go/src/runtime/time.go:59 +0xfc
main.(*buildStatus).pingDashboard(0xc209d97880)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/dash.go:143 +0x865
created by main.(*buildStatus).start
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:908 +0x5d
goroutine 631667 [sleep]:
time.Sleep(0xdf8475800)
/Users/crawshaw/go/src/runtime/time.go:59 +0xfc
main.(*buildStatus).pingDashboard(0xc209a0a000)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/dash.go:143 +0x865
created by main.(*buildStatus).start
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:908 +0x5d
goroutine 631659 [sleep]:
time.Sleep(0xdf8475800)
/Users/crawshaw/go/src/runtime/time.go:59 +0xfc
main.(*buildStatus).pingDashboard(0xc209d97dc0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/dash.go:143 +0x865
created by main.(*buildStatus).start
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:908 +0x5d
goroutine 553596 [semacquire, 163 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.(*reverseBuildletPool).WriteHTMLStatus(0xc470a0, 0x7f59fd68b6e8, 0xc209e980e0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:191 +0x16a
main.handleStatus(0x7f59fd69b578, 0xc20a0f0dc0, 0xc2094d24e0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/status.go:60 +0xf61
net/http.HandlerFunc.ServeHTTP(0xabb148, 0x7f59fd69b578, 0xc20a0f0dc0, 0xc2094d24e0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20a0f0dc0, 0xc2094d24e0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2083deba0, 0x7f59fd69b578, 0xc20a0f0dc0, 0xc2094d24e0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc20a48f0e0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 603112 [semacquire, 105 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.handleReverse(0x7f59fd69b578, 0xc20a422f20, 0xc2098b45b0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:333 +0xc5d
net/http.HandlerFunc.ServeHTTP(0xabb138, 0x7f59fd69b578, 0xc20a422f20, 0xc2098b45b0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20a422f20, 0xc2098b45b0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2084517a0, 0x7f59fd69b578, 0xc20a422f20, 0xc2098b45b0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc20a3cfae0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 552488 [semacquire, 165 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.(*reverseBuildletPool).WriteHTMLStatus(0xc470a0, 0x7f59fd68b6e8, 0xc2091563f0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:191 +0x16a
main.handleStatus(0x7f59fd69b578, 0xc20a009760, 0xc209bc5450)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/status.go:60 +0xf61
net/http.HandlerFunc.ServeHTTP(0xabb148, 0x7f59fd69b578, 0xc20a009760, 0xc209bc5450)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20a009760, 0xc209bc5450)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2083deba0, 0x7f59fd69b578, 0xc20a009760, 0xc209bc5450)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc2089225a0)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 631579 [sleep]:
time.Sleep(0xdf8475800)
/Users/crawshaw/go/src/runtime/time.go:59 +0xfc
main.(*buildStatus).pingDashboard(0xc209d96a80)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/dash.go:143 +0x865
created by main.(*buildStatus).start
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/coordinator.go:908 +0x5d
goroutine 606947 [semacquire, 93 minutes]:
sync.runtime_Semacquire(0xc470ac)
/Users/crawshaw/go/src/runtime/sema.go:43 +0x2d
sync.(*Mutex).Lock(0xc470a8)
/Users/crawshaw/go/src/sync/mutex.go:82 +0x1c7
main.(*reverseBuildletPool).WriteHTMLStatus(0xc470a0, 0x7f59fd68b6e8, 0xc20a5759d0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/reverse.go:191 +0x16a
main.handleStatus(0x7f59fd69b578, 0xc20a4b0160, 0xc20a33dad0)
/Users/crawshaw/src/golang.org/x/build/cmd/coordinator/status.go:60 +0xf61
net/http.HandlerFunc.ServeHTTP(0xabb148, 0x7f59fd69b578, 0xc20a4b0160, 0xc20a33dad0)
/Users/crawshaw/go/src/net/http/server.go:1317 +0x41
net/http.(*ServeMux).ServeHTTP(0xc208048720, 0x7f59fd69b578, 0xc20a4b0160, 0xc20a33dad0)
/Users/crawshaw/go/src/net/http/server.go:1593 +0x180
net/http.serverHandler.ServeHTTP(0xc2083deba0, 0x7f59fd69b578, 0xc20a4b0160, 0xc20a33dad0)
/Users/crawshaw/go/src/net/http/server.go:1755 +0x1a1
net/http.(*conn).serve(0xc20a6b3400)
/Users/crawshaw/go/src/net/http/server.go:1256 +0xbf6
created by net/http.(*Server).Serve
/Users/crawshaw/go/src/net/http/server.go:1803 +0x3df
goroutine 631619 [sleep]:
time.Sleep(0xdf8475800)
/Users/crawshaw/go/src/runtime/time.go:59 +0xfc
main.(*buildStatus).pingDashboard(0xc209d97500)
/Users/crawshaw/src/golang.org/x/build/cmd

created by main.(*buildStatus).start
/Users/crawshaw/src/golang.org/x/build/cmd

rax    0x3d
rbx    0xc2080e8640
rcx    0xffffffffffffffff
rdx    0x0
rdi    0x60f1
rsi    0xc208123d24
rbp    0x0
rsp    0xc208123c88
r8     0x0
r9     0x0
r10    0xc208467440
r11    0x212
r12    0xc473e0
r13    0xc20803bf80
r14    0x0
r15    0x0
rip    0x70abe8
rflags 0x212
cs     0x33
fs     0x0
gs     0x0
@bradfitz
Copy link
Contributor

@bradfitz bradfitz commented May 5, 2015

I suspect your new reverse mode is leaking fds.

We used to have a count+list of open fds on the home page to debug this in the past but it was removed. Add it back so we can watch it grow over time, if that's indeed the case?
@golang golang locked and limited conversation to collaborators Jun 25, 2016
Sign up for free to subscribe to this conversation on GitHub. Already have an account? Sign in.
Assignees
No one assigned
Labels
FrozenDueToAge
Projects
None yet
Milestone
No milestone
Linked pull requests

Successfully merging a pull request may close this issue.

None yet
3 participants
@bradfitz @crawshaw @gopherbot
You can’t perform that action at this time.