x/build/revdial: hung revdial buildlets

[bradfitz]

A bunch of the ARM buildlets disappeared.

I investigated, SIGQUIT'ing a couple of them. The interesting part is this Lock, hung forever:

Oct 01 22:57:16 buildlet-prep buildlet-stage0[19647]: goroutine 54 [semacquire, 14409 minutes]:
Oct 01 22:57:16 buildlet-prep buildlet-stage0[19647]: sync.runtime_Semacquire(0x106a17ec)
Oct 01 22:57:16 buildlet-prep buildlet-stage0[19647]: /home/bradfitz/go/src/runtime/sema.go:43 +0x24
Oct 01 22:57:16 buildlet-prep buildlet-stage0[19647]: sync.(*Mutex).Lock(0x106a17e8)
Oct 01 22:57:16 buildlet-prep buildlet-stage0[19647]: /home/bradfitz/go/src/sync/mutex.go:82 +0x1fc
Oct 01 22:57:16 buildlet-prep buildlet-stage0[19647]: golang.org/x/build/revdial.(*conn).Write(0x1076ca80, 0x106ef00
0, 0x1000, 0x1000, 0x0, 0x0, 0x0)
Oct 01 22:57:16 buildlet-prep buildlet-stage0[19647]: /home/bradfitz/src/golang.org/x/build/revdial/revdial.go:304 +
0x1ec
Oct 01 22:57:16 buildlet-prep buildlet-stage0[19647]: net/http.checkConnErrorWriter.Write(0x1066a000, 0x106ef000, 0x

/cc @adg

[bradfitz]

More:

 SIGQUIT: quit
 PC=0x84168 m=0

goroutine 0 [idle]:
 runtime.epollwait(0x4, 0xbea80564, 0x80, 0xffffffff, 0x1, 0xffffffff, 0x1, 0x0, 0x0, 0x0, ...)
 /home/bradfitz/go/src/runtime/sys_linux_arm.s:487 +0x18
 runtime.netpoll(0x46d701, 0x0)
 /home/bradfitz/go/src/runtime/netpoll_epoll.go:68 +0x9c
 runtime.findrunnable(0x10611400, 0x0)
 /home/bradfitz/go/src/runtime/proc1.go:1513 +0x5c4
 runtime.schedule()
 /home/bradfitz/go/src/runtime/proc1.go:1640 +0x274
 runtime.park_m(0x10600ee0)
 /home/bradfitz/go/src/runtime/proc1.go:1699 +0x16c
 runtime.mcall(0x46d990)
 /home/bradfitz/go/src/runtime/asm_arm.s:178 +0x5c

 goroutine 1 [chan receive, 38 minutes]:
 golang.org/x/build/revdial.(*Listener).Accept(0x10691a60, 0x0, 0x0, 0x0, 0x0)
 /home/bradfitz/src/golang.org/x/build/revdial/revdial.go:432 +0x54
 net/http.(*Server).Serve(0x10641e80, 0x40263bc8, 0x10691a60, 0x0, 0x0)
 /home/bradfitz/go/src/net/http/server.go:1889 +0x88
 main.dialCoordinator(0x0, 0x0)
 /home/bradfitz/src/golang.org/x/build/cmd/buildlet/reverse.go:117 +0x19c0
 main.main()
 /home/bradfitz/src/golang.org/x/build/cmd/buildlet/buildlet.go:163 +0xbb4

goroutine 15 [semacquire, 34 minutes]:
 sync.runtime_Semacquire(0x10a520d4)
 /home/bradfitz/go/src/runtime/sema.go:43 +0x24
 sync.(*Mutex).Lock(0x10a520d0)
 /home/bradfitz/go/src/sync/mutex.go:82 +0x1fc
 golang.org/x/build/revdial.(*conn).peerClose(0x10a520c0)
 /home/bradfitz/src/golang.org/x/build/revdial/revdial.go:268 +0x38
 golang.org/x/build/revdial.(*Listener).closeConn(0x10691a60, 0x4, 0x0, 0x0)
 /home/bradfitz/src/golang.org/x/build/revdial/revdial.go:472 +0xe8
 golang.org/x/build/revdial.(*Listener).onFrame(0x10691a60, 0x1060bb43, 0x4, 0x109f2000, 0x0, 0x1e00, 0x0, 0x0)
 /home/bradfitz/src/golang.org/x/build/revdial/revdial.go:515 +0x3c
 golang.org/x/build/revdial.readFrames(0x1060bb30, 0x40263be8, 0x10691a60, 0x0, 0x0)
 /home/bradfitz/src/golang.org/x/build/revdial/revdial.go:379 +0x458
 golang.org/x/build/revdial.NewListener.func1(0x1060c998, 0x10691a60)
 /home/bradfitz/src/golang.org/x/build/revdial/revdial.go:398 +0x70
 created by golang.org/x/build/revdial.NewListener
 /home/bradfitz/src/golang.org/x/build/revdial/revdial.go:412 +0x124

goroutine 65 [semacquire, 34 minutes]:
 sync.runtime_Semacquire(0x10691a6c)
 /home/bradfitz/go/src/runtime/sema.go:43 +0x24
 sync.(*Mutex).Lock(0x10691a68)
 /home/bradfitz/go/src/sync/mutex.go:82 +0x1fc
 golang.org/x/build/revdial.(*conn).Write(0x10a520c0, 0x1068f000, 0x88, 0x1000, 0x0, 0x0, 0x0)
 /home/bradfitz/src/golang.org/x/build/revdial/revdial.go:304 +0x1ec
 net/http.checkConnErrorWriter.Write(0x1066a000, 0x1068f000, 0x88, 0x1000, 0x0, 0x0, 0x0)
 /home/bradfitz/go/src/net/http/server.go:2253 +0x6c
 bufio.(*Writer).flush(0x1063ee40, 0x0, 0x0)
 /home/bradfitz/go/src/bufio/bufio.go:562 +0xb0
 bufio.(*Writer).Flush(0x1063ee40, 0x0, 0x0)
 /home/bradfitz/go/src/bufio/bufio.go:551 +0x28
 net/http.(*response).finishRequest(0x106e2080)
 /home/bradfitz/go/src/net/http/server.go:1156 +0x9c
 net/http.(*conn).serve(0x1066a000)
 /home/bradfitz/go/src/net/http/server.go:1365 +0xc30
 created by net/http.(*Server).Serve
 /home/bradfitz/go/src/net/http/server.go:1912 +0x360

goroutine 20 [select, 34 minutes]:
 golang.org/x/build/pargzip.(*Writer).startChunk(0x106de0c0, 0x10aca000, 0x100000, 0x100000)
 /home/bradfitz/src/golang.org/x/build/pargzip/pargzip.go:142 +0x1fc
 golang.org/x/build/pargzip.newChunkWriter.Write(0x106de0c0, 0x10aca000, 0x0, 0x0, 0x100000, 0x0, 0x0)
 /home/bradfitz/src/golang.org/x/build/pargzip/pargzip.go:214 +0xdc
 bufio.(*Writer).flush(0x106905a0, 0x0, 0x0)
 /home/bradfitz/go/src/bufio/bufio.go:562 +0xb0
 bufio.(*Writer).Write(0x106905a0, 0x1095e000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
 /home/bradfitz/go/src/bufio/bufio.go:598 +0x208
 golang.org/x/build/pargzip.(*Writer).Write(0x106de0c0, 0x1095e000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
 /home/bradfitz/src/golang.org/x/build/pargzip/pargzip.go:172 +0x80
 archive/tar.(*Writer).Write(0x1066e480, 0x1095e000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
 /home/bradfitz/go/src/archive/tar/writer.go:366 +0xf8
 io.copyBuffer(0x40322a48, 0x1066e480, 0x40262168, 0x10708010, 0x1095e000, 0x8000, 0x8000, 0x430000, 0x0, 0x0, ...)
 /home/bradfitz/go/src/io/io.go:383 +0x22c
 io.Copy(0x40322a48, 0x1066e480, 0x40262168, 0x10708010, 0x0, 0x0, 0x0, 0x0)
 /home/bradfitz/go/src/io/io.go:351 +0x64
 main.handleGetTGZ.func1(0x106d2360, 0x24, 0x40262500, 0x106d25d0, 0x0, 0x0, 0x0, 0x0)
 /home/bradfitz/src/golang.org/x/build/cmd/buildlet/buildlet.go:404 +0x700
 path/filepath.walk(0x106d2360, 0x24, 0x40262500, 0x106d25d0, 0x109d3d10, 0x0, 0x0)
 /home/bradfitz/go/src/path/filepath/path.go:349 +0x5c
 path/filepath.walk(0x106d4140, 0x1e, 0x40262500, 0x1060a480, 0x109d3d10, 0x0, 0x0)
 /home/bradfitz/go/src/path/filepath/path.go:374 +0x3dc
 path/filepath.walk(0x106d4060, 0x14, 0x40262500, 0x1060a450, 0x109d3d10, 0x0, 0x0)
 /home/bradfitz/go/src/path/filepath/path.go:374 +0x3dc
 path/filepath.walk(0x106c7b30, 0xf, 0x40262500, 0x106a1bc0, 0x109d3d10, 0x0, 0x0)
 /home/bradfitz/go/src/path/filepath/path.go:374 +0x3dc
 path/filepath.walk(0x10694040, 0xb, 0x40262500, 0x1060a360, 0x109d3d10, 0x0, 0x0)
 /home/bradfitz/go/src/path/filepath/path.go:374 +0x3dc
 path/filepath.Walk(0x10694040, 0xb, 0x109d3d10, 0x0, 0x0)
 /home/bradfitz/go/src/path/filepath/path.go:396 +0xb8
 main.handleGetTGZ(0x40263d38, 0x106e2000, 0x10a56000)
 /home/bradfitz/src/golang.org/x/build/cmd/buildlet/buildlet.go:372 +0x3cc
 net/http.HandlerFunc.ServeHTTP(0x38f65c, 0x40263d38, 0x106e2000, 0x10a56000)
 /home/bradfitz/go/src/net/http/server.go:1422 +0x34
 main.requirePasswordHandler.ServeHTTP(0x40263768, 0x38f65c, 0x0, 0x0, 0x40263d38, 0x106e2000, 0x10a56000)
 /home/bradfitz/src/golang.org/x/build/cmd/buildlet/buildlet.go:1097 +0xb0
 main.(*requirePasswordHandler).ServeHTTP(0x10643e20, 0x40263d38, 0x106e2000, 0x10a56000)
 <autogenerated>:17 +0xac
 net/http.(*ServeMux).ServeHTTP(0x1063e740, 0x40263d38, 0x106e2000, 0x10a56000)
 /home/bradfitz/go/src/net/http/server.go:1699 +0x164
 net/http.serverHandler.ServeHTTP(0x10641e80, 0x40263d38, 0x106e2000, 0x10a56000)
 /home/bradfitz/go/src/net/http/server.go:1862 +0x190
 net/http.(*conn).serve(0x10688000)
 /home/bradfitz/go/src/net/http/server.go:1361 +0xbfc
 created by net/http.(*Server).Serve
 /home/bradfitz/go/src/net/http/server.go:1912 +0x360

goroutine 34 [semacquire, 34 minutes]:
 sync.runtime_Syncsemacquire(0x10690748)
 /home/bradfitz/go/src/runtime/sema.go:237 +0x250
 sync.(*Cond).Wait(0x10690740)
 /home/bradfitz/go/src/sync/cond.go:62 +0xbc
 golang.org/x/build/revdial.(*conn).Read(0x106d2570, 0x107b6000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
 /home/bradfitz/src/golang.org/x/build/revdial/revdial.go:279 +0x114
 io.copyBuffer(0x402688b8, 0x1060c2b0, 0x40263cd8, 0x106d2570, 0x107b6000, 0x8000, 0x8000, 0x14b, 0x0, 0x0, ...)
 /home/bradfitz/go/src/io/io.go:381 +0x1c4
 io.Copy(0x402688b8, 0x1060c2b0, 0x40263cd8, 0x106d2570, 0x0, 0x0, 0x0, 0x0)
 /home/bradfitz/go/src/io/io.go:351 +0x64
 net/http.(*conn).closeNotify.func1(0x1060c2b0, 0x40263cd8, 0x106d2570, 0x10688000)
 /home/bradfitz/go/src/net/http/server.go:182 +0x74
 created by net/http.(*conn).closeNotify
 /home/bradfitz/go/src/net/http/server.go:188 +0x3b4

goroutine 51 [semacquire, 34 minutes]:
 sync.runtime_Semacquire(0x10691a6c)
 /home/bradfitz/go/src/runtime/sema.go:43 +0x24
 sync.(*Mutex).Lock(0x10691a68)
 /home/bradfitz/go/src/sync/mutex.go:82 +0x1fc
 golang.org/x/build/revdial.(*conn).Write(0x106d2570, 0x10655000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
 /home/bradfitz/src/golang.org/x/build/revdial/revdial.go:304 +0x1ec
 net/http.checkConnErrorWriter.Write(0x10688000, 0x10655000, 0x1000, 0x1000, 0x209fd8, 0x0, 0x0)
 /home/bradfitz/go/src/net/http/server.go:2253 +0x6c
 bufio.(*Writer).flush(0x10690780, 0x0, 0x0)
 /home/bradfitz/go/src/bufio/bufio.go:562 +0xb0
 bufio.(*Writer).Write(0x10690780, 0x108be000, 0x58105, 0x7fe00, 0x0, 0x0, 0x0)
 /home/bradfitz/go/src/bufio/bufio.go:598 +0x208
 net/http.(*chunkWriter).Write(0x106e2010, 0x108be000, 0x58105, 0x7fe00, 0x0, 0x0, 0x0)
 /home/bradfitz/go/src/net/http/server.go:274 +0x280
 bufio.(*Writer).Write(0x107700c0, 0x108be000, 0x58105, 0x7fe00, 0x0, 0x0, 0x0)
 /home/bradfitz/go/src/bufio/bufio.go:594 +0xa8
 net/http.(*response).write(0x106e2000, 0x58105, 0x108be000, 0x58105, 0x7fe00, 0x0, 0x0, 0x0, 0x0, 0x0)
 /home/bradfitz/go/src/net/http/server.go:1140 +0x1e0
 net/http.(*response).Write(0x106e2000, 0x108be000, 0x58105, 0x7fe00, 0x29aeb0, 0x0, 0x0)
 /home/bradfitz/go/src/net/http/server.go:1112 +0x60
 golang.org/x/build/pargzip.(*Writer).writeCompressedChunk(0x106de0c0, 0x106a0120, 0x0, 0x0)
 /home/bradfitz/src/golang.org/x/build/pargzip/pargzip.go:164 +0xf8
 golang.org/x/build/pargzip.(*Writer).init.func1(0x106de0c0)
 /home/bradfitz/src/golang.org/x/build/pargzip/pargzip.go:126 +0x9c
 created by golang.org/x/build/pargzip.(*Writer).init
 /home/bradfitz/src/golang.org/x/build/pargzip/pargzip.go:131 +0x198
 trap    0x0
 error   0x0
 oldmask 0x0
 r0      0xfffffffc
 r1      0xbea80564
 r2      0x80
 r3      0xffffffff
 r4      0x1
 r5      0x1
 r6      0x9f163
 r7      0xfc
 r8      0x8dafd
 r9      0x0
 r10     0x46d970
 fp      0x456048
 ip      0x106c6783
 sp      0xbea80538
 lr      0x47e80
 pc      0x84168
 cpsr    0x60000010
 fault   0x0

[gopherbot]

CL https://golang.org/cl/15530 mentions this issue.