runtime: pointer 0xc420203000 to unused region of span idx=0x101 span.base()=0xc420314000

[freeekanayaka]

This crash happens occasionally in the demo program.

The reason is a violation CGO rules for passing pointers from Go to C, as the pages[i].Fill(page.Data, uint16(page.Flags), page.Number) call stores a Go pointer directly in C memory, which is forbidden. See also:

golang/go#19135

Stacktrace:

10.204.119.242:9980: 2017/07/14 08:24:11 [INFO] demo: start
runtime: pointer 0xc420203000 to unused region of span idx=0x101 span.base()=0xc420314000
span.limit=0xc420316000 span.state=1
fatal error: found bad pointer in Go heap (incorrect use of unsafe or cgo?)

runtime stack:
runtime.throw(0x8e8a89, 0x3e)
    /usr/lib/go-1.8/src/runtime/panic.go:596 +0x95
runtime.heapBitsForObject(0xc420203000, 0x0, 0x0, 0xc41ffdd0ff, 0xc400000000,
0x7ff18c52d410, 0xc42001d228, 0x0)
    /usr/lib/go-1.8/src/runtime/mbitmap.go:433 +0x2bb
runtime.shade(0xc420203000)
    /usr/lib/go-1.8/src/runtime/mgcmark.go:1340 +0x41
runtime.gcmarkwb_m(0x7ff160000a20, 0xc42045e000)
    /usr/lib/go-1.8/src/runtime/mbarrier.go:154 +0xea
runtime.writebarrierptr_prewrite1.func1()
    /usr/lib/go-1.8/src/runtime/mbarrier.go:188 +0x64
runtime.systemstack(0xc420164ba0)
    /usr/lib/go-1.8/src/runtime/asm_amd64.s:327 +0x79
runtime.mstart()
    /usr/lib/go-1.8/src/runtime/proc.go:1132

goroutine 10 [running]:
runtime.systemstack_switch()
    /usr/lib/go-1.8/src/runtime/asm_amd64.s:281 fp=0xc420049a18 sp=0xc420049a10
runtime.writebarrierptr_prewrite1(0x7ff160000a20, 0xc42045e000)
    /usr/lib/go-1.8/src/runtime/mbarrier.go:189 +0xbf fp=0xc420049a58 sp=0xc420049a18
runtime.writebarrierptr(0x7ff160000a20, 0xc42045e000)
    /usr/lib/go-1.8/src/runtime/mbarrier.go:211 +0x4d fp=0xc420049a90 sp=0xc420049a58
github.com/dqlite/go-sqlite3x.(*ReplicationPage).Fill(0x7ff160000a20, 0xc42045e000, 0x1000,
0x1000, 0x200000006)
    /root/go/src/github.com/dqlite/go-sqlite3x/replication.go:39 +0xd6 fp=0xc420049ae0
sp=0xc420049a90
github.com/dqlite/dqlite/replication.(*FSM).applyWalFrames(0xc420016a50, 0xc420317f60, 0x0,
0x0)
    /root/go/src/github.com/dqlite/dqlite/replication/fsm.go:157 +0x1e5 fp=0xc420049b80
sp=0xc420049ae0
github.com/dqlite/dqlite/replication.(*FSM).Apply(0xc420016a50, 0xc4204e0280, 0xd01980,
0xed0fa7a01)
    /root/go/src/github.com/dqlite/dqlite/replication/fsm.go:58 +0x3d2 fp=0xc420049c68
sp=0xc420049b80
github.com/hashicorp/raft.(*Raft).runFSM(0xc420150600)
    /root/go/src/github.com/hashicorp/raft/raft.go:596 +0xc79 fp=0xc420049f90
sp=0xc420049c68
github.com/hashicorp/raft.(*Raft).(github.com/hashicorp/raft.runFSM)-fm()
    /root/go/src/github.com/hashicorp/raft/raft.go:259 +0x2a fp=0xc420049fa8 sp=0xc420049f90
github.com/hashicorp/raft.(*raftState).goFunc.func1(0xc420150600, 0xc420012a80)
    /root/go/src/github.com/hashicorp/raft/state.go:142 +0x53 fp=0xc420049fd0
sp=0xc420049fa8
runtime.goexit()
    /usr/lib/go-1.8/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc420049fd8 sp=0xc420049fd0
created by github.com/hashicorp/raft.(*raftState).goFunc
    /root/go/src/github.com/hashicorp/raft/state.go:143 +0x66

goroutine 1 [chan receive, 14 minutes]:
github.com/dqlite/dqlite.(*Driver).WaitLeadership(0xc4200172f0, 0x0, 0x0)
    /root/go/src/github.com/dqlite/dqlite/driver.go:124 +0x70
github.com/dqlite/dqlite.(*Driver).Open(0xc4200172f0, 0x8d0196, 0x7, 0xc4200ada00, 0x75e636,
0x456710, 0xc4200adab0)
    /root/go/src/github.com/dqlite/dqlite/driver.go:89 +0x2f
database/sql.(*DB).conn(0xc4201600a0, 0xcdc4c0, 0xc420070190, 0x1, 0xc4200adce0, 0x413982,
0xc420017350)
    /usr/lib/go-1.8/src/database/sql/sql.go:965 +0x146
database/sql.(*DB).begin(0xc4201600a0, 0xcdc4c0, 0xc420070190, 0x0, 0x1, 0xc4201600a0, 0x0,
0x0)
    /usr/lib/go-1.8/src/database/sql/sql.go:1360 +0x8a
database/sql.(*DB).BeginTx(0xc4201600a0, 0xcdc4c0, 0xc420070190, 0x0, 0xc4200add88,
0xc4200add78, 0x4da202)
    /usr/lib/go-1.8/src/database/sql/sql.go:1342 +0x70
database/sql.(*DB).Begin(0xc4201600a0, 0x8eb6e0, 0xc4200addd8, 0x4dab9a)
    /usr/lib/go-1.8/src/database/sql/sql.go:1356 +0x4c
main.insertForever(0xc4201600a0)
    /root/go/src/github.com/dqlite/dqlite/testdata/demo.go:105 +0x40
main.main()
    /root/go/src/github.com/dqlite/dqlite/testdata/demo.go:92 +0x7fe

goroutine 17 [syscall, 14 minutes, locked to thread]:
runtime.goexit()
    /usr/lib/go-1.8/src/runtime/asm_amd64.s:2197 +0x1

goroutine 3 [syscall, 14 minutes]:
os/signal.signal_recv(0x6e09cb)
    /usr/lib/go-1.8/src/runtime/sigqueue.go:116 +0x104
os/signal.loop()
    /usr/lib/go-1.8/src/os/signal/signal_unix.go:22 +0x22
created by os/signal.init.1
    /usr/lib/go-1.8/src/os/signal/signal_unix.go:28 +0x41

goroutine 4 [IO wait, 2 minutes]:
net.runtime_pollWait(0x7ff18c5c6f70, 0x72, 0x0)
    /usr/lib/go-1.8/src/runtime/netpoll.go:164 +0x59
net.(*pollDesc).wait(0xc420134068, 0x72, 0x0, 0xc4202452e0)
    /usr/lib/go-1.8/src/net/fd_poll_runtime.go:75 +0x38
net.(*pollDesc).waitRead(0xc420134068, 0xffffffffffffffff, 0x0)
    /usr/lib/go-1.8/src/net/fd_poll_runtime.go:80 +0x34
net.(*netFD).accept(0xc420134000, 0x0, 0xcd7240, 0xc4202452e0)
    /usr/lib/go-1.8/src/net/fd_unix.go:430 +0x1e5
net.(*TCPListener).accept(0xc42000e030, 0xc42016eed8, 0x67eb3e, 0x456710)
    /usr/lib/go-1.8/src/net/tcpsock_posix.go:136 +0x2e
net.(*TCPListener).Accept(0xc42000e030, 0x8ec6a0, 0xc4201b4000, 0xcdc540, 0xc420073a10)
    /usr/lib/go-1.8/src/net/tcpsock.go:228 +0x49
net/http.(*Server).Serve(0xc42012e160, 0xcdb980, 0xc42000e030, 0x0, 0x0)
    /usr/lib/go-1.8/src/net/http/server.go:2643 +0x228
created by main.main
    /root/go/src/github.com/dqlite/dqlite/testdata/demo.go:63 +0x4a7

goroutine 5 [chan receive, 2 minutes]:
github.com/dqlite/raft-http.(*Layer).Accept(0xc420016900, 0x8ec208, 0xc420134070, 0xcdedc0,
0xc42018c260)
    /root/go/src/github.com/dqlite/raft-http/layer.go:39 +0x5e
github.com/hashicorp/raft.(*NetworkTransport).listen(0xc420134070)
    /root/go/src/github.com/hashicorp/raft/net_transport.go:362 +0x49
created by github.com/hashicorp/raft.NewNetworkTransportWithLogger
    /root/go/src/github.com/hashicorp/raft/net_transport.go:154 +0x192

goroutine 9 [select]:
github.com/hashicorp/raft.(*Raft).runFollower(0xc420150600)
    /root/go/src/github.com/hashicorp/raft/raft.go:646 +0xa1a
github.com/hashicorp/raft.(*Raft).run(0xc420150600)
    /root/go/src/github.com/hashicorp/raft/raft.go:630 +0xa4
github.com/hashicorp/raft.(*Raft).(github.com/hashicorp/raft.run)-fm()
    /root/go/src/github.com/hashicorp/raft/raft.go:258 +0x2a
github.com/hashicorp/raft.(*raftState).goFunc.func1(0xc420150600, 0xc420012a70)
    /root/go/src/github.com/hashicorp/raft/state.go:142 +0x53
created by github.com/hashicorp/raft.(*raftState).goFunc
    /root/go/src/github.com/hashicorp/raft/state.go:143 +0x66

goroutine 11 [select]:
github.com/hashicorp/raft.(*Raft).runSnapshots(0xc420150600)
    /root/go/src/github.com/hashicorp/raft/raft.go:1744 +0x367
github.com/hashicorp/raft.(*Raft).(github.com/hashicorp/raft.runSnapshots)-fm()
    /root/go/src/github.com/hashicorp/raft/raft.go:260 +0x2a
github.com/hashicorp/raft.(*raftState).goFunc.func1(0xc420150600, 0xc420012a90)
    /root/go/src/github.com/hashicorp/raft/state.go:142 +0x53
created by github.com/hashicorp/raft.(*raftState).goFunc
    /root/go/src/github.com/hashicorp/raft/state.go:143 +0x66

goroutine 12 [chan receive, 14 minutes]:
github.com/dqlite/raft-membership.HandleChangeRequests(0xc420150600, 0xc4200100c0)
    /root/go/src/github.com/dqlite/raft-membership/handle.go:27 +0x51
created by github.com/dqlite/dqlite.NewDriver
    /root/go/src/github.com/dqlite/dqlite/driver.go:81 +0x573

goroutine 13 [chan receive, 14 minutes]:
database/sql.(*DB).connectionOpener(0xc4201600a0)
    /usr/lib/go-1.8/src/database/sql/sql.go:837 +0x4a
created by database/sql.Open
    /usr/lib/go-1.8/src/database/sql/sql.go:582 +0x212

goroutine 531 [IO wait]:
net.runtime_pollWait(0x7ff18c5c6d30, 0x72, 0xb)
    /usr/lib/go-1.8/src/runtime/netpoll.go:164 +0x59
net.(*pollDesc).wait(0xc4204f15d8, 0x72, 0xcd85c0, 0xcd4740)
    /usr/lib/go-1.8/src/net/fd_poll_runtime.go:75 +0x38
net.(*pollDesc).waitRead(0xc4204f15d8, 0xc42025a000, 0x1000)
    /usr/lib/go-1.8/src/net/fd_poll_runtime.go:80 +0x34
net.(*netFD).Read(0xc4204f1570, 0xc42025a000, 0x1000, 0x1000, 0x0, 0xcd85c0, 0xcd4740)
    /usr/lib/go-1.8/src/net/fd_unix.go:250 +0x1b7
net.(*conn).Read(0xc4201224d8, 0xc42025a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/lib/go-1.8/src/net/net.go:181 +0x70
bufio.(*Reader).fill(0xc4203cf440)
    /usr/lib/go-1.8/src/bufio/bufio.go:97 +0x117
bufio.(*Reader).ReadByte(0xc4203cf440, 0xc420198240, 0xc420489cd0, 0xc420489ca0)
    /usr/lib/go-1.8/src/bufio/bufio.go:239 +0x5b
github.com/hashicorp/raft.(*NetworkTransport).handleCommand(0xc420134070, 0xc4203cf440,
0xc4203cf4a0, 0xc4204f1650, 0x0, 0x0)
    /root/go/src/github.com/hashicorp/raft/net_transport.go:402 +0x43
github.com/hashicorp/raft.(*NetworkTransport).handleConn(0xc420134070, 0xcdedc0,
0xc4201224d8)
    /root/go/src/github.com/hashicorp/raft/net_transport.go:386 +0x221
created by github.com/hashicorp/raft.(*NetworkTransport).listen
    /root/go/src/github.com/hashicorp/raft/net_transport.go:373 +0x1d0

goroutine 1696 [IO wait]:
net.runtime_pollWait(0x7ff18c5c6eb0, 0x72, 0xa)
    /usr/lib/go-1.8/src/runtime/netpoll.go:164 +0x59
net.(*pollDesc).wait(0xc4204f0998, 0x72, 0xcd85c0, 0xcd4740)
    /usr/lib/go-1.8/src/net/fd_poll_runtime.go:75 +0x38
net.(*pollDesc).waitRead(0xc4204f0998, 0xc4203fc000, 0x1000)
    /usr/lib/go-1.8/src/net/fd_poll_runtime.go:80 +0x34
net.(*netFD).Read(0xc4204f0930, 0xc4203fc000, 0x1000, 0x1000, 0x0, 0xcd85c0, 0xcd4740)
    /usr/lib/go-1.8/src/net/fd_unix.go:250 +0x1b7
net.(*conn).Read(0xc42018c260, 0xc4203fc000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/lib/go-1.8/src/net/net.go:181 +0x70
bufio.(*Reader).fill(0xc420259ce0)
    /usr/lib/go-1.8/src/bufio/bufio.go:97 +0x117
bufio.(*Reader).ReadByte(0xc420259ce0, 0x300000002, 0xc420165a00, 0xc420035ca0)
    /usr/lib/go-1.8/src/bufio/bufio.go:239 +0x5b
github.com/hashicorp/raft.(*NetworkTransport).handleCommand(0xc420134070, 0xc420259ce0,
0xc420259d40, 0xc4204f0a10, 0x0, 0x0)
    /root/go/src/github.com/hashicorp/raft/net_transport.go:402 +0x43
github.com/hashicorp/raft.(*NetworkTransport).handleConn(0xc420134070, 0xcdedc0,
0xc42018c260)
    /root/go/src/github.com/hashicorp/raft/net_transport.go:386 +0x221
created by github.com/hashicorp/raft.(*NetworkTransport).listen
    /root/go/src/github.com/hashicorp/raft/net_transport.go:373 +0x1d0