crash when a replica remove itself from raft group

[yananzhi]

// TestReadEncounterGroupDeleteError verifies that read command should return
// RangeNotFoundError rather than a ErrGroupDeleted error.
func TestReadEncounterGroupDeleteError(t *testing.T) {
    defer leaktest.AfterTest(t)

    mtc := startMultiTestContext(t, 2)
    defer mtc.Stop()
    raftID := proto.RaftID(1)
    mtc.replicateRange(raftID, 0, 1)
    // Remove the replica from first store.
    mtc.unreplicateRange(raftID, 0, 0)
    getArgs, getResp := getArgs([]byte("a"), raftID, mtc.stores[0].StoreID())

    // Force the read command request a new lease.
    clock := mtc.clocks[0]
    getArgs.Header().Timestamp = clock.Update(clock.Now().Add(int64(storage.DefaultLeaderLeaseDuration), 0))

    // Expect get a RangeNotFoundError
    err := mtc.stores[0].ExecuteCmd(context.Background(), proto.Call{Args: getArgs, Reply: getResp})
    if _, ok := err.(*proto.RangeNotFoundError); !ok {
        t.Fatalf("expect get RangeNotFoundError, actual get %v ", err)
    }
}

panic: runtime error: invalid memory address or nil pointer dereference
[signal 0xb code=0x1 addr=0x0 pc=0xbb0a8c]

goroutine 11 [running]:
github.com/coreos/etcd/raft.(*Progress).maybeUpdate(0x0, 0x17, 0x100000001)
    /home/zyn/gopath/src/github.com/coreos/etcd/raft/progress.go:104 +0xc
github.com/coreos/etcd/raft.(*raft).appendEntry(0xc2080ae540, 0xc20812a410, 0x1, 0x1)
    /home/zyn/gopath/src/github.com/coreos/etcd/raft/raft.go:358 +0x111
github.com/coreos/etcd/raft.stepLeader(0xc2080ae540, 0x2, 0x0, 0x100000001, 0x0, 0x0, 0x0, 0xc20812a410, 0x1, 0x1, ...)
    /home/zyn/gopath/src/github.com/coreos/etcd/raft/raft.go:512 +0x2a6
github.com/coreos/etcd/raft.(*raft).Step(0xc2080ae540, 0x2, 0x0, 0x100000001, 0x0, 0x0, 0x0, 0xc20812a410, 0x1, 0x1, ...)
    /home/zyn/gopath/src/github.com/coreos/etcd/raft/raft.go:487 +0x259
github.com/coreos/etcd/raft.(*multiNode).run(0xc208053020)
    /home/zyn/gopath/src/github.com/coreos/etcd/raft/multinode.go:231 +0xbb9
created by github.com/coreos/etcd/raft.StartMultiNode
    /home/zyn/gopath/src/github.com/coreos/etcd/raft/multinode.go:56 +0xb6

A raft leader remove itself from raft group, after the peer was removed from raft, multiNode can still propose command to the raft.

[es-chow]

This issue also mentioned in #768.

[tbg]

@bdarnell still current?

[bdarnell]

I'm pretty sure this is fixed now but I want to take a closer look at the test case @yananzhi posted and add it to client_raft_test.go if we don't already have something like it.