diff --git a/server/raft.go b/server/raft.go
index 9ce0b1a467..8a080c4dc4 100644
--- a/server/raft.go
+++ b/server/raft.go
@@ -1352,6 +1352,10 @@ func (n *raft) StepDown(preferred ...string) error {
 		}
 	}
 
+	// Clear our vote state.
+	n.vote = noVote
+	n.writeTermVote()
+
 	stepdown := n.stepdown
 	prop := n.prop
 	n.Unlock()
@@ -3196,6 +3200,10 @@ func (n *raft) processAppendEntry(ae *appendEntry, sub *subscription) {
 						// Here we can become a leader but need to wait for resume of the apply channel.
 						n.lxfer = true
 					}
+				} else {
+					// Since we are here we are not the chosen one but we should clear any vote preference.
+					n.vote = noVote
+					n.writeTermVote()
 				}
 			}
 		case EntryAddPeer: