Return error if user tries to reuse RAFT id for a node that was removed.

Author: Pawan Rawal

dgraph/cmd/zero/raft.go

@@ -10,6 +10,7 @@ package zero
 import (
 	"encoding/binary"
 	"errors"
+	"log"
 	"math/rand"
 	"sync"
 	"time"
@@ -380,6 +381,18 @@ func (n *node) applyConfChange(e raftpb.Entry) {
 		n.Connect(rc.Id, rc.Addr)
 
 		m := &intern.Member{Id: rc.Id, Addr: rc.Addr, GroupId: 0}
+
+		for _, member := range n.server.membershipState().Removed {
+			// It is not recommended to reuse RAFT ids.
+			if member.GroupId == 0 && m.Id == member.Id {
+				n.DoneConfChange(cc.ID, x.ErrReuseRemovedId)
+				// Cancel configuration change.
+				cc.NodeID = raft.None
+				n.Raft().ApplyConfChange(cc)
+				return
+			}
+		}
+
 		n.server.storeZero(m)
 	}
 
@@ -429,14 +442,16 @@ func (n *node) initAndStartNode(wal *raftwal.Wal) error {
 			time.Sleep(delay)
 			ctx, cancel := context.WithTimeout(n.ctx, time.Second)
 			defer cancel()
-			// JoinCluster can block idefinitely, raft ignores conf change proposal
+			// JoinCluster can block indefinitely, raft ignores conf change proposal
 			// if it has pending configuration.
 			_, err = c.JoinCluster(ctx, n.RaftContext)
 			if err == nil {
 				break
 			}
-			if grpc.ErrorDesc(err) == conn.ErrDuplicateRaftId.Error() {
-				x.Fatalf("Error while joining cluster %v", err)
+			errorDesc := grpc.ErrorDesc(err)
+			if errorDesc == conn.ErrDuplicateRaftId.Error() ||
+				errorDesc == x.ErrReuseRemovedId.Error() {
+				log.Fatalf("Error while joining cluster: %v", errorDesc)
 			}
 			x.Printf("Error while joining cluster %v\n", err)
 			delay *= 2

dgraph/cmd/zero/zero.go

@@ -340,6 +340,14 @@ func (s *Server) Connect(ctx context.Context,
 		fmt.Println("No address provided.")
 		return &emptyConnectionState, errInvalidAddress
 	}
+
+	for _, member := range s.membershipState().Removed {
+		// It is not recommended to reuse RAFT ids.
+		if member.GroupId != 0 && m.Id == member.Id {
+			return &emptyConnectionState, x.ErrReuseRemovedId
+		}
+	}
+
 	for _, group := range s.state.Groups {
 		member, has := group.Members[m.Id]
 		if !has {
@@ -353,6 +361,7 @@ func (s *Server) Connect(ctx context.Context,
 			}
 		}
 	}
+
 	// Create a connection and check validity of the address by doing an Echo.
 	conn.Get().Connect(m.Addr)
 

wiki/content/contribute/index.md

@@ -97,11 +97,11 @@ Over years of writing big scalable systems, we are convinced that striving for s
 
 Every new source file must begin with a license header.
 
-Badger repo and files under `client/` and `x/` in Dgraph repo are licensed under the Apache license:
+Badger repo and the dgraph clients(dgo, dgraph-js, pydgraph and dgraph4j) are licensed under the Apache license:
 
 
     /*
-     * Copyright 2016-17 Dgraph Labs, Inc.
+     * Copyright 2016-2018 Dgraph Labs, Inc. and Contributors
      *
      * Licensed under the Apache License, Version 2.0 (the "License");
      * you may not use this file except in compliance with the License.
@@ -116,24 +116,15 @@ Badger repo and files under `client/` and `x/` in Dgraph repo are licensed under
      * limitations under the License.
      */
 
-All other code in the Dgraph repo is licenses under the GNU AGPL v3 license:
+All the code in the Dgraph repo is licensed under the Apache license with the Commons Clause
+restriction:
 
 
     /*
-     * Copyright (C) 2017 Dgraph Labs, Inc. and Contributors
+     * Copyright 2017-2018 Dgraph Labs, Inc. and Contributors
      *
-     * This program is free software: you can redistribute it and/or modify
-     * it under the terms of the GNU Affero General Public License as published by
-     * the Free Software Foundation, either version 3 of the License, or
-     * (at your option) any later version.
-     *
-     * This program is distributed in the hope that it will be useful,
-     * but WITHOUT ANY WARRANTY; without even the implied warranty of
-     * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-     * GNU Affero General Public License for more details.
-     *
-     * You should have received a copy of the GNU Affero General Public License
-     * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+     * This file is available under the Apache License, Version 2.0,
+     * with the Commons Clause restriction.
      */
 
 ### Signed Commits

wiki/content/deploy/index.md

@@ -1069,7 +1069,7 @@ id of the Zero node.
 {{% notice "note" %}}
 Before using the api ensure that the node is down and ensure that it doesn't come back up ever again.
 
-Remember to specify the `idx` flag while replacing the dead node or else Zero might assign it a different group.
+You should not use the same `idx` as that of a node that was removed earlier.
 {{% /notice %}}
 * `/moveTablet?tablet=name&group=2` This endpoint can be used to move a tablet to a group. Zero
   already does shard rebalancing every 8 mins, this endpoint can be used to force move a tablet.

worker/groups.go

@@ -23,6 +23,8 @@ import (
 	"sync/atomic"
 	"time"
 
+	"google.golang.org/grpc"
+
 	"golang.org/x/net/context"
 
 	"github.com/dgraph-io/badger"
@@ -94,10 +96,10 @@ func StartRaftNodes(walStore *badger.ManagedDB, bindall bool) {
 	m := &intern.Member{Id: Config.RaftId, Addr: Config.MyAddr}
 	delay := 50 * time.Millisecond
 	maxHalfDelay := 15 * time.Second
+	var err error
 	for { // Keep on retrying. See: https://github.com/dgraph-io/dgraph/issues/2289
-		var err error
 		connState, err = zc.Connect(gr.ctx, m)
-		if err == nil {
+		if err == nil || grpc.ErrorDesc(err) == x.ErrReuseRemovedId.Error() {
 			break
 		}
 		x.Printf("Error while connecting with group zero: %v", err)
@@ -106,6 +108,7 @@ func StartRaftNodes(walStore *badger.ManagedDB, bindall bool) {
 			delay *= 2
 		}
 	}
+	x.CheckfNoTrace(err)
 	if connState.GetMember() == nil || connState.GetState() == nil {
 		x.Fatalf("Unable to join cluster via dgraphzero")
 	}

x/error.go

@@ -40,6 +40,12 @@ func Checkf(err error, format string, args ...interface{}) {
 	}
 }
 
+func CheckfNoTrace(err error) {
+	if err != nil {
+		log.Fatalf(err.Error())
+	}
+}
+
 // Check2 acts as convenience wrapper around Check, using the 2nd argument as error.
 func Check2(_ interface{}, err error) {
 	Check(err)

x/x.go

@@ -12,6 +12,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net"
 	"net/http"
@@ -61,7 +62,8 @@ const (
 
 var (
 	// Useful for running multiple servers on the same machine.
-	regExpHostName = regexp.MustCompile(ValidHostnameRegex)
+	regExpHostName    = regexp.MustCompile(ValidHostnameRegex)
+	ErrReuseRemovedId = errors.New("Reusing RAFT index of a removed node.")
 )
 
 // WhiteSpace Replacer removes spaces and tabs from a string.