PD failover (#74)

* *: pd failover

Makefile

@@ -1,5 +1,6 @@
-GOENV     := GO15VENDOREXPERIMENT="1" CGO_ENABLED=0 GOOS=linux GOARCH=amd64
-GO        := $(GOENV) go
+GOENV  := GO15VENDOREXPERIMENT="1" CGO_ENABLED=0 GOOS=linux GOARCH=amd64
+GO     := $(GOENV) go
+GOTEST := go test -v -cover
 
 LDFLAGS += -X "github.com/pingcap/tidb-operator/version.BuildTS=$(shell date -u '+%Y-%m-%d %I:%M:%S')"
 LDFLAGS += -X "github.com/pingcap/tidb-operator/version.GitSHA=$(shell git rev-parse HEAD)"
@@ -39,8 +40,7 @@ e2e-build:
 	$(GOENV) ginkgo build tests/e2e
 
 test:
-	@echo "run unit tests"
-	@$(GO) test ./pkg/... -v -cover && echo success
+	@$(GOTEST) ./pkg/... && echo "\nUnit tests run successfully!"
 
 check-all: lint check-static check-shadow check-gosec megacheck errcheck
 

charts/tidb-cluster/templates/pd-configmap.yaml

@@ -32,6 +32,9 @@ data:
     fi
     source ${ANNOTATIONS} 2>/dev/null
 
+    PEER_SERVICE_DOMAIN="${HOSTNAME}.${PEER_SERVICE_NAME}.${NAMESPACE}.svc"
+    SERVICE_DOMAIN="${SERVICE_NAME}.${NAMESPACE}.svc"
+
     runmode=${runmode:-normal}
     if [[ X${runmode} == Xdebug ]]
     then
@@ -42,18 +45,35 @@ data:
     elapseTime=0
     period=1
     threshold=30
-    while true;do
-        nslookup ${HOSTNAME}.${PEER_SERVICE_NAME}.${NAMESPACE}.svc 2>/dev/null
-        [[ $? -eq 0 ]] && break
-        echo "nslookup domain ${HOSTNAME}.${PEER_SERVICE_NAME}.${NAMESPACE}.svc failed" >&2
+    while true; do
         sleep ${period}
         elapseTime=$(( elapseTime+period ))
-        if [[ ${elapseTime} -ge ${threshold} ]];then
-            echo "nslookup domain ${HOSTNAME}.${PEER_SERVICE_NAME}.${NAMESPACE}.svc timeout" >&2
+
+        if [[ ${elapseTime} -ge ${threshold} ]]
+        then
+            echo "waiting for pd cluster ready timeout" >&2
             exit 1
         fi
+
+        source ${ANNOTATIONS} 2>/dev/null
+        if nslookup ${PEER_SERVICE_DOMAIN} 2>/dev/null
+        then
+            echo "nslookup domain ${HOSTNAME}.${PEER_SERVICE_NAME}.${NAMESPACE}.svc success"
+
+            if [[ ${ORDINAL} -eq 0 ]]
+            then
+                [[ -z ${bootstrapping:-} ]] && continue
+                [[ ${bootstrapping} == "true" ]] && break
+            fi
+
+            [[ -d /var/lib/pd/member/wal ]] && break
+            wget -qO- ${SERVICE_DOMAIN}:2379/pd/api/v1/members 2>/dev/null
+            [[ $? -eq 0 ]] && break
+            echo "pd cluster is not ready now: ${SERVICE_DOMAIN}"
+        else
+            echo "nslookup domain ${PEER_SERVICE_DOMAIN} failed" >&2
+        fi
     done
-    echo "nslookup domain ${HOSTNAME}.${PEER_SERVICE_NAME}.${NAMESPACE}.svc success"
 
     ARGS="--data-dir=/var/lib/pd \
     --name=${HOSTNAME} \
@@ -64,14 +84,22 @@ data:
     --config=/etc/pd/pd.toml \
     "
 
-    if [[ ${ORDINAL} == "0" ]]
+    replicas=${replicas:-3}
+    if [[ ${ORDINAL} -eq 0 && ${bootstrapping:-} == "true" ]]
     then
         ARGS="${ARGS}--initial-cluster=${HOSTNAME}=http://${HOSTNAME}.${PEER_SERVICE_NAME}.${NAMESPACE}.svc:2380"
     else
+        if [[ ${ORDINAL} -eq 0 ]]
+        then
+          TOP=$((replicas-1))
+        else
+          TOP=$((ORDINAL-1))
+        fi
+
         ARGS="${ARGS}--join="
-        TOP=$((ORDINAL-1))
         for i in $(seq 0 ${TOP});
         do
+            [[ ${i} -eq ${ORDINAL} ]] && continue
             ARGS="${ARGS}http://${SET_NAME}-${i}.${PEER_SERVICE_NAME}.${NAMESPACE}.svc:2380"
             if [[ ${i} -lt ${TOP} ]]
             then

charts/tidb-operator/templates/controller-manager-deployment.yaml

@@ -35,6 +35,8 @@ spec:
           - /usr/local/bin/tidb-controller-manager
           - -default-storage-class-name={{ .Values.defaultStorageClassName }}
           - -cluster-scoped={{ .Values.clusterScoped }}
+          - -auto-failover={{ .Values.controllerManager.autoFailover | default false }}
+          - -pd-failover-period={{ .Values.controllerManager.pdFailoverPeriod | default "5m" }}
           - -v={{ .Values.controllerManager.logLevel }}
         env:
           - name: NAMESPACE

charts/tidb-operator/templates/controller-manager-rbac.yaml

@@ -27,7 +27,7 @@ rules:
   verbs: ["create", "get", "update"]
 - apiGroups: [""]
   resources: ["pods"]
-  verbs: ["get", "list", "watch","update"]
+  verbs: ["get", "list", "watch","update", "delete"]
 - apiGroups: [""]
   resources: ["persistentvolumes"]
   verbs: ["get", "list", "watch", "patch","update"]
@@ -84,7 +84,7 @@ rules:
   verbs: ["get", "list", "watch", "update", "delete"]
 - apiGroups: [""]
   resources: ["pods"]
-  verbs: ["get", "list", "watch"]
+  verbs: ["get", "list", "watch","update", "delete"]
 - apiGroups: ["apps"]
   resources: ["statefulsets"]
   verbs: ["*"]

charts/tidb-operator/values.yaml

@@ -26,3 +26,7 @@ controllerManager:
     requests:
       cpu: 80m
       memory: 50Mi
+  # autoFailover is whether tidb-operator should auto failover when failure occurs
+  autoFailover: false
+  # pd failover period default(5m)
+  pdFailoverPeriod: 5m

cmd/controller-manager/main.go

@@ -38,13 +38,15 @@ import (
 )
 
 var (
-	printVersion   bool
-	workers        int
-	leaseDuration  = 15 * time.Second
-	renewDuration  = 5 * time.Second
-	retryPeriod    = 3 * time.Second
-	resyncDuration = 30 * time.Second
-	waitDuration   = 5 * time.Second
+	printVersion     bool
+	workers          int
+	pdFailoverPeriod time.Duration
+	autoFailover     bool
+	leaseDuration    = 15 * time.Second
+	renewDuration    = 5 * time.Second
+	retryPeriod      = 3 * time.Second
+	resyncDuration   = 30 * time.Second
+	waitDuration     = 5 * time.Second
 )
 
 func init() {
@@ -53,6 +55,8 @@ func init() {
 	flag.IntVar(&workers, "workers", 5, "The number of workers that are allowed to sync concurrently. Larger number = more responsive management, but more CPU (and network) load")
 	flag.BoolVar(&controller.ClusterScoped, "cluster-scoped", true, "Whether tidb-operator should manage kubernetes cluster wide TiDB Clusters")
 	flag.StringVar(&controller.DefaultStorageClassName, "default-storage-class-name", "standard", "Default storage class name")
+	flag.BoolVar(&autoFailover, "auto-failover", false, "Auto failover")
+	flag.DurationVar(&pdFailoverPeriod, "pd-failover-period", time.Duration(5*time.Minute), "PD failover period default(5m)")
 
 	flag.Parse()
 }
@@ -112,7 +116,7 @@ func main() {
 		},
 	}
 
-	tcController := tidbcluster.NewController(kubeCli, cli, informerFactory, kubeInformerFactory)
+	tcController := tidbcluster.NewController(kubeCli, cli, informerFactory, kubeInformerFactory, autoFailover, pdFailoverPeriod)
 	stop := make(chan struct{})
 	defer close(stop)
 	go informerFactory.Start(stop)

pkg/apis/pingcap.com/v1alpha1/types.go

@@ -16,6 +16,7 @@ package v1alpha1
 import (
 	apps "k8s.io/api/apps/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 
 	corev1 "k8s.io/api/core/v1"
 )
@@ -165,9 +166,10 @@ type ResourceRequirement struct {
 
 // PDStatus is PD status
 type PDStatus struct {
-	Phase       MemberPhase             `json:"phase,omitempty"`
-	StatefulSet *apps.StatefulSetStatus `json:"statefulSet,omitempty"`
-	Members     map[string]PDMember     `json:"members,omitempty"`
+	Phase          MemberPhase                `json:"phase,omitempty"`
+	StatefulSet    *apps.StatefulSetStatus    `json:"statefulSet,omitempty"`
+	Members        map[string]PDMember        `json:"members,omitempty"`
+	FailureMembers map[string]PDFailureMember `json:"failureMembers,omitempty"`
 }
 
 // PDMember is PD member
@@ -178,6 +180,17 @@ type PDMember struct {
 	ID        string `json:"id"`
 	ClientURL string `json:"clientURL"`
 	Health    bool   `json:"health"`
+	// Last time the health transitioned from one to another.
+	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
+}
+
+// PDFailureMember is the pd failure member information
+type PDFailureMember struct {
+	PodName       string    `json:"podName,omitempty"`
+	MemberID      string    `json:"memberID,omitempty"`
+	PVUID         types.UID `json:"pvUID,omitempty"`
+	Replicas      int32     `json:"replicas,omitempty"`
+	MemberDeleted bool      `json:"memberDeleted,omitempty"`
 }
 
 // TiDBStatus is TiDB status