Merge branch 'master' into fix-panic

go.mod

@@ -30,7 +30,7 @@ require (
 	github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8
 	github.com/pingcap/errcode v0.0.0-20180921232412-a1a7271709d9
 	github.com/pingcap/failpoint v0.0.0-20190512135322-30cc7431d99c
-	github.com/pingcap/kvproto v0.0.0-20190822090350-11ea838aedf7
+	github.com/pingcap/kvproto v0.0.0-20190918040755-121d118fa10e
 	github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd
 	github.com/pkg/errors v0.8.1
 	github.com/prometheus/client_golang v0.8.0

go.sum

@@ -110,8 +110,8 @@ github.com/pingcap/errors v0.11.0 h1:DCJQB8jrHbQ1VVlMFIrbj2ApScNNotVmkSNplu2yUt4
 github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pingcap/failpoint v0.0.0-20190512135322-30cc7431d99c h1:hvQd3aOLKLF7xvRV6DzvPkKY4QXzfVbjU1BhW0d9yL8=
 github.com/pingcap/failpoint v0.0.0-20190512135322-30cc7431d99c/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI=
-github.com/pingcap/kvproto v0.0.0-20190822090350-11ea838aedf7 h1:tt24R4cv6GlvnmvkHNC1OrS/ETvXxbJkJ1Nrk4prtWo=
-github.com/pingcap/kvproto v0.0.0-20190822090350-11ea838aedf7/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY=
+github.com/pingcap/kvproto v0.0.0-20190918040755-121d118fa10e h1:NHsHH1L3jlQk596H5g1MhPnNitLIFxjOh9+IJcSHgnM=
+github.com/pingcap/kvproto v0.0.0-20190918040755-121d118fa10e/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY=
 github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd h1:hWDol43WY5PGhsh3+8794bFHY1bPrmu6bTalpssCrGg=
 github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd/go.mod h1:WpHUKhNZ18v116SvGrmjkA9CBhYmuUTKL+p8JC9ANEw=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

server/api/api.raml

@@ -307,7 +307,8 @@ types:
       region_id: integer
       policy:
         type: string
-        enum: [ scan, approximate ]
+        enum: [ scan, approximate, usekey ]
+      keys?: string[]
   ScatterRegionOperator:
     type: Operator
     discriminatorValue: scatter-region

server/api/operator.go

@@ -228,7 +228,18 @@ func (h *operatorHandler) Post(w http.ResponseWriter, r *http.Request) {
 			h.r.JSON(w, http.StatusBadRequest, "missing split policy")
 			return
 		}
-		if err := h.AddSplitRegionOperator(uint64(regionID), policy); err != nil {
+		var keys []string
+		if ks, ok := input["keys"]; ok {
+			for _, k := range ks.([]interface{}) {
+				key, ok := k.(string)
+				if !ok {
+					h.r.JSON(w, http.StatusBadRequest, "bad format keys")
+					return
+				}
+				keys = append(keys, key)
+			}
+		}
+		if err := h.AddSplitRegionOperator(uint64(regionID), policy, keys); err != nil {
 			h.r.JSON(w, http.StatusInternalServerError, err.Error())
 			return
 		}

server/handler.go

@@ -15,11 +15,14 @@ package server
 
 import (
 	"bytes"
+	"encoding/hex"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/pingcap/errcode"
 	"github.com/pingcap/kvproto/pkg/metapb"
+	"github.com/pingcap/kvproto/pkg/pdpb"
 	"github.com/pingcap/log"
 	"github.com/pingcap/pd/server/config"
 	"github.com/pingcap/pd/server/core"
@@ -634,7 +637,7 @@ func (h *Handler) AddMergeRegionOperator(regionID uint64, targetID uint64) error
 }
 
 // AddSplitRegionOperator adds an operator to split a region.
-func (h *Handler) AddSplitRegionOperator(regionID uint64, policy string) error {
+func (h *Handler) AddSplitRegionOperator(regionID uint64, policyStr string, keys []string) error {
 	c, err := h.getCoordinator()
 	if err != nil {
 		return err
@@ -645,7 +648,23 @@ func (h *Handler) AddSplitRegionOperator(regionID uint64, policy string) error {
 		return ErrRegionNotFound(regionID)
 	}
 
-	op := operator.CreateSplitRegionOperator("admin-split-region", region, operator.OpAdmin, policy)
+	policy, ok := pdpb.CheckPolicy_value[strings.ToUpper(policyStr)]
+	if !ok {
+		return errors.Errorf("check policy %s is not supported", policyStr)
+	}
+
+	var splitKeys [][]byte
+	if pdpb.CheckPolicy(policy) == pdpb.CheckPolicy_USEKEY {
+		for i := range keys {
+			k, err := hex.DecodeString(keys[i])
+			if err != nil {
+				return errors.Errorf("split key %s is not in hex format", keys[i])
+			}
+			splitKeys = append(splitKeys, k)
+		}
+	}
+
+	op := operator.CreateSplitRegionOperator("admin-split-region", region, operator.OpAdmin, pdpb.CheckPolicy(policy), splitKeys)
 	if ok := c.opController.AddOperator(op); !ok {
 		return errors.WithStack(ErrAddOperator)
 	}

server/schedule/operator/operator.go

@@ -352,6 +352,7 @@ func (mr MergeRegion) Influence(opInfluence OpInfluence, region *core.RegionInfo
 type SplitRegion struct {
 	StartKey, EndKey []byte
 	Policy           pdpb.CheckPolicy
+	SplitKeys        [][]byte
 }
 
 // ConfVerChanged returns true if the conf version has been changed by this step
@@ -925,11 +926,12 @@ func CreateMoveLeaderOperator(desc string, cluster Cluster, region *core.RegionI
 }
 
 // CreateSplitRegionOperator creates an operator that splits a region.
-func CreateSplitRegionOperator(desc string, region *core.RegionInfo, kind OpKind, policy string) *Operator {
+func CreateSplitRegionOperator(desc string, region *core.RegionInfo, kind OpKind, policy pdpb.CheckPolicy, keys [][]byte) *Operator {
 	step := SplitRegion{
-		StartKey: region.GetStartKey(),
-		EndKey:   region.GetEndKey(),
-		Policy:   pdpb.CheckPolicy(pdpb.CheckPolicy_value[strings.ToUpper(policy)]),
+		StartKey:  region.GetStartKey(),
+		EndKey:    region.GetEndKey(),
+		Policy:    policy,
+		SplitKeys: keys,
 	}
 	brief := fmt.Sprintf("split: region %v", region.GetID())
 	return NewOperator(desc, brief, region.GetID(), region.GetRegionEpoch(), kind, step)

server/schedule/operator_controller.go

@@ -516,6 +516,7 @@ func (oc *OperatorController) SendScheduleCommand(region *core.RegionInfo, step
 		cmd := &pdpb.RegionHeartbeatResponse{
 			SplitRegion: &pdpb.SplitRegion{
 				Policy: st.Policy,
+				Keys:   st.SplitKeys,
 			},
 		}
 		oc.hbStreams.SendMsg(region, cmd)

tests/pdctl/cluster/cluster_test.go

@@ -15,9 +15,6 @@ package cluster_test
 
 import (
 	"encoding/json"
-	"io/ioutil"
-	"os"
-	"path/filepath"
 	"strings"
 	"testing"
 
@@ -26,7 +23,6 @@ import (
 	"github.com/pingcap/pd/server"
 	"github.com/pingcap/pd/tests"
 	"github.com/pingcap/pd/tests/pdctl"
-	ctl "github.com/pingcap/pd/tools/pd-ctl/pdctl"
 )
 
 func Test(t *testing.T) {
@@ -63,16 +59,8 @@ func (s *clusterTestSuite) TestClusterAndPing(c *C) {
 	c.Assert(json.Unmarshal(output, ci), IsNil)
 	c.Assert(ci, DeepEquals, cluster.GetCluster())
 
-	fname := filepath.Join(os.TempDir(), "stdout")
-	old := os.Stdout
-	temp, _ := os.Create(fname)
-	os.Stdout = temp
-	ctl.Start([]string{"-u", pdAddr, "--cacert=ca.pem", "cluster"})
-	temp.Close()
-	os.Stdout = old
-	out, _ := ioutil.ReadFile(fname)
-	os.Remove(fname)
-	c.Assert(strings.Contains(string(out), "no such file or directory"), IsTrue)
+	echo := pdctl.GetEcho([]string{"-u", pdAddr, "--cacert=ca.pem", "cluster"})
+	c.Assert(strings.Contains(echo, "no such file or directory"), IsTrue)
 
 	// cluster status
 	args = []string{"-u", pdAddr, "cluster", "status"}

tests/pdctl/helper.go

@@ -17,6 +17,9 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
 	"sort"
 
 	"github.com/pingcap/check"
@@ -27,6 +30,7 @@ import (
 	"github.com/pingcap/pd/server/core"
 	"github.com/pingcap/pd/tests"
 	"github.com/pingcap/pd/tools/pd-ctl/pdctl"
+	ctl "github.com/pingcap/pd/tools/pd-ctl/pdctl"
 	"github.com/pingcap/pd/tools/pd-ctl/pdctl/command"
 	"github.com/spf13/cobra"
 )
@@ -132,3 +136,17 @@ func MustPutRegion(c *check.C, cluster *tests.TestCluster, regionID, storeID uin
 	c.Assert(err, check.IsNil)
 	return r
 }
+
+// GetEcho is used to get echo from stdout.
+func GetEcho(args []string) string {
+	filename := filepath.Join(os.TempDir(), "stdout")
+	old := os.Stdout
+	temp, _ := os.Create(filename)
+	os.Stdout = temp
+	ctl.Start(args)
+	temp.Close()
+	os.Stdout = old
+	out, _ := ioutil.ReadFile(filename)
+	_ = os.Remove(filename)
+	return string(out)
+}

tests/pdctl/operator/operator_test.go

@@ -195,4 +195,12 @@ func (s *operatorTestSuite) TestOperator(c *C) {
 	_, output, err = pdctl.ExecuteCommandC(cmd, args...)
 	c.Assert(err, IsNil)
 	c.Assert(strings.Contains(string(output), "scatter-region"), IsTrue)
+
+	// test echo
+	echo := pdctl.GetEcho([]string{"-u", pdAddr, "operator", "add", "scatter-region", "1"})
+	c.Assert(strings.Contains(echo, "Success!"), IsTrue)
+	echo = pdctl.GetEcho([]string{"-u", pdAddr, "operator", "remove", "1"})
+	c.Assert(strings.Contains(echo, "Success!"), IsTrue)
+	echo = pdctl.GetEcho([]string{"-u", pdAddr, "operator", "remove", "1"})
+	c.Assert(strings.Contains(echo, "Success!"), IsFalse)
 }

tests/pdctl/region/region_test.go

@@ -15,9 +15,6 @@ package region_test
 
 import (
 	"encoding/json"
-	"io/ioutil"
-	"os"
-	"path/filepath"
 	"strings"
 	"testing"
 
@@ -29,7 +26,6 @@ import (
 	"github.com/pingcap/pd/server/core"
 	"github.com/pingcap/pd/tests"
 	"github.com/pingcap/pd/tests/pdctl"
-	ctl "github.com/pingcap/pd/tools/pd-ctl/pdctl"
 )
 
 func Test(t *testing.T) {
@@ -60,16 +56,9 @@ func (s *regionTestSuite) TestRegionKeyFormat(c *C) {
 	leaderServer := cluster.GetServer(cluster.GetLeader())
 	c.Assert(leaderServer.BootstrapCluster(), IsNil)
 	pdctl.MustPutStore(c, leaderServer.GetServer(), store.Id, store.State, store.Labels)
-	fname := filepath.Join(os.TempDir(), "stdout")
-	old := os.Stdout
-	temp, _ := os.Create(fname)
-	os.Stdout = temp
-	ctl.Start([]string{"-u", url, "region", "key", "--format=raw", " "})
-	temp.Close()
-	os.Stdout = old
-	out, _ := ioutil.ReadFile(fname)
-	os.Remove(fname)
-	c.Assert(strings.Contains(string(out), "unknown flag"), IsFalse)
+
+	echo := pdctl.GetEcho([]string{"-u", url, "region", "key", "--format=raw", " "})
+	c.Assert(strings.Contains(string(echo), "unknown flag"), IsFalse)
 }
 
 func (s *regionTestSuite) TestRegion(c *C) {

tests/pdctl/scheduler/scheduler_test.go

@@ -15,6 +15,7 @@ package scheduler_test
 
 import (
 	"encoding/json"
+	"strings"
 	"testing"
 	"time"
 
@@ -131,4 +132,12 @@ func (s *schedulerTestSuite) TestScheduler(c *C) {
 	for _, scheduler := range schedulers {
 		c.Assert(expected[scheduler], Equals, true)
 	}
+
+	// test echo
+	echo := pdctl.GetEcho([]string{"-u", pdAddr, "scheduler", "add", "balance-region-scheduler"})
+	c.Assert(strings.Contains(echo, "Success!"), IsTrue)
+	echo = pdctl.GetEcho([]string{"-u", pdAddr, "scheduler", "remove", "balance-region-scheduler"})
+	c.Assert(strings.Contains(echo, "Success!"), IsTrue)
+	echo = pdctl.GetEcho([]string{"-u", pdAddr, "scheduler", "remove", "balance-region-scheduler"})
+	c.Assert(strings.Contains(echo, "Success!"), IsFalse)
 }

tools/pd-ctl/pdctl/command/operator.go

@@ -408,6 +408,7 @@ func removeOperatorCommandFunc(cmd *cobra.Command, args []string) {
 		cmd.Println(err)
 		return
 	}
+	cmd.Println("Success!")
 }
 
 func parseUint64s(args []string) ([]uint64, error) {

tools/pd-ctl/pdctl/command/scheduler.go

@@ -315,4 +315,5 @@ func removeSchedulerCommandFunc(cmd *cobra.Command, args []string) {
 		cmd.Println(err)
 		return
 	}
+	cmd.Println("Success!")
 }