Merge pull request #740 from youtube/log

Log

go/vt/topo/shard.go

@@ -475,7 +475,7 @@ func (si *ShardInfo) CheckServedTypesMigration(tabletType TabletType, cells []st
 
 	// we can't remove a type we don't have
 	if _, ok := si.ServedTypesMap[tabletType]; !ok && remove {
-		return fmt.Errorf("supplied type cannot be migrated")
+		return fmt.Errorf("supplied type %v cannot be migrated out of %#v", tabletType, si)
 	}
 
 	return nil

go/vt/wrangler/testlib/emergency_reparent_shard_test.go

@@ -20,9 +20,10 @@ import (
 )
 
 func TestEmergencyReparentShard(t *testing.T) {
-	ctx := context.Background()
 	ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
 	wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second)
+	vp := NewVtctlPipe(t, ts)
+	defer vp.Close()
 
 	// Create a master, a couple good slaves
 	oldMaster := NewFakeTablet(t, wr, "cell1", 0, topo.TYPE_MASTER)
@@ -93,7 +94,7 @@ func TestEmergencyReparentShard(t *testing.T) {
 	defer goodSlave2.StopActionLoop(t)
 
 	// run EmergencyReparentShard
-	if err := wr.EmergencyReparentShard(ctx, newMaster.Tablet.Keyspace, newMaster.Tablet.Shard, newMaster.Tablet.Alias, 10*time.Second); err != nil {
+	if err := vp.Run([]string{"EmergencyReparentShard", "-wait_slave_timeout", "10s", newMaster.Tablet.Keyspace + "/" + newMaster.Tablet.Shard, newMaster.Tablet.Alias.String()}); err != nil {
 		t.Fatalf("EmergencyReparentShard failed: %v", err)
 	}
 

go/vt/wrangler/testlib/init_shard_master_test.go

@@ -25,6 +25,8 @@ func TestInitMasterShard(t *testing.T) {
 	ctx := context.Background()
 	ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
 	wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second)
+	vp := NewVtctlPipe(t, ts)
+	defer vp.Close()
 
 	// Create a master, a couple good slaves
 	master := NewFakeTablet(t, wr, "cell1", 0, topo.TYPE_MASTER)
@@ -90,7 +92,7 @@ func TestInitMasterShard(t *testing.T) {
 	defer goodSlave2.StopActionLoop(t)
 
 	// run InitShardMaster
-	if err := wr.InitShardMaster(ctx, master.Tablet.Keyspace, master.Tablet.Shard, master.Tablet.Alias, false /*force*/, 10*time.Second); err != nil {
+	if err := vp.Run([]string{"InitShardMaster", "-wait_slave_timeout", "10s", master.Tablet.Keyspace + "/" + master.Tablet.Shard, master.Tablet.Alias.String()}); err != nil {
 		t.Fatalf("InitShardMaster failed: %v", err)
 	}
 

go/vt/wrangler/testlib/migrate_served_from_test.go

@@ -0,0 +1,175 @@
+// Copyright 2015, Google Inc. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package testlib
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	mproto "github.com/youtube/vitess/go/mysql/proto"
+	"github.com/youtube/vitess/go/sqltypes"
+	"github.com/youtube/vitess/go/vt/logutil"
+	myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto"
+	"github.com/youtube/vitess/go/vt/tabletmanager/tmclient"
+	"github.com/youtube/vitess/go/vt/topo"
+	"github.com/youtube/vitess/go/vt/wrangler"
+	"github.com/youtube/vitess/go/vt/zktopo"
+	"golang.org/x/net/context"
+)
+
+func TestMigrateServedFrom(t *testing.T) {
+	ctx := context.Background()
+	ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
+	wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second)
+	vp := NewVtctlPipe(t, ts)
+	defer vp.Close()
+
+	// create the source keyspace tablets
+	sourceMaster := NewFakeTablet(t, wr, "cell1", 10, topo.TYPE_MASTER,
+		TabletKeyspaceShard(t, "source", "0"))
+	sourceReplica := NewFakeTablet(t, wr, "cell1", 11, topo.TYPE_REPLICA,
+		TabletKeyspaceShard(t, "source", "0"))
+	sourceRdonly := NewFakeTablet(t, wr, "cell1", 12, topo.TYPE_RDONLY,
+		TabletKeyspaceShard(t, "source", "0"))
+
+	// create the destination keyspace, served form source
+	// double check it has all entries in map
+	if err := vp.Run([]string{"CreateKeyspace", "-served_from", "master:source,replica:source,rdonly:source", "dest"}); err != nil {
+		t.Fatalf("CreateKeyspace(dest) failed: %v", err)
+	}
+	ki, err := ts.GetKeyspace(ctx, "dest")
+	if err != nil {
+		t.Fatalf("GetKeyspace failed: %v", err)
+	}
+	if len(ki.ServedFromMap) != 3 {
+		t.Fatalf("bad initial dest ServedFrom: %v", ki.ServedFromMap)
+	}
+
+	// create the destination keyspace tablets
+	destMaster := NewFakeTablet(t, wr, "cell1", 20, topo.TYPE_MASTER,
+		TabletKeyspaceShard(t, "dest", "0"))
+	destReplica := NewFakeTablet(t, wr, "cell1", 21, topo.TYPE_REPLICA,
+		TabletKeyspaceShard(t, "dest", "0"))
+	destRdonly := NewFakeTablet(t, wr, "cell1", 22, topo.TYPE_RDONLY,
+		TabletKeyspaceShard(t, "dest", "0"))
+
+	// sourceRdonly will see the refresh
+	sourceRdonly.StartActionLoop(t, wr)
+	defer sourceRdonly.StopActionLoop(t)
+
+	// sourceReplica will see the refresh
+	sourceReplica.StartActionLoop(t, wr)
+	defer sourceReplica.StopActionLoop(t)
+
+	// sourceMaster will see the refresh, and has to respond to it
+	// also will be asked about its replication position.
+	sourceMaster.FakeMysqlDaemon.CurrentMasterPosition = myproto.ReplicationPosition{
+		GTIDSet: myproto.MariadbGTID{
+			Domain:   5,
+			Server:   456,
+			Sequence: 892,
+		},
+	}
+	sourceMaster.StartActionLoop(t, wr)
+	defer sourceMaster.StopActionLoop(t)
+
+	// destRdonly will see the refresh
+	destRdonly.StartActionLoop(t, wr)
+	defer destRdonly.StopActionLoop(t)
+
+	// destReplica will see the refresh
+	destReplica.StartActionLoop(t, wr)
+	defer destReplica.StopActionLoop(t)
+
+	// destMaster will see the refresh, and has to respond to it.
+	// It will also need to respond to WaitBlpPosition, saying it's already caught up.
+	destMaster.FakeMysqlDaemon.FetchSuperQueryMap = map[string]*mproto.QueryResult{
+		"SELECT pos, flags FROM _vt.blp_checkpoint WHERE source_shard_uid=0": &mproto.QueryResult{
+			Rows: [][]sqltypes.Value{
+				[]sqltypes.Value{
+					sqltypes.MakeString([]byte(myproto.EncodeReplicationPosition(sourceMaster.FakeMysqlDaemon.CurrentMasterPosition))),
+					sqltypes.MakeString([]byte("")),
+				},
+			},
+		},
+	}
+	destMaster.StartActionLoop(t, wr)
+	defer destMaster.StopActionLoop(t)
+
+	// simulate the clone, by fixing the dest shard record
+	if err := vp.Run([]string{"SourceShardAdd", "--tables", "gone1,gone2", "dest/0", "0", "source/0"}); err != nil {
+		t.Fatalf("SourceShardAdd failed: %v", err)
+	}
+
+	// migrate rdonly over
+	if err := vp.Run([]string{"MigrateServedFrom", "dest/0", "rdonly"}); err != nil {
+		t.Fatalf("MigrateServedFrom(rdonly) failed: %v", err)
+	}
+
+	// check it's gone from keyspace
+	ki, err = ts.GetKeyspace(ctx, "dest")
+	if err != nil {
+		t.Fatalf("GetKeyspace failed: %v", err)
+	}
+	if _, ok := ki.ServedFromMap[topo.TYPE_RDONLY]; len(ki.ServedFromMap) != 2 || ok {
+		t.Fatalf("bad initial dest ServedFrom: %v", ki.ServedFromMap)
+	}
+
+	// check the source shard has the right blacklisted tables
+	si, err := ts.GetShard(ctx, "source", "0")
+	if err != nil {
+		t.Fatalf("GetShard failed: %v", err)
+	}
+	if len(si.TabletControlMap) != 1 || !reflect.DeepEqual(si.TabletControlMap[topo.TYPE_RDONLY].BlacklistedTables, []string{"gone1", "gone2"}) {
+		t.Fatalf("rdonly type doesn't have right blacklisted tables")
+	}
+
+	// migrate replica over
+	if err := vp.Run([]string{"MigrateServedFrom", "dest/0", "replica"}); err != nil {
+		t.Fatalf("MigrateServedFrom(replica) failed: %v", err)
+	}
+
+	// check it's gone from keyspace
+	ki, err = ts.GetKeyspace(ctx, "dest")
+	if err != nil {
+		t.Fatalf("GetKeyspace failed: %v", err)
+	}
+	if _, ok := ki.ServedFromMap[topo.TYPE_REPLICA]; len(ki.ServedFromMap) != 1 || ok {
+		t.Fatalf("bad initial dest ServedFrom: %v", ki.ServedFromMap)
+	}
+
+	// check the source shard has the right blacklisted tables
+	si, err = ts.GetShard(ctx, "source", "0")
+	if err != nil {
+		t.Fatalf("GetShard failed: %v", err)
+	}
+	if len(si.TabletControlMap) != 2 || !reflect.DeepEqual(si.TabletControlMap[topo.TYPE_REPLICA].BlacklistedTables, []string{"gone1", "gone2"}) {
+		t.Fatalf("replica type doesn't have right blacklisted tables")
+	}
+
+	// migrate master over
+	if err := vp.Run([]string{"MigrateServedFrom", "dest/0", "master"}); err != nil {
+		t.Fatalf("MigrateServedFrom(master) failed: %v", err)
+	}
+
+	// make sure ServedFromMap is empty
+	ki, err = ts.GetKeyspace(ctx, "dest")
+	if err != nil {
+		t.Fatalf("GetKeyspace failed: %v", err)
+	}
+	if len(ki.ServedFromMap) > 0 {
+		t.Fatalf("dest keyspace still is ServedFrom: %v", ki.ServedFromMap)
+	}
+
+	// check the source shard has the right blacklisted tables
+	si, err = ts.GetShard(ctx, "source", "0")
+	if err != nil {
+		t.Fatalf("GetShard failed: %v", err)
+	}
+	if len(si.TabletControlMap) != 3 || !reflect.DeepEqual(si.TabletControlMap[topo.TYPE_MASTER].BlacklistedTables, []string{"gone1", "gone2"}) {
+		t.Fatalf("master type doesn't have right blacklisted tables")
+	}
+}

go/vt/wrangler/testlib/migrate_served_types_test.go

@@ -0,0 +1,168 @@
+// Copyright 2015, Google Inc. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package testlib
+
+import (
+	"testing"
+	"time"
+
+	mproto "github.com/youtube/vitess/go/mysql/proto"
+	"github.com/youtube/vitess/go/sqltypes"
+	"github.com/youtube/vitess/go/vt/logutil"
+	myproto "github.com/youtube/vitess/go/vt/mysqlctl/proto"
+	"github.com/youtube/vitess/go/vt/tabletmanager/tmclient"
+	"github.com/youtube/vitess/go/vt/topo"
+	"github.com/youtube/vitess/go/vt/wrangler"
+	"github.com/youtube/vitess/go/vt/zktopo"
+	"golang.org/x/net/context"
+)
+
+func checkShardServedTypes(t *testing.T, ts topo.Server, shard string, expected int) {
+	ctx := context.Background()
+	si, err := ts.GetShard(ctx, "ks", shard)
+	if err != nil {
+		t.Fatalf("GetShard failed: %v", err)
+	}
+	if len(si.ServedTypesMap) != expected {
+		t.Fatalf("shard %v has wrong served types: %#v", shard, si.ServedTypesMap)
+	}
+}
+
+func TestMigrateServedTypes(t *testing.T) {
+	ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"})
+	wr := wrangler.New(logutil.NewConsoleLogger(), ts, tmclient.NewTabletManagerClient(), time.Second)
+	vp := NewVtctlPipe(t, ts)
+	defer vp.Close()
+
+	// create the source shard
+	sourceMaster := NewFakeTablet(t, wr, "cell1", 10, topo.TYPE_MASTER,
+		TabletKeyspaceShard(t, "ks", "0"))
+	sourceReplica := NewFakeTablet(t, wr, "cell1", 11, topo.TYPE_REPLICA,
+		TabletKeyspaceShard(t, "ks", "0"))
+	sourceRdonly := NewFakeTablet(t, wr, "cell1", 12, topo.TYPE_RDONLY,
+		TabletKeyspaceShard(t, "ks", "0"))
+
+	// create the first destination shard
+	dest1Master := NewFakeTablet(t, wr, "cell1", 20, topo.TYPE_MASTER,
+		TabletKeyspaceShard(t, "ks", "-80"))
+	dest1Replica := NewFakeTablet(t, wr, "cell1", 21, topo.TYPE_REPLICA,
+		TabletKeyspaceShard(t, "ks", "-80"))
+	dest1Rdonly := NewFakeTablet(t, wr, "cell1", 22, topo.TYPE_RDONLY,
+		TabletKeyspaceShard(t, "ks", "-80"))
+
+	// create the second destination shard
+	dest2Master := NewFakeTablet(t, wr, "cell1", 30, topo.TYPE_MASTER,
+		TabletKeyspaceShard(t, "ks", "80-"))
+	dest2Replica := NewFakeTablet(t, wr, "cell1", 31, topo.TYPE_REPLICA,
+		TabletKeyspaceShard(t, "ks", "80-"))
+	dest2Rdonly := NewFakeTablet(t, wr, "cell1", 32, topo.TYPE_RDONLY,
+		TabletKeyspaceShard(t, "ks", "80-"))
+
+	// double check the shards have the right served types
+	checkShardServedTypes(t, ts, "0", 3)
+	checkShardServedTypes(t, ts, "-80", 0)
+	checkShardServedTypes(t, ts, "80-", 0)
+
+	// sourceRdonly will see the refresh
+	sourceRdonly.StartActionLoop(t, wr)
+	defer sourceRdonly.StopActionLoop(t)
+
+	// sourceReplica will see the refresh
+	sourceReplica.StartActionLoop(t, wr)
+	defer sourceReplica.StopActionLoop(t)
+
+	// sourceMaster will see the refresh, and has to respond to it
+	// also will be asked about its replication position.
+	sourceMaster.FakeMysqlDaemon.CurrentMasterPosition = myproto.ReplicationPosition{
+		GTIDSet: myproto.MariadbGTID{
+			Domain:   5,
+			Server:   456,
+			Sequence: 892,
+		},
+	}
+	sourceMaster.StartActionLoop(t, wr)
+	defer sourceMaster.StopActionLoop(t)
+
+	// dest1Rdonly will see the refresh
+	dest1Rdonly.StartActionLoop(t, wr)
+	defer dest1Rdonly.StopActionLoop(t)
+
+	// dest1Replica will see the refresh
+	dest1Replica.StartActionLoop(t, wr)
+	defer dest1Replica.StopActionLoop(t)
+
+	// dest1Master will see the refresh, and has to respond to it.
+	// It will also need to respond to WaitBlpPosition, saying it's already caught up.
+	dest1Master.FakeMysqlDaemon.FetchSuperQueryMap = map[string]*mproto.QueryResult{
+		"SELECT pos, flags FROM _vt.blp_checkpoint WHERE source_shard_uid=0": &mproto.QueryResult{
+			Rows: [][]sqltypes.Value{
+				[]sqltypes.Value{
+					sqltypes.MakeString([]byte(myproto.EncodeReplicationPosition(sourceMaster.FakeMysqlDaemon.CurrentMasterPosition))),
+					sqltypes.MakeString([]byte("")),
+				},
+			},
+		},
+	}
+	dest1Master.StartActionLoop(t, wr)
+	defer dest1Master.StopActionLoop(t)
+
+	// dest2Rdonly will see the refresh
+	dest2Rdonly.StartActionLoop(t, wr)
+	defer dest2Rdonly.StopActionLoop(t)
+
+	// dest2Replica will see the refresh
+	dest2Replica.StartActionLoop(t, wr)
+	defer dest2Replica.StopActionLoop(t)
+
+	// dest2Master will see the refresh, and has to respond to it.
+	// It will also need to respond to WaitBlpPosition, saying it's already caught up.
+	dest2Master.FakeMysqlDaemon.FetchSuperQueryMap = map[string]*mproto.QueryResult{
+		"SELECT pos, flags FROM _vt.blp_checkpoint WHERE source_shard_uid=0": &mproto.QueryResult{
+			Rows: [][]sqltypes.Value{
+				[]sqltypes.Value{
+					sqltypes.MakeString([]byte(myproto.EncodeReplicationPosition(sourceMaster.FakeMysqlDaemon.CurrentMasterPosition))),
+					sqltypes.MakeString([]byte("")),
+				},
+			},
+		},
+	}
+	dest2Master.StartActionLoop(t, wr)
+	defer dest2Master.StopActionLoop(t)
+
+	// simulate the clone, by fixing the dest shard record
+	if err := vp.Run([]string{"SourceShardAdd", "--key_range=-", "ks/-80", "0", "ks/0"}); err != nil {
+		t.Fatalf("SourceShardAdd failed: %v", err)
+	}
+	if err := vp.Run([]string{"SourceShardAdd", "--key_range=-", "ks/80-", "0", "ks/0"}); err != nil {
+		t.Fatalf("SourceShardAdd failed: %v", err)
+	}
+
+	// migrate rdonly over
+	if err := vp.Run([]string{"MigrateServedTypes", "ks/0", "rdonly"}); err != nil {
+		t.Fatalf("MigrateServedType(rdonly) failed: %v", err)
+	}
+
+	checkShardServedTypes(t, ts, "0", 2)
+	checkShardServedTypes(t, ts, "-80", 1)
+	checkShardServedTypes(t, ts, "80-", 1)
+
+	// migrate replica over
+	if err := vp.Run([]string{"MigrateServedTypes", "ks/0", "replica"}); err != nil {
+		t.Fatalf("MigrateServedType(replica) failed: %v", err)
+	}
+
+	checkShardServedTypes(t, ts, "0", 1)
+	checkShardServedTypes(t, ts, "-80", 2)
+	checkShardServedTypes(t, ts, "80-", 2)
+
+	// migrate master over
+	if err := vp.Run([]string{"MigrateServedTypes", "ks/0", "master"}); err != nil {
+		t.Fatalf("MigrateServedType(master) failed: %v", err)
+	}
+
+	checkShardServedTypes(t, ts, "0", 0)
+	checkShardServedTypes(t, ts, "-80", 3)
+	checkShardServedTypes(t, ts, "80-", 3)
+}