diff --git a/.gitignore b/.gitignore index fff60c139df..6b9bbeb787f 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,10 @@ *.swp tags +# Eclipse files +.project +.settings/ + # cbson & other C build dirs **/build @@ -21,3 +25,7 @@ third_party/mysql java/vtocc-client/target java/vtocc-jdbc-driver/target third_party/acolyte + +# intellij files +*.iml +.idea diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000000..26ba8f466cf --- /dev/null +++ b/.travis.yml @@ -0,0 +1,21 @@ +language: go +go: + - 1.4 +env: + global: + - MYSQL_FLAVOR=MariaDB + matrix: + - MAKE_TARGET=java_vtgate_client_test + - MAKE_TARGET=unit_test_goveralls + - MAKE_TARGET=small_integration_test + - MAKE_TARGET=medium_integration_test + - MAKE_TARGET=large_integration_test + - MAKE_TARGET=queryservice_test + - MAKE_TARGET=unit_test +before_install: + - bash -v travis/dependencies.sh +install: + - bash -v bootstrap.sh +script: + - source dev.env + - travis_retry make $MAKE_TARGET diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json new file mode 100644 index 00000000000..41cbb1bedcb --- /dev/null +++ b/Godeps/Godeps.json @@ -0,0 +1,14 @@ +{ + "ImportPath": "github.com/youtube/vitess", + "GoVersion": "go1.4.1", + "Packages": [ + "./go/vt/etcdtopo" + ], + "Deps": [ + { + "ImportPath": "github.com/coreos/go-etcd/etcd", + "Comment": "v0.2.0-rc1-127-g6fe04d5", + "Rev": "6fe04d580dfb71c9e34cbce2f4df9eefd1e1241e" + } + ] +} diff --git a/Godeps/Readme b/Godeps/Readme new file mode 100644 index 00000000000..4cdaa53d56d --- /dev/null +++ b/Godeps/Readme @@ -0,0 +1,5 @@ +This directory tree is generated automatically by godep. + +Please do not edit. + +See https://github.com/tools/godep for more information. diff --git a/Godeps/_workspace/.gitignore b/Godeps/_workspace/.gitignore new file mode 100644 index 00000000000..f037d684ef2 --- /dev/null +++ b/Godeps/_workspace/.gitignore @@ -0,0 +1,2 @@ +/pkg +/bin diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/add_child.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/add_child.go new file mode 100644 index 00000000000..7122be049e2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/add_child.go @@ -0,0 +1,23 @@ +package etcd + +// Add a new directory with a random etcd-generated key under the given path. +func (c *Client) AddChildDir(key string, ttl uint64) (*Response, error) { + raw, err := c.post(key, "", ttl) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +// Add a new file with a random etcd-generated key under the given path. +func (c *Client) AddChild(key string, value string, ttl uint64) (*Response, error) { + raw, err := c.post(key, value, ttl) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/add_child_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/add_child_test.go new file mode 100644 index 00000000000..26223ff1c85 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/add_child_test.go @@ -0,0 +1,73 @@ +package etcd + +import "testing" + +func TestAddChild(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("fooDir", true) + c.Delete("nonexistentDir", true) + }() + + c.CreateDir("fooDir", 5) + + _, err := c.AddChild("fooDir", "v0", 5) + if err != nil { + t.Fatal(err) + } + + _, err = c.AddChild("fooDir", "v1", 5) + if err != nil { + t.Fatal(err) + } + + resp, err := c.Get("fooDir", true, false) + // The child with v0 should proceed the child with v1 because it's added + // earlier, so it should have a lower key. + if !(len(resp.Node.Nodes) == 2 && (resp.Node.Nodes[0].Value == "v0" && resp.Node.Nodes[1].Value == "v1")) { + t.Fatalf("AddChild 1 failed. There should be two chlidren whose values are v0 and v1, respectively."+ + " The response was: %#v", resp) + } + + // Creating a child under a nonexistent directory should succeed. + // The directory should be created. + resp, err = c.AddChild("nonexistentDir", "foo", 5) + if err != nil { + t.Fatal(err) + } +} + +func TestAddChildDir(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("fooDir", true) + c.Delete("nonexistentDir", true) + }() + + c.CreateDir("fooDir", 5) + + _, err := c.AddChildDir("fooDir", 5) + if err != nil { + t.Fatal(err) + } + + _, err = c.AddChildDir("fooDir", 5) + if err != nil { + t.Fatal(err) + } + + resp, err := c.Get("fooDir", true, false) + // The child with v0 should proceed the child with v1 because it's added + // earlier, so it should have a lower key. + if !(len(resp.Node.Nodes) == 2 && (len(resp.Node.Nodes[0].Nodes) == 0 && len(resp.Node.Nodes[1].Nodes) == 0)) { + t.Fatalf("AddChildDir 1 failed. There should be two chlidren whose values are v0 and v1, respectively."+ + " The response was: %#v", resp) + } + + // Creating a child under a nonexistent directory should succeed. + // The directory should be created. + resp, err = c.AddChildDir("nonexistentDir", 5) + if err != nil { + t.Fatal(err) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client.go new file mode 100644 index 00000000000..f6ae5486173 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client.go @@ -0,0 +1,435 @@ +package etcd + +import ( + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" + "os" + "path" + "time" +) + +// See SetConsistency for how to use these constants. +const ( + // Using strings rather than iota because the consistency level + // could be persisted to disk, so it'd be better to use + // human-readable values. + STRONG_CONSISTENCY = "STRONG" + WEAK_CONSISTENCY = "WEAK" +) + +const ( + defaultBufferSize = 10 +) + +type Config struct { + CertFile string `json:"certFile"` + KeyFile string `json:"keyFile"` + CaCertFile []string `json:"caCertFiles"` + DialTimeout time.Duration `json:"timeout"` + Consistency string `json:"consistency"` +} + +type Client struct { + config Config `json:"config"` + cluster *Cluster `json:"cluster"` + httpClient *http.Client + persistence io.Writer + cURLch chan string + // CheckRetry can be used to control the policy for failed requests + // and modify the cluster if needed. + // The client calls it before sending requests again, and + // stops retrying if CheckRetry returns some error. The cases that + // this function needs to handle include no response and unexpected + // http status code of response. + // If CheckRetry is nil, client will call the default one + // `DefaultCheckRetry`. + // Argument cluster is the etcd.Cluster object that these requests have been made on. + // Argument numReqs is the number of http.Requests that have been made so far. + // Argument lastResp is the http.Responses from the last request. + // Argument err is the reason of the failure. + CheckRetry func(cluster *Cluster, numReqs int, + lastResp http.Response, err error) error +} + +// NewClient create a basic client that is configured to be used +// with the given machine list. +func NewClient(machines []string) *Client { + config := Config{ + // default timeout is one second + DialTimeout: time.Second, + // default consistency level is STRONG + Consistency: STRONG_CONSISTENCY, + } + + client := &Client{ + cluster: NewCluster(machines), + config: config, + } + + client.initHTTPClient() + client.saveConfig() + + return client +} + +// NewTLSClient create a basic client with TLS configuration +func NewTLSClient(machines []string, cert, key, caCert string) (*Client, error) { + // overwrite the default machine to use https + if len(machines) == 0 { + machines = []string{"https://127.0.0.1:4001"} + } + + config := Config{ + // default timeout is one second + DialTimeout: time.Second, + // default consistency level is STRONG + Consistency: STRONG_CONSISTENCY, + CertFile: cert, + KeyFile: key, + CaCertFile: make([]string, 0), + } + + client := &Client{ + cluster: NewCluster(machines), + config: config, + } + + err := client.initHTTPSClient(cert, key) + if err != nil { + return nil, err + } + + err = client.AddRootCA(caCert) + + client.saveConfig() + + return client, nil +} + +// NewClientFromFile creates a client from a given file path. +// The given file is expected to use the JSON format. +func NewClientFromFile(fpath string) (*Client, error) { + fi, err := os.Open(fpath) + if err != nil { + return nil, err + } + + defer func() { + if err := fi.Close(); err != nil { + panic(err) + } + }() + + return NewClientFromReader(fi) +} + +// NewClientFromReader creates a Client configured from a given reader. +// The configuration is expected to use the JSON format. +func NewClientFromReader(reader io.Reader) (*Client, error) { + c := new(Client) + + b, err := ioutil.ReadAll(reader) + if err != nil { + return nil, err + } + + err = json.Unmarshal(b, c) + if err != nil { + return nil, err + } + if c.config.CertFile == "" { + c.initHTTPClient() + } else { + err = c.initHTTPSClient(c.config.CertFile, c.config.KeyFile) + } + + if err != nil { + return nil, err + } + + for _, caCert := range c.config.CaCertFile { + if err := c.AddRootCA(caCert); err != nil { + return nil, err + } + } + + return c, nil +} + +// Override the Client's HTTP Transport object +func (c *Client) SetTransport(tr *http.Transport) { + c.httpClient.Transport = tr +} + +// initHTTPClient initializes a HTTP client for etcd client +func (c *Client) initHTTPClient() { + tr := &http.Transport{ + Dial: c.dial, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + c.httpClient = &http.Client{Transport: tr} +} + +// initHTTPClient initializes a HTTPS client for etcd client +func (c *Client) initHTTPSClient(cert, key string) error { + if cert == "" || key == "" { + return errors.New("Require both cert and key path") + } + + tlsCert, err := tls.LoadX509KeyPair(cert, key) + if err != nil { + return err + } + + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{tlsCert}, + InsecureSkipVerify: true, + } + + tr := &http.Transport{ + TLSClientConfig: tlsConfig, + Dial: c.dial, + } + + c.httpClient = &http.Client{Transport: tr} + return nil +} + +// SetPersistence sets a writer to which the config will be +// written every time it's changed. +func (c *Client) SetPersistence(writer io.Writer) { + c.persistence = writer +} + +// SetConsistency changes the consistency level of the client. +// +// When consistency is set to STRONG_CONSISTENCY, all requests, +// including GET, are sent to the leader. This means that, assuming +// the absence of leader failures, GET requests are guaranteed to see +// the changes made by previous requests. +// +// When consistency is set to WEAK_CONSISTENCY, other requests +// are still sent to the leader, but GET requests are sent to a +// random server from the server pool. This reduces the read +// load on the leader, but it's not guaranteed that the GET requests +// will see changes made by previous requests (they might have not +// yet been committed on non-leader servers). +func (c *Client) SetConsistency(consistency string) error { + if !(consistency == STRONG_CONSISTENCY || consistency == WEAK_CONSISTENCY) { + return errors.New("The argument must be either STRONG_CONSISTENCY or WEAK_CONSISTENCY.") + } + c.config.Consistency = consistency + return nil +} + +// Sets the DialTimeout value +func (c *Client) SetDialTimeout(d time.Duration) { + c.config.DialTimeout = d +} + +// AddRootCA adds a root CA cert for the etcd client +func (c *Client) AddRootCA(caCert string) error { + if c.httpClient == nil { + return errors.New("Client has not been initialized yet!") + } + + certBytes, err := ioutil.ReadFile(caCert) + if err != nil { + return err + } + + tr, ok := c.httpClient.Transport.(*http.Transport) + + if !ok { + panic("AddRootCA(): Transport type assert should not fail") + } + + if tr.TLSClientConfig.RootCAs == nil { + caCertPool := x509.NewCertPool() + ok = caCertPool.AppendCertsFromPEM(certBytes) + if ok { + tr.TLSClientConfig.RootCAs = caCertPool + } + tr.TLSClientConfig.InsecureSkipVerify = false + } else { + ok = tr.TLSClientConfig.RootCAs.AppendCertsFromPEM(certBytes) + } + + if !ok { + err = errors.New("Unable to load caCert") + } + + c.config.CaCertFile = append(c.config.CaCertFile, caCert) + c.saveConfig() + + return err +} + +// SetCluster updates cluster information using the given machine list. +func (c *Client) SetCluster(machines []string) bool { + success := c.internalSyncCluster(machines) + return success +} + +func (c *Client) GetCluster() []string { + return c.cluster.Machines +} + +// SyncCluster updates the cluster information using the internal machine list. +func (c *Client) SyncCluster() bool { + return c.internalSyncCluster(c.cluster.Machines) +} + +// internalSyncCluster syncs cluster information using the given machine list. +func (c *Client) internalSyncCluster(machines []string) bool { + for _, machine := range machines { + httpPath := c.createHttpPath(machine, path.Join(version, "machines")) + resp, err := c.httpClient.Get(httpPath) + if err != nil { + // try another machine in the cluster + continue + } else { + b, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + // try another machine in the cluster + continue + } + + // update Machines List + c.cluster.updateFromStr(string(b)) + + // update leader + // the first one in the machine list is the leader + c.cluster.switchLeader(0) + + logger.Debug("sync.machines ", c.cluster.Machines) + c.saveConfig() + return true + } + } + return false +} + +// createHttpPath creates a complete HTTP URL. +// serverName should contain both the host name and a port number, if any. +func (c *Client) createHttpPath(serverName string, _path string) string { + u, err := url.Parse(serverName) + if err != nil { + panic(err) + } + + u.Path = path.Join(u.Path, _path) + + if u.Scheme == "" { + u.Scheme = "http" + } + return u.String() +} + +// dial attempts to open a TCP connection to the provided address, explicitly +// enabling keep-alives with a one-second interval. +func (c *Client) dial(network, addr string) (net.Conn, error) { + conn, err := net.DialTimeout(network, addr, c.config.DialTimeout) + if err != nil { + return nil, err + } + + tcpConn, ok := conn.(*net.TCPConn) + if !ok { + return nil, errors.New("Failed type-assertion of net.Conn as *net.TCPConn") + } + + // Keep TCP alive to check whether or not the remote machine is down + if err = tcpConn.SetKeepAlive(true); err != nil { + return nil, err + } + + if err = tcpConn.SetKeepAlivePeriod(time.Second); err != nil { + return nil, err + } + + return tcpConn, nil +} + +func (c *Client) OpenCURL() { + c.cURLch = make(chan string, defaultBufferSize) +} + +func (c *Client) CloseCURL() { + c.cURLch = nil +} + +func (c *Client) sendCURL(command string) { + go func() { + select { + case c.cURLch <- command: + default: + } + }() +} + +func (c *Client) RecvCURL() string { + return <-c.cURLch +} + +// saveConfig saves the current config using c.persistence. +func (c *Client) saveConfig() error { + if c.persistence != nil { + b, err := json.Marshal(c) + if err != nil { + return err + } + + _, err = c.persistence.Write(b) + if err != nil { + return err + } + } + + return nil +} + +// MarshalJSON implements the Marshaller interface +// as defined by the standard JSON package. +func (c *Client) MarshalJSON() ([]byte, error) { + b, err := json.Marshal(struct { + Config Config `json:"config"` + Cluster *Cluster `json:"cluster"` + }{ + Config: c.config, + Cluster: c.cluster, + }) + + if err != nil { + return nil, err + } + + return b, nil +} + +// UnmarshalJSON implements the Unmarshaller interface +// as defined by the standard JSON package. +func (c *Client) UnmarshalJSON(b []byte) error { + temp := struct { + Config Config `json:"config"` + Cluster *Cluster `json:"cluster"` + }{} + err := json.Unmarshal(b, &temp) + if err != nil { + return err + } + + c.cluster = temp.Cluster + c.config = temp.Config + return nil +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client_test.go new file mode 100644 index 00000000000..c245e479844 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/client_test.go @@ -0,0 +1,96 @@ +package etcd + +import ( + "encoding/json" + "fmt" + "net" + "net/url" + "os" + "testing" +) + +// To pass this test, we need to create a cluster of 3 machines +// The server should be listening on 127.0.0.1:4001, 4002, 4003 +func TestSync(t *testing.T) { + fmt.Println("Make sure there are three nodes at 0.0.0.0:4001-4003") + + // Explicit trailing slash to ensure this doesn't reproduce: + // https://github.com/coreos/go-etcd/issues/82 + c := NewClient([]string{"http://127.0.0.1:4001/"}) + + success := c.SyncCluster() + if !success { + t.Fatal("cannot sync machines") + } + + for _, m := range c.GetCluster() { + u, err := url.Parse(m) + if err != nil { + t.Fatal(err) + } + if u.Scheme != "http" { + t.Fatal("scheme must be http") + } + + host, _, err := net.SplitHostPort(u.Host) + if err != nil { + t.Fatal(err) + } + if host != "127.0.0.1" { + t.Fatal("Host must be 127.0.0.1") + } + } + + badMachines := []string{"abc", "edef"} + + success = c.SetCluster(badMachines) + + if success { + t.Fatal("should not sync on bad machines") + } + + goodMachines := []string{"127.0.0.1:4002"} + + success = c.SetCluster(goodMachines) + + if !success { + t.Fatal("cannot sync machines") + } else { + fmt.Println(c.cluster.Machines) + } + +} + +func TestPersistence(t *testing.T) { + c := NewClient(nil) + c.SyncCluster() + + fo, err := os.Create("config.json") + if err != nil { + t.Fatal(err) + } + defer func() { + if err := fo.Close(); err != nil { + panic(err) + } + }() + + c.SetPersistence(fo) + err = c.saveConfig() + if err != nil { + t.Fatal(err) + } + + c2, err := NewClientFromFile("config.json") + if err != nil { + t.Fatal(err) + } + + // Verify that the two clients have the same config + b1, _ := json.Marshal(c) + b2, _ := json.Marshal(c2) + + if string(b1) != string(b2) { + t.Fatalf("The two configs should be equal!") + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/cluster.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/cluster.go new file mode 100644 index 00000000000..aaa20546e32 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/cluster.go @@ -0,0 +1,51 @@ +package etcd + +import ( + "net/url" + "strings" +) + +type Cluster struct { + Leader string `json:"leader"` + Machines []string `json:"machines"` +} + +func NewCluster(machines []string) *Cluster { + // if an empty slice was sent in then just assume HTTP 4001 on localhost + if len(machines) == 0 { + machines = []string{"http://127.0.0.1:4001"} + } + + // default leader and machines + return &Cluster{ + Leader: machines[0], + Machines: machines, + } +} + +// switchLeader switch the current leader to machines[num] +func (cl *Cluster) switchLeader(num int) { + logger.Debugf("switch.leader[from %v to %v]", + cl.Leader, cl.Machines[num]) + + cl.Leader = cl.Machines[num] +} + +func (cl *Cluster) updateFromStr(machines string) { + cl.Machines = strings.Split(machines, ", ") +} + +func (cl *Cluster) updateLeader(leader string) { + logger.Debugf("update.leader[%s,%s]", cl.Leader, leader) + cl.Leader = leader +} + +func (cl *Cluster) updateLeaderFromURL(u *url.URL) { + var leader string + if u.Scheme == "" { + leader = "http://" + u.Host + } else { + leader = u.Scheme + "://" + u.Host + } + cl.updateLeader(leader) +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_delete.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_delete.go new file mode 100644 index 00000000000..11131bb7602 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_delete.go @@ -0,0 +1,34 @@ +package etcd + +import "fmt" + +func (c *Client) CompareAndDelete(key string, prevValue string, prevIndex uint64) (*Response, error) { + raw, err := c.RawCompareAndDelete(key, prevValue, prevIndex) + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +func (c *Client) RawCompareAndDelete(key string, prevValue string, prevIndex uint64) (*RawResponse, error) { + if prevValue == "" && prevIndex == 0 { + return nil, fmt.Errorf("You must give either prevValue or prevIndex.") + } + + options := Options{} + if prevValue != "" { + options["prevValue"] = prevValue + } + if prevIndex != 0 { + options["prevIndex"] = prevIndex + } + + raw, err := c.delete(key, options) + + if err != nil { + return nil, err + } + + return raw, err +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_delete_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_delete_test.go new file mode 100644 index 00000000000..223e50f2916 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_delete_test.go @@ -0,0 +1,46 @@ +package etcd + +import ( + "testing" +) + +func TestCompareAndDelete(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("foo", true) + }() + + c.Set("foo", "bar", 5) + + // This should succeed an correct prevValue + resp, err := c.CompareAndDelete("foo", "bar", 0) + if err != nil { + t.Fatal(err) + } + if !(resp.PrevNode.Value == "bar" && resp.PrevNode.Key == "/foo" && resp.PrevNode.TTL == 5) { + t.Fatalf("CompareAndDelete 1 prevNode failed: %#v", resp) + } + + resp, _ = c.Set("foo", "bar", 5) + // This should fail because it gives an incorrect prevValue + _, err = c.CompareAndDelete("foo", "xxx", 0) + if err == nil { + t.Fatalf("CompareAndDelete 2 should have failed. The response is: %#v", resp) + } + + // This should succeed because it gives an correct prevIndex + resp, err = c.CompareAndDelete("foo", "", resp.Node.ModifiedIndex) + if err != nil { + t.Fatal(err) + } + if !(resp.PrevNode.Value == "bar" && resp.PrevNode.Key == "/foo" && resp.PrevNode.TTL == 5) { + t.Fatalf("CompareAndSwap 3 prevNode failed: %#v", resp) + } + + c.Set("foo", "bar", 5) + // This should fail because it gives an incorrect prevIndex + resp, err = c.CompareAndDelete("foo", "", 29817514) + if err == nil { + t.Fatalf("CompareAndDelete 4 should have failed. The response is: %#v", resp) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_swap.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_swap.go new file mode 100644 index 00000000000..bb4f90643ac --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_swap.go @@ -0,0 +1,36 @@ +package etcd + +import "fmt" + +func (c *Client) CompareAndSwap(key string, value string, ttl uint64, + prevValue string, prevIndex uint64) (*Response, error) { + raw, err := c.RawCompareAndSwap(key, value, ttl, prevValue, prevIndex) + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +func (c *Client) RawCompareAndSwap(key string, value string, ttl uint64, + prevValue string, prevIndex uint64) (*RawResponse, error) { + if prevValue == "" && prevIndex == 0 { + return nil, fmt.Errorf("You must give either prevValue or prevIndex.") + } + + options := Options{} + if prevValue != "" { + options["prevValue"] = prevValue + } + if prevIndex != 0 { + options["prevIndex"] = prevIndex + } + + raw, err := c.put(key, value, ttl, options) + + if err != nil { + return nil, err + } + + return raw, err +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_swap_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_swap_test.go new file mode 100644 index 00000000000..14a1b00f5a7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/compare_and_swap_test.go @@ -0,0 +1,57 @@ +package etcd + +import ( + "testing" +) + +func TestCompareAndSwap(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("foo", true) + }() + + c.Set("foo", "bar", 5) + + // This should succeed + resp, err := c.CompareAndSwap("foo", "bar2", 5, "bar", 0) + if err != nil { + t.Fatal(err) + } + if !(resp.Node.Value == "bar2" && resp.Node.Key == "/foo" && resp.Node.TTL == 5) { + t.Fatalf("CompareAndSwap 1 failed: %#v", resp) + } + + if !(resp.PrevNode.Value == "bar" && resp.PrevNode.Key == "/foo" && resp.PrevNode.TTL == 5) { + t.Fatalf("CompareAndSwap 1 prevNode failed: %#v", resp) + } + + // This should fail because it gives an incorrect prevValue + resp, err = c.CompareAndSwap("foo", "bar3", 5, "xxx", 0) + if err == nil { + t.Fatalf("CompareAndSwap 2 should have failed. The response is: %#v", resp) + } + + resp, err = c.Set("foo", "bar", 5) + if err != nil { + t.Fatal(err) + } + + // This should succeed + resp, err = c.CompareAndSwap("foo", "bar2", 5, "", resp.Node.ModifiedIndex) + if err != nil { + t.Fatal(err) + } + if !(resp.Node.Value == "bar2" && resp.Node.Key == "/foo" && resp.Node.TTL == 5) { + t.Fatalf("CompareAndSwap 3 failed: %#v", resp) + } + + if !(resp.PrevNode.Value == "bar" && resp.PrevNode.Key == "/foo" && resp.PrevNode.TTL == 5) { + t.Fatalf("CompareAndSwap 3 prevNode failed: %#v", resp) + } + + // This should fail because it gives an incorrect prevIndex + resp, err = c.CompareAndSwap("foo", "bar3", 5, "", 29817514) + if err == nil { + t.Fatalf("CompareAndSwap 4 should have failed. The response is: %#v", resp) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/debug.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/debug.go new file mode 100644 index 00000000000..0f777886bae --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/debug.go @@ -0,0 +1,55 @@ +package etcd + +import ( + "fmt" + "io/ioutil" + "log" + "strings" +) + +var logger *etcdLogger + +func SetLogger(l *log.Logger) { + logger = &etcdLogger{l} +} + +func GetLogger() *log.Logger { + return logger.log +} + +type etcdLogger struct { + log *log.Logger +} + +func (p *etcdLogger) Debug(args ...interface{}) { + msg := "DEBUG: " + fmt.Sprint(args...) + p.log.Println(msg) +} + +func (p *etcdLogger) Debugf(f string, args ...interface{}) { + msg := "DEBUG: " + fmt.Sprintf(f, args...) + // Append newline if necessary + if !strings.HasSuffix(msg, "\n") { + msg = msg + "\n" + } + p.log.Print(msg) +} + +func (p *etcdLogger) Warning(args ...interface{}) { + msg := "WARNING: " + fmt.Sprint(args...) + p.log.Println(msg) +} + +func (p *etcdLogger) Warningf(f string, args ...interface{}) { + msg := "WARNING: " + fmt.Sprintf(f, args...) + // Append newline if necessary + if !strings.HasSuffix(msg, "\n") { + msg = msg + "\n" + } + p.log.Print(msg) +} + +func init() { + // Default logger uses the go default log. + SetLogger(log.New(ioutil.Discard, "go-etcd", log.LstdFlags)) +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/debug_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/debug_test.go new file mode 100644 index 00000000000..97f6d1110bc --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/debug_test.go @@ -0,0 +1,28 @@ +package etcd + +import ( + "testing" +) + +type Foo struct{} +type Bar struct { + one string + two int +} + +// Tests that logs don't panic with arbitrary interfaces +func TestDebug(t *testing.T) { + f := &Foo{} + b := &Bar{"asfd", 3} + for _, test := range []interface{}{ + 1234, + "asdf", + f, + b, + } { + logger.Debug(test) + logger.Debugf("something, %s", test) + logger.Warning(test) + logger.Warningf("something, %s", test) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/delete.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/delete.go new file mode 100644 index 00000000000..b37accd7db3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/delete.go @@ -0,0 +1,40 @@ +package etcd + +// Delete deletes the given key. +// +// When recursive set to false, if the key points to a +// directory the method will fail. +// +// When recursive set to true, if the key points to a file, +// the file will be deleted; if the key points to a directory, +// then everything under the directory (including all child directories) +// will be deleted. +func (c *Client) Delete(key string, recursive bool) (*Response, error) { + raw, err := c.RawDelete(key, recursive, false) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +// DeleteDir deletes an empty directory or a key value pair +func (c *Client) DeleteDir(key string) (*Response, error) { + raw, err := c.RawDelete(key, false, true) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +func (c *Client) RawDelete(key string, recursive bool, dir bool) (*RawResponse, error) { + ops := Options{ + "recursive": recursive, + "dir": dir, + } + + return c.delete(key, ops) +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/delete_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/delete_test.go new file mode 100644 index 00000000000..5904971556d --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/delete_test.go @@ -0,0 +1,81 @@ +package etcd + +import ( + "testing" +) + +func TestDelete(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("foo", true) + }() + + c.Set("foo", "bar", 5) + resp, err := c.Delete("foo", false) + if err != nil { + t.Fatal(err) + } + + if !(resp.Node.Value == "") { + t.Fatalf("Delete failed with %s", resp.Node.Value) + } + + if !(resp.PrevNode.Value == "bar") { + t.Fatalf("Delete PrevNode failed with %s", resp.Node.Value) + } + + resp, err = c.Delete("foo", false) + if err == nil { + t.Fatalf("Delete should have failed because the key foo did not exist. "+ + "The response was: %v", resp) + } +} + +func TestDeleteAll(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("foo", true) + c.Delete("fooDir", true) + }() + + c.SetDir("foo", 5) + // test delete an empty dir + resp, err := c.DeleteDir("foo") + if err != nil { + t.Fatal(err) + } + + if !(resp.Node.Value == "") { + t.Fatalf("DeleteAll 1 failed: %#v", resp) + } + + if !(resp.PrevNode.Dir == true && resp.PrevNode.Value == "") { + t.Fatalf("DeleteAll 1 PrevNode failed: %#v", resp) + } + + c.CreateDir("fooDir", 5) + c.Set("fooDir/foo", "bar", 5) + _, err = c.DeleteDir("fooDir") + if err == nil { + t.Fatal("should not able to delete a non-empty dir with deletedir") + } + + resp, err = c.Delete("fooDir", true) + if err != nil { + t.Fatal(err) + } + + if !(resp.Node.Value == "") { + t.Fatalf("DeleteAll 2 failed: %#v", resp) + } + + if !(resp.PrevNode.Dir == true && resp.PrevNode.Value == "") { + t.Fatalf("DeleteAll 2 PrevNode failed: %#v", resp) + } + + resp, err = c.Delete("foo", true) + if err == nil { + t.Fatalf("DeleteAll should have failed because the key foo did not exist. "+ + "The response was: %v", resp) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/error.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/error.go new file mode 100644 index 00000000000..7e692872472 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/error.go @@ -0,0 +1,48 @@ +package etcd + +import ( + "encoding/json" + "fmt" +) + +const ( + ErrCodeEtcdNotReachable = 501 +) + +var ( + errorMap = map[int]string{ + ErrCodeEtcdNotReachable: "All the given peers are not reachable", + } +) + +type EtcdError struct { + ErrorCode int `json:"errorCode"` + Message string `json:"message"` + Cause string `json:"cause,omitempty"` + Index uint64 `json:"index"` +} + +func (e EtcdError) Error() string { + return fmt.Sprintf("%v: %v (%v) [%v]", e.ErrorCode, e.Message, e.Cause, e.Index) +} + +func newError(errorCode int, cause string, index uint64) *EtcdError { + return &EtcdError{ + ErrorCode: errorCode, + Message: errorMap[errorCode], + Cause: cause, + Index: index, + } +} + +func handleError(b []byte) error { + etcdErr := new(EtcdError) + + err := json.Unmarshal(b, etcdErr) + if err != nil { + logger.Warningf("cannot unmarshal etcd error: %v", err) + return err + } + + return etcdErr +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/get.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/get.go new file mode 100644 index 00000000000..976bf07fd74 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/get.go @@ -0,0 +1,27 @@ +package etcd + +// Get gets the file or directory associated with the given key. +// If the key points to a directory, files and directories under +// it will be returned in sorted or unsorted order, depending on +// the sort flag. +// If recursive is set to false, contents under child directories +// will not be returned. +// If recursive is set to true, all the contents will be returned. +func (c *Client) Get(key string, sort, recursive bool) (*Response, error) { + raw, err := c.RawGet(key, sort, recursive) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +func (c *Client) RawGet(key string, sort, recursive bool) (*RawResponse, error) { + ops := Options{ + "recursive": recursive, + "sorted": sort, + } + + return c.get(key, ops) +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/get_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/get_test.go new file mode 100644 index 00000000000..279c4e26f8b --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/get_test.go @@ -0,0 +1,131 @@ +package etcd + +import ( + "reflect" + "testing" +) + +// cleanNode scrubs Expiration, ModifiedIndex and CreatedIndex of a node. +func cleanNode(n *Node) { + n.Expiration = nil + n.ModifiedIndex = 0 + n.CreatedIndex = 0 +} + +// cleanResult scrubs a result object two levels deep of Expiration, +// ModifiedIndex and CreatedIndex. +func cleanResult(result *Response) { + // TODO(philips): make this recursive. + cleanNode(result.Node) + for i, _ := range result.Node.Nodes { + cleanNode(result.Node.Nodes[i]) + for j, _ := range result.Node.Nodes[i].Nodes { + cleanNode(result.Node.Nodes[i].Nodes[j]) + } + } +} + +func TestGet(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("foo", true) + }() + + c.Set("foo", "bar", 5) + + result, err := c.Get("foo", false, false) + + if err != nil { + t.Fatal(err) + } + + if result.Node.Key != "/foo" || result.Node.Value != "bar" { + t.Fatalf("Get failed with %s %s %v", result.Node.Key, result.Node.Value, result.Node.TTL) + } + + result, err = c.Get("goo", false, false) + if err == nil { + t.Fatalf("should not be able to get non-exist key") + } +} + +func TestGetAll(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("fooDir", true) + }() + + c.CreateDir("fooDir", 5) + c.Set("fooDir/k0", "v0", 5) + c.Set("fooDir/k1", "v1", 5) + + // Return kv-pairs in sorted order + result, err := c.Get("fooDir", true, false) + + if err != nil { + t.Fatal(err) + } + + expected := Nodes{ + &Node{ + Key: "/fooDir/k0", + Value: "v0", + TTL: 5, + }, + &Node{ + Key: "/fooDir/k1", + Value: "v1", + TTL: 5, + }, + } + + cleanResult(result) + + if !reflect.DeepEqual(result.Node.Nodes, expected) { + t.Fatalf("(actual) %v != (expected) %v", result.Node.Nodes, expected) + } + + // Test the `recursive` option + c.CreateDir("fooDir/childDir", 5) + c.Set("fooDir/childDir/k2", "v2", 5) + + // Return kv-pairs in sorted order + result, err = c.Get("fooDir", true, true) + + cleanResult(result) + + if err != nil { + t.Fatal(err) + } + + expected = Nodes{ + &Node{ + Key: "/fooDir/childDir", + Dir: true, + Nodes: Nodes{ + &Node{ + Key: "/fooDir/childDir/k2", + Value: "v2", + TTL: 5, + }, + }, + TTL: 5, + }, + &Node{ + Key: "/fooDir/k0", + Value: "v0", + TTL: 5, + }, + &Node{ + Key: "/fooDir/k1", + Value: "v1", + TTL: 5, + }, + } + + cleanResult(result) + + if !reflect.DeepEqual(result.Node.Nodes, expected) { + t.Fatalf("(actual) %v != (expected) %v", result.Node.Nodes, expected) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/options.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/options.go new file mode 100644 index 00000000000..701c9b35b97 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/options.go @@ -0,0 +1,72 @@ +package etcd + +import ( + "fmt" + "net/url" + "reflect" +) + +type Options map[string]interface{} + +// An internally-used data structure that represents a mapping +// between valid options and their kinds +type validOptions map[string]reflect.Kind + +// Valid options for GET, PUT, POST, DELETE +// Using CAPITALIZED_UNDERSCORE to emphasize that these +// values are meant to be used as constants. +var ( + VALID_GET_OPTIONS = validOptions{ + "recursive": reflect.Bool, + "consistent": reflect.Bool, + "sorted": reflect.Bool, + "wait": reflect.Bool, + "waitIndex": reflect.Uint64, + } + + VALID_PUT_OPTIONS = validOptions{ + "prevValue": reflect.String, + "prevIndex": reflect.Uint64, + "prevExist": reflect.Bool, + "dir": reflect.Bool, + } + + VALID_POST_OPTIONS = validOptions{} + + VALID_DELETE_OPTIONS = validOptions{ + "recursive": reflect.Bool, + "dir": reflect.Bool, + "prevValue": reflect.String, + "prevIndex": reflect.Uint64, + } +) + +// Convert options to a string of HTML parameters +func (ops Options) toParameters(validOps validOptions) (string, error) { + p := "?" + values := url.Values{} + + if ops == nil { + return "", nil + } + + for k, v := range ops { + // Check if the given option is valid (that it exists) + kind := validOps[k] + if kind == reflect.Invalid { + return "", fmt.Errorf("Invalid option: %v", k) + } + + // Check if the given option is of the valid type + t := reflect.TypeOf(v) + if kind != t.Kind() { + return "", fmt.Errorf("Option %s should be of %v kind, not of %v kind.", + k, kind, t.Kind()) + } + + values.Set(k, fmt.Sprintf("%v", v)) + } + + p += values.Encode() + return p, nil +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests.go new file mode 100644 index 00000000000..fa6d36a9f2e --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/requests.go @@ -0,0 +1,396 @@ +package etcd + +import ( + "errors" + "fmt" + "io" + "io/ioutil" + "math/rand" + "net/http" + "net/url" + "path" + "strings" + "sync" + "time" +) + +// Errors introduced by handling requests +var ( + ErrRequestCancelled = errors.New("sending request is cancelled") +) + +type RawRequest struct { + Method string + RelativePath string + Values url.Values + Cancel <-chan bool +} + +// NewRawRequest returns a new RawRequest +func NewRawRequest(method, relativePath string, values url.Values, cancel <-chan bool) *RawRequest { + return &RawRequest{ + Method: method, + RelativePath: relativePath, + Values: values, + Cancel: cancel, + } +} + +// getCancelable issues a cancelable GET request +func (c *Client) getCancelable(key string, options Options, + cancel <-chan bool) (*RawResponse, error) { + logger.Debugf("get %s [%s]", key, c.cluster.Leader) + p := keyToPath(key) + + // If consistency level is set to STRONG, append + // the `consistent` query string. + if c.config.Consistency == STRONG_CONSISTENCY { + options["consistent"] = true + } + + str, err := options.toParameters(VALID_GET_OPTIONS) + if err != nil { + return nil, err + } + p += str + + req := NewRawRequest("GET", p, nil, cancel) + resp, err := c.SendRequest(req) + + if err != nil { + return nil, err + } + + return resp, nil +} + +// get issues a GET request +func (c *Client) get(key string, options Options) (*RawResponse, error) { + return c.getCancelable(key, options, nil) +} + +// put issues a PUT request +func (c *Client) put(key string, value string, ttl uint64, + options Options) (*RawResponse, error) { + + logger.Debugf("put %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.Leader) + p := keyToPath(key) + + str, err := options.toParameters(VALID_PUT_OPTIONS) + if err != nil { + return nil, err + } + p += str + + req := NewRawRequest("PUT", p, buildValues(value, ttl), nil) + resp, err := c.SendRequest(req) + + if err != nil { + return nil, err + } + + return resp, nil +} + +// post issues a POST request +func (c *Client) post(key string, value string, ttl uint64) (*RawResponse, error) { + logger.Debugf("post %s, %s, ttl: %d, [%s]", key, value, ttl, c.cluster.Leader) + p := keyToPath(key) + + req := NewRawRequest("POST", p, buildValues(value, ttl), nil) + resp, err := c.SendRequest(req) + + if err != nil { + return nil, err + } + + return resp, nil +} + +// delete issues a DELETE request +func (c *Client) delete(key string, options Options) (*RawResponse, error) { + logger.Debugf("delete %s [%s]", key, c.cluster.Leader) + p := keyToPath(key) + + str, err := options.toParameters(VALID_DELETE_OPTIONS) + if err != nil { + return nil, err + } + p += str + + req := NewRawRequest("DELETE", p, nil, nil) + resp, err := c.SendRequest(req) + + if err != nil { + return nil, err + } + + return resp, nil +} + +// SendRequest sends a HTTP request and returns a Response as defined by etcd +func (c *Client) SendRequest(rr *RawRequest) (*RawResponse, error) { + + var req *http.Request + var resp *http.Response + var httpPath string + var err error + var respBody []byte + + var numReqs = 1 + + checkRetry := c.CheckRetry + if checkRetry == nil { + checkRetry = DefaultCheckRetry + } + + cancelled := make(chan bool, 1) + reqLock := new(sync.Mutex) + + if rr.Cancel != nil { + cancelRoutine := make(chan bool) + defer close(cancelRoutine) + + go func() { + select { + case <-rr.Cancel: + cancelled <- true + logger.Debug("send.request is cancelled") + case <-cancelRoutine: + return + } + + // Repeat canceling request until this thread is stopped + // because we have no idea about whether it succeeds. + for { + reqLock.Lock() + c.httpClient.Transport.(*http.Transport).CancelRequest(req) + reqLock.Unlock() + + select { + case <-time.After(100 * time.Millisecond): + case <-cancelRoutine: + return + } + } + }() + } + + // If we connect to a follower and consistency is required, retry until + // we connect to a leader + sleep := 25 * time.Millisecond + maxSleep := time.Second + + for attempt := 0; ; attempt++ { + if attempt > 0 { + select { + case <-cancelled: + return nil, ErrRequestCancelled + case <-time.After(sleep): + sleep = sleep * 2 + if sleep > maxSleep { + sleep = maxSleep + } + } + } + + logger.Debug("Connecting to etcd: attempt ", attempt+1, " for ", rr.RelativePath) + + if rr.Method == "GET" && c.config.Consistency == WEAK_CONSISTENCY { + // If it's a GET and consistency level is set to WEAK, + // then use a random machine. + httpPath = c.getHttpPath(true, rr.RelativePath) + } else { + // Else use the leader. + httpPath = c.getHttpPath(false, rr.RelativePath) + } + + // Return a cURL command if curlChan is set + if c.cURLch != nil { + command := fmt.Sprintf("curl -X %s %s", rr.Method, httpPath) + for key, value := range rr.Values { + command += fmt.Sprintf(" -d %s=%s", key, value[0]) + } + c.sendCURL(command) + } + + logger.Debug("send.request.to ", httpPath, " | method ", rr.Method) + + req, err := func() (*http.Request, error) { + reqLock.Lock() + defer reqLock.Unlock() + + if rr.Values == nil { + if req, err = http.NewRequest(rr.Method, httpPath, nil); err != nil { + return nil, err + } + } else { + body := strings.NewReader(rr.Values.Encode()) + if req, err = http.NewRequest(rr.Method, httpPath, body); err != nil { + return nil, err + } + + req.Header.Set("Content-Type", + "application/x-www-form-urlencoded; param=value") + } + return req, nil + }() + + if err != nil { + return nil, err + } + + resp, err = c.httpClient.Do(req) + defer func() { + if resp != nil { + resp.Body.Close() + } + }() + + // If the request was cancelled, return ErrRequestCancelled directly + select { + case <-cancelled: + return nil, ErrRequestCancelled + default: + } + + numReqs++ + + // network error, change a machine! + if err != nil { + logger.Debug("network error: ", err.Error()) + lastResp := http.Response{} + if checkErr := checkRetry(c.cluster, numReqs, lastResp, err); checkErr != nil { + return nil, checkErr + } + + c.cluster.switchLeader(attempt % len(c.cluster.Machines)) + continue + } + + // if there is no error, it should receive response + logger.Debug("recv.response.from ", httpPath) + + if validHttpStatusCode[resp.StatusCode] { + // try to read byte code and break the loop + respBody, err = ioutil.ReadAll(resp.Body) + if err == nil { + logger.Debug("recv.success ", httpPath) + break + } + // ReadAll error may be caused due to cancel request + select { + case <-cancelled: + return nil, ErrRequestCancelled + default: + } + + if err == io.ErrUnexpectedEOF { + // underlying connection was closed prematurely, probably by timeout + // TODO: empty body or unexpectedEOF can cause http.Transport to get hosed; + // this allows the client to detect that and take evasive action. Need + // to revisit once code.google.com/p/go/issues/detail?id=8648 gets fixed. + respBody = []byte{} + break + } + } + + // if resp is TemporaryRedirect, set the new leader and retry + if resp.StatusCode == http.StatusTemporaryRedirect { + u, err := resp.Location() + + if err != nil { + logger.Warning(err) + } else { + // Update cluster leader based on redirect location + // because it should point to the leader address + c.cluster.updateLeaderFromURL(u) + logger.Debug("recv.response.relocate ", u.String()) + } + resp.Body.Close() + continue + } + + if checkErr := checkRetry(c.cluster, numReqs, *resp, + errors.New("Unexpected HTTP status code")); checkErr != nil { + return nil, checkErr + } + resp.Body.Close() + } + + r := &RawResponse{ + StatusCode: resp.StatusCode, + Body: respBody, + Header: resp.Header, + } + + return r, nil +} + +// DefaultCheckRetry defines the retrying behaviour for bad HTTP requests +// If we have retried 2 * machine number, stop retrying. +// If status code is InternalServerError, sleep for 200ms. +func DefaultCheckRetry(cluster *Cluster, numReqs int, lastResp http.Response, + err error) error { + + if numReqs >= 2*len(cluster.Machines) { + return newError(ErrCodeEtcdNotReachable, + "Tried to connect to each peer twice and failed", 0) + } + + code := lastResp.StatusCode + if code == http.StatusInternalServerError { + time.Sleep(time.Millisecond * 200) + + } + + logger.Warning("bad response status code", code) + return nil +} + +func (c *Client) getHttpPath(random bool, s ...string) string { + var machine string + if random { + machine = c.cluster.Machines[rand.Intn(len(c.cluster.Machines))] + } else { + machine = c.cluster.Leader + } + + fullPath := machine + "/" + version + for _, seg := range s { + fullPath = fullPath + "/" + seg + } + + return fullPath +} + +// buildValues builds a url.Values map according to the given value and ttl +func buildValues(value string, ttl uint64) url.Values { + v := url.Values{} + + if value != "" { + v.Set("value", value) + } + + if ttl > 0 { + v.Set("ttl", fmt.Sprintf("%v", ttl)) + } + + return v +} + +// convert key string to http path exclude version +// for example: key[foo] -> path[keys/foo] +// key[/] -> path[keys/] +func keyToPath(key string) string { + p := path.Join("keys", key) + + // corner case: if key is "/" or "//" ect + // path join will clear the tailing "/" + // we need to add it back + if p == "keys" { + p = "keys/" + } + + return p +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/response.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/response.go new file mode 100644 index 00000000000..1fe9b4e8711 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/response.go @@ -0,0 +1,89 @@ +package etcd + +import ( + "encoding/json" + "net/http" + "strconv" + "time" +) + +const ( + rawResponse = iota + normalResponse +) + +type responseType int + +type RawResponse struct { + StatusCode int + Body []byte + Header http.Header +} + +var ( + validHttpStatusCode = map[int]bool{ + http.StatusCreated: true, + http.StatusOK: true, + http.StatusBadRequest: true, + http.StatusNotFound: true, + http.StatusPreconditionFailed: true, + http.StatusForbidden: true, + } +) + +// Unmarshal parses RawResponse and stores the result in Response +func (rr *RawResponse) Unmarshal() (*Response, error) { + if rr.StatusCode != http.StatusOK && rr.StatusCode != http.StatusCreated { + return nil, handleError(rr.Body) + } + + resp := new(Response) + + err := json.Unmarshal(rr.Body, resp) + + if err != nil { + return nil, err + } + + // attach index and term to response + resp.EtcdIndex, _ = strconv.ParseUint(rr.Header.Get("X-Etcd-Index"), 10, 64) + resp.RaftIndex, _ = strconv.ParseUint(rr.Header.Get("X-Raft-Index"), 10, 64) + resp.RaftTerm, _ = strconv.ParseUint(rr.Header.Get("X-Raft-Term"), 10, 64) + + return resp, nil +} + +type Response struct { + Action string `json:"action"` + Node *Node `json:"node"` + PrevNode *Node `json:"prevNode,omitempty"` + EtcdIndex uint64 `json:"etcdIndex"` + RaftIndex uint64 `json:"raftIndex"` + RaftTerm uint64 `json:"raftTerm"` +} + +type Node struct { + Key string `json:"key, omitempty"` + Value string `json:"value,omitempty"` + Dir bool `json:"dir,omitempty"` + Expiration *time.Time `json:"expiration,omitempty"` + TTL int64 `json:"ttl,omitempty"` + Nodes Nodes `json:"nodes,omitempty"` + ModifiedIndex uint64 `json:"modifiedIndex,omitempty"` + CreatedIndex uint64 `json:"createdIndex,omitempty"` +} + +type Nodes []*Node + +// interfaces for sorting +func (ns Nodes) Len() int { + return len(ns) +} + +func (ns Nodes) Less(i, j int) bool { + return ns[i].Key < ns[j].Key +} + +func (ns Nodes) Swap(i, j int) { + ns[i], ns[j] = ns[j], ns[i] +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_curl_chan_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_curl_chan_test.go new file mode 100644 index 00000000000..756e317815a --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_curl_chan_test.go @@ -0,0 +1,42 @@ +package etcd + +import ( + "fmt" + "testing" +) + +func TestSetCurlChan(t *testing.T) { + c := NewClient(nil) + c.OpenCURL() + + defer func() { + c.Delete("foo", true) + }() + + _, err := c.Set("foo", "bar", 5) + if err != nil { + t.Fatal(err) + } + + expected := fmt.Sprintf("curl -X PUT %s/v2/keys/foo -d value=bar -d ttl=5", + c.cluster.Leader) + actual := c.RecvCURL() + if expected != actual { + t.Fatalf(`Command "%s" is not equal to expected value "%s"`, + actual, expected) + } + + c.SetConsistency(STRONG_CONSISTENCY) + _, err = c.Get("foo", false, false) + if err != nil { + t.Fatal(err) + } + + expected = fmt.Sprintf("curl -X GET %s/v2/keys/foo?consistent=true&recursive=false&sorted=false", + c.cluster.Leader) + actual = c.RecvCURL() + if expected != actual { + t.Fatalf(`Command "%s" is not equal to expected value "%s"`, + actual, expected) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_update_create.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_update_create.go new file mode 100644 index 00000000000..e2840cf3567 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_update_create.go @@ -0,0 +1,137 @@ +package etcd + +// Set sets the given key to the given value. +// It will create a new key value pair or replace the old one. +// It will not replace a existing directory. +func (c *Client) Set(key string, value string, ttl uint64) (*Response, error) { + raw, err := c.RawSet(key, value, ttl) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +// SetDir sets the given key to a directory. +// It will create a new directory or replace the old key value pair by a directory. +// It will not replace a existing directory. +func (c *Client) SetDir(key string, ttl uint64) (*Response, error) { + raw, err := c.RawSetDir(key, ttl) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +// CreateDir creates a directory. It succeeds only if +// the given key does not yet exist. +func (c *Client) CreateDir(key string, ttl uint64) (*Response, error) { + raw, err := c.RawCreateDir(key, ttl) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +// UpdateDir updates the given directory. It succeeds only if the +// given key already exists. +func (c *Client) UpdateDir(key string, ttl uint64) (*Response, error) { + raw, err := c.RawUpdateDir(key, ttl) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +// Create creates a file with the given value under the given key. It succeeds +// only if the given key does not yet exist. +func (c *Client) Create(key string, value string, ttl uint64) (*Response, error) { + raw, err := c.RawCreate(key, value, ttl) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +// CreateInOrder creates a file with a key that's guaranteed to be higher than other +// keys in the given directory. It is useful for creating queues. +func (c *Client) CreateInOrder(dir string, value string, ttl uint64) (*Response, error) { + raw, err := c.RawCreateInOrder(dir, value, ttl) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +// Update updates the given key to the given value. It succeeds only if the +// given key already exists. +func (c *Client) Update(key string, value string, ttl uint64) (*Response, error) { + raw, err := c.RawUpdate(key, value, ttl) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() +} + +func (c *Client) RawUpdateDir(key string, ttl uint64) (*RawResponse, error) { + ops := Options{ + "prevExist": true, + "dir": true, + } + + return c.put(key, "", ttl, ops) +} + +func (c *Client) RawCreateDir(key string, ttl uint64) (*RawResponse, error) { + ops := Options{ + "prevExist": false, + "dir": true, + } + + return c.put(key, "", ttl, ops) +} + +func (c *Client) RawSet(key string, value string, ttl uint64) (*RawResponse, error) { + return c.put(key, value, ttl, nil) +} + +func (c *Client) RawSetDir(key string, ttl uint64) (*RawResponse, error) { + ops := Options{ + "dir": true, + } + + return c.put(key, "", ttl, ops) +} + +func (c *Client) RawUpdate(key string, value string, ttl uint64) (*RawResponse, error) { + ops := Options{ + "prevExist": true, + } + + return c.put(key, value, ttl, ops) +} + +func (c *Client) RawCreate(key string, value string, ttl uint64) (*RawResponse, error) { + ops := Options{ + "prevExist": false, + } + + return c.put(key, value, ttl, ops) +} + +func (c *Client) RawCreateInOrder(dir string, value string, ttl uint64) (*RawResponse, error) { + return c.post(dir, value, ttl) +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_update_create_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_update_create_test.go new file mode 100644 index 00000000000..ced0f06e7be --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/set_update_create_test.go @@ -0,0 +1,241 @@ +package etcd + +import ( + "testing" +) + +func TestSet(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("foo", true) + }() + + resp, err := c.Set("foo", "bar", 5) + if err != nil { + t.Fatal(err) + } + if resp.Node.Key != "/foo" || resp.Node.Value != "bar" || resp.Node.TTL != 5 { + t.Fatalf("Set 1 failed: %#v", resp) + } + if resp.PrevNode != nil { + t.Fatalf("Set 1 PrevNode failed: %#v", resp) + } + + resp, err = c.Set("foo", "bar2", 5) + if err != nil { + t.Fatal(err) + } + if !(resp.Node.Key == "/foo" && resp.Node.Value == "bar2" && resp.Node.TTL == 5) { + t.Fatalf("Set 2 failed: %#v", resp) + } + if resp.PrevNode.Key != "/foo" || resp.PrevNode.Value != "bar" || resp.Node.TTL != 5 { + t.Fatalf("Set 2 PrevNode failed: %#v", resp) + } +} + +func TestUpdate(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("foo", true) + c.Delete("nonexistent", true) + }() + + resp, err := c.Set("foo", "bar", 5) + + if err != nil { + t.Fatal(err) + } + + // This should succeed. + resp, err = c.Update("foo", "wakawaka", 5) + if err != nil { + t.Fatal(err) + } + + if !(resp.Action == "update" && resp.Node.Key == "/foo" && resp.Node.TTL == 5) { + t.Fatalf("Update 1 failed: %#v", resp) + } + if !(resp.PrevNode.Key == "/foo" && resp.PrevNode.Value == "bar" && resp.Node.TTL == 5) { + t.Fatalf("Update 1 prevValue failed: %#v", resp) + } + + // This should fail because the key does not exist. + resp, err = c.Update("nonexistent", "whatever", 5) + if err == nil { + t.Fatalf("The key %v did not exist, so the update should have failed."+ + "The response was: %#v", resp.Node.Key, resp) + } +} + +func TestCreate(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("newKey", true) + }() + + newKey := "/newKey" + newValue := "/newValue" + + // This should succeed + resp, err := c.Create(newKey, newValue, 5) + if err != nil { + t.Fatal(err) + } + + if !(resp.Action == "create" && resp.Node.Key == newKey && + resp.Node.Value == newValue && resp.Node.TTL == 5) { + t.Fatalf("Create 1 failed: %#v", resp) + } + if resp.PrevNode != nil { + t.Fatalf("Create 1 PrevNode failed: %#v", resp) + } + + // This should fail, because the key is already there + resp, err = c.Create(newKey, newValue, 5) + if err == nil { + t.Fatalf("The key %v did exist, so the creation should have failed."+ + "The response was: %#v", resp.Node.Key, resp) + } +} + +func TestCreateInOrder(t *testing.T) { + c := NewClient(nil) + dir := "/queue" + defer func() { + c.DeleteDir(dir) + }() + + var firstKey, secondKey string + + resp, err := c.CreateInOrder(dir, "1", 5) + if err != nil { + t.Fatal(err) + } + + if !(resp.Action == "create" && resp.Node.Value == "1" && resp.Node.TTL == 5) { + t.Fatalf("Create 1 failed: %#v", resp) + } + + firstKey = resp.Node.Key + + resp, err = c.CreateInOrder(dir, "2", 5) + if err != nil { + t.Fatal(err) + } + + if !(resp.Action == "create" && resp.Node.Value == "2" && resp.Node.TTL == 5) { + t.Fatalf("Create 2 failed: %#v", resp) + } + + secondKey = resp.Node.Key + + if firstKey >= secondKey { + t.Fatalf("Expected first key to be greater than second key, but %s is not greater than %s", + firstKey, secondKey) + } +} + +func TestSetDir(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("foo", true) + c.Delete("fooDir", true) + }() + + resp, err := c.CreateDir("fooDir", 5) + if err != nil { + t.Fatal(err) + } + if !(resp.Node.Key == "/fooDir" && resp.Node.Value == "" && resp.Node.TTL == 5) { + t.Fatalf("SetDir 1 failed: %#v", resp) + } + if resp.PrevNode != nil { + t.Fatalf("SetDir 1 PrevNode failed: %#v", resp) + } + + // This should fail because /fooDir already points to a directory + resp, err = c.CreateDir("/fooDir", 5) + if err == nil { + t.Fatalf("fooDir already points to a directory, so SetDir should have failed."+ + "The response was: %#v", resp) + } + + _, err = c.Set("foo", "bar", 5) + if err != nil { + t.Fatal(err) + } + + // This should succeed + // It should replace the key + resp, err = c.SetDir("foo", 5) + if err != nil { + t.Fatal(err) + } + if !(resp.Node.Key == "/foo" && resp.Node.Value == "" && resp.Node.TTL == 5) { + t.Fatalf("SetDir 2 failed: %#v", resp) + } + if !(resp.PrevNode.Key == "/foo" && resp.PrevNode.Value == "bar" && resp.PrevNode.TTL == 5) { + t.Fatalf("SetDir 2 failed: %#v", resp) + } +} + +func TestUpdateDir(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("fooDir", true) + }() + + resp, err := c.CreateDir("fooDir", 5) + if err != nil { + t.Fatal(err) + } + + // This should succeed. + resp, err = c.UpdateDir("fooDir", 5) + if err != nil { + t.Fatal(err) + } + + if !(resp.Action == "update" && resp.Node.Key == "/fooDir" && + resp.Node.Value == "" && resp.Node.TTL == 5) { + t.Fatalf("UpdateDir 1 failed: %#v", resp) + } + if !(resp.PrevNode.Key == "/fooDir" && resp.PrevNode.Dir == true && resp.PrevNode.TTL == 5) { + t.Fatalf("UpdateDir 1 PrevNode failed: %#v", resp) + } + + // This should fail because the key does not exist. + resp, err = c.UpdateDir("nonexistentDir", 5) + if err == nil { + t.Fatalf("The key %v did not exist, so the update should have failed."+ + "The response was: %#v", resp.Node.Key, resp) + } +} + +func TestCreateDir(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("fooDir", true) + }() + + // This should succeed + resp, err := c.CreateDir("fooDir", 5) + if err != nil { + t.Fatal(err) + } + + if !(resp.Action == "create" && resp.Node.Key == "/fooDir" && + resp.Node.Value == "" && resp.Node.TTL == 5) { + t.Fatalf("CreateDir 1 failed: %#v", resp) + } + if resp.PrevNode != nil { + t.Fatalf("CreateDir 1 PrevNode failed: %#v", resp) + } + + // This should fail, because the key is already there + resp, err = c.CreateDir("fooDir", 5) + if err == nil { + t.Fatalf("The key %v did exist, so the creation should have failed."+ + "The response was: %#v", resp.Node.Key, resp) + } +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/version.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/version.go new file mode 100644 index 00000000000..b3d05df70bc --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/version.go @@ -0,0 +1,3 @@ +package etcd + +const version = "v2" diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/watch.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/watch.go new file mode 100644 index 00000000000..aa8d3df301c --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/watch.go @@ -0,0 +1,103 @@ +package etcd + +import ( + "errors" +) + +// Errors introduced by the Watch command. +var ( + ErrWatchStoppedByUser = errors.New("Watch stopped by the user via stop channel") +) + +// If recursive is set to true the watch returns the first change under the given +// prefix since the given index. +// +// If recursive is set to false the watch returns the first change to the given key +// since the given index. +// +// To watch for the latest change, set waitIndex = 0. +// +// If a receiver channel is given, it will be a long-term watch. Watch will block at the +//channel. After someone receives the channel, it will go on to watch that +// prefix. If a stop channel is given, the client can close long-term watch using +// the stop channel. +func (c *Client) Watch(prefix string, waitIndex uint64, recursive bool, + receiver chan *Response, stop chan bool) (*Response, error) { + logger.Debugf("watch %s [%s]", prefix, c.cluster.Leader) + if receiver == nil { + raw, err := c.watchOnce(prefix, waitIndex, recursive, stop) + + if err != nil { + return nil, err + } + + return raw.Unmarshal() + } + defer close(receiver) + + for { + raw, err := c.watchOnce(prefix, waitIndex, recursive, stop) + + if err != nil { + return nil, err + } + + resp, err := raw.Unmarshal() + + if err != nil { + return nil, err + } + + waitIndex = resp.Node.ModifiedIndex + 1 + receiver <- resp + } +} + +func (c *Client) RawWatch(prefix string, waitIndex uint64, recursive bool, + receiver chan *RawResponse, stop chan bool) (*RawResponse, error) { + + logger.Debugf("rawWatch %s [%s]", prefix, c.cluster.Leader) + if receiver == nil { + return c.watchOnce(prefix, waitIndex, recursive, stop) + } + + for { + raw, err := c.watchOnce(prefix, waitIndex, recursive, stop) + + if err != nil { + return nil, err + } + + resp, err := raw.Unmarshal() + + if err != nil { + return nil, err + } + + waitIndex = resp.Node.ModifiedIndex + 1 + receiver <- raw + } +} + +// helper func +// return when there is change under the given prefix +func (c *Client) watchOnce(key string, waitIndex uint64, recursive bool, stop chan bool) (*RawResponse, error) { + + options := Options{ + "wait": true, + } + if waitIndex > 0 { + options["waitIndex"] = waitIndex + } + if recursive { + options["recursive"] = true + } + + resp, err := c.getCancelable(key, options, stop) + + if err == ErrRequestCancelled { + return nil, ErrWatchStoppedByUser + } + + return resp, err +} diff --git a/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/watch_test.go b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/watch_test.go new file mode 100644 index 00000000000..43e1dfeb81f --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/go-etcd/etcd/watch_test.go @@ -0,0 +1,119 @@ +package etcd + +import ( + "fmt" + "runtime" + "testing" + "time" +) + +func TestWatch(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("watch_foo", true) + }() + + go setHelper("watch_foo", "bar", c) + + resp, err := c.Watch("watch_foo", 0, false, nil, nil) + if err != nil { + t.Fatal(err) + } + if !(resp.Node.Key == "/watch_foo" && resp.Node.Value == "bar") { + t.Fatalf("Watch 1 failed: %#v", resp) + } + + go setHelper("watch_foo", "bar", c) + + resp, err = c.Watch("watch_foo", resp.Node.ModifiedIndex+1, false, nil, nil) + if err != nil { + t.Fatal(err) + } + if !(resp.Node.Key == "/watch_foo" && resp.Node.Value == "bar") { + t.Fatalf("Watch 2 failed: %#v", resp) + } + + routineNum := runtime.NumGoroutine() + + ch := make(chan *Response, 10) + stop := make(chan bool, 1) + + go setLoop("watch_foo", "bar", c) + + go receiver(ch, stop) + + _, err = c.Watch("watch_foo", 0, false, ch, stop) + if err != ErrWatchStoppedByUser { + t.Fatalf("Watch returned a non-user stop error") + } + + if newRoutineNum := runtime.NumGoroutine(); newRoutineNum != routineNum { + t.Fatalf("Routine numbers differ after watch stop: %v, %v", routineNum, newRoutineNum) + } +} + +func TestWatchAll(t *testing.T) { + c := NewClient(nil) + defer func() { + c.Delete("watch_foo", true) + }() + + go setHelper("watch_foo/foo", "bar", c) + + resp, err := c.Watch("watch_foo", 0, true, nil, nil) + if err != nil { + t.Fatal(err) + } + if !(resp.Node.Key == "/watch_foo/foo" && resp.Node.Value == "bar") { + t.Fatalf("WatchAll 1 failed: %#v", resp) + } + + go setHelper("watch_foo/foo", "bar", c) + + resp, err = c.Watch("watch_foo", resp.Node.ModifiedIndex+1, true, nil, nil) + if err != nil { + t.Fatal(err) + } + if !(resp.Node.Key == "/watch_foo/foo" && resp.Node.Value == "bar") { + t.Fatalf("WatchAll 2 failed: %#v", resp) + } + + ch := make(chan *Response, 10) + stop := make(chan bool, 1) + + routineNum := runtime.NumGoroutine() + + go setLoop("watch_foo/foo", "bar", c) + + go receiver(ch, stop) + + _, err = c.Watch("watch_foo", 0, true, ch, stop) + if err != ErrWatchStoppedByUser { + t.Fatalf("Watch returned a non-user stop error") + } + + if newRoutineNum := runtime.NumGoroutine(); newRoutineNum != routineNum { + t.Fatalf("Routine numbers differ after watch stop: %v, %v", routineNum, newRoutineNum) + } +} + +func setHelper(key, value string, c *Client) { + time.Sleep(time.Second) + c.Set(key, value, 100) +} + +func setLoop(key, value string, c *Client) { + time.Sleep(time.Second) + for i := 0; i < 10; i++ { + newValue := fmt.Sprintf("%s_%v", value, i) + c.Set(key, newValue, 100) + time.Sleep(time.Second / 10) + } +} + +func receiver(c chan *Response, stop chan bool) { + for i := 0; i < 10; i++ { + <-c + } + stop <- true +} diff --git a/Makefile b/Makefile index 9d3f792d4ca..770191ce76f 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,16 @@ MAKEFLAGS = -s all: build test +# Values to be burned into the binary at build-time. +LDFLAGS = "\ + -X github.com/youtube/vitess/go/vt/servenv.buildHost '$$(hostname)'\ + -X github.com/youtube/vitess/go/vt/servenv.buildUser '$$(whoami)'\ + -X github.com/youtube/vitess/go/vt/servenv.buildGitRev '$$(git rev-parse HEAD)'\ + -X github.com/youtube/vitess/go/vt/servenv.buildTime '$$(LC_ALL=C date)'\ +" + build: - go install ./go/... + godep go install -ldflags ${LDFLAGS} ./go/... # Set VT_TEST_FLAGS to pass flags to python tests. # For example, verbose output: export VT_TEST_FLAGS=-v @@ -21,16 +29,27 @@ clean: rm -rf java/vtocc-client/target java/vtocc-jdbc-driver/target third_party/acolyte unit_test: - go test ./go/... + godep go test ./go/... # Run the code coverage tools, compute aggregate. # If you want to improve in a directory, run: # go test -coverprofile=coverage.out && go tool cover -html=coverage.out unit_test_cover: - go test -cover ./go/... | misc/parse_cover.py + godep go test -cover ./go/... | misc/parse_cover.py unit_test_race: - go test -race ./go/... + godep go test -race ./go/... + +# Run coverage and upload to coveralls.io. +# Requires the secret COVERALLS_TOKEN env variable to be set. +unit_test_goveralls: + go list -f '{{if len .TestGoFiles}}godep go test -coverprofile={{.Dir}}/.coverprofile {{.ImportPath}}{{end}}' ./go/... | xargs -i sh -c {} + gover ./go/ + # Travis doesn't set the token for forked pull requests, so skip + # upload if COVERALLS_TOKEN is unset. + if ! [ -z "$$COVERALLS_TOKEN" ]; then \ + goveralls -coverprofile=gover.coverprofile -repotoken $$COVERALLS_TOKEN; \ + fi queryservice_test: echo $$(date): Running test/queryservice_test.py... @@ -55,35 +74,60 @@ site_integration_test_files = \ zkocc_test.py # These tests should be run by developers after making code changes. -integration_test_files = \ - binlog.py \ - clone.py \ - initial_sharding_bytes.py \ +# Integration tests are grouped into 3 suites. +# - small: under 30 secs +# - medium: 30 secs - 1 min +# - large: over 1 min +small_integration_test_files = \ initial_sharding.py \ - keyrange_test.py \ + initial_sharding_bytes.py \ + vertical_split.py \ + vertical_split_vtgate.py \ + schema.py \ keyspace_test.py \ + keyrange_test.py \ mysqlctl.py \ - reparent.py \ - resharding_bytes.py \ - resharding.py \ - rowcache_invalidator.py \ - secure.py \ - schema.py \ sharded.py \ + secure.py \ + binlog.py \ + clone.py \ + update_stream.py + +medium_integration_test_files = \ tabletmanager.py \ - update_stream.py \ - vertical_split.py \ - vertical_split_vtgate.py \ + reparent.py \ vtdb_test.py \ vtgate_utils_test.py \ + rowcache_invalidator.py + +large_integration_test_files = \ vtgatev2_test.py \ zkocc_test.py +# The following tests are considered too flaky to be included +# in the continous integration test suites +ci_skip_integration_test_files = \ + resharding_bytes.py \ + resharding.py + +# Run the following tests after making worker changes +worker_integration_test_files = \ + binlog.py \ + resharding.py \ + resharding_bytes.py \ + vertical_split.py \ + vertical_split_vtgate.py \ + initial_sharding.py \ + initial_sharding_bytes.py + .ONESHELL: SHELL = /bin/bash -integration_test: + +# function to execute a list of integration test files +# exits on first failure +define run_integration_tests cd test ; \ - for t in $(integration_test_files) ; do \ + for t in $1 ; do \ echo $$(date): Running test/$$t... ; \ output=$$(time ./$$t $$VT_TEST_FLAGS 2>&1) ; \ if [[ $$? != 0 ]]; then \ @@ -92,62 +136,38 @@ integration_test: fi ; \ echo ; \ done +endef + +small_integration_test: + $(call run_integration_tests, $(small_integration_test_files)) + +medium_integration_test: + $(call run_integration_tests, $(medium_integration_test_files)) + +large_integration_test: + $(call run_integration_tests, $(large_integration_test_files)) + +ci_skip_integration_test: + $(call run_integration_tests, $(ci_skip_integration_test_files)) + +worker_test: + godep go test ./go/vt/worker/ + $(call run_integration_tests, $(worker_integration_test_files)) + +integration_test: small_integration_test medium_integration_test large_integration_test ci_skip_integration_test site_integration_test: - cd test ; \ - for t in $(site_integration_test_files) ; do \ - echo $$(date): Running test/$$t... ; \ - output=$$(time ./$$t $$VT_TEST_FLAGS 2>&1) ; \ - if [[ $$? != 0 ]]; then \ - echo "$$output" >&2 ; \ - exit 1 ; \ - fi ; \ - echo ; \ - done + $(call run_integration_tests, $(site_integration_test_files)) # this rule only works if bootstrap.sh was successfully ran in ./java java_test: cd java && mvn verify -bson: - bsongen -file ./go/mysql/proto/structs.go -type QueryResult -o ./go/mysql/proto/query_result_bson.go - bsongen -file ./go/mysql/proto/structs.go -type Field -o ./go/mysql/proto/field_bson.go - bsongen -file ./go/mysql/proto/structs.go -type Charset -o ./go/mysql/proto/charset_bson.go - bsongen -file ./go/vt/key/key.go -type KeyRange -o ./go/vt/key/key_range_bson.go - bsongen -file ./go/vt/key/key.go -type KeyspaceId -o ./go/vt/key/keyspace_id_bson.go - bsongen -file ./go/vt/key/key.go -type KeyspaceIdType -o ./go/vt/key/keyspace_id_type_bson.go - bsongen -file ./go/vt/tabletserver/proto/structs.go -type Query -o ./go/vt/tabletserver/proto/query_bson.go - bsongen -file ./go/vt/tabletserver/proto/structs.go -type Session -o ./go/vt/tabletserver/proto/session_bson.go - bsongen -file ./go/vt/tabletserver/proto/structs.go -type BoundQuery -o ./go/vt/tabletserver/proto/bound_query_bson.go - bsongen -file ./go/vt/tabletserver/proto/structs.go -type QueryList -o ./go/vt/tabletserver/proto/query_list_bson.go - bsongen -file ./go/vt/tabletserver/proto/structs.go -type QueryResultList -o ./go/vt/tabletserver/proto/query_result_list_bson.go - bsongen -file ./go/vt/vtgate/proto/vtgate_proto.go -type Query -o ./go/vt/vtgate/proto/query_bson.go - bsongen -file ./go/vt/vtgate/proto/vtgate_proto.go -type QueryShard -o ./go/vt/vtgate/proto/query_shard_bson.go - bsongen -file ./go/vt/vtgate/proto/vtgate_proto.go -type BatchQueryShard -o ./go/vt/vtgate/proto/batch_query_shard_bson.go - bsongen -file ./go/vt/vtgate/proto/vtgate_proto.go -type KeyspaceIdQuery -o ./go/vt/vtgate/proto/keyspace_id_query_bson.go - bsongen -file ./go/vt/vtgate/proto/vtgate_proto.go -type KeyRangeQuery -o ./go/vt/vtgate/proto/key_range_query_bson.go - bsongen -file ./go/vt/vtgate/proto/vtgate_proto.go -type EntityId -o ./go/vt/vtgate/proto/entity_id_bson.go - bsongen -file ./go/vt/vtgate/proto/vtgate_proto.go -type EntityIdsQuery -o ./go/vt/vtgate/proto/entity_ids_query_bson.go - bsongen -file ./go/vt/vtgate/proto/vtgate_proto.go -type KeyspaceIdBatchQuery -o ./go/vt/vtgate/proto/keyspace_id_batch_query_bson.go - bsongen -file ./go/vt/vtgate/proto/vtgate_proto.go -type Session -o ./go/vt/vtgate/proto/session_bson.go - bsongen -file ./go/vt/vtgate/proto/vtgate_proto.go -type ShardSession -o ./go/vt/vtgate/proto/shard_session_bson.go - bsongen -file ./go/vt/vtgate/proto/vtgate_proto.go -type QueryResult -o ./go/vt/vtgate/proto/query_result_bson.go - bsongen -file ./go/vt/topo/srvshard.go -type SrvShard -o ./go/vt/topo/srvshard_bson.go - bsongen -file ./go/vt/topo/srvshard.go -type SrvKeyspace -o ./go/vt/topo/srvkeyspace_bson.go - bsongen -file ./go/vt/topo/srvshard.go -type KeyspacePartition -o ./go/vt/topo/keyspace_partition_bson.go - bsongen -file ./go/vt/topo/tablet.go -type TabletType -o ./go/vt/topo/tablet_type_bson.go - bsongen -file ./go/vt/topo/toporeader.go -type GetSrvKeyspaceNamesArgs -o ./go/vt/topo/get_srv_keyspace_names_args_bson.go - bsongen -file ./go/vt/topo/toporeader.go -type GetSrvKeyspaceArgs -o ./go/vt/topo/get_srv_keyspace_args_bson.go - bsongen -file ./go/vt/topo/toporeader.go -type SrvKeyspaceNames -o ./go/vt/topo/srv_keyspace_names_bson.go - bsongen -file ./go/vt/topo/toporeader.go -type GetEndPointsArgs -o ./go/vt/topo/get_end_points_args_bson.go - bsongen -file ./go/vt/binlog/proto/binlog_player.go -type BlpPosition -o ./go/vt/binlog/proto/blp_position_bson.go - bsongen -file ./go/vt/binlog/proto/binlog_player.go -type BlpPositionList -o ./go/vt/binlog/proto/blp_position_list_bson.go - bsongen -file ./go/vt/binlog/proto/binlog_transaction.go -type BinlogTransaction -o ./go/vt/binlog/proto/binlog_transaction_bson.go - bsongen -file ./go/vt/binlog/proto/binlog_transaction.go -type Statement -o ./go/vt/binlog/proto/statement_bson.go - bsongen -file ./go/vt/binlog/proto/stream_event.go -type StreamEvent -o ./go/vt/binlog/proto/stream_event_bson.go - bsongen -file ./go/zk/zkocc_structs.go -type ZkPath -o ./go/zk/zkpath_bson.go - bsongen -file ./go/zk/zkocc_structs.go -type ZkPathV -o ./go/zk/zkpathv_bson.go - bsongen -file ./go/zk/zkocc_structs.go -type ZkStat -o ./go/zk/zkstat_bson.go - bsongen -file ./go/zk/zkocc_structs.go -type ZkNode -o ./go/zk/zknode_bson.go - bsongen -file ./go/zk/zkocc_structs.go -type ZkNodeV -o ./go/zk/zknodev_bson.go +java_vtgate_client_test: + mvn -f java/vtgate-client/pom.xml clean verify + +v3_test: + cd test && ./vtgatev3_test.py +bson: + go generate ./go/... diff --git a/README.md b/README.md index 504d00358f9..f5631a93f4b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Vitess +# Vitess
[![Build Status](https://travis-ci.org/youtube/vitess.svg?branch=master)](https://travis-ci.org/youtube/vitess/builds) [![Coverage Status](https://coveralls.io/repos/youtube/vitess/badge.png)](https://coveralls.io/r/youtube/vitess)
Vitess is a set of servers and tools meant to facilitate scaling of MySQL databases for the web. It's been developed since 2011, and is currently used as diff --git a/bootstrap.sh b/bootstrap.sh index c43e0ad915c..2b6b6bbc46d 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -9,6 +9,17 @@ if [ ! -f bootstrap.sh ]; then exit 1 fi +if [ "$USER" == "root" ]; then + echo "Vitess cannot run as root. Please bootstrap with a non-root user." + exit 1 +fi + +go version 2>&1 >/dev/null +if [ $? != 0 ]; then + echo "Go is not installed or is not on \$PATH" + exit 1 +fi + . ./dev.env mkdir -p $VTROOT/dist @@ -38,16 +49,42 @@ ln -nfs $VTTOP/third_party/go/launchpad.net $VTROOT/src go install launchpad.net/gozk/zookeeper go get code.google.com/p/goprotobuf/proto -go get code.google.com/p/go.net/context -go get code.google.com/p/go.tools/cmd/goimports +go get golang.org/x/net/context +go get golang.org/x/tools/cmd/goimports go get github.com/golang/glog +go get github.com/golang/lint/golint +go get github.com/tools/godep + +# goversion_min returns true if major.minor go version is at least some value. +function goversion_min() { + [[ "$(go version)" =~ go([0-9]+)\.([0-9]+) ]] + gotmajor=${BASH_REMATCH[1]} + gotminor=${BASH_REMATCH[2]} + [[ "$1" =~ ([0-9]+)\.([0-9]+) ]] + wantmajor=${BASH_REMATCH[1]} + wantminor=${BASH_REMATCH[2]} + [ "$gotmajor" -lt "$wantmajor" ] && return 1 + [ "$gotmajor" -gt "$wantmajor" ] && return 0 + [ "$gotminor" -lt "$wantminor" ] && return 1 + return 0 +} + +# Packages for uploading code coverage to coveralls.io. +# The cover tool needs to be installed into the Go toolchain, so it will fail +# if Go is installed somewhere that requires root access. However, this tool +# is optional, so we should hide any errors to avoid confusion. +if goversion_min 1.4; then + go get golang.org/x/tools/cmd/cover &> /dev/null +else + go get code.google.com/p/go.tools/cmd/cover &> /dev/null +fi +go get github.com/modocache/gover +go get github.com/mattn/goveralls ln -snf $VTTOP/config $VTROOT/config ln -snf $VTTOP/data $VTROOT/data ln -snf $VTTOP/py $VTROOT/py-vtdb ln -snf $VTTOP/go/zk/zkctl/zksrv.sh $VTROOT/bin/zksrv.sh -ln -snf $VTTOP/test/vthook-copy_snapshot_from_storage.sh $VTROOT/vthook/copy_snapshot_from_storage -ln -snf $VTTOP/test/vthook-copy_snapshot_to_storage.sh $VTROOT/vthook/copy_snapshot_to_storage ln -snf $VTTOP/test/vthook-test.sh $VTROOT/vthook/test.sh # install mysql @@ -127,4 +164,6 @@ fi # create pre-commit hooks echo "creating git pre-commit hooks" ln -sf $VTTOP/misc/git/pre-commit $VTTOP/.git/hooks/pre-commit -echo "source dev.env in your shell to complete the setup." + +echo +echo "bootstrap finished - run 'source dev.env' in your shell before building." diff --git a/config/mycnf/default-fast.cnf b/config/mycnf/default-fast.cnf index 833c259595d..53f38151431 100644 --- a/config/mycnf/default-fast.cnf +++ b/config/mycnf/default-fast.cnf @@ -61,5 +61,5 @@ tmpdir = {{.TmpDir}} tmp_table_size = 32M transaction-isolation = REPEATABLE-READ # READ-COMMITTED would be better, but mysql 5.1 disables this with statement based replication -# READ-UNCOMMTTED might be better +# READ-UNCOMMITTED might be better lower_case_table_names = 1 diff --git a/config/mycnf/default.cnf b/config/mycnf/default.cnf index ddd40998f9b..b332c2ec47a 100644 --- a/config/mycnf/default.cnf +++ b/config/mycnf/default.cnf @@ -11,14 +11,14 @@ default-storage-engine = innodb expire_logs_days = 3 innodb_additional_mem_pool_size = 32M innodb_autoextend_increment = 64 -innodb_buffer_pool_size = 64M +innodb_buffer_pool_size = 32M innodb_data_file_path = ibdata1:10M:autoextend innodb_data_home_dir = {{.InnodbDataHomeDir}} innodb_file_per_table innodb_flush_log_at_trx_commit = 2 innodb_flush_method = O_DIRECT innodb_lock_wait_timeout = 20 -innodb_log_buffer_size = 64M +innodb_log_buffer_size = 8M innodb_log_file_size = 64M innodb_log_files_in_group = 2 innodb_log_group_home_dir = {{.InnodbLogGroupHomeDir}} @@ -61,5 +61,5 @@ tmpdir = {{.TmpDir}} tmp_table_size = 32M transaction-isolation = REPEATABLE-READ # READ-COMMITTED would be better, but mysql 5.1 disables this with statement based replication -# READ-UNCOMMTTED might be better +# READ-UNCOMMITTED might be better lower_case_table_names = 1 diff --git a/data/bootstrap/upgrade.sh b/data/bootstrap/upgrade.sh index 241b52a5e9d..7f3bafa3799 100755 --- a/data/bootstrap/upgrade.sh +++ b/data/bootstrap/upgrade.sh @@ -10,18 +10,23 @@ fi mysql_port=33306 tablet_uid=99999 +logdir=$VTDATAROOT/tmp tablet_dir=$VTDATAROOT/vt_0000099999 +mysqlctl_args="-log_dir $logdir -db-config-dba-uname vt_dba -db-config-dba-charset utf8 -tablet_uid $tablet_uid -mysql_port $mysql_port" + set -e +mkdir -p $logdir + echo Starting mysqld -mysqlctl -tablet_uid=$tablet_uid -mysql_port=$mysql_port init -skip_schema +mysqlctl $mysqlctl_args init -skip_schema echo Running mysql_upgrade mysql_upgrade --socket=$tablet_dir/mysql.sock --user=vt_dba echo Stopping mysqld -mysqlctl -tablet_uid=$tablet_uid -mysql_port=$mysql_port shutdown +mysqlctl $mysqlctl_args shutdown newfile=mysql-db-dir_$(cat $tablet_dir/data/mysql_upgrade_info).tbz @@ -30,4 +35,4 @@ echo Creating new bootstrap file: $newfile mv $tablet_dir/data.tbz ./$newfile echo Removing tablet directory -rm -r $tablet_dir \ No newline at end of file +rm -r $tablet_dir diff --git a/data/test/sqlparser_test/parse_pass.sql b/data/test/sqlparser_test/parse_pass.sql index 826c07c9ab4..56b8e37058a 100644 --- a/data/test/sqlparser_test/parse_pass.sql +++ b/data/test/sqlparser_test/parse_pass.sql @@ -61,6 +61,7 @@ select /* and */ 1 from t where a = b and a = c select /* or */ 1 from t where a = b or a = c select /* not */ 1 from t where not a = b select /* exists */ 1 from t where exists (select 1 from t) +select /* keyrange */ 1 from t where keyrange(1, 2) select /* (boolean) */ 1 from t where not (a = b) select /* in value list */ 1 from t where a in (b, c) select /* in select */ 1 from t where a in (select 1 from t) diff --git a/data/test/vtgate/dml_cases.txt b/data/test/vtgate/dml_cases.txt index 3e2aeb1b148..0c13946bbdc 100644 --- a/data/test/vtgate/dml_cases.txt +++ b/data/test/vtgate/dml_cases.txt @@ -3,10 +3,12 @@ { "ID":"NoPlan", "Reason":"table nouser not found", - "Table":null, + "Table": "", "Original":"update nouser set val = 1", "Rewritten":"", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } @@ -15,10 +17,12 @@ { "ID":"NoPlan", "Reason":"table nouser not found", - "Table":null, + "Table": "", "Original":"delete from nouser", "Rewritten":"", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } @@ -30,7 +34,9 @@ "Table":"main1", "Original": "update main1 set val = 1", "Rewritten":"update main1 set val = 1", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } @@ -42,7 +48,9 @@ "Table":"main1", "Original": "delete from main1", "Rewritten":"delete from main1", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } @@ -50,11 +58,13 @@ "update user set val = 1" { "ID": "NoPlan", - "Reason": "too complex", + "Reason": "update has multi-shard where clause", "Table": "user", "Original": "update user set val = 1", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -62,63 +72,83 @@ "delete from user" { "ID": "NoPlan", - "Reason": "too complex", + "Reason": "delete has multi-shard where clause", "Table": "user", "Original": "delete from user", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } # update by primary keyspace id "update user set val = 1 where id = 1" { - "ID": "UpdateSingleShardKey", + "ID": "UpdateEqual", "Reason": "", "Table": "user", "Original": "update user set val = 1 where id = 1", "Rewritten": "update user set val = 1 where id = 1", - "Index": { - "Type": 0, - "Column": "id", - "Name": "user_index", - "From": "", - "To": "", - "Owner": "user", - "IsAutoInc": true - }, + "Subquery": "", + "Vindex": "user_index", + "Col": "id", "Values": 1 } # delete from by primary keyspace id "delete from user where id = 1" { - "ID": "DeleteSingleShardKey", + "ID": "DeleteEqual", "Reason": "", "Table": "user", "Original": "delete from user where id = 1", "Rewritten": "delete from user where id = 1", - "Index": { - "Type": 0, - "Column": "id", - "Name": "user_index", - "From": "", - "To": "", - "Owner": "user", - "IsAutoInc": true - }, + "Subquery": "select id, name from user where id = 1 for update", + "Vindex": "user_index", + "Col": "id", "Values": 1 } +# update KEYRANGE +"update user set val = 1 where keyrange(1, 2)" +{ + "ID": "NoPlan", + "Reason": "update has multi-shard where clause", + "Table": "user", + "Original": "update user set val = 1 where keyrange(1, 2)", + "Rewritten": "", + "Subquery": "", + "Vindex": "", + "Col": "", + "Values": null +} + +# delete KEYRANGE +"delete from user where keyrange(1, 2)" +{ + "ID": "NoPlan", + "Reason": "delete has multi-shard where clause", + "Table": "user", + "Original": "delete from user where keyrange(1, 2)", + "Rewritten": "", + "Subquery": "", + "Vindex": "", + "Col": "", + "Values": null +} + # update with primary id through IN clause "update user set val = 1 where id in (1, 2)" { "ID": "NoPlan", - "Reason": "too complex", + "Reason": "update has multi-shard where clause", "Table": "user", "Original": "update user set val = 1 where id in (1, 2)", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -126,11 +156,41 @@ "delete from user where id in (1, 2)" { "ID": "NoPlan", - "Reason": "too complex", + "Reason": "delete has multi-shard where clause", "Table": "user", "Original": "delete from user where id in (1, 2)", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", + "Values": null +} + +# update with non-unique key +"update user set val = 1 where name = 'foo'" +{ + "ID": "NoPlan", + "Reason": "update has multi-shard where clause", + "Table": "user", + "Original": "update user set val = 1 where name = 'foo'", + "Rewritten": "", + "Subquery": "", + "Vindex": "", + "Col": "", + "Values": null +} + +# delete from with primary id through IN clause +"delete from user where name = 'foo'" +{ + "ID": "NoPlan", + "Reason": "delete has multi-shard where clause", + "Table": "user", + "Original": "delete from user where name = 'foo'", + "Rewritten": "", + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -138,11 +198,13 @@ "update user set val = 1 where user_id = 1" { "ID": "NoPlan", - "Reason": "too complex", + "Reason": "update has multi-shard where clause", "Table": "user", "Original": "update user set val = 1 where user_id = 1", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -150,51 +212,55 @@ "delete from user where user_id = 1" { "ID": "NoPlan", - "Reason": "too complex", + "Reason": "delete has multi-shard where clause", "Table": "user", "Original": "delete from user where user_id = 1", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } # update by lookup "update music set val = 1 where id = 1" { - "ID": "UpdateSingleLookup", + "ID": "UpdateEqual", "Reason": "", "Table": "music", "Original": "update music set val = 1 where id = 1", "Rewritten": "update music set val = 1 where id = 1", - "Index": { - "Type": 1, - "Column": "id", - "Name": "music_user_map", - "From": "music_id", - "To": "user_id", - "Owner": "music", - "IsAutoInc": true - }, + "Subquery": "", + "Vindex": "music_user_map", + "Col": "id", "Values": 1 } # delete from by lookup "delete from music where id = 1" { - "ID": "DeleteSingleLookup", + "ID": "DeleteEqual", "Reason": "", "Table": "music", "Original": "delete from music where id = 1", "Rewritten": "delete from music where id = 1", - "Index": { - "Type": 1, - "Column": "id", - "Name": "music_user_map", - "From": "music_id", - "To": "user_id", - "Owner": "music", - "IsAutoInc": true - }, + "Subquery": "select id from music where id = 1 for update", + "Vindex": "music_user_map", + "Col": "id", + "Values": 1 +} + +# delete from, no owned vindexes +"delete from music_extra where user_id = 1" +{ + "ID": "DeleteEqual", + "Reason": "", + "Table": "music_extra", + "Original": "delete from music_extra where user_id = 1", + "Rewritten": "delete from music_extra where user_id = 1", + "Subquery": "", + "Vindex": "user_index", + "Col": "user_id", "Values": 1 } @@ -202,11 +268,13 @@ "update music set val = 1 where id in (1, 2)" { "ID": "NoPlan", - "Reason": "too complex", + "Reason": "update has multi-shard where clause", "Table": "music", "Original": "update music set val = 1 where id in (1, 2)", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -214,11 +282,13 @@ "delete from music where id in (1, 2)" { "ID": "NoPlan", - "Reason": "too complex", + "Reason": "delete has multi-shard where clause", "Table": "music", "Original": "delete from music where id in (1, 2)", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -230,6 +300,8 @@ "Table": "music", "Original": "update music set id = 1 where id = 1", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } diff --git a/data/test/vtgate/insert_cases.txt b/data/test/vtgate/insert_cases.txt index 931b160f136..ecba1d45823 100644 --- a/data/test/vtgate/insert_cases.txt +++ b/data/test/vtgate/insert_cases.txt @@ -6,7 +6,9 @@ "Table":"main1", "Original":"insert into main1 values(1, 2)", "Rewritten":"insert into main1 values (1, 2)", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } @@ -18,7 +20,9 @@ "Table": "user", "Original":"insert into user values(1, 2, 3)", "Rewritten":"", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -30,7 +34,9 @@ "Table":"user", "Original":"insert into user(id) select 1 from dual", "Rewritten":"", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } @@ -42,7 +48,9 @@ "Table":"user", "Original":"insert into user(id) values (1), (2)", "Rewritten":"", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } @@ -54,7 +62,9 @@ "Table":"user", "Original":"insert into user(id) values (select 1 from dual)", "Rewritten":"", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } @@ -66,78 +76,92 @@ "Table":"user", "Original":"insert into user(id) values (1, 2)", "Rewritten":"", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } -# insert ShardKey owned autoinc +# insert with one vindex "insert into user(id) values (1)" { "ID":"InsertSharded", "Reason":"", "Table":"user", "Original":"insert into user(id) values (1)", - "Rewritten":"insert into user(id) values (:_id)", - "Index":null, - "Values":[1] + "Rewritten":"insert into user(id, name) values (:_id, :_name)", + "Subquery": "", + "Vindex": "", + "Col": "", + "Values":[1, null] } -# insert ShardKey owned autoinc +# insert with non vindex "insert into user(nonid) values (2)" { "ID":"InsertSharded", "Reason":"", "Table":"user", "Original":"insert into user(nonid) values (2)", - "Rewritten":"insert into user(nonid, id) values (2, :_id)", - "Index":null, - "Values":[null] + "Rewritten":"insert into user(nonid, id, name) values (2, :_id, :_name)", + "Subquery": "", + "Vindex": "", + "Col": "", + "Values":[null, null] } -# insert Lookup owned no-autoinc -"insert into music(user_id, id) values(1, 2)" +# insert with all vindexes supplied +"insert into user(nonid, name, id) values (2, 'foo', 1)" { "ID":"InsertSharded", "Reason":"", - "Table":"music", - "Original":"insert into music(user_id, id) values(1, 2)", - "Rewritten":"insert into music(user_id, id) values (1, :_id)", - "Index":null, - "Values":[1, 2] + "Table":"user", + "Original":"insert into user(nonid, name, id) values (2, 'foo', 1)", + "Rewritten":"insert into user(nonid, name, id) values (2, :_name, :_id)", + "Subquery": "", + "Vindex": "", + "Col": "", + "Values":[1,"Zm9v"] } -# insert unowned -"insert into music_extra(music_id, user_id) values(1, 2)" +# insert invalid index value +"insert into music_extra(music_id, user_id) values(1, 1.1)" { - "ID":"InsertSharded", - "Reason":"", + "ID":"NoPlan", + "Reason":"could not convert val: 1.1, pos: 1: strconv.ParseUint: parsing \"1.1\": invalid syntax", "Table":"music_extra", - "Original":"insert into music_extra(music_id, user_id) values(1, 2)", - "Rewritten":"insert into music_extra(music_id, user_id) values (1, 2)", - "Index":null, - "Values":[2, 1] + "Original":"insert into music_extra(music_id, user_id) values(1, 1.1)", + "Rewritten":"", + "Subquery": "", + "Vindex": "", + "Col": "", + "Values":null } -# insert missing index value -"insert into music_extra(music_id) values(1)" +# insert invalid index value +"insert into music_extra(music_id, user_id) values(1, id)" { "ID":"NoPlan", - "Reason":"must supply value for indexed column: user_id", + "Reason":"could not convert val: id, pos: 1: id is not a value", "Table":"music_extra", - "Original":"insert into music_extra(music_id) values(1)", + "Original":"insert into music_extra(music_id, user_id) values(1, id)", "Rewritten":"", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } -# insert invalid index value -"insert into music_extra(music_id, user_id) values(1, 1.1)" +# insert invalid table +"insert into noexist(music_id, user_id) values(1, 1.1)" { "ID":"NoPlan", - "Reason":"could not convert val: 1.1, pos: 1", - "Table":"music_extra", - "Original":"insert into music_extra(music_id, user_id) values(1, 1.1)", + "Reason":"table noexist not found", + "Table":"", + "Original":"insert into noexist(music_id, user_id) values(1, 1.1)", "Rewritten":"", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } diff --git a/data/test/vtgate/router_test.json b/data/test/vtgate/router_test.json deleted file mode 100644 index 1dfb8e77e35..00000000000 --- a/data/test/vtgate/router_test.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "Keyspaces": { - "TestRouter": { - "ShardingScheme": 1, - "Indexes": { - "user_index": { - "Type": 0, - "Owner": "user", - "IsAutoInc": true - }, - "music_user_map": { - "Type": 1, - "From": "music_id", - "To": "user_id", - "Owner": "music", - "IsAutoInc": true - } - }, - "Tables": { - "user": { - "IndexColumns": [ - { - "Column": "id", - "IndexName": "user_index" - } - ] - }, - "user_extra": { - "IndexColumns": [ - { - "Column": "user_id", - "IndexName": "user_index" - } - ] - }, - "music": { - "IndexColumns": [ - { - "Column": "user_id", - "IndexName": "user_index" - }, - { - "Column": "id", - "IndexName": "music_user_map" - } - ] - }, - "music_extra": { - "IndexColumns": [ - { - "Column": "user_id", - "IndexName": "user_index" - }, - { - "Column": "music_id", - "IndexName": "music_user_map" - } - ] - } - } - } - } -} diff --git a/data/test/vtgate/schema_test.json b/data/test/vtgate/schema_test.json index 78d5c57fee8..8db6b09e48f 100644 --- a/data/test/vtgate/schema_test.json +++ b/data/test/vtgate/schema_test.json @@ -1,68 +1,77 @@ { "Keyspaces": { "user": { - "ShardingScheme": 1, - "Indexes": { + "Sharded": true, + "Vindexes": { "user_index": { - "Type": 0, - "Owner": "user", - "IsAutoInc": true + "Type": "hash", + "Owner": "user" }, "music_user_map": { - "Type": 1, - "From": "music_id", - "To": "user_id", - "Owner": "music", - "IsAutoInc": true + "Type": "lookup", + "Owner": "music" + }, + "name_user_map": { + "Type": "multi", + "Owner": "user" } }, - "Tables": { + "Classes": { "user": { - "IndexColumns": [ + "ColVindexes": [ + { + "Col": "id", + "Name": "user_index" + }, { - "Column": "id", - "IndexName": "user_index" + "Col": "name", + "Name": "name_user_map" } ] }, "user_extra": { - "IndexColumns": [ + "ColVindexes": [ { - "Column": "user_id", - "IndexName": "user_index" + "Col": "user_id", + "Name": "user_index" } ] }, "music": { - "IndexColumns": [ + "ColVindexes": [ { - "Column": "user_id", - "IndexName": "user_index" + "Col": "user_id", + "Name": "user_index" }, { - "Column": "id", - "IndexName": "music_user_map" + "Col": "id", + "Name": "music_user_map" } ] }, "music_extra": { - "IndexColumns": [ + "ColVindexes": [ { - "Column": "user_id", - "IndexName": "user_index" + "Col": "user_id", + "Name": "user_index" }, { - "Column": "music_id", - "IndexName": "music_user_map" + "Col": "music_id", + "Name": "music_user_map" } ] } + }, + "Tables": { + "user": "user", + "user_extra": "user_extra", + "music": "music", + "music_extra": "music_extra" } }, "main": { - "ShardingScheme": 0, "Tables": { - "main1": {} + "main1": "" } } } diff --git a/data/test/vtgate/select_cases.txt b/data/test/vtgate/select_cases.txt index 2f7c6603796..46767560304 100644 --- a/data/test/vtgate/select_cases.txt +++ b/data/test/vtgate/select_cases.txt @@ -3,10 +3,12 @@ { "ID":"NoPlan", "Reason":"syntax error at position 4 near the", - "Table":null, + "Table": "", "Original":"the quick brown fox", "Rewritten":"", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } @@ -14,11 +16,13 @@ "select * from user union select * from user" { "ID":"NoPlan", - "Reason":"too complex", - "Table":null, + "Reason":"cannot build a plan for this construct", + "Table": "", "Original":"select * from user union select * from user", "Rewritten":"", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } @@ -26,11 +30,13 @@ "set a=1" { "ID":"NoPlan", - "Reason":"too complex", - "Table":null, + "Reason":"cannot build a plan for this construct", + "Table": "", "Original":"set a=1", "Rewritten":"", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } @@ -38,11 +44,13 @@ "create table a()" { "ID":"NoPlan", - "Reason":"too complex", - "Table":null, + "Reason":"cannot build a plan for this construct", + "Table": "", "Original":"create table a()", "Rewritten":"", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } @@ -50,11 +58,13 @@ "explain select * from user" { "ID":"NoPlan", - "Reason":"too complex", - "Table":null, + "Reason":"cannot build a plan for this construct", + "Table": "", "Original":"explain select * from user", "Rewritten":"", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } @@ -63,10 +73,12 @@ { "ID":"NoPlan", "Reason":"complex table expression", - "Table":null, + "Table": "", "Original":"select * from (select 2 from dual)", "Rewritten":"", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } @@ -75,10 +87,12 @@ { "ID":"NoPlan", "Reason":"table nouser not found", - "Table":null, + "Table": "", "Original":"select * from nouser where id = 1", "Rewritten":"", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } @@ -87,10 +101,12 @@ { "ID":"NoPlan", "Reason":"complex table expression", - "Table":null, + "Table": "", "Original":"select * from music, user where id = 1", "Rewritten":"", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } @@ -99,10 +115,12 @@ { "ID":"NoPlan", "Reason":"complex table expression", - "Table":null, + "Table": "", "Original":"select * from (user) where id = 1", "Rewritten":"", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } @@ -111,10 +129,12 @@ { "ID":"SelectUnsharded", "Reason":"", - "Table":"main1", + "Table": "main1", "Original":"select * from main1", - "Rewritten":"select * from main1", - "Index":null, + "Rewritten":"", + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } @@ -122,11 +142,13 @@ "select * from user" { "ID": "SelectScatter", - "Reason": "no where clause", + "Reason": "", "Table": "user", "Original":"select * from user", "Rewritten": "select * from user", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -138,7 +160,9 @@ "Table": "user", "Original":"select * from user where id in (select * from music)", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -150,7 +174,9 @@ "Table": "user", "Original":"select * from user where not (id in (select * from music))", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -162,7 +188,9 @@ "Table": "user", "Original":"select * from user where id between (select 1 from dual) and 2", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -174,7 +202,9 @@ "Table": "user", "Original":"select * from user where (select 1 from dual) is null", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -186,7 +216,9 @@ "Table": "user", "Original":"select * from user where exists (select 1 from dual)", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -198,7 +230,9 @@ "Table": "user", "Original":"select * from user where 1+1 = (select 1 from dual)", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -210,7 +244,9 @@ "Table": "user", "Original":"select * from user where id = -(select 1 from dual)", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -222,7 +258,9 @@ "Table": "user", "Original":"select * from user where id = func(1, (select 1 from dual))", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -230,11 +268,13 @@ "select * from user where id = func(1)" { "ID":"SelectScatter", - "Reason":"no index match", - "Table":"user", + "Reason":"", + "Table": "user", "Original":"select * from user where id = func(1)", "Rewritten":"select * from user where id = func(1)", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } @@ -246,7 +286,9 @@ "Table": "user", "Original":"select * from user where id = case (select 1 from dual) when a=b then c end", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -258,7 +300,9 @@ "Table": "user", "Original":"select * from user where id = case aa when a = b then c else (select 1 from dual) end", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -270,7 +314,9 @@ "Table": "user", "Original":"select * from user where id = case aa when (select 1 from dual) = b then c else d end", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -282,7 +328,9 @@ "Table": "user", "Original":"select * from user where id = case aa when a = b then (select 1 from dual) else d end", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -290,11 +338,13 @@ "select * from user where id = case aa when a = b then c end" { "ID":"SelectScatter", - "Reason":"no index match", - "Table":"user", + "Reason":"", + "Table": "user", "Original":"select * from user where id = case aa when a = b then c end", "Rewritten": "select * from user where id = case aa when a = b then c end", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } @@ -302,83 +352,97 @@ "select * from user where 1 = id" { "ID":"SelectScatter", - "Reason":"no index match", - "Table":"user", + "Reason":"", + "Table": "user", "Original":"select * from user where 1 = id", "Rewritten":"select * from user where 1 = id", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } # select by primary keyspace id "select * from user where id = 1" { - "ID": "SelectSingleShardKey", + "ID": "SelectEqual", "Reason": "", "Table": "user", "Original":"select * from user where id = 1", "Rewritten": "select * from user where id = 1", - "Index": { - "Type": 0, - "Column": "id", - "Name": "user_index", - "From": "", - "To": "", - "Owner": "user", - "IsAutoInc": true - }, + "Subquery": "", + "Vindex": "user_index", + "Col": "id", "Values": 1 } +# select by primary keyspace id unsigned value +"select * from user where id = 9223372036854775808" +{ + "ID": "SelectEqual", + "Reason": "", + "Table": "user", + "Original":"select * from user where id = 9223372036854775808", + "Rewritten": "select * from user where id = 9223372036854775808", + "Subquery": "", + "Vindex": "user_index", + "Col": "id", + "Values": 9223372036854775808 +} + +# select by non-unique index +"select * from user where name = 'foo'" +{ + "ID": "SelectEqual", + "Reason": "", + "Table": "user", + "Original":"select * from user where name = 'foo'", + "Rewritten": "select * from user where name = 'foo'", + "Subquery": "", + "Vindex": "name_user_map", + "Col": "name", + "Values": "Zm9v" +} + # select by primary keyspace id, invalid value "select * from user where id = 1.1" { "ID":"SelectScatter", - "Reason":"no index match", - "Table":"user", + "Reason":"", + "Table": "user", "Original":"select * from user where id = 1.1", "Rewritten":"select * from user where id = 1.1", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } # select with primary keyspace id through bind var "select * from user where id = :id" { - "ID": "SelectSingleShardKey", + "ID": "SelectEqual", "Reason": "", "Table": "user", "Original":"select * from user where id = :id", "Rewritten": "select * from user where id = :id", - "Index": { - "Type": 0, - "Column": "id", - "Name": "user_index", - "From": "", - "To": "", - "Owner": "user", - "IsAutoInc": true - }, + "Subquery": "", + "Vindex": "user_index", + "Col": "id", "Values": ":id" } # select with primary id through IN clause "select * from user where id in (1, 2)" { - "ID": "SelectMultiShardKey", + "ID": "SelectIN", "Reason": "", "Table": "user", "Original":"select * from user where id in (1, 2)", "Rewritten": "select * from user where id in ::_vals", - "Index": { - "Type": 0, - "Column": "id", - "Name": "user_index", - "From": "", - "To": "", - "Owner": "user", - "IsAutoInc": true - }, + "Subquery": "", + "Vindex": "user_index", + "Col": "id", "Values": [ 1, 2 @@ -389,11 +453,13 @@ "select * from user where id in (1+1, 2)" { "ID":"SelectScatter", - "Reason":"no index match", - "Table":"user", + "Reason":"", + "Table": "user", "Original":"select * from user where id in (1+1, 2)", "Rewritten":"select * from user where id in (1+1, 2)", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } @@ -401,103 +467,83 @@ "select * from user where id in (1.1, 2)" { "ID":"SelectScatter", - "Reason":"no index match", - "Table":"user", + "Reason":"", + "Table": "user", "Original":"select * from user where id in (1.1, 2)", "Rewritten":"select * from user where id in (1.1, 2)", - "Index":null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values":null } -# select with no index match +# select with no vindex match "select * from user where user_id = 1" { "ID": "SelectScatter", - "Reason": "no index match", + "Reason": "", "Table": "user", "Original":"select * from user where user_id = 1", "Rewritten": "select * from user where user_id = 1", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } # select with primary id with different column name "select * from user_extra where user_id = 1" { - "ID": "SelectSingleShardKey", + "ID": "SelectEqual", "Reason": "", "Table": "user_extra", "Original":"select * from user_extra where user_id = 1", "Rewritten": "select * from user_extra where user_id = 1", - "Index": { - "Type": 0, - "Column": "user_id", - "Name": "user_index", - "From": "", - "To": "", - "Owner": "user", - "IsAutoInc": true - }, + "Subquery": "", + "Vindex": "user_index", + "Col": "user_id", "Values": 1 } -# select with primary id when there's more than one index +# select with primary id when there's more than one vindex "select * from music where user_id = 1" { - "ID": "SelectSingleShardKey", + "ID": "SelectEqual", "Reason": "", "Table": "music", "Original":"select * from music where user_id = 1", "Rewritten": "select * from music where user_id = 1", - "Index": { - "Type": 0, - "Column": "user_id", - "Name": "user_index", - "From": "", - "To": "", - "Owner": "user", - "IsAutoInc": true - }, + "Subquery": "", + "Vindex": "user_index", + "Col": "user_id", "Values": 1 } # select by lookup "select * from music where id = 1" { - "ID": "SelectSingleLookup", + "ID": "SelectEqual", "Reason": "", "Table": "music", "Original":"select * from music where id = 1", "Rewritten": "select * from music where id = 1", - "Index": { - "Type": 1, - "Column": "id", - "Name": "music_user_map", - "From": "music_id", - "To": "user_id", - "Owner": "music", - "IsAutoInc": true - }, + "Subquery": "", + "Vindex": "music_user_map", + "Col": "id", "Values": 1 } # select by lookup with IN clause "select * from music where id in (1, 2)" { - "ID": "SelectMultiLookup", + "ID": "SelectIN", "Reason": "", "Table": "music", "Original":"select * from music where id in (1, 2)", "Rewritten": "select * from music where id in ::_vals", - "Index": { - "Type": 1, - "Column": "id", - "Name": "music_user_map", - "From": "music_id", - "To": "user_id", - "Owner": "music", - "IsAutoInc": true - }, + "Subquery": "", + "Vindex": "music_user_map", + "Col": "id", "Values": [ 1, 2 @@ -507,20 +553,14 @@ # select by lookup with IN clause and bind vars "select * from music where id in (:a, 2)" { - "ID": "SelectMultiLookup", + "ID": "SelectIN", "Reason": "", "Table": "music", "Original":"select * from music where id in (:a, 2)", "Rewritten": "select * from music where id in ::_vals", - "Index": { - "Type": 1, - "Column": "id", - "Name": "music_user_map", - "From": "music_id", - "To": "user_id", - "Owner": "music", - "IsAutoInc": true - }, + "Subquery": "", + "Vindex": "music_user_map", + "Col": "id", "Values": [ ":a", 2 @@ -530,40 +570,28 @@ # select by lookup with list bind var "select * from music where id in ::list" { - "ID": "SelectMultiLookup", + "ID": "SelectIN", "Reason": "", "Table": "music", "Original":"select * from music where id in ::list", "Rewritten": "select * from music where id in ::_vals", - "Index": { - "Type": 1, - "Column": "id", - "Name": "music_user_map", - "From": "music_id", - "To": "user_id", - "Owner": "music", - "IsAutoInc": true - }, + "Subquery": "", + "Vindex": "music_user_map", + "Col": "id", "Values": "::list" } # select by lookup if there's no primary key "select * from music_extra where music_id = 1" { - "ID": "SelectSingleLookup", + "ID": "SelectEqual", "Reason": "", "Table": "music_extra", "Original":"select * from music_extra where music_id = 1", "Rewritten": "select * from music_extra where music_id = 1", - "Index": { - "Type": 1, - "Column": "music_id", - "Name": "music_user_map", - "From": "music_id", - "To": "user_id", - "Owner": "music", - "IsAutoInc": true - }, + "Subquery": "", + "Vindex": "music_user_map", + "Col": "music_id", "Values": 1 } @@ -571,31 +599,27 @@ "select * from user where id = 1 and var = 2 or var = 3" { "ID": "SelectScatter", - "Reason": "no index match", + "Reason": "", "Table": "user", "Original":"select * from user where id = 1 and var = 2 or var = 3", "Rewritten": "select * from user where id = 1 and var = 2 or var = 3", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } # select with acceptable parenthesized OR clause at end "select * from user where id = 1 and (var = 2 or var = 3)" { - "ID": "SelectSingleShardKey", + "ID": "SelectEqual", "Reason": "", "Table": "user", "Original":"select * from user where id = 1 and (var = 2 or var = 3)", "Rewritten": "select * from user where id = 1 and (var = 2 or var = 3)", - "Index": { - "Type": 0, - "Column": "id", - "Name": "user_index", - "From": "", - "To": "", - "Owner": "user", - "IsAutoInc": true - }, + "Subquery": "", + "Vindex": "user_index", + "Col": "id", "Values": 1 } @@ -603,43 +627,279 @@ "select * from user where var = 2 or var = 3 and id = 1" { "ID": "SelectScatter", - "Reason": "no index match", + "Reason": "", "Table": "user", "Original":"select * from user where var = 2 or var = 3 and id = 1", "Rewritten": "select * from user where var = 2 or var = 3 and id = 1", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } # select with acceptable parenthesized OR clause at beginning "select * from user where (var = 2 or var = 3) and id = 1" { - "ID": "SelectSingleShardKey", + "ID": "SelectEqual", "Reason": "", "Table": "user", "Original":"select * from user where (var = 2 or var = 3) and id = 1", "Rewritten": "select * from user where (var = 2 or var = 3) and id = 1", - "Index": { - "Type": 0, - "Column": "id", - "Name": "user_index", - "From": "", - "To": "", - "Owner": "user", - "IsAutoInc": true - }, + "Subquery": "", + "Vindex": "user_index", + "Col": "id", "Values": 1 } +# select with KEYRANGE +"select * from user where keyrange(1, 2)" +{ + "ID": "SelectKeyrange", + "Reason": "", + "Table": "user", + "Original":"select * from user where keyrange(1, 2)", + "Rewritten": "select * from user", + "Subquery": "", + "Vindex": "", + "Col": "", + "Values": [1, 2] +} + +# select with KEYRANGE parenthesized +"select * from user where (keyrange(1, 2))" +{ + "ID": "SelectKeyrange", + "Reason": "", + "Table": "user", + "Original":"select * from user where (keyrange(1, 2))", + "Rewritten": "select * from user", + "Subquery": "", + "Vindex": "", + "Col": "", + "Values": [1, 2] +} + +# select KEYRANGE in AND lhs +"select * from user where keyrange(1, 2) and a = 1" +{ + "ID": "SelectKeyrange", + "Reason": "", + "Table": "user", + "Original":"select * from user where keyrange(1, 2) and a = 1", + "Rewritten": "select * from user where a = 1", + "Subquery": "", + "Vindex": "", + "Col": "", + "Values": [1, 2] +} + +# select KEYRANGE in AND rhs +"select * from user where a = 1 and keyrange(1, 2)" +{ + "ID": "SelectKeyrange", + "Reason": "", + "Table": "user", + "Original":"select * from user where a = 1 and keyrange(1, 2)", + "Rewritten": "select * from user where a = 1", + "Subquery": "", + "Vindex": "", + "Col": "", + "Values": [1, 2] +} + +# select KEYRANGE in AND lhs parenthesized +"select * from user where (keyrange(1, 2)) and a = 1" +{ + "ID": "SelectKeyrange", + "Reason": "", + "Table": "user", + "Original":"select * from user where (keyrange(1, 2)) and a = 1", + "Rewritten": "select * from user where a = 1", + "Subquery": "", + "Vindex": "", + "Col": "", + "Values": [1, 2] +} + +# select KEYRANGE in AND rhs parenthesized +"select * from user where a = 1 and (keyrange(1, 2))" +{ + "ID": "SelectKeyrange", + "Reason": "", + "Table": "user", + "Original":"select * from user where a = 1 and (keyrange(1, 2))", + "Rewritten": "select * from user where a = 1", + "Subquery": "", + "Vindex": "", + "Col": "", + "Values": [1, 2] +} + +# select KEYRANGE in AND lhs double parenthesized +"select * from user where ((keyrange(1, 2))) and a = 1" +{ + "ID": "SelectKeyrange", + "Reason": "", + "Table": "user", + "Original":"select * from user where ((keyrange(1, 2))) and a = 1", + "Rewritten": "select * from user where a = 1", + "Subquery": "", + "Vindex": "", + "Col": "", + "Values": [1, 2] +} + +# select KEYRANGE in AND rhs double parenthesized +"select * from user where a = 1 and ((keyrange(1, 2)))" +{ + "ID": "SelectKeyrange", + "Reason": "", + "Table": "user", + "Original":"select * from user where a = 1 and ((keyrange(1, 2)))", + "Rewritten": "select * from user where a = 1", + "Subquery": "", + "Vindex": "", + "Col": "", + "Values": [1, 2] +} + +# select KEYRANGE in double AND double parenthesized lhs +"select * from user where (b = 2 and (keyrange(1, 2))) and a = 1" +{ + "ID": "SelectKeyrange", + "Reason": "", + "Table": "user", + "Original":"select * from user where (b = 2 and (keyrange(1, 2))) and a = 1", + "Rewritten": "select * from user where (b = 2) and a = 1", + "Subquery": "", + "Vindex": "", + "Col": "", + "Values": [1, 2] +} + +# select KEYRANGE in double AND double parenthesized rhs +"select * from user where a = 1 and (b = 2 and (keyrange(1, 2)))" +{ + "ID": "SelectKeyrange", + "Reason": "", + "Table": "user", + "Original":"select * from user where a = 1 and (b = 2 and (keyrange(1, 2)))", + "Rewritten": "select * from user where a = 1 and (b = 2)", + "Subquery": "", + "Vindex": "", + "Col": "", + "Values": [1, 2] +} + +# select with KEYRANGE syntax error +"select * from user where keyrange(1+1, 2)" +{ + "ID": "NoPlan", + "Reason": "syntax error at position 37", + "Table": "", + "Original":"select * from user where keyrange(1+1, 2)", + "Rewritten": "", + "Subquery": "", + "Vindex": "", + "Col": "", + "Values": null +} + +# select KEYRANGE innvalid lhs +"select * from user where keyrange(1.1, 2)" +{ + "ID": "NoPlan", + "Reason": "invalid keyrange: strconv.ParseUint: parsing \"1.1\": invalid syntax", + "Table": "user", + "Original":"select * from user where keyrange(1.1, 2)", + "Rewritten": "", + "Subquery": "", + "Vindex": "", + "Col": "", + "Values": null +} + +# select KEYRANGE innvalid lhs of AND +"select * from user where keyrange(1.1, 2) and a = 1" +{ + "ID": "NoPlan", + "Reason": "invalid keyrange: strconv.ParseUint: parsing \"1.1\": invalid syntax", + "Table": "user", + "Original":"select * from user where keyrange(1.1, 2) and a = 1", + "Rewritten": "", + "Subquery": "", + "Vindex": "", + "Col": "", + "Values": null +} + +# select KEYRANGE innvalid in parenthesized +"select * from user where (keyrange(1.1, 2))" +{ + "ID": "NoPlan", + "Reason": "invalid keyrange: strconv.ParseUint: parsing \"1.1\": invalid syntax", + "Table": "user", + "Original":"select * from user where (keyrange(1.1, 2))", + "Rewritten": "", + "Subquery": "", + "Vindex": "", + "Col": "", + "Values": null +} + +# select KEYRANGE innvalid rhs of AND +"select * from user where a = 1 and keyrange(1.1, 2)" +{ + "ID": "NoPlan", + "Reason": "invalid keyrange: strconv.ParseUint: parsing \"1.1\": invalid syntax", + "Table": "user", + "Original":"select * from user where a = 1 and keyrange(1.1, 2)", + "Rewritten": "", + "Subquery": "", + "Vindex": "", + "Col": "", + "Values": null +} + +# select KEYRANGE ininvalid rhs +"select * from user where keyrange(1, 2.2)" +{ + "ID": "NoPlan", + "Reason": "invalid keyrange: strconv.ParseUint: parsing \"2.2\": invalid syntax", + "Table": "user", + "Original":"select * from user where keyrange(1, 2.2)", + "Rewritten": "", + "Subquery": "", + "Vindex": "", + "Col": "", + "Values": null +} + # aggregates in select, simple "select count(*) from user where id in (1, 2)" { "ID": "NoPlan", - "Reason": "too complex", + "Reason": "multi-shard query has post-processing constructs", "Table": "user", "Original":"select count(*) from user where id in (1, 2)", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", + "Values": null +} + +# aggregates in select, non-unique vindex +"select count(*) from user where name = 'foo'" +{ + "ID": "NoPlan", + "Reason": "multi-shard query has post-processing constructs", + "Table": "user", + "Original":"select count(*) from user where name = 'foo'", + "Rewritten": "", + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -647,11 +907,13 @@ "select a = 1 and count(*) = 1 from user where id in (1, 2)" { "ID": "NoPlan", - "Reason": "too complex", + "Reason": "multi-shard query has post-processing constructs", "Table": "user", "Original":"select a = 1 and count(*) = 1 from user where id in (1, 2)", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -659,11 +921,13 @@ "select a = 1 or count(*) = 1 from user where id in (1, 2)" { "ID": "NoPlan", - "Reason": "too complex", + "Reason": "multi-shard query has post-processing constructs", "Table": "user", "Original":"select a = 1 or count(*) = 1 from user where id in (1, 2)", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -671,11 +935,13 @@ "select (not count(*) = 1) from user where id in (1, 2)" { "ID": "NoPlan", - "Reason": "too complex", + "Reason": "multi-shard query has post-processing constructs", "Table": "user", "Original":"select (not count(*) = 1) from user where id in (1, 2)", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -683,11 +949,13 @@ "select count(*) between 1 and 2 from user where id in (1, 2)" { "ID": "NoPlan", - "Reason": "too complex", + "Reason": "multi-shard query has post-processing constructs", "Table": "user", "Original":"select count(*) between 1 and 2 from user where id in (1, 2)", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -695,51 +963,41 @@ "select count(*) is null from user where id in (1, 2)" { "ID": "NoPlan", - "Reason": "too complex", + "Reason": "multi-shard query has post-processing constructs", "Table": "user", "Original":"select count(*) is null from user where id in (1, 2)", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } -# aggregates in select, EXISTS (cannot aggregate) +# subquery in select list, EXISTS (cannot aggregate) "select exists (select 1 from dual) from user where id in (1, 2)" { - "ID":"SelectMultiShardKey", + "ID":"SelectIN", "Reason":"", - "Table":"user", + "Table": "user", "Original":"select exists (select 1 from dual) from user where id in (1, 2)", "Rewritten":"select exists (select 1 from dual) from user where id in ::_vals", - "Index":{ - "Type":0, - "Column":"id", - "Name":"user_index", - "From":"", - "To":"", - "Owner":"user", - "IsAutoInc":true - }, + "Subquery": "", + "Vindex": "user_index", + "Col": "id", "Values":[1,2] } -# aggregates in select, subquery +# subquery in select in select list "select (select 1 from dual) from user where id in (1, 2)" { - "ID":"SelectMultiShardKey", + "ID":"SelectIN", "Reason":"", - "Table":"user", + "Table": "user", "Original":"select (select 1 from dual) from user where id in (1, 2)", "Rewritten": "select (select 1 from dual) from user where id in ::_vals", - "Index":{ - "Type":0, - "Column":"id", - "Name":"user_index", - "From":"", - "To":"", - "Owner":"user", - "IsAutoInc":true - }, + "Subquery": "", + "Vindex": "user_index", + "Col": "id", "Values":[1,2] } @@ -747,11 +1005,13 @@ "select count(*)+1 from user where id in (1, 2)" { "ID": "NoPlan", - "Reason": "too complex", + "Reason": "multi-shard query has post-processing constructs", "Table": "user", "Original":"select count(*)+1 from user where id in (1, 2)", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -759,11 +1019,13 @@ "select -count(*) from user where id in (1, 2)" { "ID": "NoPlan", - "Reason": "too complex", + "Reason": "multi-shard query has post-processing constructs", "Table": "user", "Original":"select -count(*) from user where id in (1, 2)", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -771,31 +1033,27 @@ "select fun(1, count(*)) from user where id in (1, 2)" { "ID": "NoPlan", - "Reason": "too complex", + "Reason": "multi-shard query has post-processing constructs", "Table": "user", "Original":"select fun(1, count(*)) from user where id in (1, 2)", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } # aggregates in select, non-aggregate function "select fun(*) from user where id in (1, 2)" { - "ID": "SelectMultiShardKey", + "ID": "SelectIN", "Reason": "", "Table": "user", "Original":"select fun(*) from user where id in (1, 2)", "Rewritten": "select fun(*) from user where id in ::_vals", - "Index": { - "Type": 0, - "Column": "id", - "Name": "user_index", - "From": "", - "To": "", - "Owner": "user", - "IsAutoInc": true - }, + "Subquery": "", + "Vindex": "user_index", + "Col": "id", "Values": [ 1, 2 @@ -806,11 +1064,13 @@ "select case count(*) when a = b then d end from user where id in (1, 2)" { "ID": "NoPlan", - "Reason": "too complex", + "Reason": "multi-shard query has post-processing constructs", "Table": "user", "Original":"select case count(*) when a = b then d end from user where id in (1, 2)", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -818,11 +1078,13 @@ "select case a when a = b then d else count(*) end from user where id in (1, 2)" { "ID": "NoPlan", - "Reason": "too complex", + "Reason": "multi-shard query has post-processing constructs", "Table": "user", "Original":"select case a when a = b then d else count(*) end from user where id in (1, 2)", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -830,11 +1092,13 @@ "select case a when count(*) = b then d else e end from user where id in (1, 2)" { "ID": "NoPlan", - "Reason": "too complex", + "Reason": "multi-shard query has post-processing constructs", "Table": "user", "Original":"select case a when count(*) = b then d else e end from user where id in (1, 2)", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } @@ -842,31 +1106,27 @@ "select case a when a = b then count(*) else e end from user where id in (1, 2)" { "ID": "NoPlan", - "Reason": "too complex", + "Reason": "multi-shard query has post-processing constructs", "Table": "user", "Original":"select case a when a = b then count(*) else e end from user where id in (1, 2)", "Rewritten": "", - "Index": null, + "Subquery": "", + "Vindex": "", + "Col": "", "Values": null } # aggregates in select, no aggregates "select case a when a = b then d else e end from user where id in (1, 2)" { - "ID": "SelectMultiShardKey", + "ID": "SelectIN", "Reason": "", "Table": "user", "Original":"select case a when a = b then d else e end from user where id in (1, 2)", "Rewritten": "select case a when a = b then d else e end from user where id in ::_vals", - "Index": { - "Type": 0, - "Column": "id", - "Name": "user_index", - "From": "", - "To": "", - "Owner": "user", - "IsAutoInc": true - }, + "Subquery": "", + "Vindex": "user_index", + "Col": "id", "Values": [ 1, 2 diff --git a/doc/GettingStarted.md b/doc/GettingStarted.md index b96d74f32af..135b3c8054f 100644 --- a/doc/GettingStarted.md +++ b/doc/GettingStarted.md @@ -3,18 +3,20 @@ If you run into issues or have questions, you can use our mailing list: vitess@g ## Dependencies -* We currently develop on Ubuntu 12.04 and 14.04. -* You'll need some kind of Java Runtime (for ZooKeeper). +* We currently develop on Ubuntu 14.04 (Trusty) and Debian 7.0 (Wheezy). +* You'll need some kind of Java Runtime if you use ZooKeeper. We use OpenJDK (*sudo apt-get install openjdk-7-jre*). -* [Go](http://golang.org) 1.2+: Needed for building Vitess. -* [MariaDB](https://mariadb.org/): We currently develop with version 10.0.13. +* [Go](http://golang.org) 1.3+: Needed for building Vitess. +* [MariaDB](https://mariadb.org/): We currently develop with version 10.0.14. Other 10.0.x versions may also work. -* [ZooKeeper](http://zookeeper.apache.org/): By default, Vitess - uses Zookeeper as the lock service. It is possible to plug in - something else as long as the new service supports the - necessary API functions. +* [ZooKeeper](http://zookeeper.apache.org/) + or [etcd](https://github.com/coreos/etcd) 0.4.6: + By default, Vitess uses ZooKeeper as the lock service. + We also have a plugin for etcd. See the Building section below. + It is possible to plug in something else as long as the new service supports + the necessary API functions. * [Memcached](http://memcached.org): Used for the rowcache. -* [Python](http://python.org): For the client and testing. +* [Python](http://python.org) 2.7: For the client and testing. ## Building @@ -24,6 +26,13 @@ If you run into issues or have questions, you can use our mailing list: vitess@g You can use any installation method (src/bin/rpm/deb), but be sure to include the client development headers (**libmariadbclient-dev**). +ZooKeeper 3.3.5 is included by default. If you plan to use it, you don't need +to install anything else. + +If you want to use etcd instead, install +[etcd 0.4.6](https://github.com/coreos/etcd/releases/tag/v0.4.6) +and make sure the "etcd" command is on your path. + Then download and build Vitess. Note that the value of MYSQL_FLAVOR is case-sensitive. If the mysql_config command from libmariadbclient-dev is not on the PATH, you'll need to *export VT_MYSQL_ROOT=/path/to/mariadb* before running bootstrap.sh, @@ -42,6 +51,12 @@ make build ## Testing +If you want to use etcd, set the following environment variable: + +``` sh +export VT_TEST_FLAGS='--topo-server-flavor=etcd' +``` + The full set of tests included in the default _make_ and _make test_ targets is intended for use by Vitess developers to verify code changes. These tests simulate a small cluster by launching many servers on the local @@ -92,10 +107,7 @@ This could indicate that no Java Runtime is installed. Some of the larger tests use up to 4GB of temporary space on disk. ## Setting up a cluster -TODO: Expand on all sections -### Setup zookeeper -### Start a MySql instance -### Start vttablet -### Start vtgate -### Write a client -### Test + +Once you have a successful `make build`, you can proceed to start up a +[local cluster](https://github.com/youtube/vitess/tree/master/examples/local) +for testing. diff --git a/doc/LifeOfAQuery.md b/doc/LifeOfAQuery.md new file mode 100644 index 00000000000..33a0480ab99 --- /dev/null +++ b/doc/LifeOfAQuery.md @@ -0,0 +1,129 @@ +Life of A Query +===================== + +* [From Client to VtGate](#from-client-to-vtgate) +* [From VtGate to VtTablet](#from-vtgate-to-vttablet) +* [From VtTablet to MySQL](#from-vttablet-to-mysql) +* [Put it all together](#put-it-all-together) +* [TopoServer](#toposerver) +* [Streaming Query](#streaming-query) +* [Scatter Query](#scatter-query) +* [Misc](#misc) + * [Rpc Server Code Path (VtGate)](#rpc-server-code-path-vtgate) + * [VtGate to VtTablet Code Path](#vtgate-to-vttablet-code-path) + * [VtTablet to MySQL Code Path](#vttablet-to-mysql-code-path) + +A query means a request for information from database and it involves four componenets in the case of Vitess, including client application, VtGate, VtTablet and MySQL instance. This doc explains interaction happens between and within components. + +![](./life_of_a_query.png) + +At a very high level, as the graph shows, first client sends a query to VtGate. VtGate then resolves the query and routes it to the right VtTablets. For each VtTablet that receives the query, it does necessary validations and passes the query to underlying MySQL instance. After gathering results from MySQL, VtTablet sends response back to VtGate. Once VtGate receives response from all VtTablets, it sends the combined result to client. In the presence of VtTablet errors, VtGate will retry the query if errors are recoverable and it only fails the query if either errors are unrecoverable or maximum retry times has been reached. + +## From Client to VtGate + +A client application first sends a bson rpc with an embedded sql query to VtGate. VtGate's rpc server unmarshals this rpc request, call appropriate VtGate method and return its result back to client. As following graph shows, VtGate has a rpc server that listens to localhost:port/\_bson\_rpc\_ for http requests and localhost:port/\_bson\_rpc\_/auth for https requests. + +![](./life_of_a_query_client_to_vtgate.png) + +VtGate keeps an in-memory table that stores all available rpc methods for each service, e.g. VtGate uses "VTGate" as its service name and most its methods defined in [go/vt/vtgate/vtgate.go](../go/vt/vtgate/vtgate.go) are used to serve rpc request [go/rpcplus/server.go](../go/rpcplus/server.go). + +## From VtGate to VtTablet + +![](./life_of_a_query_vtgate_to_vttablet.png) + +After receiving a rpc call from client and one of its Execute* method being invoked, VtGate needs to figure out which shards should receive the query and send query to each of them. In addition, VtGate talks to topo server to get necessary information to create a VtTablet connection for each shard. At this point, VtGate is able to send query to the right VtTablets in parallel. VtGate also does retry if timeout happens or some VtTablets return recoverable errors. + +Internally, VtGate has a ScatterConn instance and uses it to execute queries across multiple ShardConn connections. A ScatterConn performs the query on selected shards in parallel. It first obtains a ShardConn connection for every shard and sends query use ShardConn's execute method. If the requested session is in a transaction, it will open a new transactions on the connection, and updates the Session with the transaction id. If the session already contains a transaction id for the shard, it reuses it. If there are any unrecoverable errors during a transaction, it rolls back the transaction for all shards. + +A ShardConn object represents a load balanced connection to a group of VtTablets that belong the same shard. ShardConn can be concurrently used across goroutines. + +## From VtTablet to MySQL + +![](./life_of_a_query_vttablet_to_mysql.png) + +Once received a rpc call from VtGate, VtTablet do a few checks before passing query to MySQL. It first validates the current VtTablet state including sessions id, then generates a query plan and applies predefined query rules and do ACL check. It also checks whether the query hits row cache and returns result immediately if so. In addition, VtTablet consolidates duplicate queries from executing simultaneously and shares results between them. At this point, VtTablet has no way but pass the query down to MySQL layer and waits for the result. + +## Put it all together + +![](./life_of_a_query_all.png) + +## TopoServer + +A topo server stores information to help VtGate navigate query to the right VtTablets. It contains keyspace to shards mappings, keyspace id to shard mapping and ports that a VtTablet listens to (EndPoint). VtGates caches those information in the memory and periodically do updates if there are changes happened in the topo server. + +## Streaming Query + +In general speaking, a streaming query means query results will be returned as a stream. In Vitess's case, both VtGate and VtTablet will send result back as soon as it is available. VtTablet by default will collect a fixed number of rows returned from MySQL, send them back to VtGate and repeats the above step until all rows have been returned. + +## Scatter Query + +A scatter query, as its name indicates, will hit multiple shards. In Vitess, a scatter query is recognized once VtGate determines a query need to hit multiple VtTablets. VtGate then sends query to these VtTablets, assembles the result after receiving all response and returns the combined result to the client. + +## Misc + +### Rpc Server Code Path (VtGate) + +Init a rpc server + +``` +go/cmd/vtgate/vtgate.go: main() -> + go/vt/servenv/servenv.go: RunDefault() -> // use the port specified in command line "--port" + go/vt/servenv/run.go: Run(port int) -> + go/vt/servenv/rpc.go: ServeRPC() -> // set up rpc server + go/rpcwrap/bsonrpc/codec.go: ServeRPC() -> // set up bson rpc server + go/rpcwrap/rpcwrap.go: ServeRPC("bson", NewServerCodec) -> // common code to register rpc server +``` + +ServeRPC("bson", NewServerCodec) register a rpcHandler instance whose ServeHTTP(http.ResponseWriter, *http.Request) will be called for every http request + +Rpc server handle http request + +``` +go/rpcwrap/rpcwrap.go rpcHandler.ServeHTTP -> +go/rpcwrap/rpcwrap.go rpcHandler.server.ServeCodecWithContext -> +go/rpcplus/server.go Server.ServeCodecWithContext(context.Context, ServerCodec) (note: rpcHandler uses a global DefaultServer instance defined in the sever.go) -> +go/rpcplus/server.go Server.readRequest(ServeCodec) will use a given codec to extract (service, methodType, request, request arguments, reply value, keep reading), go/rpcplus/server.go +Finally we do "service.call(..)" with parameters provided in the request. In the current setup, service.call will always call some method in VtGate (go/vt/vtgate/vtgate.go). +``` + +### VtGate to VtTablet Code Path + +Here is the code path for a query with keyspace id. + +``` +go/vt/vtgate/vtgate.go VTGate.ExecuteKeyspaceIds(context.Context, *proto.KeyspaceIdQuery, *proto.QueryResult) -> + go/vt/vtgate/resolver.go resolver.ExecuteKeyspaceIds(context.Context, *proto.KeyspaceIdQuery) -> + go/vt/vtgate/resolver.go resolver.Execute -> + go/vt/vtgate/scatter_conn.go ScatterConn.Execute -> + go/vt/vtgate/scatter_conn.go ScatterConn.multiGo -> + go/vt/vtgate/scatter_conn.go ScatterConn.getConnection -> + go/vt/vtgate/shard_conn.go ShardConn.Execute -> + go/vt/vtgate/shard_conn.go ShardConn.withRetry -> + go/vt/vtgate/shard_conn.go ShardConn.getConn -> + go/vt/tabletserver/tabletconn/tablet_conn.go tabletconn.GetDialer -> + go/vt/tabletserver/tabletconn/tablet_conn.go tabletconn.TabletConn.Execute -> + go/vt/tabletserver/gorpctabletconn/conn.go TabletBson.Execute -> + go/vt/tabletserver/gorpctabletconn/conn.go TabletBson.rpcClient.Call -> + go/rpcplus/client.go rpcplus.Client.Call -> + go/rpcplus/client.go rpcplus.Client.Go -> + go/rpcplus/client.go rpcplus.Client.send +``` + +### VtTablet to MySQL Code Path + +Here is the code path for a select query. + +``` +go/vt/tabletserver/sqlquery.go SqlQuery.Execute -> +go/vt/tabletserver/query_executor.go QueryExecutor.Execute -> +go/vt/tabletserver/query_executor.go QueryExecutor.execSelect -> +go/vt/tabletserver/request_context.go RequestContext.getConn -> // QueryExecutor composes a RequestContext +go/vt/tabletserver/request_context.go RequestContext.fullFetch -> +go/vt/tabletserver/request_context.go RequestContext.execSQL -> +go/vt/tabletserver/request_context.go RequestContext.execSQLNoPanic -> +go/vt/tabletserver/request_context.go RequestContext.execSQLOnce -> +go/vt/dbconnpool/connection_pool.go PoolConnection.ExecuteFetch (current implementation is in DBConnection) -> +go/vt/dbconnpool/connection.go PooledConnection.DBConnection.ExecuteFetch -> +go/mysql/mysql.go mysql.Connection.ExecuteFetch -> +go/mysql/mysql.go mysql.Connection.fetchAll +``` diff --git a/doc/Reparenting.md b/doc/Reparenting.md index 525322ec8ab..b2b27dd09aa 100644 --- a/doc/Reparenting.md +++ b/doc/Reparenting.md @@ -15,29 +15,21 @@ They are triggered by using the 'vtctl ReparentShard' command. See the help for ## External Reparents In this part, we assume another tool has been reparenting our servers. We then trigger the -'vtctl ShardExternallyReparented' command. +'vtctl TabletExternallyReparented' command. The flow for that command is as follows: - the shard is locked in the global topology server. - we read the Shard object from the global topology server. -- we read all the tablets in the replication graph for the shard. We also check the new master is in the map. Note we allow partial reads here, so if a data center is down, as long as the data center containing the new master is up, we keep going. -- we call the 'SlaveWasPromoted' remote action on the new master. This remote action makes sure the new master is not a MySQL slave of another server (the 'show slave status' command should not return anything, meaning 'reset slave' should ave been called). -- for every host in the replication graph, we call the 'SlaveWasRestarted' action. It takes as paremeter the address of the new master. On each slave, it executes a 'show slave status'. If the master matches the new master, we update the topology server record for that tablet with the new master, and the replication graph for that tablet as well. If it doesn't match, we keep the old record in the replication graph (pointing at whatever master was there before). We optionally Scrap tablets that bad (disabled by default). -- if a smaller percentage than a configurable value of the slaves works (80% be default), we stop here. +- we read all the tablets in the replication graph for the shard. Note we allow partial reads here, so if a data center is down, as long as the data center containing the new master is up, we keep going. +- the new master performs a 'SlaveWasPromoted' action. This remote action makes sure the new master is not a MySQL slave of another server (the 'show slave status' command should not return anything, meaning 'reset slave' should ave been called). +- for every host in the replication graph, we call the 'SlaveWasRestarted' action. It takes as paremeter the address of the new master. On each slave, we update the topology server record for that tablet with the new master, and the replication graph for that tablet as well. +- for the old master, if it doesn't successfully return from 'SlaveWasRestarted', we change its type to 'spare' (so a dead old master doesn't interfere). - we then update the Shard object with the new master. - we rebuild the serving graph for that shard. This will update the 'master' record for sure, and also keep all the tablets that have successfully reparented. -Optional Flags: -- -accept-success-percents=80: will declare success if more than that many slaves can be reparented -- -continue_on_unexpected_master=false: if a slave has the wrong master, we'll just log the error and keep going -- -scrap-stragglers=false: will scrap bad hosts - Failure cases: - The global topology server has to be available for locking and modification during this operation. If not, the operation will just fail. -- If a single topology server is down in one data center 9and it's nto the master data center), the tablets in that data center will be ignored by the reparent. Provided it doesn't trigger the 80% threshold, this is not a big deal. When the topology server comes back up, just re-run 'vtctl InitTablet' on the tablets, and that will fix their master record. -- If scrap-straggler is false (the default), a tablet that has the wrong master will be kept in the replication graph with its original master. When we rebuild the serving graph, that tablet won't be added, as it doesn't have the right master. -- if more than 20% of the tablets fails, we don't update the Shard object, and don't rebuild. We assume something is seriously wrong, and it might be our process, not the servers. Figuring out the cause and re-running 'vtctl ShardExternallyReparented' should work. -- if for some reasons none of the slaves report the right master (replication is going through a proxy for instance, and the master address is not what the clients are showing in 'show slave status'), the result is pretty bad. All slaves are kept in the replication graph, but with their old (incorrect) master. Next time a Shard rebuild happens, all the servers will disappear. At that point, fixing the issue and then re-parenting will work. +- If a single topology server is down in one data center (and it's nto the master data center), the tablets in that data center will be ignored by the reparent. When the topology server comes back up, just re-run 'vtctl InitTablet' on the tablets, and that will fix their master record. ## Reparenting And Serving Graph diff --git a/doc/ReplicationGraph.md b/doc/ReplicationGraph.md index 8dc81767a92..17baa238ad9 100644 --- a/doc/ReplicationGraph.md +++ b/doc/ReplicationGraph.md @@ -1,7 +1,7 @@ # Replication Graph The replication graph contains the mysql replication information for a shard. Currently, we only support one layer -of replication (a single master wiht multiple slaves), but the design doesn't preclude us from supporting +of replication (a single master with multiple slaves), but the design doesn't preclude us from supporting hierarchical replication later on. ## Master @@ -14,7 +14,7 @@ When creating a master (using 'vtctl InitTablet ... master'), we make sure Maste The slaves are added to the ShardReplication object present on each local topology server. So for slaves, the replication graph is colocated in the same cell as the tablets themselves. This makes disaster recovery much easier: -when loosing a data center, the replication graph for other data centers is not lost. +when losing a data center, the replication graph for other data centers is not lost. When creating a slave (using 'vtctl InitTablet ... replica' for instance), we get the master record (if not specified) from the MasterAlias of the Shard. We then add an entry in the ReplicationLinks list of the ShardReplication object for the tablet’s cell (we create ShardReplication if it doesn’t exist yet). diff --git a/doc/SchemaManagement.md b/doc/SchemaManagement.md index 1bcf76c1c71..bcfe1ed2874 100644 --- a/doc/SchemaManagement.md +++ b/doc/SchemaManagement.md @@ -7,25 +7,36 @@ The schema is the list of tables and how to create them. It is managed by vtctl. The following vtctl commands exist to look at the schema, and validate it's the same on all databases. ``` -GetSchema{{queryInfo[0]}} | +{{queryInfo[1]}} | +
{{field[0]}} | +
---|
{{cell}} | +
Waiting for database connection...
+/env +
" + val + "
"); + }); + } + + var handleSubmission = function(e) { + e.preventDefault(); + var entryValue = entryContentElement.val() + if (entryValue.length > 0) { + entriesElement.append("...
"); + $.getJSON("rpush/guestbook/" + entryValue, appendGuestbookEntries); + } + return false; + } + + // colors = purple, blue, red, green, yellow + var colors = ["#549", "#18d", "#d31", "#2a4", "#db1"]; + var randomColor = colors[Math.floor(5 * Math.random())]; + (function setElementsColor(color) { + headerTitleElement.css("color", color); + entryContentElement.css("box-shadow", "inset 0 0 0 2px " + color); + submitElement.css("background-color", color); + })(randomColor); + + submitElement.click(handleSubmission); + formElement.submit(handleSubmission); + hostAddressElement.append(document.URL); + + // Poll every second. + (function fetchGuestbook() { + $.getJSON("lrange/guestbook").done(appendGuestbookEntries).always( + function() { + setTimeout(fetchGuestbook, 1000); + }); + })(); +}); diff --git a/examples/kubernetes/guestbook/static/style.css b/examples/kubernetes/guestbook/static/style.css new file mode 100644 index 00000000000..fd1c393fb08 --- /dev/null +++ b/examples/kubernetes/guestbook/static/style.css @@ -0,0 +1,61 @@ +body, input { + color: #123; + font-family: "Gill Sans", sans-serif; +} + +div { + overflow: hidden; + padding: 1em 0; + position: relative; + text-align: center; +} + +h1, h2, p, input, a { + font-weight: 300; + margin: 0; +} + +h1 { + color: #BDB76B; + font-size: 3.5em; +} + +h2 { + color: #999; +} + +form { + margin: 0 auto; + max-width: 50em; + text-align: center; +} + +input { + border: 0; + border-radius: 1000px; + box-shadow: inset 0 0 0 2px #BDB76B; + display: inline; + font-size: 1.5em; + margin-bottom: 1em; + outline: none; + padding: .5em 5%; + width: 55%; +} + +form a { + background: #BDB76B; + border: 0; + border-radius: 1000px; + color: #FFF; + font-size: 1.25em; + font-weight: 400; + padding: .75em 2em; + text-decoration: none; + text-transform: uppercase; + white-space: normal; +} + +p { + font-size: 1.5em; + line-height: 1.5; +} diff --git a/examples/kubernetes/vtctld-down.sh b/examples/kubernetes/vtctld-down.sh index bb915d7f51e..44b398a6a4b 100755 --- a/examples/kubernetes/vtctld-down.sh +++ b/examples/kubernetes/vtctld-down.sh @@ -1,10 +1,12 @@ #!/bin/bash # This is an example script that stops vtctld. -# It assumes that kubernetes/cluster/kubecfg.sh is in the path. + +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh echo "Deleting vtctld pod..." -kubecfg.sh delete pods/vtctld +$KUBECTL delete pod vtctld echo "Deleting vtctld service..." -kubecfg.sh delete services/vtctld +$KUBECTL delete service vtctld diff --git a/examples/kubernetes/vtctld-pod.yaml b/examples/kubernetes/vtctld-pod.yaml index 7767138016c..b75fbd09efc 100644 --- a/examples/kubernetes/vtctld-pod.yaml +++ b/examples/kubernetes/vtctld-pod.yaml @@ -17,13 +17,9 @@ desiredState: - sh - "-c" - >- - echo "{\"test_cell\":\"$SERVICE_HOST:2181\",\"global\":\"$SERVICE_HOST:2181\"}" > /vt/zk-client-conf.json && mkdir -p $VTDATAROOT/tmp && chown -R vitess /vt && - su -p -c "/vt/bin/vtctld -debug -templates $VTTOP/go/cmd/vtctld/templates -log_dir $VTDATAROOT/tmp -port 15000" vitess - env: - - name: ZK_CLIENT_CONFIG - value: /vt/zk-client-conf.json + su -p -c "/vt/bin/vtctld -debug -templates $VTTOP/go/cmd/vtctld/templates -log_dir $VTDATAROOT/tmp -alsologtostderr -port 15000 -topo_implementation etcd -etcd_global_addrs http://$ETCD_GLOBAL_SERVICE_HOST:$ETCD_GLOBAL_SERVICE_PORT" vitess volumes: - name: syslog source: {hostDir: {path: /dev/log}} diff --git a/examples/kubernetes/vtctld-service.yaml b/examples/kubernetes/vtctld-service.yaml index b4f1fe1d2b5..0da91dd6350 100644 --- a/examples/kubernetes/vtctld-service.yaml +++ b/examples/kubernetes/vtctld-service.yaml @@ -3,6 +3,7 @@ kind: Service id: vtctld port: 15000 containerPort: 15000 +createExternalLoadBalancer: true selector: name: vtctld labels: diff --git a/examples/kubernetes/vtctld-up.sh b/examples/kubernetes/vtctld-up.sh index 950241d0426..d8360af69c1 100755 --- a/examples/kubernetes/vtctld-up.sh +++ b/examples/kubernetes/vtctld-up.sh @@ -1,12 +1,14 @@ #!/bin/bash # This is an example script that starts vtctld. -# It assumes that kubernetes/cluster/kubecfg.sh is in the path. set -e +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + echo "Creating vtctld service..." -kubecfg.sh -c vtctld-service.yaml create services +$KUBECTL create -f vtctld-service.yaml echo "Creating vtctld pod..." -kubecfg.sh -c vtctld-pod.yaml create pods +$KUBECTL create -f vtctld-pod.yaml diff --git a/examples/kubernetes/vtgate-controller.yaml b/examples/kubernetes/vtgate-controller.yaml index f60e317c169..87e0d5d8d39 100644 --- a/examples/kubernetes/vtgate-controller.yaml +++ b/examples/kubernetes/vtgate-controller.yaml @@ -1,6 +1,6 @@ apiVersion: v1beta1 kind: ReplicationController -id: vtgateController +id: vtgate desiredState: replicas: 3 replicaSelector: {name: vtgate} @@ -8,7 +8,7 @@ desiredState: desiredState: manifest: version: v1beta1 - id: vtgateController + id: vtgate containers: - name: vtgate image: vitess/root @@ -21,13 +21,15 @@ desiredState: - sh - "-c" - >- - echo "{\"test_cell\":\"$SERVICE_HOST:2181\",\"global\":\"$SERVICE_HOST:2181\"}" > /vt/zk-client-conf.json && mkdir -p $VTDATAROOT/tmp && chown -R vitess /vt && - su -p -c "/vt/bin/vtgate -log_dir $VTDATAROOT/tmp -port 15001 -cell test_cell" vitess - env: - - name: ZK_CLIENT_CONFIG - value: /vt/zk-client-conf.json + su -p -c "/vt/bin/vtgate + -topo_implementation etcd + -etcd_global_addrs http://$ETCD_GLOBAL_SERVICE_HOST:$ETCD_GLOBAL_SERVICE_PORT + -log_dir $VTDATAROOT/tmp + -alsologtostderr + -port 15001 + -cell test" vitess volumes: - name: syslog source: {hostDir: {path: /dev/log}} diff --git a/examples/kubernetes/vtgate-down.sh b/examples/kubernetes/vtgate-down.sh index 9b168a03838..df7a794c769 100755 --- a/examples/kubernetes/vtgate-down.sh +++ b/examples/kubernetes/vtgate-down.sh @@ -1,13 +1,12 @@ #!/bin/bash # This is an example script that stops vtgate. -# It assumes that kubernetes/cluster/kubecfg.sh is in the path. -echo "Deleting pods created by vtgate replicationController..." -kubecfg.sh stop vtgateController +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh -echo "Deleting vtgate replicationController..." -kubecfg.sh delete replicationControllers/vtgateController +echo "Stopping vtgate replicationController..." +$KUBECTL stop replicationController vtgate echo "Deleting vtgate service..." -kubecfg.sh delete services/vtgate +$KUBECTL delete service vtgate diff --git a/examples/kubernetes/vtgate-up.sh b/examples/kubernetes/vtgate-up.sh index bba9ab0a3ba..f751a5a60e3 100755 --- a/examples/kubernetes/vtgate-up.sh +++ b/examples/kubernetes/vtgate-up.sh @@ -1,12 +1,14 @@ #!/bin/bash # This is an example script that starts a vtgate replicationController. -# It assumes that kubernetes/cluster/kubecfg.sh is in the path. set -e +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + echo "Creating vtgate service..." -kubecfg.sh -c vtgate-service.yaml create services +$KUBECTL create -f vtgate-service.yaml echo "Creating vtgate replicationController..." -kubecfg.sh -c vtgate-controller.yaml create replicationControllers +$KUBECTL create -f vtgate-controller.yaml diff --git a/examples/kubernetes/vttablet-down.sh b/examples/kubernetes/vttablet-down.sh index db9c4fc330f..84bdd4582e7 100755 --- a/examples/kubernetes/vttablet-down.sh +++ b/examples/kubernetes/vttablet-down.sh @@ -1,11 +1,14 @@ #!/bin/bash # This is an example script that tears down the vttablet pods started by -# vttablet-up.sh. It assumes that kubernetes/cluster/kubecfg.sh is in the path. +# vttablet-up.sh. -# Create the pods for shard-0 -cell=test_cell -keyspace=test_keyspace +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + +# Delete the pods for shard-0 +cell='test' +keyspace='test_keyspace' shard=0 uid_base=100 @@ -14,5 +17,5 @@ for uid_index in 0 1 2; do printf -v alias '%s-%010d' $cell $uid echo "Deleting pod for tablet $alias..." - kubecfg.sh delete pods/vttablet-$uid + $KUBECTL delete pod vttablet-$uid done diff --git a/examples/kubernetes/vttablet-pod-template.yaml b/examples/kubernetes/vttablet-pod-template.yaml index 70efb2af4d6..a6681009aba 100644 --- a/examples/kubernetes/vttablet-pod-template.yaml +++ b/examples/kubernetes/vttablet-pod-template.yaml @@ -19,63 +19,47 @@ desiredState: - >- set -e - echo "{\"test_cell\":\"$SERVICE_HOST:2181\",\"global\":\"$SERVICE_HOST:2181\"}" > /vt/zk-client-conf.json - - log_file=$VTDATAROOT/tmp/vttablet.log - mysql_socket="$VTDATAROOT/{{tablet_subdir}}/mysql.sock" - hostname=$(hostname -i) - mkdir -p $VTDATAROOT/tmp chown -R vitess /vt - su -p -s /bin/bash -c "/vt/bin/vtctlclient - -log_dir $VTDATAROOT/tmp - -server $SERVICE_HOST:15000 - InitTablet -force -parent - -port {{port}} - -hostname $hostname - -keyspace {{keyspace}} - -shard {{shard}} - {{alias}} {{type}} - &>> $log_file" vitess - while [ ! -e $mysql_socket ]; do - echo "[$(date)] waiting for $mysql_socket" >> $log_file ; + echo "[$(date)] waiting for $mysql_socket" ; sleep 1 ; done su -p -s /bin/bash -c "mysql -u vt_dba -S $mysql_socket - -e 'CREATE DATABASE IF NOT EXISTS vt_{{keyspace}}' - &>> $log_file" vitess + -e 'CREATE DATABASE IF NOT EXISTS vt_{{keyspace}}'" vitess su -p -s /bin/bash -c "/vt/bin/vttablet + -topo_implementation etcd + -etcd_global_addrs http://$ETCD_GLOBAL_SERVICE_HOST:$ETCD_GLOBAL_SERVICE_PORT -log_dir $VTDATAROOT/tmp + -alsologtostderr -port {{port}} -tablet-path {{alias}} - -tablet_hostname $hostname + -tablet_hostname $(hostname -i) + -init_keyspace {{keyspace}} + -init_shard {{shard}} -target_tablet_type replica -mysqlctl_socket $VTDATAROOT/mysqlctl.sock + -db-config-app-uname vt_app -db-config-app-dbname vt_{{keyspace}} + -db-config-app-charset utf8 + -db-config-dba-uname vt_dba -db-config-dba-dbname vt_{{keyspace}} + -db-config-dba-charset utf8 + -db-config-repl-uname vt_repl -db-config-repl-dbname vt_{{keyspace}} + -db-config-repl-charset utf8 + -db-config-filtered-uname vt_filtered -db-config-filtered-dbname vt_{{keyspace}} + -db-config-filtered-charset utf8 -enable-rowcache -rowcache-bin /usr/bin/memcached - -rowcache-socket $VTDATAROOT/{{tablet_subdir}}/memcache.sock - &>> $log_file" vitess - env: - - name: ZK_CLIENT_CONFIG - value: /vt/zk-client-conf.json - ports: - - # We only publish this port so we can access the tablet's status - # page from outside the Kubernetes cluster. You can unpublish it if - # you don't care about that. - name: vttablet - containerPort: {{port}} - hostPort: {{port}} + -rowcache-socket $VTDATAROOT/{{tablet_subdir}}/memcache.sock" vitess - name: mysql image: vitess/root volumeMounts: @@ -87,13 +71,28 @@ desiredState: - sh - "-c" - >- - echo "{\"test_cell\":\"$SERVICE_HOST:2181\",\"global\":\"$SERVICE_HOST:2181\"}" > /vt/zk-client-conf.json && mkdir -p $VTDATAROOT/tmp && - chown -R vitess /vt && - su -p -c "/vt/bin/mysqlctld -log_dir $VTDATAROOT/tmp -tablet_uid {{uid}} -socket_file $VTDATAROOT/mysqlctl.sock -bootstrap_archive mysql-db-dir_10.0.13-MariaDB.tbz" vitess + chown -R vitess /vt + + su -p -c "/vt/bin/mysqlctld + -log_dir $VTDATAROOT/tmp + -alsologtostderr + -tablet_uid {{uid}} + -socket_file $VTDATAROOT/mysqlctl.sock + -db-config-app-uname vt_app + -db-config-app-dbname vt_{{keyspace}} + -db-config-app-charset utf8 + -db-config-dba-uname vt_dba + -db-config-dba-dbname vt_{{keyspace}} + -db-config-dba-charset utf8 + -db-config-repl-uname vt_repl + -db-config-repl-dbname vt_{{keyspace}} + -db-config-repl-charset utf8 + -db-config-filtered-uname vt_filtered + -db-config-filtered-dbname vt_{{keyspace}} + -db-config-filtered-charset utf8 + -bootstrap_archive mysql-db-dir_10.0.13-MariaDB.tbz" vitess env: - - name: ZK_CLIENT_CONFIG - value: /vt/zk-client-conf.json - name: EXTRA_MY_CNF value: /vt/config/mycnf/master_mariadb.cnf volumes: @@ -103,6 +102,6 @@ desiredState: source: {emptyDir: {}} labels: name: vttablet - keyspace: {{keyspace}} - shard: {{shard}} - tabletAlias: {{alias}} + keyspace: "{{keyspace}}" + shard: "{{shard}}" + tablet: "{{alias}}" diff --git a/examples/kubernetes/vttablet-up.sh b/examples/kubernetes/vttablet-up.sh index f62cf8edb91..f77377e2a5a 100755 --- a/examples/kubernetes/vttablet-up.sh +++ b/examples/kubernetes/vttablet-up.sh @@ -1,16 +1,18 @@ #!/bin/bash # This is an example script that creates a single shard vttablet deployment. -# It assumes that kubernetes/cluster/kubecfg.sh is in the path. set -e +script_root=`dirname "${BASH_SOURCE}"` +source $script_root/env.sh + # Create the pods for shard-0 -cell=test_cell -keyspace=test_keyspace +cell='test' +keyspace='test_keyspace' shard=0 uid_base=100 -port_base=15000 +port=15002 echo "Creating $keyspace.shard-$shard pods in cell $cell..." for uid_index in 0 1 2; do @@ -18,30 +20,16 @@ for uid_index in 0 1 2; do printf -v alias '%s-%010d' $cell $uid printf -v tablet_subdir 'vt_%010d' $uid - # It's not strictly necessary to assign a unique port to every tablet since - # Kubernetes gives every pod its own IP address. However, Kubernetes currently - # doesn't provide routing from the internet into a particular pod, so we have - # to publish a port to the host if we want to access each tablet's status page - # from a workstation. As a result, we need tablets to have unique ports or - # else Kubernetes will be unable to schedule more than one tablet per host. - port=$[$port_base + $uid] - - if [ "$uid_index" == "0" ]; then - type=master - else - type=replica - fi - echo "Creating pod for tablet $alias..." # Expand template variables sed_script="" - for var in alias cell uid keyspace shard type port tablet_subdir; do + for var in alias cell uid keyspace shard port tablet_subdir; do sed_script+="s/{{$var}}/${!var}/g;" done - # Instantiate template and send to kubecfg. + # Instantiate template and send to kubectl. cat vttablet-pod-template.yaml | \ sed -e "$sed_script" | \ - kubecfg.sh -c - create pods + $KUBECTL create -f - done diff --git a/examples/kubernetes/zk-client-service.yaml b/examples/kubernetes/zk-client-service.yaml deleted file mode 100644 index c4e274a7c4f..00000000000 --- a/examples/kubernetes/zk-client-service.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1beta1 -kind: Service -id: zk-client -port: 2181 -containerPort: 2181 -selector: - name: zk -labels: - name: zk diff --git a/examples/kubernetes/zk-down.sh b/examples/kubernetes/zk-down.sh deleted file mode 100755 index 1c2d8b06b74..00000000000 --- a/examples/kubernetes/zk-down.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -# This is an example script that tears down the ZooKeeper servers started by -# zk-up.sh. It assumes that kubernetes/cluster/kubecfg.sh is in the path. - -# Delete pods. -for zkid in 1 2 3; do - echo "Deleting zk$zkid pod..." - kubecfg.sh delete pods/zk$zkid -done - -# Delete client service. -echo "Deleting zk-client service..." -kubecfg.sh delete services/zk-client - -# Delete leader and election services. -for zkid in 1 2 3; do - echo "Deleting zk$zkid-leader service..." - kubecfg.sh delete services/zk$zkid-leader - - echo "Deleting zk$zkid-election service..." - kubecfg.sh delete services/zk$zkid-election -done diff --git a/examples/kubernetes/zk-pod-template.yaml b/examples/kubernetes/zk-pod-template.yaml deleted file mode 100644 index ea23e8fedb8..00000000000 --- a/examples/kubernetes/zk-pod-template.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: v1beta1 -kind: Pod -id: zk{{zkid}} -desiredState: - manifest: - version: v1beta1 - id: zk{{zkid}} - containers: - - name: zk{{zkid}} - image: vitess/root - volumeMounts: - - name: syslog - mountPath: /dev/log - - name: vtdataroot - mountPath: /vt/vtdataroot - command: - - sh - - "-c" - - >- - mkdir -p $VTDATAROOT/tmp && - chown -R vitess /vt && - su -p -c "/vt/bin/zkctld -zk.myid {{zkid}} -zk.cfg {{zkcfg}} -log_dir $VTDATAROOT/tmp" vitess - volumes: - - name: syslog - source: {hostDir: {path: /dev/log}} - - name: vtdataroot - source: {emptyDir: {}} -labels: - name: zk - zkid: {{zkid}} diff --git a/examples/kubernetes/zk-service-template.yaml b/examples/kubernetes/zk-service-template.yaml deleted file mode 100644 index 392a8e03e4a..00000000000 --- a/examples/kubernetes/zk-service-template.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1beta1 -kind: Service -id: zk{{zkid}}-{{svc}} -port: {{port}}{{zkid}} -containerPort: {{port}}{{zkid}} -selector: - name: zk - zkid: {{zkid}} -labels: - name: zk - zkid: {{zkid}} diff --git a/examples/kubernetes/zk-up.sh b/examples/kubernetes/zk-up.sh deleted file mode 100755 index 4c6b13e74ea..00000000000 --- a/examples/kubernetes/zk-up.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# This is an example script that creates a quorum of ZooKeeper servers. -# It assumes that kubernetes/cluster/kubecfg.sh is in the path. - -set -e - -# List of all servers in the quorum. -zkcfg=(\ - '1@$SERVICE_HOST:28881:38881:2181' \ - '2@$SERVICE_HOST:28882:38882:2181' \ - '3@$SERVICE_HOST:28883:38883:2181' \ - ) -printf -v zkcfg ",%s" "${zkcfg[@]}" -zkcfg=${zkcfg:1} - -# Create the client service, which will load-balance across all replicas. -echo "Creating zk services..." -kubecfg.sh -c zk-client-service.yaml create services - -# Create a service for the leader and election ports of each replica. -# This is necessary because ZooKeeper servers need to know how to specifically -# contact replica N (not just "any replica") in order to create a quorum. -# We also have to append the zkid of each server to the port number, because -# every service in Kubernetes needs a unique port number (for now). - -ports=( 2888 3888 ) -svcs=( leader election ) - -for zkid in 1 2 3; do - for i in 0 1; do - port=${ports[$i]} - svc=${svcs[$i]} - - cat zk-service-template.yaml | \ - sed -e "s/{{zkid}}/$zkid/g" -e "s/{{port}}/$port/g" -e "s/{{svc}}/$svc/g" | \ - kubecfg.sh -c - create services - done -done - -# Create the pods. -echo "Creating zk pods..." -for zkid in 1 2 3; do - cat zk-pod-template.yaml | \ - sed -e "s/{{zkid}}/$zkid/g" -e "s/{{zkcfg}}/$zkcfg/g" | \ - kubecfg.sh -c - create pods -done diff --git a/examples/local/README.md b/examples/local/README.md new file mode 100644 index 00000000000..9e58cb1611e --- /dev/null +++ b/examples/local/README.md @@ -0,0 +1,192 @@ +# Local Vitess Cluster + +This directory contains example scripts to bring up a Vitess cluster on your +local machine, which may be useful for experimentation. These scripts can +also serve as a starting point for configuring Vitess into your preferred +deployment strategy or toolset. + +## Requirements + +You should have completed a successful `make build` after following the +[Getting Started](https://github.com/youtube/vitess/blob/master/doc/GettingStarted.md) +guide. + +## Configuration + +Before starting, set `VTROOT` to the base of the Vitess tree that you built. +For example, if you ran `make build` while in +`$HOME/vt/src/github.com/youtube/vitess`, then you should set: +`export VTROOT=$HOME/vt` + +Also set `VTDATAROOT` to the directory where you want data files and logs to +be stored. For example: `export VTDATAROOT=$HOME/vtdataroot` + +Alternatively, if you are testing on the same machine you just used for +building, you can source `dev.env` as described the Getting Started guide. + +## Starting ZooKeeper + +The way servers in a Vitess cluster find each other is by looking for dynamic +configuration stored in a distributed lock service. For this example, we will +use ZooKeeper. The following script creates a small ZooKeeper cluster. + +``` +vitess/examples/local$ ./zk-up.sh +Starting zk servers... +Waiting for zk servers to be ready... +``` + +Once we have a ZooKeeper cluster, we only need to tell each Vitess process how +to connect to ZooKeeper. Then they can find all the other Vitess processes +by coordinating via ZooKeeper. The way we tell them how to find ZooKeeper is by +setting the `ZK_CLIENT_CONFIG` environment variable to the path to the file +`zk-client-conf.json`, which contains ZK server addresses for each cell. + +## Starting vtctld + +The vtctld server provides a web interface to view all the coordination +information stored in ZooKeeper. + +``` +vitess/examples/local$ ./vtctld-up.sh +Starting vtctld... +Access vtctld at http://localhost:15000 +``` + +There won't be anything there yet, but the menu should come up, +verifying that vtctld is running. + +## Issuing commands with vtctlclient + +The vtctld server also accepts commands from the vtctlclient tool, +which is used to administer the cluster. + +``` +# list available commands +vitess/examples/local$ $VTROOT/bin/vtctlclient -server localhost:15000 +``` + +## Starting vttablets + +For this example, we will bring up 3 tablets. + +``` +vitess/examples/local$ ./vttablet-up.sh +Starting MySQL for tablet test-0000000100... +Starting vttablet for test-0000000100... +Access tablet test-0000000100 at http://localhost:15100/debug/status +Starting MySQL for tablet test-0000000101... +Starting vttablet for test-0000000101... +Access tablet test-0000000101 at http://localhost:15101/debug/status +Starting MySQL for tablet test-0000000102... +Starting vttablet for test-0000000102... +Access tablet test-0000000102 at http://localhost:15102/debug/status +``` + +Once they are up, go back to the vtctld web page and click on the +*DbTopology Tool*. You should see all three tablets listed. If you click the +address of one of the tablets, you'll see the coordination data stored in +ZooKeeper. At the top of that page, there is also `[status]` link, which takes +you to the debug page generated by the tablet itself. + +By bringing up tablets into a previously empty keyspace, we effectively just +created a new shard. To initialize the keyspace for the new shard, we need to +perform a keyspace rebuild: + +``` +$ $VTROOT/bin/vtctlclient -server localhost:15000 RebuildKeyspaceGraph test_keyspace +``` + +Note that most vtctlclient commands produce no output on success. + +## Electing a master vttablet + +The vttablets have all been started as replicas, but there is no master yet. +When we pick a master vttablet, Vitess will also take care of connecting the +other replicas' mysqld instances to start replicating from the master mysqld. + +Since this is the first time we're starting up the shard, there is no existing +replication happening, so we use the -force flag on ReparentShard to skip the +usual validation of each tablet's replication state. + +``` +$ $VTROOT/bin/vtctlclient -server localhost:15000 ReparentShard -force test_keyspace/0 test-0000000100 +``` + +Once this is done, you should see one master and two replicas in vtctld's web +interface. You can also check this on the command line with vtctlclient: + +``` +$ $VTROOT/bin/vtctlclient -server localhost:15000 ListAllTablets test +test-0000000100 test_keyspace 0 master localhost:15100 localhost:33100 [] +test-0000000101 test_keyspace 0 replica localhost:15101 localhost:33101 [] +test-0000000102 test_keyspace 0 replica localhost:15102 localhost:33102 [] +``` + +## Creating a table + +The vtctl tool can manage schema across all tablets in a keyspace. +To create the table defined in `create_test_table.sql`: + +``` +vitess/examples/local$ $VTROOT/bin/vtctlclient -server localhost:15000 ApplySchemaKeyspace -simple -sql "$(cat create_test_table.sql)" test_keyspace +``` + +# Starting vtgate + +Clients send queries to Vitess through vtgate, which routes them to the +correct vttablet behind the scenes. In a real deployment, you would likely +run multiple vtgate instances to share the load. For this local example, +we only need one. + +``` +vitess/examples/local$ ./vtgate-up.sh +``` + +## Creating a client app + +The file `client.py` contains a simple example app that connects to vtgate +and executes some queries. To run it, you need to add the Vitess Python +packages to your `PYTHONPATH`. Or you can use the wrapper script `client.sh` +that temporarily sets up the environment and then runs `client.py`. + +``` +vitess/examples/local$ ./client.sh --server=localhost:15001 +Inserting into master... +Reading from master... +(1L, 'V is for speed') +Reading from replica... +(1L, 'V is for speed') +``` + +## Tearing down the cluster + +Assuming your `VTDATAROOT` directory is something that you use just for this, +you can kill all the processes we've started like this: + +``` +# look for processes we started +$ pgrep -u $USER -f -l $VTDATAROOT + +# if that list looks right, then kill them +$ pkill -u $USER -f $VTDATAROOT +``` + +To start over, you should also clear out the contents of `VTDATAROOT`: + +``` +$ cd $VTDATAROOT +/home/user/vt/vtdataroot$ rm -rf * +``` + +## Troubleshooting + +If anything goes wrong, check the logs in your `$VTDATAROOT/tmp` directory +for error messages. There are also some tablet-specific logs, as well as +MySQL logs in the various `$VTDATAROOT/vt_*` directories. + +If you need help diagnosing a problem, send a message to our +[mailing list](https://groups.google.com/forum/#!forum/vitess). +In addition to any errors you see at the command-line, it would also help to +upload an archive of your `$VTDATAROOT` directory to a file sharing service +and provide a link to it. diff --git a/examples/kubernetes/client.py b/examples/local/client.py old mode 100755 new mode 100644 similarity index 73% rename from examples/kubernetes/client.py rename to examples/local/client.py index 913a010d39e..20db25bab73 --- a/examples/kubernetes/client.py +++ b/examples/local/client.py @@ -6,12 +6,10 @@ from vtdb import keyrange_constants from vtdb import vtgatev2 from vtdb import vtgate_cursor -from vtdb import topology from zk import zkocc # Constants and params UNSHARDED = [keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE)] -cursorclass = vtgate_cursor.VTGateCursor # Parse args parser = argparse.ArgumentParser() @@ -19,20 +17,14 @@ parser.add_argument('--timeout', dest='timeout', type=float, default='10.0') args = parser.parse_args() -vtgate_addrs = {"_vt": [args.server]} +vtgate_addrs = {"vt": [args.server]} # Connect conn = vtgatev2.connect(vtgate_addrs, args.timeout) -# Read topology -# This is a temporary work-around until the VTGate V2 client is topology-free. -topoconn = zkocc.ZkOccConnection(args.server, 'test_cell', args.timeout) -topology.read_topology(topoconn) -topoconn.close() - # Insert something. print('Inserting into master...') -cursor = conn.cursor(cursorclass, conn, 'test_keyspace', 'master', +cursor = conn.cursor('test_keyspace', 'master', keyranges=UNSHARDED, writable=True) cursor.begin() cursor.execute( @@ -52,7 +44,7 @@ # Read from a replica. # Note that this may be behind master due to replication lag. print('Reading from replica...') -cursor = conn.cursor(cursorclass, conn, 'test_keyspace', 'replica', +cursor = conn.cursor('test_keyspace', 'replica', keyranges=UNSHARDED) cursor.execute('SELECT * FROM test_table', {}) for row in cursor.fetchall(): diff --git a/examples/local/client.sh b/examples/local/client.sh new file mode 100755 index 00000000000..09dc573a374 --- /dev/null +++ b/examples/local/client.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# This is a wrapper script that sets up the environment for client.py. + +set -e + +hostname=`hostname -f` + +# We expect to find zk-client-conf.json in the same folder as this script. +script_root=`dirname "${BASH_SOURCE}"` + +# Set up environment. +for pkg in `find $VTROOT/dist -name site-packages`; do + export PYTHONPATH=$pkg:$PYTHONPATH +done + +export PYTHONPATH=$VTROOT/py-vtdb:$PYTHONPATH + +exec env python $script_root/client.py $* diff --git a/examples/local/create_test_table.sql b/examples/local/create_test_table.sql new file mode 100644 index 00000000000..e6660fea1c6 --- /dev/null +++ b/examples/local/create_test_table.sql @@ -0,0 +1,6 @@ +CREATE TABLE test_table ( + id BIGINT AUTO_INCREMENT, + msg VARCHAR(250), + PRIMARY KEY (id) +) ENGINE=InnoDB + diff --git a/examples/local/vtctld-up.sh b/examples/local/vtctld-up.sh new file mode 100755 index 00000000000..c112435fa99 --- /dev/null +++ b/examples/local/vtctld-up.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# This is an example script that starts vtctld. + +set -e + +port=15000 + +hostname=`hostname -f` + +# We expect to find zk-client-conf.json in the same folder as this script. +script_root=`dirname "${BASH_SOURCE}"` + +# Set up environment. +export VTTOP=$VTROOT/src/github.com/youtube/vitess +export LD_LIBRARY_PATH=$VTROOT/dist/vt-zookeeper-3.3.5/lib:$LD_LIBRARY_PATH +export ZK_CLIENT_CONFIG=$script_root/zk-client-conf.json +mkdir -p $VTDATAROOT/tmp + +echo "Starting vtctld..." +$VTROOT/bin/vtctld -debug -templates $VTTOP/go/cmd/vtctld/templates \ + -log_dir $VTDATAROOT/tmp -port $port > $VTDATAROOT/tmp/vtctld.out 2>&1 & +disown -a + +echo "Access vtctld at http://$hostname:$port" diff --git a/examples/local/vtgate-up.sh b/examples/local/vtgate-up.sh new file mode 100755 index 00000000000..faaaa9d25d6 --- /dev/null +++ b/examples/local/vtgate-up.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# This is an example script that starts a single vtgate. + +set -e + +cell='test' +port=15001 + +hostname=`hostname -f` + +# We expect to find zk-client-conf.json in the same folder as this script. +script_root=`dirname "${BASH_SOURCE}"` + +# Set up environment. +export LD_LIBRARY_PATH=$VTROOT/dist/vt-zookeeper-3.3.5/lib:$LD_LIBRARY_PATH +export ZK_CLIENT_CONFIG=$script_root/zk-client-conf.json +mkdir -p $VTDATAROOT/tmp + +# Start vtgate. +$VTROOT/bin/vtgate -log_dir $VTDATAROOT/tmp -port $port -cell $cell \ + > $VTDATAROOT/tmp/vtgate.out 2>&1 & +echo "Access vtgate at http://$hostname:$port/debug/status" + +disown -a diff --git a/examples/local/vttablet-up.sh b/examples/local/vttablet-up.sh new file mode 100755 index 00000000000..919e51847c1 --- /dev/null +++ b/examples/local/vttablet-up.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +# This is an example script that creates a single shard vttablet deployment. + +set -e + +cell='test' +keyspace='test_keyspace' +shard=0 +uid_base=100 +port_base=15100 +mysql_port_base=33100 + +hostname=`hostname -f` + +# We expect to find zk-client-conf.json in the same folder as this script. +script_root=`dirname "${BASH_SOURCE}"` + +dbconfig_flags="\ + -db-config-app-uname vt_app \ + -db-config-app-dbname vt_$keyspace \ + -db-config-app-charset utf8 \ + -db-config-dba-uname vt_dba \ + -db-config-dba-charset utf8 \ + -db-config-repl-uname vt_repl \ + -db-config-repl-dbname vt_$keyspace \ + -db-config-repl-charset utf8 \ + -db-config-filtered-uname vt_filtered \ + -db-config-filtered-dbname vt_$keyspace \ + -db-config-filtered-charset utf8" + +# Set up environment. +export LD_LIBRARY_PATH=$VTROOT/dist/vt-zookeeper-3.3.5/lib:$LD_LIBRARY_PATH +export ZK_CLIENT_CONFIG=$script_root/zk-client-conf.json +export EXTRA_MY_CNF=$VTROOT/config/mycnf/master_mariadb.cnf +mkdir -p $VTDATAROOT/tmp + +# Try to find mysqld_safe on PATH. +if [ -z "$VT_MYSQL_ROOT" ]; then + mysql_path=`which mysqld_safe` + if [ -z "$mysql_path" ]; then + echo "Can't guess location of mysqld_safe. Please set VT_MYSQL_ROOT so it can be found at \$VT_MYSQL_ROOT/bin/mysqld_safe." + exit 1 + fi + export VT_MYSQL_ROOT=$(dirname `dirname $mysql_path`) +fi + +# Look for memcached. +memcached_path=`which memcached` +if [ -z "$memcached_path" ]; then + echo "Can't find memcached. Please make sure it is available in PATH." + exit 1 +fi + +# Start 3 vttablets. +for uid_index in 0 1 2; do + uid=$[$uid_base + $uid_index] + port=$[$port_base + $uid_index] + mysql_port=$[$mysql_port_base + $uid_index] + printf -v alias '%s-%010d' $cell $uid + printf -v tablet_dir 'vt_%010d' $uid + + echo "Starting MySQL for tablet $alias..." + $VTROOT/bin/mysqlctl -log_dir $VTDATAROOT/tmp -tablet_uid $uid $dbconfig_flags \ + -mysql_port $mysql_port \ + init -bootstrap_archive mysql-db-dir_10.0.13-MariaDB.tbz + + $VT_MYSQL_ROOT/bin/mysql -u vt_dba -S $VTDATAROOT/$tablet_dir/mysql.sock \ + -e "CREATE DATABASE IF NOT EXISTS vt_$keyspace" + + echo "Starting vttablet for $alias..." + $VTROOT/bin/vttablet -log_dir $VTDATAROOT/tmp -port $port $dbconfig_flags \ + -tablet-path $alias \ + -init_keyspace $keyspace \ + -init_shard $shard \ + -target_tablet_type replica \ + -enable-rowcache \ + -rowcache-bin $memcached_path \ + -rowcache-socket $VTDATAROOT/$tablet_dir/memcache.sock \ + > $VTDATAROOT/$tablet_dir/vttablet.out 2>&1 & + + echo "Access tablet $alias at http://$hostname:$port/debug/status" +done + +disown -a diff --git a/examples/local/zk-client-conf.json b/examples/local/zk-client-conf.json new file mode 100644 index 00000000000..02aa41d2b26 --- /dev/null +++ b/examples/local/zk-client-conf.json @@ -0,0 +1,4 @@ +{ + "test": "localhost:21811,localhost:21812,localhost:21813", + "global": "localhost:21811,localhost:21812,localhost:21813" +} diff --git a/examples/local/zk-up.sh b/examples/local/zk-up.sh new file mode 100755 index 00000000000..be88f56fabc --- /dev/null +++ b/examples/local/zk-up.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# This is an example script that creates a quorum of ZooKeeper servers. + +set -e + +hostname=`hostname -f` + +# Each ZooKeeper server needs a list of all servers in the quorum. +# Since we're running them all locally, we need to give them unique ports. +# In a real deployment, these should be on different machines, and their +# respective hostnames should be given. +zkcfg=(\ + "1@$hostname:28881:38881:21811" \ + "2@$hostname:28882:38882:21812" \ + "3@$hostname:28883:38883:21813" \ + ) +printf -v zkcfg ",%s" "${zkcfg[@]}" +zkcfg=${zkcfg:1} + +# Set up environment. +export LD_LIBRARY_PATH=$VTROOT/dist/vt-zookeeper-3.3.5/lib:$LD_LIBRARY_PATH +mkdir -p $VTDATAROOT/tmp + +# Start ZooKeeper servers. +# The "zkctl init" command won't return until the server is able to contact its +# peers, so we need to start them all in the background and then wait for them. +echo "Starting zk servers..." +for zkid in 1 2 3; do + $VTROOT/bin/zkctl -zk.myid $zkid -zk.cfg $zkcfg -log_dir $VTDATAROOT/tmp init \ + > $VTDATAROOT/tmp/zkctl_$zkid.out 2>&1 & +done + +# Wait for all the zkctl commands to return. +echo "Waiting for zk servers to be ready..." +wait diff --git a/go/cgzip/cgzip_test.go b/go/cgzip/cgzip_test.go index 26b500c4784..99d0290215d 100644 --- a/go/cgzip/cgzip_test.go +++ b/go/cgzip/cgzip_test.go @@ -25,18 +25,26 @@ func newPrettyTimer(name string) *prettyTimer { } func (pt *prettyTimer) stopAndPrintCompress(t *testing.T, size, processed int) { - durationMs := int(int64(time.Now().Sub(pt.before)) / 1000) + duration := time.Since(pt.before) t.Log(pt.name + ":") t.Log(" size :", size) - t.Log(" time :", durationMs, "ms") - t.Log(" speed:", processed*1000/durationMs, "KB/s") + t.Log(" time :", duration.String()) + if duration != 0 { + t.Logf(" speed: %.0f KB/s", float64(processed)/duration.Seconds()/1024.0) + } else { + t.Log(" processed:", processed, "B") + } } func (pt *prettyTimer) stopAndPrintUncompress(t *testing.T, processed int) { - durationMs := int(int64(time.Now().Sub(pt.before)) / 1000) + duration := time.Since(pt.before) t.Log(" " + pt.name + ":") - t.Log(" time :", durationMs, "ms") - t.Log(" speed:", processed*1000/durationMs, "KB/s") + t.Log(" time :", duration.String()) + if duration != 0 { + t.Logf(" speed: %.0f KB/s", float64(processed)/duration.Seconds()/1024.0) + } else { + t.Log(" processed:", processed, "B") + } } func compareCompressedBuffer(t *testing.T, source []byte, compressed *bytes.Buffer) { diff --git a/go/cmd/bsongen/bsongen_test.go b/go/cmd/bsongen/bsongen_test.go index d3ccd0d0b88..0efab6289ca 100644 --- a/go/cmd/bsongen/bsongen_test.go +++ b/go/cmd/bsongen/bsongen_test.go @@ -6,7 +6,7 @@ package main import ( "bytes" - "fmt" + "errors" "io/ioutil" "os" "os/exec" @@ -17,23 +17,40 @@ import ( ) func TestValidFiles(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode.") + } + inputs := testfiles.Glob("bson_test/input*.go") for _, input := range inputs { b, err := ioutil.ReadFile(input) if err != nil { - t.Fatal(err) + t.Fatalf("ioutil.ReadFile error: %v", err) } - out, err := generateCode(string(b), "MyType") + want, err := ioutil.ReadFile(strings.Replace(input, "input", "output", 1)) if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - return + t.Fatalf("ioutil.ReadFile error: %v", err) } - want, err := ioutil.ReadFile(strings.Replace(input, "input", "output", 1)) + + out, err := generateCode(string(b), "MyType") if err != nil { - t.Fatal(err) + t.Fatalf("generateCode error: %v", err) } + // goimports is flaky. So, let's not test that part. - d, err := diff(skip_imports(want), skip_imports(out)) + want, err = skipImports(want) + if err != nil { + t.Fatalf("skipImports error: %v", err) + } + out, err = skipImports(out) + if err != nil { + t.Fatalf("skipImports error: %v", err) + } + + d, err := diff(want, out) + if err != nil { + t.Fatalf("diff error: %v", err) + } if len(d) != 0 { t.Errorf("Unexpected output for %s:\n%s", input, string(d)) if testing.Verbose() { @@ -71,18 +88,16 @@ func diff(b1, b2 []byte) (data []byte, err error) { return } -func skip_imports(b []byte) []byte { - buf := bytes.NewBuffer(b) - for { - line, err := buf.ReadBytes('\n') - if err != nil { - return b[:0] - } - if len(line) == 0 || line[0] != ')' { - continue - } - return b[len(b)-buf.Len():] +func skipImports(b []byte) ([]byte, error) { + begin := bytes.Index(b, []byte("\nimport (\n")) + if begin < 0 { + return nil, errors.New("couldn't find beginning of import block") + } + end := bytes.Index(b, []byte("\n)\n")) + if end < 0 { + return nil, errors.New("couldn't find end of imports block") } + return append(b[:begin], b[end+3:]...), nil } var invalidInputs = []struct{ title, input, err string }{ diff --git a/go/cmd/mysqlctl/mysqlctl.go b/go/cmd/mysqlctl/mysqlctl.go index ddc8e10ff00..dc6a933b264 100644 --- a/go/cmd/mysqlctl/mysqlctl.go +++ b/go/cmd/mysqlctl/mysqlctl.go @@ -8,13 +8,12 @@ package main import ( "flag" "fmt" - "net/url" "os" - "strings" log "github.com/golang/glog" + "github.com/youtube/vitess/go/exit" + "github.com/youtube/vitess/go/netutil" "github.com/youtube/vitess/go/vt/dbconfigs" - "github.com/youtube/vitess/go/vt/key" "github.com/youtube/vitess/go/vt/logutil" "github.com/youtube/vitess/go/vt/mysqlctl" ) @@ -22,204 +21,121 @@ import ( var ( port = flag.Int("port", 6612, "vtocc port") mysqlPort = flag.Int("mysql_port", 3306, "mysql port") - tabletUid = flag.Uint("tablet_uid", 41983, "tablet uid") + tabletUID = flag.Uint("tablet_uid", 41983, "tablet uid") mysqlSocket = flag.String("mysql_socket", "", "path to the mysql socket") tabletAddr string ) -func initCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) { +func initCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) error { waitTime := subFlags.Duration("wait_time", mysqlctl.MysqlWaitTime, "how long to wait for startup") bootstrapArchive := subFlags.String("bootstrap_archive", "mysql-db-dir.tbz", "name of bootstrap archive within vitess/data/bootstrap directory") skipSchema := subFlags.Bool("skip_schema", false, "don't apply initial schema") subFlags.Parse(args) if err := mysqld.Init(*waitTime, *bootstrapArchive, *skipSchema); err != nil { - log.Fatalf("failed init mysql: %v", err) + return fmt.Errorf("failed init mysql: %v", err) } + return nil } -func multisnapshotCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) { - concurrency := subFlags.Int("concurrency", 8, "how many compression jobs to run simultaneously") - spec := subFlags.String("spec", "-", "shard specification") - tablesString := subFlags.String("tables", "", "dump only this comma separated list of regexp for tables") - excludeTablesString := subFlags.String("exclude_tables", "", "do not dump this comma separated list of regexp for tables") - skipSlaveRestart := subFlags.Bool("skip_slave_restart", false, "after the snapshot is done, do not restart slave replication") - maximumFilesize := subFlags.Uint64("maximum_file_size", 128*1024*1024, "the maximum size for an uncompressed data file") - keyType := subFlags.String("key_type", "uint64", "type of the key column") - subFlags.Parse(args) - if subFlags.NArg() != 2 { - log.Fatalf("action multisnapshot requires{{.}}+{{end}} + +{{with .Actions}} +
{{.Output}}+ + + + + diff --git a/go/cmd/vtctld/topo_data.go b/go/cmd/vtctld/topo_data.go new file mode 100644 index 00000000000..be5220ddb88 --- /dev/null +++ b/go/cmd/vtctld/topo_data.go @@ -0,0 +1,373 @@ +package main + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + "sync" + "time" + + "github.com/youtube/vitess/go/vt/topo" +) + +// This file includes the support for serving topo data to an ajax-based +// front-end. There are three ways we collect data: +// - reading topology records that don't change too often, caching them +// with a somewhat big TTL, and have a 'flush' command in case it's needed. +// (list of cells, tablets in a cell/shard, ...) +// - subscribing to topology change channels (serving graph) +// - establishing streaming connections to vttablets to get up-to-date +// health reports. + +// VersionedObject is the interface implemented by objects in the versioned +// object cache VersionedObjectCache object. +type VersionedObject interface { + GetVersion() int + SetVersion(int) + Reset() +} + +// BaseVersionedObject is the base implementation for VersionedObject +// that handles the version stuff. It handles GetVersion and SetVersion. +// Inherited types still need to implement Reset(). +type BaseVersionedObject struct { + Version int +} + +// GetVersion is part of the VersionedObject interface +func (bvo *BaseVersionedObject) GetVersion() int { + return bvo.Version +} + +// SetVersion is part of the VersionedObject interface +func (bvo *BaseVersionedObject) SetVersion(version int) { + bvo.Version = version +} + +// GetVersionedObjectFunc is the function the cache will call to get +// the object itself. +type GetVersionedObjectFunc func() (VersionedObject, error) + +// VersionedObjectCache is the main cache object. Just needs a method to get +// the content. +type VersionedObjectCache struct { + getObject GetVersionedObjectFunc + + mu sync.Mutex + timestamp time.Time + versionedObject VersionedObject + result []byte +} + +// NewVersionedObjectCache returns a new VersionedObjectCache object. +func NewVersionedObjectCache(getObject GetVersionedObjectFunc) *VersionedObjectCache { + return &VersionedObjectCache{ + getObject: getObject, + } +} + +// Get returns the versioned value from the cache. +func (voc *VersionedObjectCache) Get() ([]byte, error) { + voc.mu.Lock() + defer voc.mu.Unlock() + + now := time.Now() + if now.Sub(voc.timestamp) < 5*time.Minute { + return voc.result, nil + } + + newObject, err := voc.getObject() + if err != nil { + return nil, err + } + if voc.versionedObject != nil { + // we already have an object, check if it's equal + newObject.SetVersion(voc.versionedObject.GetVersion()) + if reflect.DeepEqual(newObject, voc.versionedObject) { + voc.timestamp = now + return voc.result, nil + } + + // it's not equal increment the version + newObject.SetVersion(voc.versionedObject.GetVersion() + 1) + } else { + newObject.SetVersion(1) + } + + // we have a new object, update the version, and save it + voc.result, err = json.MarshalIndent(newObject, "", " ") + if err != nil { + return nil, err + } + voc.versionedObject = newObject + voc.timestamp = now + return voc.result, nil +} + +// Flush will flush the cache, and force a version increment, so +// clients will reload. +func (voc *VersionedObjectCache) Flush() { + voc.mu.Lock() + defer voc.mu.Unlock() + + // we reset timestamp and content, so the Version will increase again + // and force a client refresh, even if the data is the same. + voc.timestamp = time.Time{} + if voc.versionedObject != nil { + voc.versionedObject.Reset() + } +} + +// VersionedObjectCacheFactory knows how to construct a VersionedObjectCache +// from the key +type VersionedObjectCacheFactory func(key string) *VersionedObjectCache + +// VersionedObjectCacheMap is a map of VersionedObjectCache protected +// by a mutex. +type VersionedObjectCacheMap struct { + factory VersionedObjectCacheFactory + + mu sync.Mutex + cacheMap map[string]*VersionedObjectCache +} + +// NewVersionedObjectCacheMap returns a new VersionedObjectCacheFactory +// that uses the factory method to create individual VersionedObjectCache. +func NewVersionedObjectCacheMap(factory VersionedObjectCacheFactory) *VersionedObjectCacheMap { + return &VersionedObjectCacheMap{ + factory: factory, + cacheMap: make(map[string]*VersionedObjectCache), + } +} + +// Get finds the right VersionedObjectCache and returns its value +func (vocm *VersionedObjectCacheMap) Get(key string) ([]byte, error) { + vocm.mu.Lock() + voc, ok := vocm.cacheMap[key] + if !ok { + voc = vocm.factory(key) + vocm.cacheMap[key] = voc + } + vocm.mu.Unlock() + + return voc.Get() +} + +// Flush will flush the entire cache +func (vocm *VersionedObjectCacheMap) Flush() { + vocm.mu.Lock() + defer vocm.mu.Unlock() + for _, voc := range vocm.cacheMap { + voc.Flush() + } +} + +// KnownCells contains the cached result of topo.Server.GetKnownCells +type KnownCells struct { + BaseVersionedObject + + // Cells is the list of Known Cells for this topology + Cells []string +} + +// Reset is part of the VersionedObject interface +func (kc *KnownCells) Reset() { + kc.Cells = nil +} + +func newKnownCellsCache(ts topo.Server) *VersionedObjectCache { + return NewVersionedObjectCache(func() (VersionedObject, error) { + cells, err := ts.GetKnownCells() + if err != nil { + return nil, err + } + return &KnownCells{ + Cells: cells, + }, nil + }) +} + +// Keyspaces contains the cached result of topo.Server.GetKeyspaces +type Keyspaces struct { + BaseVersionedObject + + // Keyspaces is the list of Keyspaces for this topology + Keyspaces []string +} + +// Reset is part of the VersionedObject interface +func (k *Keyspaces) Reset() { + k.Keyspaces = nil +} + +func newKeyspacesCache(ts topo.Server) *VersionedObjectCache { + return NewVersionedObjectCache(func() (VersionedObject, error) { + keyspaces, err := ts.GetKeyspaces() + if err != nil { + return nil, err + } + return &Keyspaces{ + Keyspaces: keyspaces, + }, nil + }) +} + +// Keyspace contains the cached results of topo.Server.GetKeyspace +type Keyspace struct { + BaseVersionedObject + + // KeyspaceName is the name of this keyspace + KeyspaceName string + + // Keyspace is the topo value of this keyspace + Keyspace *topo.Keyspace +} + +// Reset is part of the VersionedObject interface +func (k *Keyspace) Reset() { + k.KeyspaceName = "" + k.Keyspace = nil +} + +func newKeyspaceCache(ts topo.Server) *VersionedObjectCacheMap { + return NewVersionedObjectCacheMap(func(key string) *VersionedObjectCache { + return NewVersionedObjectCache(func() (VersionedObject, error) { + k, err := ts.GetKeyspace(key) + if err != nil { + return nil, err + } + return &Keyspace{ + KeyspaceName: k.KeyspaceName(), + Keyspace: k.Keyspace, + }, nil + }) + }) +} + +// ShardNames contains the cached results of topo.Server.GetShardNames +type ShardNames struct { + BaseVersionedObject + + // KeyspaceName is the name of the keyspace this result applies to + KeyspaceName string + + // ShardNames is the list of shard names for this keyspace + ShardNames []string +} + +// Reset is part of the VersionedObject interface +func (s *ShardNames) Reset() { + s.KeyspaceName = "" + s.ShardNames = nil +} + +func newShardNamesCache(ts topo.Server) *VersionedObjectCacheMap { + return NewVersionedObjectCacheMap(func(key string) *VersionedObjectCache { + return NewVersionedObjectCache(func() (VersionedObject, error) { + sn, err := ts.GetShardNames(key) + if err != nil { + return nil, err + } + return &ShardNames{ + KeyspaceName: key, + ShardNames: sn, + }, nil + }) + }) +} + +// Shard contains the cached results of topo.Server.GetShard +// the map key is keyspace/shard +type Shard struct { + BaseVersionedObject + + // KeyspaceName is the keyspace for this shard + KeyspaceName string + + // ShardName is the name for this shard + ShardName string + + // Shard is the topo value of this shard + Shard *topo.Shard +} + +// Reset is part of the VersionedObject interface +func (s *Shard) Reset() { + s.KeyspaceName = "" + s.ShardName = "" + s.Shard = nil +} + +func newShardCache(ts topo.Server) *VersionedObjectCacheMap { + return NewVersionedObjectCacheMap(func(key string) *VersionedObjectCache { + return NewVersionedObjectCache(func() (VersionedObject, error) { + + keyspace, shard, err := topo.ParseKeyspaceShardString(key) + if err != nil { + return nil, err + } + s, err := ts.GetShard(keyspace, shard) + if err != nil { + return nil, err + } + return &Shard{ + KeyspaceName: s.Keyspace(), + ShardName: s.ShardName(), + Shard: s.Shard, + }, nil + }) + }) +} + +// CellShardTablets contains the cached results of the list of tablets +// in a cell / keyspace / shard. +// The map key is cell/keyspace/shard. +type CellShardTablets struct { + BaseVersionedObject + + // Cell is the name of the cell + Cell string + + // KeyspaceName is the keyspace for this shard + KeyspaceName string + + // ShardName is the name for this shard + ShardName string + + // TabletAliases is the array os tablet aliases + TabletAliases []topo.TabletAlias +} + +// Reset is part of the VersionedObject interface +func (cst *CellShardTablets) Reset() { + cst.Cell = "" + cst.KeyspaceName = "" + cst.ShardName = "" + cst.TabletAliases = nil +} + +func newCellShardTabletsCache(ts topo.Server) *VersionedObjectCacheMap { + return NewVersionedObjectCacheMap(func(key string) *VersionedObjectCache { + return NewVersionedObjectCache(func() (VersionedObject, error) { + parts := strings.Split(key, "/") + if len(parts) != 3 { + return nil, fmt.Errorf("Invalid shard tablets path: %v", key) + } + sr, err := ts.GetShardReplication(parts[0], parts[1], parts[2]) + if err != nil { + return nil, err + } + result := &CellShardTablets{ + Cell: parts[0], + KeyspaceName: parts[1], + ShardName: parts[2], + TabletAliases: make([]topo.TabletAlias, len(sr.ReplicationLinks)), + } + for i, rl := range sr.ReplicationLinks { + result.TabletAliases[i] = rl.TabletAlias + } + return result, nil + }) + }) +} diff --git a/go/cmd/vtctld/topo_data_test.go b/go/cmd/vtctld/topo_data_test.go new file mode 100644 index 00000000000..8e34c8d4a6b --- /dev/null +++ b/go/cmd/vtctld/topo_data_test.go @@ -0,0 +1,328 @@ +package main + +import ( + "encoding/json" + "reflect" + "testing" + "time" + + "github.com/youtube/vitess/go/vt/topo" + "github.com/youtube/vitess/go/vt/zktopo" +) + +func testVersionedObjectCache(t *testing.T, voc *VersionedObjectCache, vo VersionedObject, expectedVO VersionedObject) { + result, err := voc.Get() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if err := json.Unmarshal(result, vo); err != nil { + t.Fatalf("bad json: %v", err) + } + if vo.GetVersion() != 1 { + t.Fatalf("Got wrong initial version: %v", vo.GetVersion()) + } + expectedVO.SetVersion(1) + if !reflect.DeepEqual(vo, expectedVO) { + t.Fatalf("Got bad result: %#v expected: %#v", vo, expectedVO) + } + + result2, err := voc.Get() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !reflect.DeepEqual(result, result2) { + t.Fatalf("Bad content from cache: %v != %v", string(result), string(result2)) + } + + // force a re-get with same content, version shouldn't change + voc.timestamp = time.Time{} + result2, err = voc.Get() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !reflect.DeepEqual(result, result2) { + t.Fatalf("Bad content from cache: %v != %v", string(result), string(result2)) + } + + // force a reget with different content, version should change + voc.timestamp = time.Time{} + voc.versionedObject.Reset() // poking inside the object here + result, err = voc.Get() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if err := json.Unmarshal(result, vo); err != nil { + t.Fatalf("bad json: %v", err) + } + if vo.GetVersion() != 2 { + t.Fatalf("Got wrong second version: %v", vo.GetVersion()) + } + expectedVO.SetVersion(2) + if !reflect.DeepEqual(vo, expectedVO) { + t.Fatalf("Got bad result: %#v expected: %#v", vo, expectedVO) + } + + // force a flush and see the version increase again + voc.Flush() + result, err = voc.Get() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if err := json.Unmarshal(result, vo); err != nil { + t.Fatalf("bad json: %v", err) + } + if vo.GetVersion() != 3 { + t.Fatalf("Got wrong third version: %v", vo.GetVersion()) + } + expectedVO.SetVersion(3) + if !reflect.DeepEqual(vo, expectedVO) { + t.Fatalf("Got bad result: %#v expected: %#v", vo, expectedVO) + } +} + +func testVersionedObjectCacheMap(t *testing.T, vocm *VersionedObjectCacheMap, key string, vo VersionedObject, expectedVO VersionedObject) { + result, err := vocm.Get(key) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if err := json.Unmarshal(result, vo); err != nil { + t.Fatalf("bad json: %v", err) + } + if vo.GetVersion() != 1 { + t.Fatalf("Got wrong initial version: %v", vo.GetVersion()) + } + expectedVO.SetVersion(1) + if !reflect.DeepEqual(vo, expectedVO) { + t.Fatalf("Got bad result: %#v expected: %#v", vo, expectedVO) + } + + result2, err := vocm.Get(key) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !reflect.DeepEqual(result, result2) { + t.Fatalf("Bad content from cache: %v != %v", string(result), string(result2)) + } + + // force a re-get with same content, version shouldn't change + vocm.cacheMap[key].timestamp = time.Time{} + result2, err = vocm.Get(key) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !reflect.DeepEqual(result, result2) { + t.Fatalf("Bad content from cache: %v != %v", string(result), string(result2)) + } + + // force a reget with different content, version should change + vocm.cacheMap[key].timestamp = time.Time{} + vocm.cacheMap[key].versionedObject.Reset() // poking inside the object here + result, err = vocm.Get(key) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if err := json.Unmarshal(result, vo); err != nil { + t.Fatalf("bad json: %v", err) + } + if vo.GetVersion() != 2 { + t.Fatalf("Got wrong second version: %v", vo.GetVersion()) + } + expectedVO.SetVersion(2) + if !reflect.DeepEqual(vo, expectedVO) { + t.Fatalf("Got bad result: %#v expected: %#v", vo, expectedVO) + } + + // force a flush and see the version increase again + vocm.Flush() + result, err = vocm.Get(key) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if err := json.Unmarshal(result, vo); err != nil { + t.Fatalf("bad json: %v", err) + } + if vo.GetVersion() != 3 { + t.Fatalf("Got wrong third version: %v", vo.GetVersion()) + } + expectedVO.SetVersion(3) + if !reflect.DeepEqual(vo, expectedVO) { + t.Fatalf("Got bad result: %#v expected: %#v", vo, expectedVO) + } +} + +func TestKnownCellsCache(t *testing.T) { + ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"}) + kcc := newKnownCellsCache(ts) + var kc KnownCells + expectedKc := KnownCells{ + Cells: []string{"cell1", "cell2"}, + } + + testVersionedObjectCache(t, kcc, &kc, &expectedKc) +} + +func TestKeyspacesCache(t *testing.T) { + ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"}) + if err := ts.CreateKeyspace("ks1", &topo.Keyspace{}); err != nil { + t.Fatalf("CreateKeyspace failed: %v", err) + } + if err := ts.CreateKeyspace("ks2", &topo.Keyspace{}); err != nil { + t.Fatalf("CreateKeyspace failed: %v", err) + } + kc := newKeyspacesCache(ts) + var k Keyspaces + expectedK := Keyspaces{ + Keyspaces: []string{"ks1", "ks2"}, + } + + testVersionedObjectCache(t, kc, &k, &expectedK) +} + +func TestKeyspaceCache(t *testing.T) { + ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"}) + if err := ts.CreateKeyspace("ks1", &topo.Keyspace{ + ShardingColumnName: "sharding_key", + }); err != nil { + t.Fatalf("CreateKeyspace failed: %v", err) + } + if err := ts.CreateKeyspace("ks2", &topo.Keyspace{ + SplitShardCount: 10, + }); err != nil { + t.Fatalf("CreateKeyspace failed: %v", err) + } + kc := newKeyspaceCache(ts) + var k Keyspace + + expectedK := Keyspace{ + KeyspaceName: "ks1", + Keyspace: &topo.Keyspace{ + ShardingColumnName: "sharding_key", + }, + } + testVersionedObjectCacheMap(t, kc, "ks1", &k, &expectedK) + + expectedK = Keyspace{ + KeyspaceName: "ks2", + Keyspace: &topo.Keyspace{ + SplitShardCount: 10, + }, + } + testVersionedObjectCacheMap(t, kc, "ks2", &k, &expectedK) +} + +func TestShardNamesCache(t *testing.T) { + ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"}) + if err := ts.CreateKeyspace("ks1", &topo.Keyspace{ + ShardingColumnName: "sharding_key", + }); err != nil { + t.Fatalf("CreateKeyspace failed: %v", err) + } + if err := ts.CreateShard("ks1", "s1", &topo.Shard{ + Cells: []string{"cell1", "cell2"}, + }); err != nil { + t.Fatalf("CreateShard failed: %v", err) + } + if err := ts.CreateShard("ks1", "s2", &topo.Shard{ + MasterAlias: topo.TabletAlias{ + Cell: "cell1", + Uid: 12, + }, + }); err != nil { + t.Fatalf("CreateShard failed: %v", err) + } + snc := newShardNamesCache(ts) + var sn ShardNames + + expectedSN := ShardNames{ + KeyspaceName: "ks1", + ShardNames: []string{"s1", "s2"}, + } + testVersionedObjectCacheMap(t, snc, "ks1", &sn, &expectedSN) +} + +func TestShardCache(t *testing.T) { + ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"}) + if err := ts.CreateKeyspace("ks1", &topo.Keyspace{ + ShardingColumnName: "sharding_key", + }); err != nil { + t.Fatalf("CreateKeyspace failed: %v", err) + } + if err := ts.CreateShard("ks1", "s1", &topo.Shard{ + Cells: []string{"cell1", "cell2"}, + }); err != nil { + t.Fatalf("CreateShard failed: %v", err) + } + if err := ts.CreateShard("ks1", "s2", &topo.Shard{ + MasterAlias: topo.TabletAlias{ + Cell: "cell1", + Uid: 12, + }, + }); err != nil { + t.Fatalf("CreateShard failed: %v", err) + } + sc := newShardCache(ts) + var s Shard + + expectedS := Shard{ + KeyspaceName: "ks1", + ShardName: "s1", + Shard: &topo.Shard{ + Cells: []string{"cell1", "cell2"}, + }, + } + testVersionedObjectCacheMap(t, sc, "ks1/s1", &s, &expectedS) + + expectedS = Shard{ + KeyspaceName: "ks1", + ShardName: "s2", + Shard: &topo.Shard{ + MasterAlias: topo.TabletAlias{ + Cell: "cell1", + Uid: 12, + }, + }, + } + testVersionedObjectCacheMap(t, sc, "ks1/s2", &s, &expectedS) +} + +func TestCellShardTabletsCache(t *testing.T) { + ts := zktopo.NewTestServer(t, []string{"cell1", "cell2"}) + if err := ts.UpdateShardReplicationFields("cell1", "ks1", "s1", func(sr *topo.ShardReplication) error { + sr.ReplicationLinks = []topo.ReplicationLink{ + topo.ReplicationLink{ + TabletAlias: topo.TabletAlias{ + Cell: "cell1", + Uid: 12, + }, + }, + topo.ReplicationLink{ + TabletAlias: topo.TabletAlias{ + Cell: "cell1", + Uid: 13, + }, + }, + } + return nil + }); err != nil { + t.Fatalf("UpdateShardReplicationFields failed: %v", err) + } + cstc := newCellShardTabletsCache(ts) + var cst CellShardTablets + + expectedCST := CellShardTablets{ + Cell: "cell1", + KeyspaceName: "ks1", + ShardName: "s1", + TabletAliases: []topo.TabletAlias{ + topo.TabletAlias{ + Cell: "cell1", + Uid: 12, + }, + topo.TabletAlias{ + Cell: "cell1", + Uid: 13, + }, + }, + } + testVersionedObjectCacheMap(t, cstc, "cell1/ks1/s1", &cst, &expectedCST) +} diff --git a/go/cmd/vtctld/vtctld.go b/go/cmd/vtctld/vtctld.go index fa82f8301e0..6465eb7f992 100644 --- a/go/cmd/vtctld/vtctld.go +++ b/go/cmd/vtctld/vtctld.go @@ -1,32 +1,26 @@ package main import ( - "bytes" - "encoding/json" "flag" "fmt" - "html/template" - "io" "net/http" - "path" - "reflect" "strings" - "time" - "code.google.com/p/go.net/context" + "golang.org/x/net/context" log "github.com/golang/glog" "github.com/youtube/vitess/go/acl" - "github.com/youtube/vitess/go/vt/logutil" "github.com/youtube/vitess/go/vt/servenv" + "github.com/youtube/vitess/go/vt/tabletmanager/tmclient" "github.com/youtube/vitess/go/vt/topo" "github.com/youtube/vitess/go/vt/topotools" "github.com/youtube/vitess/go/vt/wrangler" ) var ( - templateDir = flag.String("templates", "", "directory containing templates") - debug = flag.Bool("debug", false, "recompile templates for every request") + templateDir = flag.String("templates", "", "directory containing templates") + debug = flag.Bool("debug", false, "recompile templates for every request") + schemaEditorDir = flag.String("schema-editor-dir", "", "directory containing schema_editor/") ) func init() { @@ -34,298 +28,18 @@ func init() { servenv.InitServiceMapForBsonRpcService("vtctl") } -// FHtmlize writes data to w as debug HTML (using definition lists). -func FHtmlize(w io.Writer, data interface{}) { - v := reflect.Indirect(reflect.ValueOf(data)) - typ := v.Type() - switch typ.Kind() { - case reflect.Struct: - fmt.Fprintf(w, "
Polling health information from {{github_com_youtube_vitess_health_html_name}}.
{{.Time.Format "Jan 2, 2006 at 15:04:05 (MST)"}} | -- {{ if eq "unhealthy" .Class}} - unhealthy: {{.Error}} - {{else if eq "unhappy" .Class}} - unhappy (reasons: {{range $key, $value := .Result}}{{$key}}: {{$value}} {{end}}) - {{else}} - healthy - {{end}} + | {{.HTML}} |
The following flags are also supported, but their use is very strongly discouraged:
-