Merge branch 'master' of https://github.com/youtube/vitess

* 'master' of https://github.com/youtube/vitess: (48 commits)
  vtgate/buffer: When evicting entries from the buffer, do not wait for the request to finish.
  vtgate/buffer: Fix incrementing the BufferRequestsInFlightMax status variable.
  vtgate/buffer: Buffer each request only once.
  workflow: Add horizontal resharding workflow.
  vtgate/buffer: Fix lost lock when returning early.
  Added transaction-throttler and connected it to tabletserver. It uses vitess/go/vt/throttler with healthchecks to throttle transactions based on replication lag.
  1. Replaced the thread_throttler throttle() code with the golang.org/x/time/rate token-bucket implementation. 2. Changed max_replication_lag_module to not increase the rate limit if the current rate did not approach it.
  preseving batch error in jdbc (vitessio#2476)
  Adding 'vtctl UpdateCellInfo' command.
  Rebuilding webapp.
  Initial cluster sandbox code. (vitessio#2413)
  Properly building the sub-workflows.
  Adding a UI test for topo validator.
  Fixing MySQL flavr env variable.
  docs: Link to original Markdown files for pull requests.
  key: Add more examples for 512 shard setup for EvenShardsKeyRange enumeration function.
  Update repo in docker orchestrator build.sh file (vitessio#2468)
  test: vtgate_buffer.py: Refactor test into a base class.
  test: vtgate_buffer.py: Log when calling vtctl TabletExternallyReparented.
  test: vtgate_buffer.py: Log MySQL commands executed during external reparent.
  ...

bootstrap.sh

@@ -248,6 +248,7 @@ mkdir -p $VTTOP/.git/hooks
 ln -sf $VTTOP/misc/git/pre-commit $VTTOP/.git/hooks/pre-commit
 ln -sf $VTTOP/misc/git/prepare-commit-msg.bugnumber $VTTOP/.git/hooks/prepare-commit-msg
 ln -sf $VTTOP/misc/git/commit-msg.bugnumber $VTTOP/.git/hooks/commit-msg
+(cd $VTTOP && git config core.hooksPath $VTTOP/.git/hooks)
 
 # Download chromedriver
 echo "Installing selenium and chromedriver"

data/test/tabletserver/exec_cases.txt

@@ -43,6 +43,24 @@
   "FullQuery": "select * from a limit 5"
 }
 
+# limit with offset arg
+"select * from a limit 10, 5"
+{
+  "PlanID": "PASS_SELECT",
+  "TableName": "a",
+  "FieldQuery": "select * from a where 1 != 1",
+  "FullQuery": "select * from a limit 10, 5"
+}
+
+# limit with offset keyword
+"select * from a limit 5 offset 10"
+{
+  "PlanID": "PASS_SELECT",
+  "TableName": "a",
+  "FieldQuery": "select * from a where 1 != 1",
+  "FullQuery": "select * from a limit 10, 5"
+}
+
 # cross-db
 "select * from a.b"
 {
@@ -189,6 +207,16 @@
   "PKValues": ["0"]
 }
 
+# default without INTO
+"insert d(id) values(1)"
+{
+  "PlanID": "INSERT_PK",
+  "TableName": "d",
+  "FullQuery": "insert into d(id) values (1)",
+  "OuterQuery": "insert into d(id) values (1)",
+  "PKValues": ["0"]
+}
+
 # mismatch
 "insert into a (eid, id) values (1)"
 "column count doesn't match value count"

dev.env

@@ -33,6 +33,7 @@ done
 
 export PYTHONPATH=$(prepend_path $PYTHONPATH $VTROOT/py-vtdb)
 export PYTHONPATH=$(prepend_path $PYTHONPATH $VTTOP/test)
+export PYTHONPATH=$(prepend_path $PYTHONPATH $VTTOP/test/cluster/sandbox)
 
 selenium_dist=$VTROOT/dist/selenium
 export PYTHONPATH=$(prepend_path $PYTHONPATH $selenium_dist)

doc/VitessApi.md

@@ -808,17 +808,17 @@ ExecuteOptions is passed around for all Execute calls.
 #### Properties
 
 | Name |Description |
-| :-------- | :--------
-| <code>include_event_token</code> <br>bool| If set, we will try to include an EventToken with the responses. |
+| :-------- | :-------- 
+| <code>include_event_token</code> <br>bool| This used to be exclude_field_names, which was replaced by IncludedFields enum below If set, we will try to include an EventToken with the responses. |
 | <code>compare_event_token</code> <br>[EventToken](#query.eventtoken)| EventToken is a structure that describes a point in time in a replication stream on one shard. The most recent known replication position can be retrieved from vttablet when executing a query. It is also sent with the replication streams from the binlog service. |
-| <code>included_fields</code> <br>[IncludedFields](#query.includedfields) | The IncludedFields allows you to specify which Field metadata returned by a query. This is an optimization for high-QPS queries where the client knows what it's getting. |
+| <code>included_fields</code> <br>[IncludedFields](#executeoptions.includedfields)| Controls what fields are returned in Field message responses from mysql, i.e. field name, table name, etc. This is an optimization for high-QPS queries where the client knows what it's getting |
 
-### query.IncludedFields
+#### Enums
 
-IncludedFields allows you to specify which Field metadata returned by a query. This is an optimization for high-QPS queries where the client knows what it's getting.
+##### ExecuteOptions.IncludedFields
 
 | Name |Value |Description |
-| :-------- | :-------- | :--------
+| :-------- | :-------- | :-------- 
 | <code>TYPE_AND_NAME</code> | <code>0</code> |  |
 | <code>TYPE_ONLY</code> | <code>1</code> |  |
 | <code>ALL</code> | <code>2</code> |  |
@@ -833,6 +833,14 @@ Field describes a single column returned by a query
 | :-------- | :-------- 
 | <code>name</code> <br>string| name of the field as returned by mysql C API |
 | <code>type</code> <br>[Type](#query.type)| vitess-defined type. Conversion function is in sqltypes package. |
+| <code>table</code> <br>string| Remaining fields from mysql C API. These fields are only populated when ExecuteOptions.included_fields is set to IncludedFields.ALL. |
+| <code>org_table</code> <br>string| |
+| <code>database</code> <br>string| |
+| <code>org_name</code> <br>string| |
+| <code>column_length</code> <br>uint32| column_length is really a uint32. All 32 bits can be used. |
+| <code>charset</code> <br>uint32| charset is actually a uint16. Only the lower 16 bits are used. |
+| <code>decimals</code> <br>uint32| decimals is actualy a uint8. Only the lower 8 bits are used. |
+| <code>flags</code> <br>uint32| flags is actually a uint16. Only the lower 16 bits are used. |
 
 ### query.QueryResult
 

docker/orchestrator/build.sh

@@ -5,14 +5,14 @@ set -e
 tmpdir=`mktemp -d`
 
 script="go get github.com/youtube/vitess/go/cmd/vtctlclient && \
-  git clone https://github.com/enisoc/orchestrator.git src/github.com/outbrain/orchestrator && \
-  go install github.com/outbrain/orchestrator/go/cmd/orchestrator"
+  git clone https://github.com/github/orchestrator.git src/github.com/github/orchestrator && \
+  go install github.com/github/orchestrator/go/cmd/orchestrator"
 
 echo "Building orchestrator..."
-docker run -ti --name=vt_orc_build golang:1.6 bash -c "$script"
+docker run -ti --name=vt_orc_build golang:1.7 bash -c "$script"
 docker cp vt_orc_build:/go/bin/orchestrator $tmpdir
 docker cp vt_orc_build:/go/bin/vtctlclient $tmpdir
-docker cp vt_orc_build:/go/src/github.com/outbrain/orchestrator/resources $tmpdir
+docker cp vt_orc_build:/go/src/github.com/github/orchestrator/resources $tmpdir
 docker rm vt_orc_build
 
 echo "Building Docker image..."

docs/README.md

@@ -1,6 +1,11 @@
+# vitess.io Website Data
+
 If you're browsing this directory on GitHub, please ignore it.
 
 It contains the files which are serving www.vitess.io.
 
 If you're looking for documentation, you should find everything on www.vitess.io.
 
+# Changing the Website
+
+If you want to send a pull request to update the website, please edit the original Markdown files at https://github.com/youtube/vitess/tree/master/doc.

examples/local/vtgate-up.sh

@@ -7,6 +7,7 @@ set -e
 cell='test'
 web_port=15001
 grpc_port=15991
+mysql_server_port=15306
 
 script_root=`dirname "${BASH_SOURCE}"`
 source $script_root/env.sh
@@ -17,6 +18,7 @@ $VTROOT/bin/vtgate \
   -log_dir $VTDATAROOT/tmp \
   -port $web_port \
   -grpc_port $grpc_port \
+  -mysql_server_port $mysql_server_port \
   -cell $cell \
   -cells_to_watch $cell \
   -tablet_types_to_wait MASTER,REPLICA \

go/cmd/mysqlctld/mysqlctld.go

@@ -100,10 +100,10 @@ func main() {
 	defer servenv.Close()
 
 	// Take mysqld down with us on SIGTERM before entering lame duck.
-	servenv.OnTerm(func() {
+	servenv.OnTermSync(func() {
 		log.Infof("mysqlctl received SIGTERM, shutting down mysqld first")
 		ctx := context.Background()
-		mysqld.Shutdown(ctx, false)
+		mysqld.Shutdown(ctx, true)
 	})
 
 	// Start RPC server and wait for SIGTERM.

go/cmd/vtctld/plugin_grpcvtworkerclient.go

@@ -0,0 +1,11 @@
+// Copyright 2015, Google Inc. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+// Imports and registers the gRPC vtworker client.
+
+import (
+	_ "github.com/youtube/vitess/go/vt/worker/grpcvtworkerclient"
+)

go/cmd/vtgateclienttest/goclienttest/echo.go

@@ -96,7 +96,7 @@ var (
 	eventTokenEcho = "timestamp:876543 shard:\"-80\" position:\"test_position\" "
 
 	options = &querypb.ExecuteOptions{
-		IncludedFields: querypb.ExecuteOptions_TYPE_ONLY,
+		IncludedFields:    querypb.ExecuteOptions_TYPE_ONLY,
 		IncludeEventToken: true,
 		CompareEventToken: eventToken,
 	}

go/flagutil/flagutil.go

@@ -3,7 +3,7 @@
 package flagutil
 
 import (
-	_ "flag"
+	"flag"
 	"sort"
 	"strings"
 )
@@ -58,6 +58,13 @@ func (value StringListValue) String() string {
 
 }
 
+// StringListVar defines a []string flag with the specified name, value and usage
+// string. The argument 'p' points to a []string in which to store the value of the flag.
+func StringListVar(p *[]string, name string, defaultValue []string, usage string) {
+	*p = defaultValue
+	flag.Var((*StringListValue)(p), name, usage)
+}
+
 // StringMapValue is a map[string]string flag. It accepts a
 // comma-separated list of key value pairs, of the form key:value. The
 // keys cannot contain colons.

go/mysql/mysql.go

@@ -231,7 +231,7 @@ func (conn *Connection) ExecuteFetch(query string, maxrows int, wantfields bool)
 		return nil, &sqldb.SQLError{
 			Num:     0,
 			Message: fmt.Sprintf("Row count exceeded %d", maxrows),
-			Query:   string(query),
+			Query:   query,
 		}
 	}
 	if wantfields {
@@ -270,12 +270,12 @@ func (conn *Connection) Fields() (fields []*querypb.Field, err error) {
 	fields = make([]*querypb.Field, nfields)
 	fvals := make([]querypb.Field, nfields)
 	for i := 0; i < nfields; i++ {
-		fvals[i].Name = copy_string(cfields[i].name_length, cfields[i].name)
-		fvals[i].OrgName = copy_string(cfields[i].org_name_length, cfields[i].org_name)
-		fvals[i].Table = copy_string(cfields[i].table_length, cfields[i].table)
-		fvals[i].OrgTable = copy_string(cfields[i].org_table_length, cfields[i].org_table)
-		fvals[i].Database = copy_string(cfields[i].db_length, cfields[i].db)
-		fvals[i].ColumnLength = uint64(cfields[i].length)
+		fvals[i].Name = copyString(cfields[i].name_length, cfields[i].name)
+		fvals[i].OrgName = copyString(cfields[i].org_name_length, cfields[i].org_name)
+		fvals[i].Table = copyString(cfields[i].table_length, cfields[i].table)
+		fvals[i].OrgTable = copyString(cfields[i].org_table_length, cfields[i].org_table)
+		fvals[i].Database = copyString(cfields[i].db_length, cfields[i].db)
+		fvals[i].ColumnLength = uint32(cfields[i].length)
 		fvals[i].Flags = uint32(cfields[i].flags)
 		fvals[i].Charset = uint32(cfields[i].charsetnr)
 		fvals[i].Decimals = uint32(cfields[i].decimals)
@@ -289,11 +289,11 @@ func (conn *Connection) Fields() (fields []*querypb.Field, err error) {
 	return fields, nil
 }
 
-func copy_string(length C.uint, field *C.char) string {
-	return string(copy_bytes(length, field))
+func copyString(length C.uint, field *C.char) string {
+	return string(copyBytes(length, field))
 }
 
-func copy_bytes(length C.uint, field *C.char) []byte {
+func copyBytes(length C.uint, field *C.char) []byte {
 	return (*[maxSize]byte)(unsafe.Pointer(field))[:length]
 }
 
@@ -382,43 +382,6 @@ func (conn *Connection) lastError(query string) error {
 	}
 }
 
-// ReadPacket reads a raw packet from the MySQL connection.
-//
-// A MySQL packet is "a single SQL statement sent to the MySQL server, a
-// single row that is sent to the client, or a binary log event sent from a
-// master replication server to a slave." -MySQL 5.1 Reference Manual
-func (conn *Connection) ReadPacket() ([]byte, error) {
-	length := C.vt_cli_safe_read(&conn.c)
-	if length == 0 {
-		return nil, conn.lastError("ReadPacket()")
-	}
-
-	return C.GoBytes(unsafe.Pointer(conn.c.mysql.net.read_pos), C.int(length)), nil
-}
-
-// SendCommand sends a raw command to the MySQL server.
-func (conn *Connection) SendCommand(command uint32, data []byte) error {
-	var ret C.my_bool
-	if data == nil {
-		ret = C.vt_simple_command(&conn.c, command, nil, 0, 1)
-	} else {
-		ret = C.vt_simple_command(&conn.c, command, (*C.uchar)(unsafe.Pointer(&data[0])), C.ulong(len(data)), 1)
-	}
-	if ret != 0 {
-		return conn.lastError(fmt.Sprintf("SendCommand(%#v, %#v)", command, data))
-	}
-	return nil
-}
-
-// Shutdown invokes the low-level shutdown call on the socket associated with
-// a MySQL connection to stop ongoing communication. This is necessary when a
-// thread is blocked in a MySQL I/O call, such as  ReadPacket(), and another
-// thread wants to cancel the operation. We can't use mysql_close() because it
-// isn't thread-safe.
-func (conn *Connection) Shutdown() {
-	C.vt_shutdown(&conn.c)
-}
-
 func cfree(str *C.char) {
 	if str != nil {
 		C.free(unsafe.Pointer(str))

go/mysql/vtmysql.c

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 #include "vtmysql.h"
-#include "vtmysql_internals.h"
 
 // All functions must call mysql_thread_init before calling mysql. This is
 // because the go runtime controls thread creation, and we don't control
@@ -146,38 +145,3 @@ const char *vt_sqlstate(VT_CONN *conn) {
   mysql_thread_init();
   return mysql_sqlstate(conn->mysql);
 }
-
-my_bool vt_simple_command(
-    VT_CONN *conn,
-    enum enum_server_command command,
-    const unsigned char *arg,
-    unsigned long arg_length,
-    my_bool skip_check)
-{
-  mysql_thread_init();
-  return simple_command(conn->mysql, command, arg, arg_length, skip_check);
-}
-
-unsigned long vt_cli_safe_read(VT_CONN *conn) {
-  unsigned long len;
-
-  mysql_thread_init();
-
-#if MYSQL_VERSION_ID >= 50700 && MYSQL_VERSION_ID < 100000
-  // MySQL 5.7
-  len = cli_safe_read(conn->mysql, NULL);
-#else
-  // MySQL 5.6 and MariaDB
-  len = cli_safe_read(conn->mysql);
-#endif // MYSQL_VERSION_ID >= 50700 && MYSQL_VERSION_ID < 100000
-
-  return len == packet_error ? 0 : len;
-}
-
-void vt_shutdown(VT_CONN *conn) {
-  mysql_thread_init();
-
-  // Shut down the underlying socket of a MYSQL connection object.
-  if (conn->mysql && conn->mysql->net.vio)
-    vio_socket_shutdown(conn->mysql->net.vio, 2 /* SHUT_RDWR */);
-}

go/mysql/vtmysql.h

@@ -51,19 +51,3 @@ extern unsigned long vt_thread_id(VT_CONN *conn);
 extern unsigned int vt_errno(VT_CONN *conn);
 extern const char *vt_error(VT_CONN *conn);
 extern const char *vt_sqlstate(VT_CONN *conn);
-
-// vt_simple_command: Calls MySQL simple_command macro to send raw commands.
-my_bool vt_simple_command(
-    VT_CONN *conn,
-    enum enum_server_command command,
-    const unsigned char *arg,
-    unsigned long arg_length,
-    my_bool skip_check);
-
-// vt_cli_safe_read: Calls cli_safe_read and returns the number of bytes read,
-// or 0 if there was an error.
-unsigned long vt_cli_safe_read(VT_CONN *conn);
-
-// vt_shutdown: Kill a MySQL connection at the socket level, to unblock
-// a thread that is waiting on a read call.
-void vt_shutdown(VT_CONN *conn);