Add webdriver support for new vtctld2 UI

Makefile

@@ -8,9 +8,9 @@ MAKEFLAGS = -s
 # Since we are not using this Makefile for compilation, limiting parallelism will not increase build time.
 .NOTPARALLEL:
 
-.PHONY: all build test clean unit_test unit_test_cover unit_test_race integration_test proto proto_banner site_test site_integration_test docker_bootstrap docker_test docker_unit_test java_test php_test reshard_tests
+.PHONY: all build build_web test clean unit_test unit_test_cover unit_test_race integration_test proto proto_banner site_test site_integration_test docker_bootstrap docker_test docker_unit_test java_test php_test reshard_tests
 
-all: build test
+all: build build_web test
 
 # Set a custom value for -p, the number of packages to be built/tested in parallel.
 # This is currently only used by our Travis CI test configuration.
@@ -27,6 +27,10 @@ ifdef VT_MYSQL_ROOT
   endif
 endif
 
+build_web:
+	echo $$(date): Building web artifacts
+	cd web/vtctld2 && ./build.sh
+
 build:
 ifndef NOBANNER
 	echo $$(date): Building source tree

bootstrap.sh

@@ -207,6 +207,12 @@ echo "creating git pre-commit hooks"
 mkdir -p $VTTOP/.git/hooks
 ln -sf $VTTOP/misc/git/pre-commit $VTTOP/.git/hooks/pre-commit
 
+# Download node
+curl -sL https://nodejs.org/dist/v6.3.1/node-v6.3.1-linux-x64.tar.xz > node_linux64.xz
+tar xf node_linux64.xz -C $VTROOT/dist
+mv $VTROOT/dist/node-v6.3.1-linux-x64 $VTROOT/dist/node
+rm node_linux64.xz
+
 # Download chromedriver
 echo "Installing selenium and chromedriver"
 selenium_dist=$VTROOT/dist/selenium

config/mycnf/vtcombo.cnf

@@ -0,0 +1 @@
+max_connections = 5000

dev.env

@@ -53,6 +53,9 @@ export PATH=$(prepend_path $PATH $VTROOT/dist/mysql/bin)
 # Add chromedriver to path for selenium tests
 export PATH=$(prepend_path $PATH $VTROOT/dist/chromedriver)
 
+# Node path
+export PATH=$(prepend_path $PATH $VTROOT/dist/node/bin)
+
 # GOROOT sanity
 go_bin=`which go`
 go_env=`go env | grep GOROOT | cut -f 2 -d\"`

go/cmd/vtcombo/main.go

@@ -10,7 +10,10 @@
 package main
 
 import (
+	"encoding/json"
 	"flag"
+	"os"
+	"path"
 	"strings"
 	"time"
 
@@ -76,6 +79,30 @@ func main() {
 	// set discoverygateway flag to default value
 	flag.Set("cells_to_watch", strings.Join(tpb.Cells, ","))
 
+	flag.Set("cell", tpb.Cells[0])
+
+	// create zk client config file
+	config := path.Join(os.Getenv("VTDATAROOT"), "vt_0000000001/tmp/test-zk-client-conf.json")
+	cellmap := make(map[string]string)
+	for _, cell := range tpb.Cells {
+		cellmap[cell] = "localhost"
+	}
+	b, err := json.Marshal(cellmap)
+	if err != nil {
+		log.Errorf("failed to marshal json: %v", err)
+	}
+
+	f, err := os.Create(config)
+	if err != nil {
+		log.Errorf("failed to create zk config file: %v", err)
+	}
+	defer f.Close()
+	_, err = f.WriteString(string(b[:]))
+	if err != nil {
+		log.Errorf("failed to write to zk config file: %v", err)
+	}
+	os.Setenv("ZK_CLIENT_CONFIG", config)
+
 	// register topo server
 	zkconn := fakezk.NewConn()
 	topo.RegisterServer("fakezk", zktopo.NewServer(zkconn))

go/cmd/vtcombo/tablet_map.go

@@ -158,10 +158,6 @@ func initTabletMap(ts topo.Server, tpb *vttestpb.VTTestTopology, mysqld mysqlctl
 						// 2 replicas in order to ensure the master cell has a master and a replica
 						replicas = 2
 					}
-					rdonlys := int(kpb.RdonlyCount)
-					if rdonlys == 0 {
-						rdonlys = 1
-					}
 
 					if cell == tpb.Cells[0] {
 						replicas--
@@ -181,7 +177,7 @@ func initTabletMap(ts topo.Server, tpb *vttestpb.VTTestTopology, mysqld mysqlctl
 						uid++
 					}
 
-					for i := 0; i < rdonlys; i++ {
+					for i := 0; i < int(kpb.RdonlyCount); i++ {
 						// create a rdonly slave
 						if err := createTablet(ctx, ts, cell, uid, keyspace, shard, dbname, topodatapb.TabletType_RDONLY, mysqld, dbcfgs); err != nil {
 							return err

py/vttest/local_database.py

@@ -19,7 +19,8 @@ def __init__(self,
                mysql_only,
                init_data_options,
                web_dir=None,
-               default_schema_dir=None):
+               default_schema_dir=None,
+               extra_my_cnf=None):
     """Initializes an object of this class.
 
     Args:
@@ -36,6 +37,7 @@ def __init__(self,
           flag in run_local_database.py
       default_schema_dir: a directory to use if no keyspace is found in the
           schema_dir directory.
+      extra_my_cnf: additional cnf file to use for the EXTRA_MY_CNF var.
     """
 
     self.topology = topology
@@ -44,12 +46,14 @@ def __init__(self,
     self.init_data_options = init_data_options
     self.web_dir = web_dir
     self.default_schema_dir = default_schema_dir
+    self.extra_my_cnf = extra_my_cnf
 
   def setup(self):
     """Create a MySQL instance and all Vitess processes."""
     mysql_port = environment.get_port('mysql')
     self.directory = environment.get_test_directory()
-    self.mysql_db = environment.mysql_db_class(self.directory, mysql_port)
+    self.mysql_db = environment.mysql_db_class(
+        self.directory, mysql_port, self.extra_my_cnf)
 
     self.mysql_db.setup()
     self.create_databases()

py/vttest/mysql_db.py

@@ -9,9 +9,10 @@
 class MySqlDB(object):
   """A MySqlDB contains basic info about a MySQL instance."""
 
-  def __init__(self, directory, port):
+  def __init__(self, directory, port, extra_my_cnf=None):
     self._directory = directory
     self._port = port
+    self._extra_my_cnf = extra_my_cnf
 
   def setup(self, port):
     """Starts the MySQL database."""

py/vttest/mysql_db_mysqlctl.py

@@ -18,8 +18,8 @@
 class MySqlDBMysqlctl(mysql_db.MySqlDB):
   """Contains data and methods to manage a MySQL instance using mysqlctl."""
 
-  def __init__(self, directory, port):
-    super(MySqlDBMysqlctl, self).__init__(directory, port)
+  def __init__(self, directory, port, extra_my_cnf):
+    super(MySqlDBMysqlctl, self).__init__(directory, port, extra_my_cnf)
 
   def setup(self):
     cmd = [
@@ -35,7 +35,10 @@ def setup(self):
     ]
     env = os.environ
     env['VTDATAROOT'] = self._directory
-    env['EXTRA_MY_CNF'] = mysql_flavor().my_cnf()
+    my_cnf = mysql_flavor().my_cnf()
+    if self._extra_my_cnf:
+      my_cnf += ':%s' % self._extra_my_cnf
+    env['EXTRA_MY_CNF'] = my_cnf
     result = subprocess.call(cmd, env=env)
     if result != 0:
       raise Exception('mysqlctl failed', result)

py/vttest/mysql_flavor.py

@@ -1,12 +1,13 @@
 """Define abstractions for various mysql flavors.
 
-This moduleis used by mysql_db_mysqlctl.py to handle differences
-between various flavors of mysql"""
+This module is used by mysql_db_mysqlctl.py to handle differences
+between various flavors of mysql.
+"""
 
 import environment
 import logging
 import os
-import subprocess
+import sys
 
 
 # For now, vttop is only used in this module. If other people
@@ -41,7 +42,7 @@ def my_cnf(self):
 
 
 class MySQL56(MysqlFlavor):
-  """Overrides specific to MySQL 5.6"""
+  """Overrides specific to MySQL 5.6."""
 
   def my_cnf(self):
     files = [
@@ -66,11 +67,7 @@ def mysql_flavor():
 def set_mysql_flavor(flavor):
   global __mysql_flavor
 
-  if not flavor:
-    flavor = os.environ.get("MYSQL_FLAVOR", "MariaDB")
-    # The environment variable might be set, but equal to "".
-    if flavor == "":
-      flavor = "MariaDB"
+  flavor = flavor or os.environ.get("MYSQL_FLAVOR", "MariaDB") or "MariaDB"
 
   # Set the environment variable explicitly in case we're overriding it via
   # command-line flag.

py/vttest/run_local_database.py

@@ -88,7 +88,9 @@ def main(cmdline_options):
       cmdline_options.mysql_only,
       init_data_opts,
       web_dir=cmdline_options.web_dir,
-      default_schema_dir=cmdline_options.default_schema_dir) as local_db:
+      default_schema_dir=cmdline_options.default_schema_dir,
+      extra_my_cnf=os.path.join(
+          os.environ['VTTOP'], 'config/mycnf/vtcombo.cnf')) as local_db:
     print json.dumps(local_db.config())
     sys.stdout.flush()
     try:

test.go

@@ -357,7 +357,7 @@ func main() {
 	} else {
 		// Since we're sharing the working dir, do the build once for all tests.
 		log.Printf("Running make build...")
-		if out, err := exec.Command("make", "build").CombinedOutput(); err != nil {
+		if out, err := exec.Command("make", "build", "build_web").CombinedOutput(); err != nil {
 			log.Fatalf("make build failed: %v\n%s", err, out)
 		}
 	}

test/config.json

@@ -366,6 +366,17 @@
 				"webdriver"
 			]
 		},
+		"vtctld2_web": {
+			"File": "vtctld2_web_test.py",
+			"Args": [],
+			"Command": [],
+			"Manual": false,
+			"Shard": 4,
+			"RetryMax": 0,
+			"Tags": [
+				"webdriver"
+			]
+		},
 		"vtgate_utils": {
 			"File": "vtgate_utils_test.py",
 			"Args": [],

test/vtctld2_web_test.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+"""A vtctld2 webdriver test."""
+
+import logging
+import os
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions
+from selenium.webdriver.support.wait import WebDriverWait
+import unittest
+
+from vtproto import vttest_pb2
+from vttest import environment as vttest_environment
+from vttest import local_database
+from vttest import mysql_flavor
+
+import environment
+import utils
+
+
+def setUpModule():
+  try:
+    if utils.options.xvfb:
+      try:
+        # This will be killed automatically by utils.kill_sub_processes()
+        utils.run_bg(['Xvfb', ':15', '-ac'])
+        os.environ['DISPLAY'] = ':15'
+      except OSError as err:
+        # Despite running in background, utils.run_bg() will throw immediately
+        # if the Xvfb binary is not found.
+        logging.error(
+            "Can't start Xvfb (will try local DISPLAY instead): %s", err)
+  except:
+    tearDownModule()
+    raise
+
+
+def tearDownModule():
+  utils.required_teardown()
+  if utils.options.skip_teardown:
+    return
+  utils.remove_tmp_files()
+
+
+class TestVtctldWeb(unittest.TestCase):
+
+  @classmethod
+  def setUpClass(cls):
+    """Set up two keyspaces: one unsharded, one with two shards."""
+    if os.environ.get('CI') == 'true' and os.environ.get('TRAVIS') == 'true':
+      username = os.environ['SAUCE_USERNAME']
+      access_key = os.environ['SAUCE_ACCESS_KEY']
+      capabilities = {}
+      capabilities['tunnel-identifier'] = os.environ['TRAVIS_JOB_NUMBER']
+      capabilities['build'] = os.environ['TRAVIS_BUILD_NUMBER']
+      capabilities['platform'] = 'Linux'
+      capabilities['browserName'] = 'chrome'
+      hub_url = '%s:%s@localhost:4445' % (username, access_key)
+      cls.driver = webdriver.Remote(
+          desired_capabilities=capabilities,
+          command_executor='http://%s/wd/hub' % hub_url)
+    else:
+      os.environ['webdriver.chrome.driver'] = os.path.join(
+          os.environ['VTROOT'], 'dist')
+      # Only testing against Chrome for now
+      cls.driver = webdriver.Chrome()
+
+    topology = vttest_pb2.VTTestTopology()
+    topology.cells.append('test')
+    keyspace = topology.keyspaces.add(name='test_keyspace')
+    keyspace.replica_count = 2
+    keyspace.rdonly_count = 2
+    keyspace.shards.add(name='-80')
+    keyspace.shards.add(name='80-')
+    keyspace2 = topology.keyspaces.add(name='test_keyspace2')
+    keyspace2.shards.add(name='0')
+    keyspace2.replica_count = 2
+    keyspace2.rdonly_count = 1
+
+    port = environment.reserve_ports(1)
+    vttest_environment.base_port = port
+    mysql_flavor.set_mysql_flavor(None)
+
+    cls.db = local_database.LocalDatabase(
+        topology, '', False, None,
+        os.path.join(os.environ['VTTOP'], 'web/vtctld2/dist'),
+        os.path.join(os.environ['VTTOP'], 'test/vttest_schema/default'))
+    cls.db.setup()
+    cls.vtctld_addr = 'http://localhost:%d' % cls.db.config()['port']
+    utils.pause('Paused test after vtcombo was started.\n'
+                'For manual testing, connect to vtctld: %s' % cls.vtctld_addr)
+
+  @classmethod
+  def tearDownClass(cls):
+    cls.db.teardown()
+    cls.driver.quit()
+
+  def _get_keyspaces(self):
+    """Get list of all present keyspaces."""
+    wait = WebDriverWait(self.driver, 10)
+    wait.until(expected_conditions.visibility_of_element_located(
+        (By.TAG_NAME, 'vt-dashboard')))
+    dashboard_content = self.driver.find_element_by_tag_name('vt-dashboard')
+    return [ks.text for ks in
+            dashboard_content.find_elements_by_tag_name('md-card-title')]
+
+  def test_dashboard(self):
+    logging.info('Testing dashboard view')
+
+    logging.info('Fetching main vtctld page: %s', self.vtctld_addr)
+    self.driver.get(self.vtctld_addr)
+
+    keyspace_names = self._get_keyspaces()
+    logging.info('Keyspaces: %s', ', '.join(keyspace_names))
+    self.assertListEqual(['test_keyspace', 'test_keyspace2'], keyspace_names)
+
+
+def add_test_options(parser):
+  parser.add_option(
+      '--no-xvfb', action='store_false', dest='xvfb', default=True,
+      help='Use local DISPLAY instead of headless Xvfb mode.')
+
+
+if __name__ == '__main__':
+  utils.main(test_options=add_test_options)