Add mysql replication support for gerrit

Change-Id: I5149888256ab46871a8278294a89492998457816

jobs/gerrit_backup/templates/run_backup

@@ -9,6 +9,7 @@ export PATH=$GIT_DIR/bin:$PATH
 LAST_BAK_FILE=$REPO_DIR/last_backup_time
 interval=<%= properties.gerrit_backup.interval || 6 %>
 max_backup=<%= properties.gerrit_backup.max_backup || 20 %>
+repo_host=<%= properties.gerrit_backup.repo_host || "reviews.cloudfoundry.org" %>
 while true
 do
   [ -f $TERMINATE_BACKUP ] && rm $TERMINATE_BACKUP && exit 1
@@ -28,7 +29,7 @@ do
     do
       if [ ! -z "$repo" ]; then
         git_path="ssh://ci-bot@reviews.cloudfoundry.org:29418/${repo}.git"
-        [ ! -d $repo_dir/${repo}.git ] && echo git clone --mirror $git_path $repo_dir/${repo}.git
+        [ ! -d $repo_dir/${repo}.git ] && git clone --mirror $git_path $repo_dir/${repo}.git
       fi
     done
     echo $now > $LAST_BAK_FILE

jobs/gerrit_mysql/monit

@@ -1,5 +1,5 @@
 check process gerrit_mysql
   with pidfile /var/vcap/sys/run/mysqld/mysqld.pid
-  start program "/var/vcap/packages/mysql/libexec/mysql.server start /var/vcap/jobs/gerrit_mysql/config/my.cnf"
-  stop program "/var/vcap/packages/mysql/libexec/mysql.server stop /var/vcap/jobs/gerrit_mysql/config/my.cnf"
+  start program "/var/vcap/jobs/gerrit_mysql/bin/mysql_ctl start"
+  stop program "/var/vcap/jobs/gerrit_mysql/bin/mysql_ctl stop"
   group vcap

jobs/gerrit_mysql/spec

@@ -4,6 +4,10 @@ templates:
   my.cnf.erb: config/my.cnf
   my.bootstrap.erb: bin/my.bootstrap
   my.configure.erb: bin/my.configure
+  mysql_ctl: bin/mysql_ctl
+  setup_slave: bin/setup_slave
+  id_rsa.erb: config/id_rsa
+  id_rsa.pub.erb: config/id_rsa.pub
 packages:
   - mysql
 

jobs/gerrit_mysql/templates/id_rsa.erb

@@ -0,0 +1 @@
+<%= properties.jenkins.rsa_keys.private %>

jobs/gerrit_mysql/templates/id_rsa.pub.erb

@@ -0,0 +1 @@
+<%= properties.jenkins.rsa_keys.public %>

jobs/gerrit_mysql/templates/my.cnf.erb

@@ -72,8 +72,10 @@ query_cache_size = 16M
 # The following can be used as easy to replay backup logs or for replication.
 # note: if you are setting up a replication slave, see README.Debian about
 # other settings you may need to change.
-#server-id = 1
-#log_bin = /var/log/mysql/mysql-bin.log
+<% if properties.gerrit_mysql.slaves_ip %>
+server-id = 1
+log_bin = /var/vcap/sys/log/mysql/mysql-bin.log
+<% end %>
 expire_logs_days = 10
 max_binlog_size = 100M
 #binlog_do_db = include_database_name

jobs/gerrit_mysql/templates/mysql_ctl

@@ -0,0 +1,27 @@
+#!/bin/bash
+STORE_DIR=/var/vcap/store/mysql
+LOG_DIR=/var/vcap/sys/log/mysql
+RUN_DIR=/var/vcap/sys/run/mysqld
+MYSQL_DIR=/var/vcap/packages/mysql
+JOB_DIR=/var/vcap/jobs/gerrit_mysql
+CONFIG_DIR=$JOB_DIR/config
+PIDFILE=$RUN_DIR/mysqld.pid
+
+case $1 in
+
+  start)
+    $MYSQL_DIR/libexec/mysql.server start $CONFIG_DIR/my.cnf
+
+    su - vcap -c "$JOB_DIR/bin/setup_slave"
+    ;;
+
+  stop)
+    echo "Stopping Gerrit_mysql"
+    $MYSQL_DIR/libexec/mysql.server stop $CONFIG_DIR/my.cnf
+    rm $PIDFILE
+    ;;
+
+  *)
+  echo "Usage: mysql_ctl {start|stop}" ;;
+esac
+exit 0

jobs/gerrit_mysql/templates/setup_slave

@@ -0,0 +1,52 @@
+#!/bin/bash
+<% if properties.gerrit_mysql.slaves_ip %>
+STORE_DIR=/var/vcap/store/mysql
+MYSQL_DIR=/var/vcap/packages/mysql
+RUN_DIR=/var/vcap/sys/run/mysqld
+JOB_DIR=/var/vcap/jobs/gerrit_mysql
+CONFIG_DIR=$JOB_DIR/config
+PIDFILE=$RUN_DIR/mysqld.pid
+MYSQL_SOCK=$RUN_DIR/mysqld.sock
+DB_DUMP_FILE=$STORE_DIR/reviewdb.sql
+
+MYSQL_SLAVE_JOB_DIR=/var/vcap/jobs/gerrit_mysql_slave
+
+export LD_LIBRARY_PATH=$MYSQL_DIR/lib/mysql
+export PATH=$MYSQL_DIR/bin:$PATH
+
+# Passwdless ssh host key pair(for mysql slave)
+mkdir -p /home/vcap/.ssh
+chmod 0700 /home/vcap/.ssh
+
+cp $CONFIG_DIR/{id_rsa,id_rsa.pub} /home/vcap/.ssh
+chmod 0600 /home/vcap/.ssh/{id_rsa,id_rsa.pub}
+
+address=<%= properties.gerrit_mysql.address %>
+user=<%= properties.gerrit_mysql.user %>
+password=<%= properties.gerrit_mysql.password %>
+repl_user=<%= properties.gerrit_mysql.repl_user || "gerrit2_repl" %>
+repl_password=<%= properties.gerrit_mysql.repl_password %>
+dbname=<%= properties.gerrit_mysql.dbname||"reviewdb" %>
+
+echo "CREATE USER '$repl_user' IDENTIFIED BY '$repl_password';
+GRANT REPLICATION SLAVE ON *.* TO '$repl_user';
+FLUSH TABLES WITH READ LOCK;" | mysql -uroot -S $MYSQL_SOCK
+
+mysqldump -h $address reviewdb > $DB_DUMP_FILE -u $user -p$password
+master_info=`echo "SHOW MASTER STATUS" | mysql -uroot -S $MYSQL_SOCK | tail -1`
+
+echo "UNLOCK TABLES" | mysql -uroot -S $MYSQL_SOCK
+
+log_file=`echo $master_info | awk '{print $1}'`
+log_pos=`echo $master_info | awk '{print $2}'`
+for ip_addr in <%= properties.gerrit_mysql.slaves_ip.join(' ') %>
+do
+  ssh-keygen -f /home/vcap/.ssh/known_hosts -R $ip_addr
+  ssh-keyscan $ip_addr >> /home/vcap/.ssh/known_hosts
+
+  mysql -h $ip_addr reviewdb < $DB_DUMP_FILE -u $user -p$password
+
+  ssh vcap@$ip_addr "$MYSQL_SLAVE_JOB_DIR/bin/config_slave $address $log_file $log_pos"
+  ssh vcap@$ip_addr "sudo /var/vcap/bosh/bin/monit restart mysql_slave"
+done
+<% end %>

jobs/gerrit_mysql_slave/monit

@@ -0,0 +1,5 @@
+check process gerrit_mysql_slave
+  with pidfile /var/vcap/sys/run/mysqld/mysqld.pid
+  start program "/var/vcap/jobs/gerrit_mysql_slave/bin/mysql_slave_ctl start"
+  stop program "/var/vcap/jobs/gerrit_mysql_slave/bin/mysql_slave_ctl stop"
+  group vcap

jobs/gerrit_mysql_slave/spec

@@ -0,0 +1,14 @@
+---
+name: gerrit_mysql_slave
+templates:
+  my.cnf.erb: config/my.cnf
+  my.bootstrap.erb: bin/my.bootstrap
+  my.configure.erb: bin/my.configure
+  mysql_slave_ctl: bin/mysql_slave_ctl
+  config_slave: bin/config_slave
+  id_rsa.erb: config/id_rsa
+  id_rsa.pub.erb: config/id_rsa.pub
+  authorized_keys.erb: config/authorized_keys
+packages:
+  - mysql
+

jobs/gerrit_mysql_slave/templates/authorized_keys.erb

@@ -0,0 +1 @@
+<%= properties.jenkins.rsa_keys.public %>

jobs/gerrit_mysql_slave/templates/config_slave

@@ -0,0 +1,28 @@
+#!/bin/bash
+RUN_DIR=/var/vcap/sys/run/mysqld
+MYSQL_DIR=/var/vcap/packages/mysql
+PIDFILE=$RUN_DIR/mysqld.pid
+MYSQL_SOCK=$RUN_DIR/mysqld.sock
+
+export LD_LIBRARY_PATH=$MYSQL_DIR/lib/mysql
+export PATH=$MYSQL_DIR/bin:$PATH
+
+master_address=$1
+log_file=$2
+log_pos=$3
+user=<%= properties.gerrit_mysql.user %>
+password=<%= properties.gerrit_mysql.password %>
+repl_user=<%= properties.gerrit_mysql.repl_user || "gerrit2_repl" %>
+repl_password=<%= properties.gerrit_mysql.repl_password %>
+dbname=<%= properties.gerrit_mysql.dbname || "reviewdb" %>
+
+echo "STOP SLAVE;
+CHANGE MASTER TO
+MASTER_HOST='$master_address',
+MASTER_USER='$repl_user',
+MASTER_PASSWORD='$password',
+MASTER_LOG_FILE='$log_file',
+MASTER_LOG_POS=$log_pos,
+MASTER_CONNECT_RETRY=10;
+START SLAVE;" | mysql -uroot -S $MYSQL_SOCK
+

jobs/gerrit_mysql_slave/templates/id_rsa.erb

@@ -0,0 +1 @@
+<%= properties.jenkins_slave.rsa_keys.private %>

jobs/gerrit_mysql_slave/templates/id_rsa.pub.erb

@@ -0,0 +1 @@
+<%= properties.jenkins_slave.rsa_keys.public %>

jobs/gerrit_mysql_slave/templates/my.bootstrap.erb

@@ -0,0 +1,16 @@
+export LD_LIBRARY_PATH=/var/vcap/packages/mysql/lib/mysql
+export PATH=/var/vcap/packages/mysql/bin
+
+hostname=`/bin/hostname`
+password=<%= properties.gerrit_mysql.password %>
+address=<%= properties.gerrit_mysql.address %>
+user=<%= properties.gerrit_mysql.user %>
+dbname=<%= properties.gerrit_mysql.dbname||"reviewdb" %>
+
+# TEMPORARY HACK
+echo "${address} ${hostname}" >> /etc/hosts
+
+/var/vcap/packages/mysql/bin/mysqladmin -u root -S /var/vcap/sys/run/mysqld/mysqld.sock
+/var/vcap/packages/mysql/bin/mysqladmin -u root -h ${hostname} -S /var/vcap/sys/run/mysqld/mysqld.sock
+
+

jobs/gerrit_mysql_slave/templates/my.cnf.erb

@@ -0,0 +1,115 @@
+# This will be passed to all mysql clients
+# It has been reported that passwords should be enclosed with ticks/quotes
+# escpecially if they contain "#" chars...
+# Remember to edit /etc/mysql/debian.cnf when changing the socket location.
+[client]
+port = <%= properties.gerrit_mysql.port||3306 %>
+socket = /var/vcap/sys/run/mysqld/mysqld.sock
+
+# Here is entries for some specific programs
+# The following values assume you have at least 32M ram
+
+# This was formally known as [safe_mysqld]. Both versions are currently parsed.
+[mysqld_safe]
+socket = /var/vcap/sys/run/mysqld/mysqld.sock
+nice = 0
+
+[mysqld]
+#
+# * Basic Settings
+#
+user = vcap
+pid-file = /var/vcap/sys/run/mysqld/mysqld.pid
+socket = /var/vcap/sys/run/mysqld/mysqld.sock
+port = <%= properties.gerrit_mysql.port||3306 %>
+basedir = /usr
+datadir = /var/vcap/store/mysql
+tmpdir = /tmp
+language = /var/vcap/packages//mysql/share/mysql/english
+skip-external-locking
+#
+# For compatibility to other Debian packages that still use
+# libmysqlclient10 and libmysqlclient12.
+old_passwords = 1
+#
+# Instead of skip-networking the default is now to listen only on
+# localhost which is more compatible and is not less secure.
+# bind-address = $slave_ip
+#
+# * Fine Tuning
+#
+key_buffer = 16M
+max_allowed_packet = 16M
+thread_stack = 128K
+thread_cache_size = 8
+# This replaces the startup script and checks MyISAM tables if needed
+# the first time they are touched
+myisam-recover = BACKUP
+#max_connections = 100
+#table_cache = 64
+#thread_concurrency = 10
+#
+# * Query Cache Configuration
+#
+query_cache_limit = 1M
+query_cache_size = 16M
+#
+# * Logging and Replication
+#
+# Both location gets rotated by the cronjob.
+# Be aware that this log type is a performance killer.
+# As of 5.1 you can enable the at runtime!
+#log_type = FILE
+#general_log = /var/log/mysql/mysql.log
+#
+# Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf.
+#
+# Here you can see queries with especially long duration
+#log_slow_queries = /var/log/mysql/mysql-slow.log
+#long_query_time = 2
+#log-queries-not-using-indexes
+#
+# The following can be used as easy to replay backup logs or for replication.
+# note: if you are setting up a replication slave, see README.Debian about
+# other settings you may need to change.
+# server-id = $slave_id
+log_bin = /var/vcap/sys/log/mysql/mysql-bin.log
+expire_logs_days = 10
+max_binlog_size = 100M
+#binlog_do_db = include_database_name
+#binlog_ignore_db = include_database_name
+#
+# * InnoDB
+#
+# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/.
+# Read the manual for more InnoDB related options. There are many!
+#
+# * Security Features
+#
+# Read the manual, too, if you want chroot!
+# chroot = /var/lib/mysql/
+#
+# For generating SSL certificates I recommend the OpenSSL GUI "tinyca".
+#
+# ssl-ca=/etc/mysql/cacert.pem
+# ssl-cert=/etc/mysql/server-cert.pem
+# ssl-key=/etc/mysql/server-key.pem
+
+
+
+[mysqldump]
+quick
+quote-names
+max_allowed_packet = 16M
+
+[mysql]
+#no-auto-rehash # faster start of mysql but no tab completition
+
+[isamchk]
+key_buffer = 16M
+
+#
+# * IMPORTANT: Additional settings that can override those from this file!
+# The files must end with '.cnf', otherwise they'll be ignored.
+#
+#!includedir /etc/mysql/conf.d/

jobs/gerrit_mysql_slave/templates/my.configure.erb

@@ -0,0 +1,11 @@
+export LD_LIBRARY_PATH=/var/vcap/packages/mysql/lib/mysql
+export PATH=/var/vcap/packages/mysql/bin
+
+hostname=`/bin/hostname`
+password=<%= properties.gerrit_mysql.password %>
+address=<%= properties.gerrit_mysql.address %>
+user=<%= properties.gerrit_mysql.user %>
+dbname=<%= properties.gerrit_mysql.dbname||"reviewdb" %>
+
+echo "create database ${dbname};\ngrant all on ${dbname}.* to ${user}@'%' identified by '${password}';\nflush privileges;\n" | /var/vcap/packages/mysql/bin/mysql -uroot -S /var/vcap/sys/run/mysqld/mysqld.sock
+

jobs/gerrit_mysql_slave/templates/mysql_slave_ctl

@@ -0,0 +1,59 @@
+#!/bin/bash
+STORE_DIR=/var/vcap/store/mysql
+LOG_DIR=/var/vcap/sys/log/mysql
+RUN_DIR=/var/vcap/sys/run/mysqld
+JOB_DIR=/var/vcap/jobs/gerrit_mysql_slave
+MYSQL_DIR=/var/vcap/packages/mysql
+CONFIG_DIR=$JOB_DIR/config
+PIDFILE=$RUN_DIR/mysqld.pid
+
+case $1 in
+
+  start)
+    mkdir -p /home/vcap/.ssh
+    chmod 0700 /home/vcap/.ssh
+
+    cp $CONFIG_DIR/{id_rsa,id_rsa.pub,authorized_keys} /home/vcap/.ssh
+    chmod 0600 /home/vcap/.ssh/{id_rsa,id_rsa.pub,authorized_keys}
+    chown -R vcap:vcap /home/vcap/.ssh
+
+    # Refresh host key for slave
+    su - vcap -c 'ssh-keygen -f /home/vcap/.ssh/known_hosts -R <%= properties.gerrit_mysql.address %>'
+    su - vcap -c 'ssh-keyscan <%= properties.gerrit_mysql.address %> >> /home/vcap/.ssh/known_hosts'
+
+    slave_id=2
+    for ip_addr in <%= properties.gerrit_mysql.slaves_ip.join(' ') %>
+    do
+      ifconfig eth0 | grep $ip_addr
+      [ $? == 0 ] && break
+      slave_id=`expr $slave_id + 1`
+    done
+
+    cat $CONFIG_DIR/my.cnf | sed "
+/^server-id = / d" | sed "
+/server-id =/i \
+server-id = $slave_id
+" | sed "
+/^bind-address = / d" | sed "
+/bind-address =/i \
+bind-address = $ip_addr
+"> /tmp/my.cnf
+    mv /tmp/my.cnf $CONFIG_DIR/my.cnf
+
+    cat /etc/sudoers | grep "vcap  ALL=(ALL) NOPASSWD: ALL" > /dev/null 2>&1
+    [ $? != 0 ] && echo "vcap  ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+
+    $MYSQL_DIR/libexec/mysql.server start $CONFIG_DIR/my.cnf
+
+    ;;
+
+  stop)
+    echo "Stopping Gerrit_mysql_slave"
+    $MYSQL_DIR/libexec/mysql.server stop $CONFIG_DIR/my.cnf
+    rm $PIDFILE
+    ;;
+
+  *)
+  echo "Usage: mysql_slave_ctl {start|stop}" ;;
+esac
+exit 0