Scripts to support the virtual flexible architecture. See README for …

…details.

vfa/README

@@ -0,0 +1,76 @@
+
+#   Copyright 2013 Palominodb
+# 
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+# 
+#        http://www.apache.org/licenses/LICENSE-2.0
+# 
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+
+DBA related scripts for managing client infrastructure
+
+Installation instructions:
+
+o To get this repo without a key from github run the following:
+
+  ssh <username>@host
+  mkdir git
+  cd git
+  git clone https://github.com/dturner-palominodb/dba.git
+
+
+o Run create_perf_tarball.sh from git/dba.
+  It will create /tmp/perf_tarball.tgz which
+  can be copied to any host.
+
+  cd ~/git/dba
+  ./create_perf_tarball.sh
+
+o Copy the file to db dest_host:/tmp
+
+  scp /tmp/perf_tarball_20120820043003.tgz <hostname>:/tmp
+
+o On the dest host open the tar file and run
+  install_perf_tarball.sh.
+
+  cd /tmp
+  tar xzvf perf_tarball.tgz
+  cd perf_tarball
+  ./install_perf_tarball.sh
+
+o Test that the script is working, type dba. This will source the vfa_lib.sh script.
+  Your prompt should change. IE
+
+  [dturner@host02-01 ~]$ dba
+  [dturner@host02-01:3306 ~]#
+
+o Files of interest
+
+  /etc/vfatab
+
+  The contents of the above file should resemble the following if you have multiple instances. Otherwise,
+  only one line for 3306
+
+  /etc/my-3306.cnf:3306:Y:Y    
+  /etc/my-3307.cnf:3307:Y:N    
+  /etc/my-3310.cnf:3310:Y:N    
+
+o /etc/init.d/mysql
+  /etc/init.d/mysqld
+
+  Both are symlinks to /usr/local/palominodb/scripts/mysqld. This is a modified mysqld script that is used to work
+  with multiple database instances. By default it will start or stop all instances. However, if a port is passed 
+  it will start or stop the individual instance.
+
+  /etc/init.d/mysql start 3307
+
+
+

vfa/call_pt-osc.sh

@@ -0,0 +1,140 @@
+#!/bin/bash
+# author: dturner@palominodb.com
+# 
+# repo : wget --no-check-certificate https://raw.github.com/dturner-palominodb/dba/master/call_pt-osc.sh
+# Must be run locally
+# Notes:
+#      mkdir 20120101_call_pt-osc
+#  
+#      Create files for each table changed.
+#      20120101_call_pt-osc/alter01.sql
+#      20120101_call_pt-osc/alter02.sql
+#      Within each file the first line is the table and second on are the 
+#      ddl statements.
+#
+#      IE:
+#         entries
+#         modify `current_tags` varchar(2047) DEFAULT NULL,
+#         modify `position` int(11) DEFAULT NULL
+#
+#      Create .call_pt-osc with schema_filter=theschema
+#
+#
+#       The script can be told to sleep with a sleep file.
+#       IE: touch /tmp/call_pt-osc.sleep
+#       And worst case use /tmp/call_pt-osc.stop to tell
+#       the script to exit.
+#
+#   Copyright 2013 Palominodb
+# 
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+# 
+#        http://www.apache.org/licenses/LICENSE-2.0
+# 
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+if [ -z $1 ];then
+  echo "Error: usage call_pt-osc.sh [modification_date] [port]"
+  exit 1
+else
+  # This is used to ensure previous modifications are not run.
+  modification_date=$1
+  modification_dir=${modification_date}_call_pt-osc
+  if [ ! -d ${modification_dir} ];then
+    echo "Creating ${modification_dir} for the alter.sql files"
+    mkdir ${modification_dir} 2> /dev/null
+    exit 0
+  fi
+fi
+
+if [ -z $2 ];then
+  port=3306
+else
+  port=$1
+fi
+
+source /usr/local/palominodb/scripts/vfa_lib.sh ${port}
+
+sleep_file=/tmp/call_pt-osc.sleep
+stop_file=/tmp/call_pt-osc.stop
+
+if [ -e .call_pt-osc ];then
+  source .call_pt-osc
+else
+  schema_filter="theschema"
+fi
+
+schema_list=`mysql ${socket} -sNe 'show databases' |grep ${schema_filter}`
+
+# Remove these when running on some hosts
+socket="--socket=$(get_socket ${port})"
+dsn_socket=",S=$(get_socket ${port})"
+echo "dsn_socket=${dsn_socket}"
+
+
+logfile="/tmp/call_pt-osc.log"
+
+# exec > >(tee $logfile) 2>&1
+
+for file in $(ls ${modification_dir}/alter*.sql)
+# for file in $(ls call_pt-osc.file*.${modification_date})
+do
+
+  table=`head -1 ${file}`
+  stmt=`tail -n +2 ${file}`  
+  echo "table=$table"
+  # if no schema_list check the files for the schemas
+  if [ -z "${schema_list}" ];then
+     if [[ $table =~ "." ]];then
+       schema_list=`echo $table |cut -d. -f1`
+       table=`echo $table |cut -d. -f2`
+     else 
+       echo "Error: no schema specified for changes."
+       exit 1
+     fi
+  else
+    echo "${schema_list}"
+    echo "${schema_list}" | wc -l
+  fi
+
+  for schema in ${schema_list}
+  do
+    if [ -e ${stop_file} ];then
+      echo "Stop file ${stop_file} found. Exiting."
+      exit 0
+    fi
+
+    while [ -e ${sleep_file} ]
+    do
+      echo "Sleep file ${sleep_file} found."
+      echo "Sleeping 60s."
+      sleep 60
+    done
+
+    echo "schema=$schema"
+    echo "time pt-online-schema-change -u root --critical-load=Threads_Running=150 --alter \"${stmt}\" --execute D=${schema},t=${table}${dsn_socket}"
+    time pt-online-schema-change -u root --critical-load=Threads_Running=150 --alter "${stmt}" --execute D=${schema},t=${table}${dsn_socket}
+    # break
+  done
+
+done 2>&1 | tee $logfile
+
+if [ `grep -i error ${logfile} | wc -l` -gt 0 ];then
+  echo 
+  echo "========================="
+  echo "Error found in ${logfile}"
+  grep -i error ${logfile}
+  echo "========================="
+else
+  echo 
+  echo "========================="
+  echo "Completed without Error."
+  echo "========================="
+fi

vfa/cleanup_osc.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+#   Copyright 2013 Palominodb
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+
+source /usr/local/palominodb/scripts/vfa_lib.sh ''
+
+port=$1
+port=${port:=3306}
+
+out_file=/tmp/cleanup_osc_${port}.sql
+echo "set sql_log_bin=0;" > ${out_file}
+
+stmt="select concat(table_schema,'.', table_name) from information_schema.tables where table_name like '__osc%'"
+
+
+table_list=`mysql --socket=$(get_socket $port) -sNe "$stmt"`
+
+for table in ${table_list}
+do
+  echo "drop table ${table};" >> ${out_file}
+
+done
+
+cat ${out_file}
+
+echo "mysql -vvv < ${out_file}"

vfa/create_db_in_aws.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+#   Copyright 2013 Palominodb
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+sleep_time=1
+
+echo "Note: This is a destructive script. It will completely remove the database on 3306"
+echo "and create a new one. Sleeping ${sleep_time}s in case you want to control-c out."
+sleep ${sleep_time}
+echo
+
+
+# Check db to see if it is production.
+# 
+other_dir_count=`find /var/lib/mysql -mindepth 1 -type d  |egrep -v "/var/lib/mysql/mysql|/var/lib/mysql/test" |wc -l`
+
+if [ ${other_dir_count} -gt 0 ];then
+  echo "Error: directories other than mysql found. Possible production host."
+  echo "If the host is not production remove the other directories and"
+  echo "run $0 again."
+  find /var/lib/mysql -mindepth 1 -type d  |grep -v "/var/lib/mysql/mysql" 
+  exit 1
+fi
+
+echo "Killing mysql"
+echo
+pkill -9 -f mysqld
+
+echo "Removing database and binary log files."
+echo
+rm -rf /var/lib/mysql/*
+rm -f /data/mysql/bin-log/*
+
+echo "Creating the database."
+echo
+mysql_install_db
+
+chown -R mysql:mysql /var/lib/mysql
+chown -R mysql:mysql /data/mysql/bin-log/
+
+/etc/init.d/mysql start
+
+
+mysql -e 'select "Completed Database Creation."'

vfa/create_perf_tarball.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+#   Copyright 2013 Palominodb
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+tarball_dir=/tmp/perf_tarball
+date=`date +"%Y%m%d%H%M%S"`
+tarball_file=perf_tarball_${date}.tgz
+perf_dest_dir="/tmp"
+
+# Removing previous versions of perf_tarball.
+rm -vf ${perf_dest_dir}/perf_tarball_[0-9][0-9]*.tgz
+
+mkdir -p /tmp/perf_tarball 2> /dev/null
+
+# Deprecated. Removing after there is a replacement
+# cd ${HOME}/git/ServerAudit/scripts/
+# cp call_pt-stalk.sh    ${tarball_dir}
+# cp gen_stalk_report.sh ${tarball_dir}
+
+cd ${HOME}/git/dba/
+cp * ${tarball_dir}
+
+cd /tmp
+tar czvf ${tarball_file} perf_tarball
+echo ${tarball_file} 
+