Initial commit

.gitignore

@@ -0,0 +1,15 @@
+# Executable
+replication_booster
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+
+# Compiled Dynamic libraries
+*.so
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a

Makefile

@@ -0,0 +1,26 @@
+TARGET= replication_booster
+
+SRCS= replication_booster.cc prefetch_worker.cc options.cc check_local.cc
+OBJS= replication_booster.o prefetch_worker.o options.o check_local.o
+
+DEST= /usr/local/bin
+CXX= g++
+CXXFLAGS= -c -Wall -O3 
+LFLAGS= -Wall -O3 
+LIBS= -lmysqlclient_r -lboost_regex -lreplication
+DEBUG= -g -pg
+
+all: $(TARGET)
+
+$(TARGET): $(OBJS)
+	$(CXX) $(LFLAGS) $(OBJS) $(LIBS) -o $(TARGET)
+
+$(TARGET).o: $(SRCS)
+	$(CXX) $(CXXFLAGS) -I/usr/include/mysql $(SRCS)
+
+clean:
+	rm -f *.o $(TARGET)
+
+install: $(TARGET)
+	install -s $(TARGET) $(DEST)
+

README

@@ -0,0 +1,20 @@
+Replication Booster -- A Tool for Prefetching MySQL Slave Relay Logs, to make SQL Thread faster
+
+How to use:
+* Install boost if not installed (recommended version is 1.39 or higher)
+ (On RHEL/CentOS5, you can get boost 1.39+ from ATrpms repository)
+* Install Binlog API (https://code.launchpad.net/mysql-replication-listener)
+* make
+* make install
+* Run replication_booster
+
+
+Limitations:
+* This project has just been started and code quality and performance should be improved more.
+* Replication Booster uses Binlog API. Binlog API is currently (Oct 2011) pre-alpha so make sure to intensively test by yourself, though Replication Booster uses limited features of Binlog API. For example, you may fail to build Replication Booster when Binlog API interface has been changed.
+
+License: GNU General Public License version 2
+
+Author: DeNA Co.,Ltd. <Yoshinori.Matsunobu@gmail.com>
+
+

check_local.cc

@@ -0,0 +1,77 @@
+/**
+ *   Replication Booster -- A Tool for Prefetching MySQL Slave Relay Logs
+ *   Copyright (C) 2011 DeNA Co.,Ltd.
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the Free Software
+ *   Foundation, Inc.,
+ *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+**/
+
+#include <stdio.h>      
+#include <sys/types.h>
+#include <ifaddrs.h>
+#include <netinet/in.h> 
+#include <string.h> 
+#include <arpa/inet.h>
+#include <netdb.h>
+#include <sys/socket.h>
+#include "replication_booster.h"
+
+int check_local(const char *hostname_or_ip)
+{
+  struct ifaddrs *idaddr=NULL;
+  struct hostent* hp;
+  struct sockaddr_in addr;
+  bool is_local= false;
+  char ip_address[INET_ADDRSTRLEN];
+
+  if ((hp = gethostbyname(hostname_or_ip)) == 0)
+  {
+    print_log("Error: %d -- gethostbyname(%s)", h_errno, hostname_or_ip);
+    return 1;
+  }
+  if (hp->h_addrtype == AF_INET)
+  {
+    memset((char*)&addr, 0, sizeof(addr));
+    memcpy((char*)&addr.sin_addr, hp->h_addr_list[0], hp->h_length);
+    strcpy(ip_address, inet_ntoa(addr.sin_addr));
+    DBUG_PRINT("Address: %s", ip_address);
+  }
+  getifaddrs(&idaddr);
+
+  for (struct ifaddrs *ifa = idaddr; ifa != NULL; ifa = ifa->ifa_next)
+  {
+    if (ifa->ifa_addr->sa_family==AF_INET)
+    { 
+      void *tmp=&((struct sockaddr_in *)ifa->ifa_addr)->sin_addr;
+      char buf[INET_ADDRSTRLEN];
+      inet_ntop(AF_INET, tmp, buf, INET_ADDRSTRLEN);
+      DBUG_PRINT("%s IP Address %s", ifa->ifa_name, buf); 
+      if (!strcmp(buf, ip_address))
+        is_local= true;
+    }
+  }
+  if (idaddr != NULL)
+    freeifaddrs(idaddr);
+
+  if (is_local)
+  {
+    return 0;
+  } else {
+    print_log("ERROR: Target hostname %s is not local address! Set local address.", hostname_or_ip);
+     return 1;
+  }
+}
+

options.cc

@@ -0,0 +1,116 @@
+/**
+ *   Replication Booster -- A Tool for Prefetching MySQL Slave Relay Logs
+ *   Copyright (C) 2011 DeNA Co.,Ltd.
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the Free Software
+ *   Foundation, Inc.,
+ *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+**/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <getopt.h>
+#include "options.h"
+
+uint opt_workers= 10;
+uint opt_skip_events= 500;
+uint opt_read_ahead_seconds= 3;
+uint opt_sleep_millis_at_read_limit= 10000; // microseconds
+char *opt_slave_user= "root";
+char *opt_slave_password= "";
+char *opt_admin_user= NULL;
+char *opt_admin_password= NULL;
+char *opt_slave_host= "localhost";
+int opt_slave_port= 3306;
+char *opt_slave_socket= NULL;
+
+struct option long_options[] =
+{
+  {"help", no_argument, 0, '?'},
+  {"version", no_argument, 0, 'v'},
+  {"threads", required_argument, 0, 't'},
+  {"offset-events", required_argument, 0, 'o'},
+  {"seconds-prefetch", required_argument, 0, 's'},
+  {"mlllis-sleep", required_argument, 0, 'm'},
+  {"user", required_argument, 0, 'u'},
+  {"password", required_argument, 0, 'p'},
+  {"admin_user", required_argument, 0, 'a'},
+  {"admin_password", required_argument, 0, 'b'},
+  {"host", required_argument, 0, 'h'},
+  {"port", required_argument, 0, 'P'},
+  {"socket", required_argument, 0, 'S'},
+  {0,0,0,0}
+};
+
+void usage()
+{
+  printf("Usage: \n");
+  printf(" replication_booster [OPTIONS]\n\n");
+  printf("Example: \n");
+  printf(" replication_booster --user=mysql_select_user --password=mysql_select_pass --admin_user=mysql_root_user --admin_password=mysql_root_password --socket=/tmp/mysql.sock \n");
+  printf("\n");
+  printf("Options (short name):\n");
+  printf(" -t, --threads=N                :Number of worker threads. Each worker thread converts binlog events and executes SELECT statements. Default is 10 (threads).\n");
+  printf(" -o, --offset-events=N          :Number of binlog events that main thread (relay log reader thread) skips initially when reading relay logs. This number should be high when you have faster storage devices such as SSD. Default is 500 (events).\n");
+  printf(" -s, --seconds-prefetch=N       :Main thread stops reading relay log events when the event's timestamp is --seconds-prefetch seconds ahead of current SQL thread's timestamp. After that the main thread starts reading relay logs from SQL threads's position again. If this value is too high, worker threads will execute many more SELECT statements than necessary. Default value is 3 (seconds).\n");
+  printf(" -m, --millis-sleep=N           :If --seconds-prefetch condition is met, main thread sleeps --millis-sleep milliseconds before starting reading relay log. Default is 10 milliseconds.\n");
+  printf(" -u, --user=mysql_user          :MySQL slave user name. This user should have at least SELECT privilege on all application tables (default: root)\n");
+  printf(" -p, --password=mysql_pwd       :MySQL slave password (default: empty)\n");
+  printf(" -a, --admin_user=mysql_user    :MySQL administration user for the slave. This user should have at least SUPER and REPLICATION CLIENT (for SHOW SLAVE STATUS) privileges. (default: root)\n");
+  printf(" -b, --admin_password=mysql_pwd :MySQL password for the administration user. (default:empty)\n");
+  printf(" -h, --host=mysql_host          :MySQL slave hostname or IP address. This must be local address. (default: localhost)\n");
+  printf(" -P, --port=mysql_port          :MySQL slave port number (default:3306)\n");
+  printf(" -S, --socket=mysql_socket      :MySQL socket file path\n");
+  exit(1);
+}
+
+void print_version()
+{
+  printf("replication_booster version %s\n", VER);
+  exit(0);
+}
+
+void get_options(int argc, char **argv)
+{
+  int c, opt_ind= 0;
+  while((c= getopt_long(argc, argv, "?vt:o:s:m:u:p:a:b:h:P:S:", long_options, &opt_ind)) != EOF)
+  {
+    switch(c)
+    {
+      case '?': usage(); break;
+      case 'v': print_version(); break;
+      case 't': opt_workers= atoi(optarg);
+        if (opt_workers < 1) opt_workers= 1;
+        break;
+      case 'o': opt_skip_events= atoi(optarg);
+        if (opt_skip_events < 0) opt_skip_events= 0;
+        break;
+      case 's': opt_read_ahead_seconds= atoi(optarg);
+        if (opt_read_ahead_seconds < 1) opt_read_ahead_seconds= 1;
+        break;
+      case 'm': opt_sleep_millis_at_read_limit= atoi(optarg)*1000; break;
+      case 'u': opt_slave_user= optarg;  break;
+      case 'p': opt_slave_password= optarg;  break;
+      case 'a': opt_admin_user= optarg;  break;
+      case 'b': opt_admin_password= optarg;  break;
+      case 'h': opt_slave_host= optarg;  break;
+      case 'P': opt_slave_port= atoi(optarg);  break;
+      case 'S': opt_slave_socket= optarg;  break;
+      default: usage();  break;
+    }
+  }
+}
+

options.h

@@ -0,0 +1,41 @@
+/**
+ *   Replication Booster -- A Tool for Prefetching MySQL Slave Relay Logs
+ *   Copyright (C) 2011 DeNA Co.,Ltd.
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation; either version 2 of the License, or
+ *   (at your option) any later version.
+ *
+ *   This program is distributed in the hope that it will be useful,
+ *   but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *   GNU General Public License for more details.
+ *
+ *   You should have received a copy of the GNU General Public License
+ *   along with this program; if not, write to the Free Software
+ *   Foundation, Inc.,
+ *   51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+**/
+
+#ifndef __options_h_
+#define __options_h_
+
+#include "replication_booster.h"
+
+extern uint opt_workers;
+extern uint opt_skip_events;
+extern uint opt_read_ahead_seconds;
+extern uint opt_sleep_millis_at_read_limit;
+extern char *opt_slave_user;
+extern char *opt_slave_password;
+extern char *opt_admin_user;
+extern char *opt_admin_password;
+extern char *opt_slave_host;
+extern int opt_slave_port;
+extern char *opt_slave_socket;
+
+void get_options(int argc, char **argv);
+
+#endif