init

.gitignore

@@ -0,0 +1 @@
+vendor

.travis.yml

@@ -0,0 +1,62 @@
+language: go
+
+go:
+  - 1.8.1
+
+sudo: required
+
+addons:
+  apt:
+    packages:
+      - unixodbc
+      - unixodbc-dev
+
+services:
+  - docker
+
+before_script:
+  - sudo service mysql stop
+  - sudo service postgresql stop
+  - docker run -d -p 5433:5433 -m 1g --name vertica jbfavre/vertica:8.1
+  - docker run -d -p 3306:3306 --name mysql -e MYSQL_ALLOW_EMPTY_PASSWORD=yes percona/percona-server:latest --binlog_format=ROW --binlog_row_image=full --server-id=1 --log-bin=/tmp/bin.log --gtid-mode=ON --enforce-gtid-consistency
+  - wget https://my.vertica.com/client_drivers/8.1.x/8.1.0-0/vertica-client-8.1.0-0.x86_64.tar.gz -O /tmp/vertica-client-8.1.0-0.x86_64.tar.gz
+  - tar -xvf /tmp/vertica-client-8.1.0-0.x86_64.tar.gz -C /
+  - printf "[Vertica]\nDescription = Vertica driver\nDriver = /opt/vertica/lib64/libverticaodbc.so" | sudo odbcinst -i -d -r
+  - printf "[Vertica]\nDescription = Vertica\nDriver = /opt/vertica/lib64/libverticaodbc.so\nDriver = Vertica" | odbcinst -i -s -r
+  - curl https://glide.sh/get | sh
+  - glide install
+
+script:
+  - make test
+
+after_success:
+  if ([ "$TRAVIS_BRANCH" == "master" ] || [ ! -z "$TRAVIS_TAG" ]) && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
+      make build;
+      make release;
+      gem install --no-ri --no-rdoc fpm;
+      gem install pleaserun;
+      make deb;
+  fi;
+
+deploy:
+  - provider: releases
+    api_key:
+      secure: fqu223Co+LgdQdgFNfrdQQBzpdLyudmwQMYCRx0/ELDla26jxmAgwYzUf1EK+3EkNtzPsZ5prCct1W7TR/gd1++2fqjlypZ2OGpEuP/B91khmrIiDy5dTGHw7D3qPsbeQLlU8pemYGwQhVEy8RpWPSJkL6FJ1VrVYxPSYdUTRDgEMO4OWPQzgXSQn9Nyqcjkiyll5PqHoczKFn+X+hDmNPSaV+1okc69k5z/dCgaMYagqb+A9yWw1pmeELCgLCuyYKP7YTZYXOZzATgflingYIywyXj5pC8i5a8fM14Wm6uePP1CVqX9uV4whQXQyR9b6ZYQHm8pWqMdOZqrRvAO++A4loPcK1RlMdQaxi5f/Fpesgvp02AjzaqvlwrDu7v4953l/RhGqTTzt8Dd8WfzDVdVXLQdrKTGAXxxD1GpbKHWJVYriMWTMcuAwmWnoFzwTs5puApAaSX/qmneKhOAQxxSK0wYvRHCxbv+Irsqc4kF9wiljThbEQtpk3W/DBBkYQ9pmzUWmt5opcYzlauObSsl+ID2cw8pq5pAa9q4ci5fqYMXlcchUmNoqsrZlJaDGAZChjrZIzVjCtBntjLzUYVF41bPyD+9Kmp3cu8lolevMAxoC6LqPSo7HIQUANS5u4p1ZPHU1fjya0nLFvsGmQ2rtYw+HrXeSkV9EV7nmAU=
+    file: repligator-linux-amd64.tar.gz
+    skip_cleanup: true
+    on:
+      repo: b13f/repligator
+      branch: master
+      tags: true
+  - provider: packagecloud
+    skip_cleanup: true
+    repository: repligator
+    username: b13f
+    package_glob: "*.deb"
+    token:
+      secure: Pu5Y0PkcpSb2pNJHN9B7hePNZhxU/HIhhWKWyGBO6VQSgCtBFmgxEfHwfba3KQKaD5uT0pSRDi9ffBYtHFJbEz2wXYK84IMQRJjml8kVAfK00g8a1ZwBRL0BXgJUlaLC/0yF9gMPuUdZOT6iDpD/z9jVfgbYGalt93bZysAyuUiyYKxnXfjC1EV9L9ocSXqAyBIZ56WJ/l5XmdLzHwpinyiPP4F67l44TGSKIwFoSUFwOA/dqprWGp50sdOz8uiBqMk9mEwhd3Ksz9Ly6eMiEDKd0EQS+nmP7yX8oMZDzo07igzBIoI7Ob1iI1TPpPTsEjwzwY3GjN+Em7JvX8QzTLlupT50PgwJYZRbhVUW/Hkrn9bt0WGyzQ8/oSpbZ0p6RWbKIKzI7gF4lmMt489MP/SkFzyNmHcMOkuHoLVZOFtmEiOW4P05VdtqNQ9EABsfklOdxZObGVu/MK+E9TGQJtrZOI8QwRz3pkptnkDkmqB4f9BBtjsKsoX2xrIhvhHf0X5WC4jKKeWdamRTKru3RZ/9luEGhTfBr5d6fqgNpD4+p4Nf8gllPryVAzQ422iHDFCDUVUQOxxnNX7hJeoyLmQnEFF0XNKEABJOoIAME4tS32zPEtxFTpkyVQ0dadwNL8iifAL6xXLGxy8owS3aFL9qtGlwMX5Js+EF9PRHTFo=
+    dist: ubuntu/trusty
+    on:
+      repo: b13f/repligator
+      branch: master
+      tags: true

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Dmitry Laptev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,39 @@
+build:
+	glide install
+	go build
+
+test:
+	go test -coverprofile=main.cr
+	cd vertica && go test -coverprofile=../vertica.cr
+	cd ddlparser && go test -coverprofile=../ddlparser.cr
+	cat main.cr > cover.profile && cat vertica.cr | tail -n +2 >> cover.profile && cat ddlparser.cr | tail -n +2 >> cover.profile
+	rm main.cr vertica.cr ddlparser.cr
+
+test-cover: test
+	go tool cover -html=cover.profile
+
+checks:
+	misspell .
+	ineffassign .
+	golint . && golint ddlparser && golint isql && golint vertica
+	gocyclo -over 12 main.go ./vertica ./ddlparser
+
+release:
+	tar -zcvf repligator-linux-amd64.tar.gz repligator
+
+deb:
+	cp repligator builds/opt/repligator/bin/repligator
+	find builds -name ".gitignore" -type f -delete
+	pleaserun --install-prefix builds/ --name repligator --user repligator --group repligator \
+	--description "MySQL to Vertica replication service" \
+	--log-directory /var/log/repligator -p sysv /opt/repligator/bin/repligator -config=/etc/repligator/config.yml
+	fpm -n repligator -v `git describe --abbrev=0 --tags | cut -d "v" -f 2` -d unixodbc -d unixodbc-dev --deb-user repligator --deb-group repligator \
+	--license MIT --vendor b13f@github.com -m b13f@github.com --description "MySQL to Vertica replication service" \
+	--url "http://github.com/b13f/repligator" --deb-no-default-config-files \
+	--pre-install builds/preinst -C builds -t deb -s dir etc opt var
+
+clean:
+	rm -f cover.profile
+	rm -f repligator
+	rm -rf builds/etc/default builds/etc/init.d
+	rm -f builds/opt/repligator/bin/repligator

PITCHME.md

@@ -0,0 +1,95 @@
+## Repligator
+#### <span style="font-family:Arial;">A  Repligator Tour</span>
+
+---
+
+## If this not about you, you may leave
+
+- You have many MySQL servers (microservices?) with tables >100m rows.
+- You not use Kafka or some other data pipelines.
+- You wanna do some queries for statistic or analytic purposes.
+- You have slaves where you run this queries and wait for hours to get result.
+- You not use ETL yet.
+
+---
+
+## Heterogeneous replication to data warehouse
+
+With Repligator you can replicate all your MySQl data to [Vertica](https://my.vertica.com/docs/8.1.x/HTML/index.htm#Authoring/ConceptsGuide/Other/ConceptsGuide.htm%3FTocPath%3DVertica%2520Concepts%7C_____0)
+<br>
+And you don't need ETL processes
+---
+
+## Requirements
+
+- MySQL servers with GTID replication enabled
+- Vertica ODBC driver installed
+- Vertica 7.2-8.1
+
+---
+
+## Install
+
+- build from source https://github.com/b13f/repligator
+- download latest release from https://github.com/b13f/repligator/releases/latest
+- for ubuntu14.04 use https://packagecloud.io/b13f/repligator
+
+---
+### Configure
+configure MySQL sources
+```
+sources:
+  - name: shard1 #unique in this config
+    type: mysql
+    server_id: 101 #unique in your mysql setup
+    host: 192.168.0.1
+    port: 3306
+    user: root
+    password:
+    timeout: 10000 #in seconds
+    try_after: 2 #in minutes
+    gtid: ccffeb16-0b05-11e7-852a-080027c2ddae:1-2
+    schemas: # when exists apply rows event only in schemas
+     - name: testing # when exists apply only rows event only in schema, ddl for all
+       sync:
+         - test # when exists apply only rows events for this tables. exclude not use
+       exclude: # when exists not apply rows events for this tables
+         - balance_oou
+         - balance_demo_oou
+       gtid: ccffeb16-0b05-11e7-852a-080027c2ddae:1-6 # you can set gtidset per schema, this schema start sync rows(!) events from this position
+```
++++
+configure Vertica destination
+```
+destination:
+  odbc: Vertica
+  host: 192.168.50.85
+  port: 5433
+  user: dbadmin
+  password: password
+  database: main
+  pack: 10000
+  flush_count: 200000
+  flush_time: 120 #seconds
+  data_dir: /opt/repligator/data
+```
++++
+configure repligator
+```
+port: 8080
+log_file: /var/log/repligator/repligator.log
+log_level: debug #panic fatal error warn info debug
+slack:
+  bot_token:
+  hook:
+  channel: "#repligator"
+  username: repligator
+  icon: ":crocodile:"
+```
+---
+
+## Run
+
+you can see current cache state at http://repligator.host:8080/info
+
+To skip not supported DDL you can use http://repligator.host:8080/skip or chat bot command `skip`

README.md

@@ -0,0 +1,65 @@
+# Repligator
+
+[![Build Status](https://travis-ci.org/b13f/repligator.svg?branch=master)](https://travis-ci.org/b13f/repligator) [![Go Report Card](https://goreportcard.com/badge/github.com/b13f/repligator)](https://goreportcard.com/report/github.com/b13f/repligator) [![GitPitch](https://gitpitch.com/assets/badge.svg)](https://gitpitch.com/b13f/repligator?grs=github&t=black)
+
+Repligator is a heterogeneous replication service heavily expired by [tungsten replicator](https://github.com/continuent/tungsten-replicator).
+For now it support replication from MySQL to Vertica.
+
+## Getting Started
+
+### Purposes
+
+Repligator is used for "no ETL" on your MySQL servers setup.
+
+You can use your MySQL in your application while havily load statistic/analytics queries can be run in Vertica setup.
+
+Or if you have sharded MySQL setup you can run queries in all of your data in Vertica.
+
+### How it works
+Repligator aggregate all MySQL replication events and periodically load it in Vertica.
+
+MySQL MUST use GTID replication with full row binlog format.
+Update events is used like delete then inserts.
+
+Repligator support some DDL statements.
+
+All events run in Vertica in transaction for easely restart replication process.
+
+### Prerequisites
+For run Repligator you need to install unixodbc and [Vertica ODBC driver](https://my.vertica.com/download/vertica/client-drivers/)
+
+### Installing
+1. build from source, clone this repo
+    ```
+    curl https://glide.sh/get | sh
+    glide install
+    go build
+    ```
+2. download latest release [here](https://github.com/b13f/repligator/releases/latest)
+3. for ubuntu14.04 use https://packagecloud.io/b13f/repligator
+
+## Usage
+For your current setup in MySQL you should 
+
+1. dump your db with --tab option of mysqldump.
+2. run `repligator -df` on your MySQL dump folder.
+3. execute result DDL in Vertica.
+4. Load dumped data to Vertica.
+5. Run Repligator with GTID position from dump. Config example at [config.yml](https://github.com/b13f/repligator/blob/master/builds/etc/repligator/config.sample.yml)
+
+You can skip Not supported DDL statements throw web interface or Slack interface.
+
+### Known issues
+If you have massive updates or deletes Vertica may perform it slowly.
+
+For some tables in Vertica 7.2 have issues for constraint rise error with delete and insert in one transactions.
+
+For some tables in Vertica 7.2 have issues with very slow delete for few rows.
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details
+
+## Acknowledgments
+
+* This couldn't be done without https://github.com/siddontang/go-mysql and https://github.com/alexbrainman/odbc

bot.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"github.com/nlopes/slack"
+)
+
+type bot struct {
+	sender  *slack.RTM
+	channel string
+}
+
+func initBot(token string) *bot {
+	api := slack.New(token)
+
+	bot := new(bot)
+
+	bot.sender = api.NewRTM()
+
+	go bot.sender.ManageConnection()
+
+	return bot
+}
+
+func (bot *bot) receive() (ch chan string) {
+	ch = make(chan string)
+
+	go func() {
+		for msg := range bot.sender.IncomingEvents {
+			switch ev := msg.Data.(type) {
+			case *slack.MessageEvent:
+				if ev.Msg.SubType == `message_deleted` {
+					continue
+				}
+				bot.channel = ev.Msg.Channel
+				ch <- ev.Msg.Text
+			default:
+			}
+		}
+	}()
+
+	return
+}
+
+func (bot *bot) send(msg string) {
+	bot.sender.SendMessage(bot.sender.NewOutgoingMessage(msg, bot.channel))
+}

builds/etc/repligator/config.sample.yml

@@ -0,0 +1,39 @@
+sources:
+  - name: shard1
+    type: mysql
+    server_id: 101
+    host: 192.168.0.1
+    port: 3306
+    user: root
+    password:
+    timeout: 10000 #in seconds
+    try_after: 2 #in minutes
+    gtid: ccffeb16-0b05-11e7-852a-080027c2ddae:1-2
+#   schemas: # when exists apply rows event only in schemas
+#    - name: testing # when exists apply only rows event only in schema, ddl for all
+#      sync:
+#        - test # when exists apply only rows events for this tables. exclude not use
+#      exclude: # when exists not apply rows events for this tables
+#        - balance_oou
+#        - balance_demo_oou
+#      gtid: ccffeb16-0b05-11e7-852a-080027c2ddae:1-6 # you can set gtidset per schema, this schema start sync rows(!) events from this position
+destination:
+  odbc: Vertica
+  host: 192.168.50.85
+  port: 5433
+  user: dbadmin
+  password: password
+  database: main
+  pack: 10000
+  flush_count: 200000
+  flush_time: 120 #seconds
+  data_dir: /opt/repligator/data
+port: 8080
+log_file: /var/log/repligator/repligator.log
+log_level: debug #panic fatal error warn info debug
+slack:
+  bot_token:
+  hook:
+  channel: "#repligator"
+  username: repligator
+  icon: ":crocodile:"

builds/opt/repligator/bin/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

builds/opt/repligator/data/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

builds/preinst

@@ -0,0 +1,3 @@
+#!/bin/sh
+addgroup --quiet --system repligator
+adduser --quiet --home /opt/repligator --system repligator