Shippify Zongji

.eslintrc

@@ -0,0 +1,18 @@
+{
+  "env": {
+    "node": true,
+    "es6": true
+  },
+  "extends": "eslint:recommended",
+  "rules": {
+    "comma-dangle": ["warn", "only-multiline"],
+    "eol-last": ["error"],
+    "keyword-spacing": ["error", { "before": true } ],
+    "no-console": "off",
+    "no-trailing-spaces": ["error", { "skipBlankLines": true }],
+    "no-unused-vars": "warn",
+    "quotes": ["warn", "single", "avoid-escape"],
+    "semi": ["error", "always"],
+    "space-before-blocks": "error"
+  }
+}

.gitignore

@@ -1,61 +1,30 @@
-# Logs
-logs
+lib-cov
+*.seed
 *.log
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-
-# Runtime data
-pids
+*.csv
+*.dat
+*.out
 *.pid
-*.seed
-*.pid.lock
-
-# Directory for instrumented libs generated by jscoverage/JSCover
-lib-cov
-
-# Coverage directory used by tools like istanbul
-coverage
-
-# nyc test coverage
-.nyc_output
-
-# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
-.grunt
-
-# Bower dependency directory (https://bower.io/)
-bower_components
+*.gz
 
-# node-waf configuration
-.lock-wscript
-
-# Compiled binary addons (https://nodejs.org/api/addons.html)
-build/Release
-
-# Dependency directories
-node_modules/
-jspm_packages/
-
-# TypeScript v1 declaration files
-typings/
-
-# Optional npm cache directory
-.npm
-
-# Optional eslint cache
-.eslintcache
-
-# Optional REPL history
-.node_repl_history
-
-# Output of 'npm pack'
-*.tgz
-
-# Yarn Integrity file
-.yarn-integrity
-
-# dotenv environment variables file
-.env
-
-# next.js build output
-.next
+pids
+logs
+results
+
+example.js
+
+npm-debug.log
+.DS_Store
+node_modules
+.idea
+*.iml
+*~
+.\#*
+
+build/
+npm-install-stamp
+*.swo
+*.swp
+ben.js
+.vagrant
+Vagrantfile

.npmignore

@@ -0,0 +1,11 @@
+.DS_Store
+.git*
+*~
+*.log
+
+test/
+build/
+npm-install-stamp
+
+*.sublime-project
+*.sublime-workspace

.travis.yml

@@ -0,0 +1,15 @@
+language: node_js
+sudo: required
+dist: trusty
+node_js:
+  - "6"
+  - "8"
+services:
+  - docker
+before_script:
+  - npm run lint
+  - docker run -p 3355:3306 --name mysql-55 -e MYSQL_ROOT_PASSWORD=numtel -d mysql:5.5 --server-id=1 --log-bin=/var/lib/mysql/mysql-bin.log --binlog-format=row
+  - docker run -p 3356:3306 --name mysql-56 -e MYSQL_ROOT_PASSWORD=numtel -d mysql:5.6 --server-id=1 --log-bin=/var/lib/mysql/mysql-bin.log --binlog-format=row
+  - docker run -p 3357:3306 --name mysql-57 -e MYSQL_ROOT_PASSWORD=numtel -d mysql:5.7 --server-id=1 --log-bin=/var/lib/mysql/mysql-bin.log --binlog-format=row
+script:
+  - ./test/travis/runner.sh

LICENSE

@@ -0,0 +1,45 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Nevill (https://github.com/nevill) and contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
+
+lib/common.js parseGeometryValue function from node-mysql package:
+
+Copyright (c) 2012 Felix Geisendörfer (felix@debuggable.com) and contributors
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.

README.md

@@ -0,0 +1,152 @@
+# ZongJi [![Build Status](https://travis-ci.org/nevill/zongji.svg?branch=master)](https://travis-ci.org/nevill/zongji)
+A MySQL binlog listener running on Node.js.
+
+ZongJi (踪迹) is pronounced as `zōng jì` in Chinese.
+
+This package is a "pure JS" implementation based on [`mysql`](https://github.com/mysqljs/mysql). Since v0.2.0, The native part (which was written in C++) has been dropped.
+
+This package has been tested to work in MySQL 5.5, 5.6, and 5.7.
+
+
+# Upcoming Release
+
+v0.4.7 is the last release which supports Node.js v4.x.
+
+The next release will be v0.5, only supports Node.js from v6.x. It uses a lot Ecmascript 6 features.
+
+## Quick Start
+
+```javascript
+var zongji = new ZongJi({ /* ... MySQL Connection Settings ... */ });
+
+// Each change to the replication log results in an event
+zongji.on('binlog', function(evt) {
+  evt.dump();
+});
+
+// Binlog must be started, optionally pass in filters
+zongji.start({
+  includeEvents: ['tablemap', 'writerows', 'updaterows', 'deleterows']
+});
+```
+
+For a complete implementation see [`example.js`](example.js)...
+
+## Installation
+
+* Requires Node.js v6+
+
+  ```bash
+  $ npm install zongji
+  ```
+
+* Enable MySQL binlog in `my.cnf`, restart MySQL server after making the changes.
+  > From [MySQL 5.6](https://dev.mysql.com/doc/refman/5.6/en/replication-options-binary-log.html), binlog checksum is enabled by default. Zongji can work with it, but it doesn't really verify it.
+
+  ```
+  # Must be unique integer from 1-2^32
+  server-id        = 1
+  # Row format required for ZongJi
+  binlog_format    = row
+  # Directory must exist. This path works for Linux. Other OS may require
+  #   different path.
+  log_bin          = /var/log/mysql/mysql-bin.log
+
+  binlog_do_db     = employees   # Optional, limit which databases to log
+  expire_logs_days = 10          # Optional, purge old logs
+  max_binlog_size  = 100M        # Optional, limit log size
+  ```
+* Create an account with replication privileges, e.g. given privileges to account `zongji` (or any account that you use to read binary logs)
+
+  ```sql
+  GRANT REPLICATION SLAVE, REPLICATION CLIENT, SELECT ON *.* TO 'zongji'@'localhost'
+  ```
+
+## ZongJi Class
+
+The `ZongJi` constructor accepts one argument of either:
+
+* An object containing MySQL connection details in the same format as used by [package mysql](https://npm.im/mysql)
+* Or, a [mysql](https://npm.im/mysql) `Connection` or `Pool` object that will be used for querying column information.
+
+If a `Connection` or `Pool` object is passed to the constructor, it will not be destroyed/ended by Zongji's `stop()` method.
+
+If there is a `dateStrings` `mysql` configuration option in the connection details or connection, `ZongJi` will follow it.
+
+Each instance includes the following methods:
+
+Method Name | Arguments | Description
+------------|-----------|------------------------
+`start`     | `options` | Start receiving replication events, see options listed below
+`stop`      | *None*    | Disconnect from MySQL server, stop receiving events
+`set`       | `options` | Change options after `start()`
+`on`        | `eventName`, `handler` | Add a listener to the `binlog` or `error` event. Each handler function accepts one argument.
+
+**Options available:**
+
+Option Name | Type | Description
+------------|------|-------------------------------
+`serverId`  | `integer` | [Unique number (1 - 2<sup>32</sup>)](http://dev.mysql.com/doc/refman/5.0/en/replication-options.html#option_mysqld_server-id) to identify this replication slave instance. Must be specified if running more than one instance of ZongJi. Must be used in `start()` method for effect.<br>**Default:** `1`
+`startAtEnd` | `boolean` | Pass `true` to only emit binlog events that occur after ZongJi's instantiation. Must be used in `start()` method for effect.<br>**Default:** `false`
+`filename` | `string` | Begin reading events from this binlog file. If specified together with `position`, will take precedence over `startAtEnd`.
+`position` | `integer` | Begin reading events from this position. Must be included with `filename`.
+`includeEvents` | `[string]` | Array of event names to include<br>**Example:** `['writerows', 'updaterows', 'deleterows']`
+`excludeEvents` | `[string]` | Array of event names to exclude<br>**Example:** `['rotate', 'tablemap']`
+`includeSchema` | `object` | Object describing which databases and tables to include (Only for row events). Use database names as the key and pass an array of table names or `true` (for the entire database).<br>**Example:** ```{ 'my_database': ['allow_table', 'another_table'], 'another_db': true }```
+`excludeSchema` | `object` | Object describing which databases and tables to exclude (Same format as `includeSchema`)<br>**Example:** ```{ 'other_db': ['disallowed_table'], 'ex_db': true }```
+
+* By default, all events and schema are emitted.
+* `excludeSchema` and `excludeEvents` take precedence over `includeSchema` and `includeEvents`, respectively.
+
+**Supported Binlog Events:**
+
+Event name  | Description
+------------|---------------
+`unknown`   | Catch any other events
+`query`     | [Insert/Update/Delete Query](http://dev.mysql.com/doc/internals/en/query-event.html)
+`intvar`    | [Autoincrement and LAST_INSERT_ID](https://dev.mysql.com/doc/internals/en/intvar-event.html)
+`rotate`    | [New Binlog file](http://dev.mysql.com/doc/internals/en/rotate-event.html) Not required to be included to rotate to new files, but it is required to be included in order to keep the `filename` and `position` properties updated with current values for [graceful restarting on errors](https://gist.github.com/numtel/5b37b2a7f47b380c1a099596c6f3db2f).
+`format`    | [Format Description](http://dev.mysql.com/doc/internals/en/format-description-event.html)
+`xid`       | [Transaction ID](http://dev.mysql.com/doc/internals/en/xid-event.html)
+`tablemap`  | Before any row event (must be included for any other row events)
+`writerows` | Rows inserted, row data array available as `rows` property on event object
+`updaterows` | Rows changed, row data array available as `rows` property on event object
+`deleterows` | Rows deleted, row data array available as `rows` property on event object
+
+**Event Methods**
+
+Neither method requires any arguments.
+
+Name   | Description
+-------|---------------------------
+`dump` | Log a description of the event to the console
+`getEventName` | Return the name of the event
+
+## Important Notes
+
+* :star2: [All types allowed by `mysql`](https://github.com/mysqljs/mysql#type-casting) are supported by this package.
+* :speak_no_evil: While 64-bit integers in MySQL (`BIGINT` type) allow values in the range of 2<sup>64</sup> (± ½ × 2<sup>64</sup> for signed values), Javascript's internal storage of numbers limits values to 2<sup>53</sup>, making the allowed range of `BIGINT` fields only `-9007199254740992` to `9007199254740992`. Unsigned 64-bit integers must also not exceed `9007199254740992`.
+* :point_right: `TRUNCATE` statement does not cause corresponding `DeleteRows` event. Use unqualified `DELETE FROM` for same effect.
+* When using fractional seconds with `DATETIME` and `TIMESTAMP` data types in MySQL > 5.6.4, only millisecond precision is available due to the limit of Javascript's `Date` object.
+
+## Run Tests
+
+* install [Docker](https://www.docker.com/community-edition#download)
+* run `docker-compose up` and then `./docker-test.sh`
+
+## Reference
+
+I learnt many things from following resources while making ZongJi.
+
+* https://github.com/mysqljs/mysql
+* https://github.com/felixge/faster-than-c/
+* http://intuitive-search.blogspot.co.uk/2011/07/binary-log-api-and-replication-listener.html
+* https://github.com/Sannis/node-mysql-libmysqlclient
+* https://kkaefer.com/node-cpp-modules/
+* http://dev.mysql.com/doc/internals/en/replication-protocol.html
+* http://www.cs.wichita.edu/~chang/lecture/cs742/program/how-mysql-c-api.html
+* https://github.com/jeremycole/mysql_binlog (Ruby implemenation of MySQL binlog parser)
+* http://dev.mysql.com/doc/internals/en/date-and-time-data-type-representation.html
+
+## License
+MIT

docker-compose.yml

@@ -0,0 +1,32 @@
+---
+version: '2'
+services:
+  mysql55:
+    image: mysql:5.5
+    command: [ "--server-id=1", "--log-bin=/var/lib/mysql/mysql-bin.log", "--binlog-format=row"]
+    networks:
+      default:
+        aliases:
+          - mysql55
+    environment:
+      MYSQL_ROOT_PASSWORD: numtel
+
+  mysql56:
+    image: mysql:5.6
+    command: [ "--server-id=1", "--log-bin=/var/lib/mysql/mysql-bin.log", "--binlog-format=row"]
+    networks:
+      default:
+        aliases:
+          - mysql56
+    environment:
+      MYSQL_ROOT_PASSWORD: numtel
+
+  mysql57:
+    image: mysql:5.7
+    command: [ "--server-id=1", "--log-bin=/var/lib/mysql/mysql-bin.log", "--binlog-format=row"]
+    networks:
+      default:
+        aliases:
+          - mysql57
+    environment:
+      MYSQL_ROOT_PASSWORD: numtel

docker-test.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+MYSQL_HOSTS="mysql55 mysql56 mysql57"
+
+for hostname in ${MYSQL_HOSTS}; do
+  echo $hostname + node 6
+  docker run -it --network=zongji_default -e MYSQL_HOST=$hostname -w /build -v $PWD:/build node:6 npm test
+  echo $hostname + node 8
+  docker run -it --network=zongji_default -e MYSQL_HOST=$hostname -w /build -v $PWD:/build node:8 npm test
+  echo $hostname + node 10
+  docker run -it --network=zongji_default -e MYSQL_HOST=$hostname -w /build -v $PWD:/build node:10 npm test
+  echo $hostname + node 12
+  docker run -it --network=zongji_default -e MYSQL_HOST=$hostname -w /build -v $PWD:/build node:12 npm test
+done