From 0930f22b2865dbeb228fd39bf6a10c4bae6f7b40 Mon Sep 17 00:00:00 2001 From: glarwood Date: Mon, 13 May 2019 16:53:31 -0700 Subject: [PATCH 001/217] fix(ByteArrayOutputStream): call composed OutputStream#write --- .../github/shyiko/mysql/binlog/io/ByteArrayOutputStream.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayOutputStream.java b/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayOutputStream.java index 91e4ca44..7154ba22 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayOutputStream.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayOutputStream.java @@ -68,6 +68,11 @@ public void write(int b) throws IOException { outputStream.write(b); } + @Override + public void write(byte[] bytes) throws IOException { + outputStream.write(bytes); + } + public byte[] toByteArray() { // todo: whole approach feels wrong if (outputStream instanceof java.io.ByteArrayOutputStream) { From 23f3239dfd5c5e8fa4c1c35b9b9764f0aa9e01a5 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Sun, 14 Jul 2019 18:28:36 +0200 Subject: [PATCH 002/217] fixes for non-strictly sequential behavior in JSON fields as of MySQL 8.0.16, there are cases (see https://github.com/zendesk/maxwell/issues/1290) in which MySQL places JSON binary data at the offset specified in the header instead of just laying it out in order. --- .../event/deserialization/json/JsonBinary.java | 3 +++ .../mysql/binlog/io/ByteArrayInputStream.java | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java index a6b5fc0e..61fe3e66 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java @@ -189,6 +189,7 @@ public JsonBinary(byte[] bytes) { public JsonBinary(ByteArrayInputStream contents) { this.reader = contents; + this.reader.mark(Integer.MAX_VALUE); } public String getString() { @@ -397,6 +398,8 @@ protected void parseObject(boolean small, JsonFormatter formatter) } } else { // Parse the value ... + this.reader.reset(); + this.reader.skip(entry.index + 1); parse(entry.type, formatter); } } diff --git a/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java b/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java index 350b8709..d899818c 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java @@ -218,4 +218,18 @@ public void skipToTheEndOfTheBlock() throws IOException { } } + @Override + public synchronized void mark(int readlimit) { + inputStream.mark(readlimit); + } + + @Override + public boolean markSupported() { + return inputStream.markSupported(); + } + + @Override + public synchronized void reset() throws IOException { + inputStream.reset(); + } } From a6a3af778c236bac4a85e347b489bb68940de99c Mon Sep 17 00:00:00 2001 From: dingxiaobo Date: Sat, 12 Oct 2019 18:53:33 +0800 Subject: [PATCH 003/217] support "caching_sha2_password" (mysql 8.0 default) --- .../shyiko/mysql/binlog/BinaryLogClient.java | 19 ++- .../binlog/network/ClientCapabilities.java | 1 + .../AuthenticateNativePasswordCommand.java | 2 +- .../command/AuthenticateSHA2Command.java | 142 ++++++++++++++++++ ... AuthenticateSecurityPasswordCommand.java} | 5 +- 5 files changed, 162 insertions(+), 7 deletions(-) create mode 100644 src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2Command.java rename src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/{AuthenticateCommand.java => AuthenticateSecurityPasswordCommand.java} (93%) diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java index c395aa7b..8d7d4af8 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java @@ -45,8 +45,9 @@ import com.github.shyiko.mysql.binlog.network.protocol.Packet; import com.github.shyiko.mysql.binlog.network.protocol.PacketChannel; import com.github.shyiko.mysql.binlog.network.protocol.ResultSetRowPacket; -import com.github.shyiko.mysql.binlog.network.protocol.command.AuthenticateCommand; import com.github.shyiko.mysql.binlog.network.protocol.command.AuthenticateNativePasswordCommand; +import com.github.shyiko.mysql.binlog.network.protocol.command.AuthenticateSHA2Command; +import com.github.shyiko.mysql.binlog.network.protocol.command.AuthenticateSecurityPasswordCommand; import com.github.shyiko.mysql.binlog.network.protocol.command.Command; import com.github.shyiko.mysql.binlog.network.protocol.command.DumpBinaryLogCommand; import com.github.shyiko.mysql.binlog.network.protocol.command.DumpBinaryLogGtidCommand; @@ -715,9 +716,11 @@ private void authenticate(GreetingPacket greetingPacket) throws IOException { usingSSLSocket = true; } } - AuthenticateCommand authenticateCommand = new AuthenticateCommand(schema, username, password, - greetingPacket.getScramble()); - authenticateCommand.setCollation(collation); + + Command authenticateCommand = "caching_sha2_password".equals(greetingPacket.getPluginProvidedData()) ? + new AuthenticateSHA2Command(schema, username, password, greetingPacket.getScramble(), collation) : + new AuthenticateSecurityPasswordCommand(schema, username, password, greetingPacket.getScramble(), collation); + channel.write(authenticateCommand, packetNumber); byte[] authenticationResult = channel.read(); if (authenticationResult[0] != (byte) 0x00 /* ok */) { @@ -728,6 +731,14 @@ private void authenticate(GreetingPacket greetingPacket) throws IOException { errorPacket.getSqlState()); } else if (authenticationResult[0] == (byte) 0xFE) { switchAuthentication(authenticationResult, usingSSLSocket); + } else if (authenticationResult[0] == (byte) 0x01) { + if (authenticationResult.length >= 2 && (authenticationResult[1] == 3) || (authenticationResult[1] == 4)) { + // 8.0 auth ok + byte[] authenticationResultSha2 = channel.read(); + logger.log(Level.FINEST, "SHA2 auth result {0}", authenticationResultSha2); + } else { + throw new AuthenticationException("Unexpected authentication result (" + authenticationResult[0] + "&" + authenticationResult[1] + ")"); + } } else { throw new AuthenticationException("Unexpected authentication result (" + authenticationResult[0] + ")"); } diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/ClientCapabilities.java b/src/main/java/com/github/shyiko/mysql/binlog/network/ClientCapabilities.java index c744d5a9..d81e2e1a 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/network/ClientCapabilities.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/ClientCapabilities.java @@ -42,6 +42,7 @@ public final class ClientCapabilities { public static final int MULTI_RESULTS = 1 << 17; /* enable/disable multi-results */ public static final int PS_MULTI_RESULTS = 1 << 18; /* multi-results in ps-protocol */ public static final int PLUGIN_AUTH = 1 << 19; /* client supports plugin authentication */ + public static final int PLUGIN_AUTH_LENENC_CLIENT_DATA = 1 << 21; public static final int SSL_VERIFY_SERVER_CERT = 1 << 30; public static final int REMEMBER_OPTIONS = 1 << 31; diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateNativePasswordCommand.java b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateNativePasswordCommand.java index f98eced0..25711af0 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateNativePasswordCommand.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateNativePasswordCommand.java @@ -29,6 +29,6 @@ public AuthenticateNativePasswordCommand(String scramble, String password) { } @Override public byte[] toByteArray() throws IOException { - return AuthenticateCommand.passwordCompatibleWithMySQL411(password, scramble); + return AuthenticateSecurityPasswordCommand.passwordCompatibleWithMySQL411(password, scramble); } } diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2Command.java b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2Command.java new file mode 100644 index 00000000..c8afca7e --- /dev/null +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2Command.java @@ -0,0 +1,142 @@ +/* + * Copyright 2018 dingxiaobo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.shyiko.mysql.binlog.network.protocol.command; + +import com.github.shyiko.mysql.binlog.io.ByteArrayOutputStream; +import com.github.shyiko.mysql.binlog.network.ClientCapabilities; + +import java.io.IOException; +import java.security.DigestException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +/** + * @author dingxiaobo + */ +public class AuthenticateSHA2Command implements Command { + + private String schema; + private String username; + private String password; + private String salt; + private int clientCapabilities; + private int collation; + + public AuthenticateSHA2Command(String schema, String username, String password, String salt, int collation) { + this.schema = schema; + this.username = username; + this.password = password; + this.salt = salt; + this.collation = collation; + } + + public void setClientCapabilities(int clientCapabilities) { + this.clientCapabilities = clientCapabilities; + } + + @Override + public byte[] toByteArray() throws IOException { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + int clientCapabilities = this.clientCapabilities; + if (clientCapabilities == 0) { + clientCapabilities |= ClientCapabilities.LONG_FLAG; + clientCapabilities |= ClientCapabilities.PROTOCOL_41; + clientCapabilities |= ClientCapabilities.SECURE_CONNECTION; + clientCapabilities |= ClientCapabilities.PLUGIN_AUTH; + clientCapabilities |= ClientCapabilities.PLUGIN_AUTH_LENENC_CLIENT_DATA; + + if (schema != null) { + clientCapabilities |= ClientCapabilities.CONNECT_WITH_DB; + } + } + buffer.writeInteger(clientCapabilities, 4); + buffer.writeInteger(0, 4); // maximum packet length + buffer.writeInteger(collation, 1); + for (int i = 0; i < 23; i++) { + buffer.write(0); + } + buffer.writeZeroTerminatedString(username); + byte[] passwordSHA1 = encodePassword(); + buffer.writeInteger(passwordSHA1.length, 1); + buffer.write(passwordSHA1); + if (schema != null) { + buffer.writeZeroTerminatedString(schema); + } + buffer.writeZeroTerminatedString("caching_sha2_password"); + + return buffer.toByteArray(); + } + + private byte[] encodePassword() { + if (password == null || "".equals(password)) { + return new byte[0]; + } + // caching_sha2_password + /* + * Server does it in 4 steps (see sql/auth/sha2_password_common.cc Generate_scramble::scramble method): + * + * SHA2(src) => digest_stage1 + * SHA2(digest_stage1) => digest_stage2 + * SHA2(digest_stage2, m_rnd) => scramble_stage1 + * XOR(digest_stage1, scramble_stage1) => scramble + */ + MessageDigest md; + try { + md = MessageDigest.getInstance("SHA-256"); + + int CACHING_SHA2_DIGEST_LENGTH = 32; + byte[] dig1 = new byte[CACHING_SHA2_DIGEST_LENGTH]; + byte[] dig2 = new byte[CACHING_SHA2_DIGEST_LENGTH]; + byte[] scramble1 = new byte[CACHING_SHA2_DIGEST_LENGTH]; + + // SHA2(src) => digest_stage1 + md.update(password.getBytes(), 0, password.getBytes().length); + md.digest(dig1, 0, CACHING_SHA2_DIGEST_LENGTH); + md.reset(); + + // SHA2(digest_stage1) => digest_stage2 + md.update(dig1, 0, dig1.length); + md.digest(dig2, 0, CACHING_SHA2_DIGEST_LENGTH); + md.reset(); + + // SHA2(digest_stage2, m_rnd) => scramble_stage1 + md.update(dig2, 0, dig1.length); + md.update(salt.getBytes(), 0, salt.getBytes().length); + md.digest(scramble1, 0, CACHING_SHA2_DIGEST_LENGTH); + + // XOR(digest_stage1, scramble_stage1) => scramble + byte[] mysqlScrambleBuff = new byte[CACHING_SHA2_DIGEST_LENGTH]; + xor(dig1, mysqlScrambleBuff, scramble1, CACHING_SHA2_DIGEST_LENGTH); + + return mysqlScrambleBuff; + } catch (NoSuchAlgorithmException ex) { + throw new RuntimeException(ex); + } catch (DigestException e) { + throw new RuntimeException(e); + } + } + + private void xor(byte[] from, byte[] to, byte[] scramble, int length) { + int pos = 0; + int scrambleLength = scramble.length; + + while (pos < length) { + to[pos] = (byte) (from[pos] ^ scramble[pos % scrambleLength]); + pos++; + } + } + +} diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateCommand.java b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSecurityPasswordCommand.java similarity index 93% rename from src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateCommand.java rename to src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSecurityPasswordCommand.java index a045fe24..d99d6317 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateCommand.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSecurityPasswordCommand.java @@ -25,7 +25,7 @@ /** * @author Stanley Shyiko */ -public class AuthenticateCommand implements Command { +public class AuthenticateSecurityPasswordCommand implements Command { private String schema; private String username; @@ -34,11 +34,12 @@ public class AuthenticateCommand implements Command { private int clientCapabilities; private int collation; - public AuthenticateCommand(String schema, String username, String password, String salt) { + public AuthenticateSecurityPasswordCommand(String schema, String username, String password, String salt, int collation) { this.schema = schema; this.username = username; this.password = password; this.salt = salt; + this.collation = collation; } public void setClientCapabilities(int clientCapabilities) { From 88de1edf637186ed1d2377d961eb836e95a6c095 Mon Sep 17 00:00:00 2001 From: Richard Burnison Date: Mon, 13 Apr 2020 11:54:50 -0400 Subject: [PATCH 004/217] Integration test idempotency. The `BinaryLogClientIntegrationTest` class did not clean-up after itself: it tried to create the `test_metameta` table unconditionally. This test would then fail if the tests were run twice. I _believe_ that this change keeps the original intent intact. --- .../shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java index 71dfb524..9896e504 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java @@ -994,6 +994,7 @@ public void execute(Statement statement) throws SQLException { @Test public void testMySQL8TableMetadata() throws Exception { + master.execute("drop table if exists test_metameta"); master.execute("create table test_metameta ( " + "a date, b date, c date, d date, e date, f date, g date, " + "h date, i date, j int)"); From 7e878a6e744d2a35c677d2ab89a5606e9d23cbb1 Mon Sep 17 00:00:00 2001 From: Richard Burnison Date: Mon, 13 Apr 2020 11:17:44 -0400 Subject: [PATCH 005/217] Add master server ID to BinaryLogClient. When the `BinaryLogClient` connects to a MySQL server, it's not currently possible to identify to which server it has connected. When connecting to a pool of replicas through a load balancer, for example, a reconnect may mean the `BinaryLogClient` is connected to a different MySQL server. When using binary log file positions instead of GTIDs, this can be critical: attempting to resume from the same binary log positions on a different MySQL server may result in missed or duplicated events. Similarly, not resuming from the same binary log positions when reconnecting to the same server is wasteful and may result in longer recovery/catch-up times. This change adds the ability to get the equivalent of `MASTER_SERVER_ID` from the `BinaryLogClient`. This can then be inspected within each of the `LifecycleListener` events. --- .../shyiko/mysql/binlog/BinaryLogClient.java | 14 ++++++++++++++ .../binlog/BinaryLogClientIntegrationTest.java | 11 +++++++++++ 2 files changed, 25 insertions(+) diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java index c395aa7b..0a860374 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java @@ -150,6 +150,7 @@ public X509Certificate[] getAcceptedIssuers() { private volatile PacketChannel channel; private volatile boolean connected; + private volatile long masterServerId = -1; private ThreadFactory threadFactory; @@ -228,6 +229,10 @@ public void setSSLMode(SSLMode sslMode) { this.sslMode = sslMode; } + public long getMasterServerId() { + return this.masterServerId; + } + /** * @return server id (65535 by default) * @see #setServerId(long) @@ -534,6 +539,7 @@ public void connect() throws IOException { if (checksumType != ChecksumType.NONE) { confirmSupportOfChecksum(checksumType); } + setMasterServerId(); if (heartbeatInterval > 0) { enableHeartbeat(); } @@ -656,6 +662,14 @@ private void enableHeartbeat() throws IOException { } } + private void setMasterServerId() throws IOException { + channel.write(new QueryCommand("select @@server_id")); + ResultSetRowPacket[] resultSet = readResultSet(); + if (resultSet.length >= 0) { + this.masterServerId = Long.parseLong(resultSet[0].getValue(0)); + } + } + private void requestBinaryLogStream() throws IOException { long serverId = blocking ? this.serverId : 0; // http://bugs.mysql.com/bug.php?id=71178 Command dumpBinaryLogCommand; diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java index 9896e504..fd45275e 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java @@ -1002,6 +1002,17 @@ public void testMySQL8TableMetadata() throws Exception { eventListener.waitFor(WriteRowsEventData.class, 1, DEFAULT_TIMEOUT); } + @Test + public void testSetMasterServerId() throws Exception { + slave.query("SELECT @@server_id", new Callback() { + @Override + public void execute(final ResultSet rs) throws SQLException { + rs.next(); + assertEquals(client.getMasterServerId(), rs.getLong("@@server_id")); + } + }); + } + @AfterMethod public void afterEachTest() throws Exception { final CountDownLatch latch = new CountDownLatch(1); From 4a7de10140c97cbd6da7bee573bf75ce005de402 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Fri, 24 Apr 2020 11:59:45 -0700 Subject: [PATCH 006/217] update pom.xml with fork info --- pom.xml | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index a3f78dcb..a2e789c2 100644 --- a/pom.xml +++ b/pom.xml @@ -2,13 +2,13 @@ 4.0.0 - com.github.shyiko + com.zendesk mysql-binlog-connector-java - 0.0.0-SNAPSHOT + 0.20.2 mysql-binlog-connector-java MySQL Binary Log connector - https://github.com/shyiko/mysql-binlog-connector-java + https://github.com/osheroff/mysql-binlog-connector-java Apache License, Version 2.0 @@ -17,9 +17,9 @@ - scm:git:git@github.com:shyiko/mysql-binlog-connector-java.git - scm:git:git@github.com:shyiko/mysql-binlog-connector-java.git - git@github.com:shyiko/mysql-binlog-connector-java.git + scm:git:git@github.com:osheroff/mysql-binlog-connector-java.git + scm:git:git@github.com:osheroff/mysql-binlog-connector-java.git + git@github.com:osheroff/mysql-binlog-connector-java.git @@ -27,6 +27,11 @@ stanley.shyiko@gmail.com Stanley Shyiko + + osheroff + ben@gimbo.net + Ben Osheroff + @@ -44,6 +49,11 @@ + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.8 + org.testng testng @@ -365,7 +375,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.7 + 1.6.8 true https://oss.sonatype.org/ From 83e28a461b07022dd70bb68728b1567904e30924 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Fri, 24 Apr 2020 12:00:29 -0700 Subject: [PATCH 007/217] add deploy notes --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 595feb03..329aa9f8 100644 --- a/README.md +++ b/README.md @@ -216,6 +216,17 @@ cd mysql-binlog-connector-java mvn # shows how to build, test, etc. project ``` +## Deployment + +First: +http://maven.apache.org/guides/mini/guide-encryption.html + +Secondly: +``` + export OSS_SONATYPE_ORG_USERNAME=username + export OSS_SONATYPE_ORG_PASSWORD={encryptedpasswordJF!#$flkj} +``` + ## Contributing In lieu of a formal styleguide, please take care to maintain the existing coding style. From 2a034d3545c0de0396f6865db4052de3c5cdc0a0 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Fri, 24 Apr 2020 12:00:51 -0700 Subject: [PATCH 008/217] add 0.22.0 notes --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd3c07ac..6a2ca264 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,13 @@ # Changelog -All notable changes to this project will be documented in this file. +All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## [0.22.0](https://github.com/shyiko/mysql-binlog-connector-java/compare/0.20.0...0.22.9) - 2020-04-24 + +- master server id is exposed in the library https://github.com/shyiko/mysql-binlog-connector-java/pull/319 +- Fixes for JSON data in mysql 8.0.16+ https://github.com/shyiko/mysql-binlog-connector-java/pull/288 +- more fixes for the bizarre azure platform https://github.com/shyiko/mysql-binlog-connector-java/pull/275 + ## [0.20.1](https://github.com/shyiko/mysql-binlog-connector-java/compare/0.20.0...0.20.1) - 2019-05-12 ### Added From 73afc83cc720c237088b0e39a69b611658d8a6a1 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Fri, 24 Apr 2020 12:53:29 -0700 Subject: [PATCH 009/217] oops, remove runtime dep on sonatype that was just for debugging --- pom.xml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index a2e789c2..ec35952c 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.zendesk mysql-binlog-connector-java - 0.20.2 + 0.22.1 mysql-binlog-connector-java MySQL Binary Log connector @@ -49,11 +49,6 @@ - - org.sonatype.plugins - nexus-staging-maven-plugin - 1.6.8 - org.testng testng From 4437b71ff35608906088eaecda6c29473a7c83ec Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Sat, 25 Apr 2020 21:51:11 -0700 Subject: [PATCH 010/217] fixes for JSON objects inside JSON objects the value's offset is given as bytes from the object's beginning, not from the beginning of the entire record, we have to account for that. --- .../binlog/event/deserialization/json/JsonBinary.java | 6 +++++- .../shyiko/mysql/binlog/io/ByteArrayInputStream.java | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java index 61fe3e66..25ea2d7f 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java @@ -320,6 +320,10 @@ protected void parse(ValueType type, JsonFormatter formatter) throws IOException */ protected void parseObject(boolean small, JsonFormatter formatter) throws IOException { + // this is terrible, but without a decent seekable InputStream the other way seemed like + // a full-on rewrite + int objectOffset = this.reader.getPosition(); + // Read the header ... int numElements = readUnsignedIndex(Integer.MAX_VALUE, small, "number of elements in"); int numBytes = readUnsignedIndex(Integer.MAX_VALUE, small, "size of"); @@ -399,7 +403,7 @@ protected void parseObject(boolean small, JsonFormatter formatter) } else { // Parse the value ... this.reader.reset(); - this.reader.skip(entry.index + 1); + this.reader.skip(objectOffset + entry.index); parse(entry.type, formatter); } } diff --git a/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java b/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java index d899818c..ff71cbee 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java @@ -27,10 +27,12 @@ public class ByteArrayInputStream extends InputStream { private InputStream inputStream; private Integer peek; + private Integer pos, markPosition; private int blockLength = -1; public ByteArrayInputStream(InputStream inputStream) { this.inputStream = inputStream; + this.pos = 0; } public ByteArrayInputStream(byte[] bytes) { @@ -189,6 +191,7 @@ public int read() throws IOException { if (result == -1) { throw new EOFException(); } + this.pos += 1; return result; } @@ -218,8 +221,13 @@ public void skipToTheEndOfTheBlock() throws IOException { } } + public int getPosition() { + return pos; + } + @Override public synchronized void mark(int readlimit) { + markPosition = pos; inputStream.mark(readlimit); } @@ -230,6 +238,7 @@ public boolean markSupported() { @Override public synchronized void reset() throws IOException { + pos = markPosition; inputStream.reset(); } } From 1dd27a6278afdd8aae519a126754ac762d6c9fac Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Sun, 26 Apr 2020 14:54:37 -0700 Subject: [PATCH 011/217] move from vagrant setup to onetimeserver setup --- pom.xml | 54 +--- .../BinaryLogClientIntegrationTest.java | 19 +- .../mysql/binlog/MysqlOnetimeServer.java | 270 ++++++++++++++++++ .../binlog/MysqlOnetimeServerOptions.java | 8 + .../json/JsonBinaryValueIntegrationTest.java | 16 +- src/test/onetimeserver | 117 ++++++++ 6 files changed, 426 insertions(+), 58 deletions(-) create mode 100644 src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java create mode 100644 src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServerOptions.java create mode 100755 src/test/onetimeserver diff --git a/pom.xml b/pom.xml index ec35952c..4a327429 100644 --- a/pom.xml +++ b/pom.xml @@ -73,6 +73,18 @@ 8.0.15 test + + com.fasterxml.jackson.core + jackson-core + 2.9.10 + test + + + com.fasterxml.jackson.core + jackson-databind + 2.9.10.3 + test + @@ -146,48 +158,6 @@ - - - org.codehaus.mojo - exec-maven-plugin - 1.1.1 - - - start-vagrant-vm - pre-integration-test - - exec - - - ${vagrant.integration.box} - ${vagrant.bin} - - up - - ${skipTests} - - - - destroy-vagrant-vm - post-integration-test - - exec - - - ${vagrant.integration.box} - ${vagrant.bin} - - destroy - --force - - ${skipTests} - - - - org.apache.maven.plugins maven-checkstyle-plugin diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java index fd45275e..7460f60e 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java @@ -114,14 +114,15 @@ public class BinaryLogClientIntegrationTest { @BeforeClass public void setUp() throws Exception { TimeZone.setDefault(TimeZone.getTimeZone("GMT")); - ResourceBundle bundle = ResourceBundle.getBundle("jdbc"); - String prefix = "jdbc.mysql.replication."; - master = new MySQLConnection(bundle.getString(prefix + "master.hostname"), - Integer.parseInt(bundle.getString(prefix + "master.port")), - bundle.getString(prefix + "master.username"), bundle.getString(prefix + "master.password")); - slave = new MySQLConnection(bundle.getString(prefix + "slave.hostname"), - Integer.parseInt(bundle.getString(prefix + "slave.port")), - bundle.getString(prefix + "slave.superUsername"), bundle.getString(prefix + "slave.superPassword")); + MysqlOnetimeServer masterServer = new MysqlOnetimeServer(); + MysqlOnetimeServer slaveServer = new MysqlOnetimeServer(); + masterServer.boot(); + slaveServer.boot(); + slaveServer.setupSlave(masterServer.getPort()); + + master = new MySQLConnection("127.0.0.1", masterServer.getPort(), "root", ""); + slave = new MySQLConnection("127.0.0.1", slaveServer.getPort(), "root", ""); + client = new BinaryLogClient(slave.hostname, slave.port, slave.username, slave.password); EventDeserializer eventDeserializer = new EventDeserializer(); eventDeserializer.setCompatibilityMode(CompatibilityMode.CHAR_AND_BINARY_AS_BYTE_ARRAY, @@ -1022,7 +1023,7 @@ public void afterEachTest() throws Exception { public void onEvent(Event event) { if (event.getHeader().getEventType() == EventType.QUERY) { EventData data = event.getData(); - if (data != null && ((QueryEventData) data).getSql().contains("_EOS_marker")) { + if (data != null && ((QueryEventData) data).getSql().toLowerCase().contains("_eos_marker")) { latch.countDown(); } } diff --git a/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java new file mode 100644 index 00000000..0810be7a --- /dev/null +++ b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java @@ -0,0 +1,270 @@ +package com.github.shyiko.mysql.binlog; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.sql.*; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import java.util.logging.Logger; +import java.util.logging.Level; + +public class MysqlOnetimeServer { + private final MysqlOnetimeServerOptions options; + public static int nextServerID = 1; + public final int SERVER_ID = MysqlOnetimeServer.nextServerID++; + private final Logger logger = Logger.getLogger(getClass().getName()); + + + private Connection connection; + private int port; + private int serverPid; + public String path; + + public static final TypeReference> MAP_STRING_OBJECT_REF = new TypeReference>() {}; + + public MysqlOnetimeServer() { + this.options = new MysqlOnetimeServerOptions(); + } + + public MysqlOnetimeServer(MysqlOnetimeServerOptions options) { + this.options = options; + } + + public void boot() throws Exception { + final String dir = System.getProperty("user.dir"); + final String xtraParams = options.extraParams == null ? "" : options.extraParams; + + // By default, MySQL doesn't run under root. However, in an environment like Docker, the root user is the + // only available user by default. By adding "--user=root" when the root user is used, we can make sure + // the tests can continue to run. + boolean isRoot = System.getProperty("user.name").equals("root"); + + String gtidParams = ""; + if ( options.gtid ) { + logger.info("In gtid test mode."); + gtidParams = + "--gtid-mode=ON " + + "--log-slave-updates=ON " + + "--enforce-gtid-consistency=true "; + } + String serverID = ""; + if ( !xtraParams.contains("--server_id") ) + serverID = "--server_id=" + options.serverID; + + String authPlugin = ""; + // if ( this.getVersion().atLeast(8, 0) ) { + if ( false ) { + authPlugin = "--default-authentication-plugin=mysql_native_password"; + } + + ProcessBuilder pb = new ProcessBuilder( + dir + "/src/test/onetimeserver", + "--mysql-version=" + this.getVersionString(), + "--log-slave-updates", + "--log-bin=master", + "--binlog_format=row", + "--innodb_flush_log_at_trx_commit=0", + serverID, + "--character-set-server=utf8", + "--sync_binlog=0", + "--default-time-zone=+00:00", + isRoot ? "--user=root" : "", + authPlugin, + gtidParams + ); + + for ( String s : xtraParams.split(" ") ) { + pb.command().add(s); + } + + Process p = pb.start(); + BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream())); + + p.waitFor(); + + final BufferedReader errReader = new BufferedReader(new InputStreamReader(p.getErrorStream())); + + new Thread() { + @Override + public void run() { + while (true) { + String l = null; + try { + l = errReader.readLine(); + } catch ( IOException e) {}; + + if (l == null) + break; + System.err.println(l); + } + } + }.start(); + + String json = reader.readLine(); + String outputFile = null; + try { + ObjectMapper mapper = new ObjectMapper(); + Map output = mapper.readValue(json, MAP_STRING_OBJECT_REF); + this.port = (Integer) output.get("port"); + this.serverPid = (Integer) output.get("server_pid"); + this.path = (String) output.get("mysql_path"); + outputFile = (String) output.get("output"); + } catch ( Exception e ) { + logger.log(Level.SEVERE, "got exception while parsing " + json, e); + throw(e); + } + + + resetConnection(); + this.connection.createStatement().executeUpdate("CREATE USER 'maxwell'@'127.0.0.1' IDENTIFIED BY 'maxwell'"); + this.connection.createStatement().executeUpdate("GRANT REPLICATION SLAVE on *.* to 'maxwell'@'127.0.0.1'"); + this.connection.createStatement().executeUpdate("GRANT ALL on *.* to 'maxwell'@'127.0.0.1'"); + this.connection.createStatement().executeUpdate("CREATE DATABASE if not exists test"); + logger.info("booted at port " + this.port + ", outputting to file " + outputFile); + + if ( options.masterServer != null ) { + this.setupSlave(options.masterServer.port); + } + } + + public void setupSlave(int masterPort) throws SQLException { + Connection master = DriverManager.getConnection("jdbc:mysql://127.0.0.1:" + masterPort + "/mysql?useSSL=false", "root", ""); + ResultSet rs = master.createStatement().executeQuery("show master status"); + if ( !rs.next() ) + throw new RuntimeException("could not get master status"); + + String file = rs.getString("File"); + Long position = rs.getLong("Position"); + rs.close(); + + String changeSQL = String.format( + "CHANGE MASTER to master_host = '127.0.0.1', master_user='maxwell', master_password='maxwell', " + + "master_log_file = '%s', master_log_pos = %d, master_port = %d", + file, position, masterPort + ); + logger.info("starting up slave: " + changeSQL); + getConnection().createStatement().execute(changeSQL); + getConnection().createStatement().execute("START SLAVE"); + + + rs.close(); + + ResultSet status = query("show slave status"); + if ( !status.next() ) + throw new RuntimeException("could not get slave status"); + + if ( status.getString("Slave_IO_Running").equals("No") + || status.getString("Slave_SQL_Running").equals("No")) { + throw new RuntimeException("could not start slave: " + dumpQuery("show slave status")); + + } + status.close(); + } + + + public String dumpQuery(String query) throws SQLException { + String result = ""; + ResultSet rs = getConnection().createStatement().executeQuery(query); + rs.next(); + for ( int i = 1 ; i <= rs.getMetaData().getColumnCount() ; i++) { + Object val = rs.getObject(i); + String asString = val == null ? "null" : val.toString(); + result = result + rs.getMetaData().getColumnName(i) + ": " + asString + "\n"; + } + return result; + } + + public void resetConnection() throws SQLException { + this.connection = getNewConnection(); + } + + public Connection getNewConnection() throws SQLException { + return DriverManager.getConnection("jdbc:mysql://127.0.0.1:" + port + "/mysql?zeroDateTimeBehavior=convertToNull&useSSL=false", "root", ""); + } + + public Connection getConnection() { + return connection; + } + + public Connection getConnection(String defaultDB) throws SQLException { + Connection conn = getNewConnection(); + conn.setCatalog(defaultDB); + return conn; + } + + public void execute(String query) throws SQLException { + Statement s = getConnection().createStatement(); + s.executeUpdate(query); + s.close(); + } + + private Connection cachedCX; + public void executeCached(String query) throws SQLException { + if ( cachedCX == null ) + cachedCX = getConnection(); + + Statement s = cachedCX.createStatement(); + s.executeUpdate(query); + s.close(); + } + + public void executeList(List queries) throws SQLException { + for (String q: queries) { + if ( q.matches("^\\s*$") ) + continue; + + execute(q); + } + } + + public void executeList(String[] schemaSQL) throws SQLException { + executeList(Arrays.asList(schemaSQL)); + } + + public void executeQuery(String sql) throws SQLException { + getConnection().createStatement().executeUpdate(sql); + } + + public ResultSet query(String sql) throws SQLException { + return getConnection().createStatement().executeQuery(sql); + } + + public int getPort() { + return port; + } + + public void shutDown() { + try { + Runtime.getRuntime().exec("kill " + this.serverPid); + } catch ( IOException e ) {} + } + + private static String getVersionString() { + String mysqlVersion = System.getenv("MYSQL_VERSION"); + return mysqlVersion == null ? "5.7" : mysqlVersion; + } + + public void waitForSlaveToBeCurrent(MysqlOnetimeServer master) throws Exception { + ResultSet ms = master.query("show master status"); + ms.next(); + String masterFile = ms.getString("File"); + Long masterPos = ms.getLong("Position"); + ms.close(); + + while ( true ) { + ResultSet rs = query("show slave status"); + rs.next(); + if ( rs.getString("Relay_Master_Log_File").equals(masterFile) && + rs.getLong("Exec_Master_Log_Pos") >= masterPos ) + return; + + Thread.sleep(200); + } + } +} diff --git a/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServerOptions.java b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServerOptions.java new file mode 100644 index 00000000..2e0ca936 --- /dev/null +++ b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServerOptions.java @@ -0,0 +1,8 @@ +package com.github.shyiko.mysql.binlog; + +public class MysqlOnetimeServerOptions { + public int serverID = MysqlOnetimeServer.nextServerID++; + public boolean gtid = false; + public MysqlOnetimeServer masterServer; + public String extraParams; +} diff --git a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java index cdf314f7..082ddb20 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java @@ -19,6 +19,7 @@ import com.github.shyiko.mysql.binlog.BinaryLogClientIntegrationTest; import com.github.shyiko.mysql.binlog.CapturingEventListener; import com.github.shyiko.mysql.binlog.CountDownEventListener; +import com.github.shyiko.mysql.binlog.MysqlOnetimeServer; import com.github.shyiko.mysql.binlog.TraceEventListener; import com.github.shyiko.mysql.binlog.TraceLifecycleListener; import com.github.shyiko.mysql.binlog.event.Event; @@ -37,7 +38,6 @@ import java.sql.SQLSyntaxErrorException; import java.sql.Statement; import java.util.List; -import java.util.ResourceBundle; import java.util.TimeZone; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -69,11 +69,12 @@ public class JsonBinaryValueIntegrationTest { @BeforeClass public void setUp() throws Exception { TimeZone.setDefault(TimeZone.getTimeZone("GMT")); - ResourceBundle bundle = ResourceBundle.getBundle("jdbc"); - String prefix = "jdbc.mysql.replication."; - master = new BinaryLogClientIntegrationTest.MySQLConnection(bundle.getString(prefix + "master.hostname"), - Integer.parseInt(bundle.getString(prefix + "master.port")), - bundle.getString(prefix + "master.username"), bundle.getString(prefix + "master.password")); + + MysqlOnetimeServer masterServer = new MysqlOnetimeServer(); + masterServer.boot(); + + master = new BinaryLogClientIntegrationTest.MySQLConnection("127.0.0.1", masterServer.getPort(), "root", ""); + client = new BinaryLogClient(master.hostname(), master.port(), master.username(), master.password()); client.setServerId(client.getServerId() - 1); // avoid clashes between BinaryLogClient instances client.setKeepAlive(false); @@ -98,6 +99,7 @@ public void execute(Statement statement) throws SQLException { }); } catch (SQLSyntaxErrorException e) { // Skip the tests altogether since MySQL is pre 5.7 + System.err.println("skipping JSON tests (pre 5.7)"); throw new org.testng.SkipException("JSON data type is not supported by current version of MySQL"); } eventListener.waitFor(EventType.QUERY, 3, DEFAULT_TIMEOUT); @@ -433,7 +435,7 @@ public void afterEachTest() throws Exception { public void onEvent(Event event) { if (event.getHeader().getEventType() == EventType.QUERY) { EventData data = event.getData(); - if (data != null && ((QueryEventData) data).getSql().contains("_EOS_marker")) { + if (data != null && ((QueryEventData) data).getSql().toLowerCase().contains("_eos_marker")) { latch.countDown(); } } diff --git a/src/test/onetimeserver b/src/test/onetimeserver new file mode 100755 index 00000000..ef4196ce --- /dev/null +++ b/src/test/onetimeserver @@ -0,0 +1,117 @@ +#!/bin/bash + +_PLATFORM=`uname -sm` +PLATFORM=${_PLATFORM/ /-} + +OS=`uname -s | tr '[:upper:]' '[:lower:]'` + +THIS_URL="https://raw.githubusercontent.com/osheroff/onetimeserver/master/onetimeserver" +WRAPPER_URL="https://raw.githubusercontent.com/osheroff/onetimeserver/master/wrapper/wrapper.c" +ONETIMESERVER_URL="https://raw.githubusercontent.com/osheroff/onetimeserver-binaries/master/onetimeserver-go/$OS/onetimeserver-go" +CACHE_DIR=$HOME/.onetimeserver/$PLATFORM + +function usage() { + echo "usage: onetimeserver [--mysql-version=VERSION|-m VERSION] [-v|--verbose] [--parent-pid=PID] [--no-clean] [--reuse PATH] [mysql_args...]" + echo " onetimeserver update" +} + +while [[ $# > 0 ]] +do + case $1 in + update) + rm -Rf $CACHE_DIR/* + curl "$THIS_URL" > $0 + echo "onetimeserver updated" + exit + ;; + --parent-pid=*) + PARENT_PID="${1#*=}" + ;; + --mysql-version=*) + MYSQL_VERSION="-mysql-version ${1#*=}" + ;; + -m) + shift + MYSQL_VERSION="-mysql-version $1" + ;; + --reuse=*) + REUSE="-reuse ${1#*=}" + ;; + -b|--block) + BLOCK="1" + ;; + --no-clean) + NO_CLEAN="-no-clean" + ;; + -d|--debug) + DEBUG="-debug" + ;; + -h|--help) + usage + exit + ;; + *) + EXTRA_ARGS="$EXTRA_ARGS $1" + ;; + esac + shift + +done + +mkdir -p $CACHE_DIR + + +cd $CACHE_DIR + +if ! [ -f $CACHE_DIR/wrapper ] +then + if [ "$DEBUG" == "-debug" ] + then + echo "downloading and compiling wrapper from $WRAPPER_URL" 1>&2 + fi + + curl -s "$WRAPPER_URL" > $CACHE_DIR/wrapper.c + gcc -g -O2 $CACHE_DIR/wrapper.c -o $CACHE_DIR/wrapper +fi + +if ! [ -f $CACHE_DIR/onetimeserver-go ] +then + if [ "$DEBUG" == "-debug" ] + then + echo "downloading main util from $ONETIMESERVER_URL" 1>&2 + fi + + curl -s $ONETIMESERVER_URL > $CACHE_DIR/onetimeserver-go + + SIZE=`du -k "$CACHE_DIR/onetimeserver-go" | cut -f1` + if [ $SIZE -lt 1000 ] + then + echo "Could not download onetimeserver-go from $ONETIMESERVER_URL" >&2 + rm -Rf $CACHE_DIR/* + exit 1 + fi + + chmod +x $CACHE_DIR/onetimeserver-go +fi + +if ! [ -z "$BLOCK" ] +then + PARENT_PID=$$ +elif [ -z "$PARENT_PID" ] +then + PARENT_PID=$PPID +fi + +if [ "$DEBUG" == "-debug" ] +then + echo "exec: " $CACHE_DIR/wrapper $DEBUG $BLOCK $MYSQL_VERSION -ppid $PARENT_PID -type mysql -- $EXTRA_ARGS 1>&2 +fi + +GO_ARGS="$DEBUG $BLOCK $MYSQL_VERSION $NO_CLEAN $REUSE -ppid $PARENT_PID -type mysql" +if [ "$BLOCK" == "1" ] +then + $CACHE_DIR/wrapper $GO_ARGS -- $EXTRA_ARGS & + sleep 2000000 +else + exec $CACHE_DIR/wrapper $GO_ARGS -- $EXTRA_ARGS +fi From 38e317dc91da6aac9a5ffb3709016a27fd3fd944 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Sun, 26 Apr 2020 14:56:41 -0700 Subject: [PATCH 012/217] add circleci config --- .circleci/config.yml | 59 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..47806d6b --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,59 @@ +version: 2.1 + +executors: + jdk11: + docker: + +jobs: + build: + docker: + - image: cimg/openjdk:11.0 + steps: + - checkout + - restore_cache: + key: dependency-cache-{{ checksum "pom.xml" }} + - run: + name: Cache m2 artifacts + command: mvn dependency:go-offline + - save_cache: + key: dependency-cache-{{ checksum "pom.xml" }} + paths: [ "~/.m2" ] + test: + docker: + - image: cimg/openjdk:11.0 + parameters: + mysql: + type: string + environment: + MYSQL_VERSION: "<< parameters.mysql >>" + JAVA_TOOL_OPTIONS: "-Xmx250m" + steps: + - checkout + - restore_cache: + key: dependency-cache-{{ checksum "pom.xml" }} + - run: + command: echo "testing under mysql $MYSQL_VERSION" + - run: + name: testit + command: mvn verify + - store_artifacts: + path: test.log + +workflows: + version: 2 + build_and_test: + jobs: + - build + - test: + name: "test-5.5" + mysql: "5.5" + requires: [ "build" ] + - test: + name: "test-5.7" + mysql: "5.7" + requires: [ "build" ] + - test: + name: "test-5.8" + mysql: "8.0" + requires: [ "build" ] + From dce6066c988a9b1a82efe130e7829a7b5c2c3c10 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Sun, 26 Apr 2020 16:01:29 -0700 Subject: [PATCH 013/217] test circleCI --- README.md | 69 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 329aa9f8..0c11b688 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,9 @@ # mysql-binlog-connector-java [![Build Status](https://travis-ci.org/shyiko/mysql-binlog-connector-java.svg?branch=master)](https://travis-ci.org/shyiko/mysql-binlog-connector-java) [![Coverage Status](https://coveralls.io/repos/shyiko/mysql-binlog-connector-java/badge.svg?branch=master)](https://coveralls.io/r/shyiko/mysql-binlog-connector-java?branch=master) [![Maven Central](https://img.shields.io/maven-central/v/com.github.shyiko/mysql-binlog-connector-java.svg)](http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22com.github.shyiko%22%20AND%20a%3A%22mysql-binlog-connector-java%22) + MySQL Binary Log connector. -Initially project was started as a fork of [open-replicator](https://code.google.com/p/open-replicator), +Initially project was started as a fork of [open-replicator](https://code.google.com/p/open-replicator), but ended up as a complete rewrite. Key differences/features: - automatic binlog filename/position | GTID resolution @@ -16,8 +17,8 @@ but ended up as a complete rewrite. Key differences/features: - no third-party dependencies - test suite over different versions of MySQL releases -> If you are looking for something similar in other languages - check out -[siddontang/go-mysql](https://github.com/siddontang/go-mysql) (Go), +> If you are looking for something similar in other languages - check out +[siddontang/go-mysql](https://github.com/siddontang/go-mysql) (Go), [noplay/python-mysql-replication](https://github.com/noplay/python-mysql-replication) (Python). ## Usage @@ -78,30 +79,30 @@ client.connect(); > By default, BinaryLogClient starts from the current (at the time of connect) master binlog position. If you wish to kick off from a specific filename or position, use `client.setBinlogFilename(filename)` + `client.setBinlogPosition(position)`. -> `client.connect()` is blocking (meaning that client will listen for events in the current thread). -`client.connect(timeout)`, on the other hand, spawns a separate thread. +> `client.connect()` is blocking (meaning that client will listen for events in the current thread). +`client.connect(timeout)`, on the other hand, spawns a separate thread. #### Controlling event deserialization -> You might need it for several reasons: -you don't want to waste time deserializing events you won't need; -there is no EventDataDeserializer defined for the event type you are interested in (or there is but it contains a bug); -you want certain type of events to be deserialized in a different way (perhaps *RowsEventData should contain table +> You might need it for several reasons: +you don't want to waste time deserializing events you won't need; +there is no EventDataDeserializer defined for the event type you are interested in (or there is but it contains a bug); +you want certain type of events to be deserialized in a different way (perhaps *RowsEventData should contain table name and not id?); etc. ```java EventDeserializer eventDeserializer = new EventDeserializer(); // do not deserialize EXT_DELETE_ROWS event data, return it as a byte array -eventDeserializer.setEventDataDeserializer(EventType.EXT_DELETE_ROWS, - new ByteArrayEventDataDeserializer()); +eventDeserializer.setEventDataDeserializer(EventType.EXT_DELETE_ROWS, + new ByteArrayEventDataDeserializer()); // skip EXT_WRITE_ROWS event data altogether -eventDeserializer.setEventDataDeserializer(EventType.EXT_WRITE_ROWS, +eventDeserializer.setEventDataDeserializer(EventType.EXT_WRITE_ROWS, new NullEventDataDeserializer()); // use custom event data deserializer for EXT_DELETE_ROWS -eventDeserializer.setEventDataDeserializer(EventType.EXT_DELETE_ROWS, +eventDeserializer.setEventDataDeserializer(EventType.EXT_DELETE_ROWS, new EventDataDeserializer() { ... }); @@ -119,7 +120,7 @@ BinaryLogClient binaryLogClient = ... ObjectName objectName = new ObjectName("mysql.binlog:type=BinaryLogClient"); mBeanServer.registerMBean(binaryLogClient, objectName); -// following bean accumulates various BinaryLogClient stats +// following bean accumulates various BinaryLogClient stats // (e.g. number of disconnects, skipped events) BinaryLogClientStatistics stats = new BinaryLogClientStatistics(binaryLogClient); ObjectName statsObjectName = new ObjectName("mysql.binlog:type=BinaryLogClientStatistics"); @@ -130,12 +131,12 @@ mBeanServer.registerMBean(stats, statsObjectName); > Introduced in 0.4.0. -TLSv1.1 & TLSv1.2 require [JDK 7](http://bugs.java.com/bugdatabase/view_bug.do?bug_id=6916074)+. -Prior to MySQL 5.7.10, MySQL supported only TLSv1 -(see [Secure Connection Protocols and Ciphers](http://dev.mysql.com/doc/refman/5.7/en/secure-connection-protocols-ciphers.html)). +TLSv1.1 & TLSv1.2 require [JDK 7](http://bugs.java.com/bugdatabase/view_bug.do?bug_id=6916074)+. +Prior to MySQL 5.7.10, MySQL supported only TLSv1 +(see [Secure Connection Protocols and Ciphers](http://dev.mysql.com/doc/refman/5.7/en/secure-connection-protocols-ciphers.html)). > To check that MySQL server is [properly configured with SSL support](http://dev.mysql.com/doc/refman/5.7/en/using-secure-connections.html) - -`mysql -h host -u root -ptypeyourpasswordmaybe -e "show global variables like 'have_%ssl';"` ("Value" +`mysql -h host -u root -ptypeyourpasswordmaybe -e "show global variables like 'have_%ssl';"` ("Value" should be "YES"). State of the current session can be determined using `\s` ("SSL" should not be blank). ```java @@ -151,23 +152,23 @@ client.setSSLMode(SSLMode.VERIFY_IDENTITY); ## Implementation notes - data of numeric types (tinyint, etc) always returned signed(!) regardless of whether column definition includes "unsigned" keyword or not. -- data of var\*/\*text/\*blob types always returned as a byte array (for var\* this is true starting from 1.0.0). +- data of var\*/\*text/\*blob types always returned as a byte array (for var\* this is true starting from 1.0.0). ## Frequently Asked Questions **Q**. How does a typical transaction look like? - -**A**. GTID event (if gtid_mode=ON) -> QUERY event with "BEGIN" as sql -> ... -> XID event | QUERY event with "COMMIT" or "ROLLBACK" as sql. -**Q**. EventData for inserted/updated/deleted rows has no information about table (except for some weird id). -How do I make sense out of it? +**A**. GTID event (if gtid_mode=ON) -> QUERY event with "BEGIN" as sql -> ... -> XID event | QUERY event with "COMMIT" or "ROLLBACK" as sql. + +**Q**. EventData for inserted/updated/deleted rows has no information about table (except for some weird id). +How do I make sense out of it? **A**. Each [WriteRowsEventData](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/event/WriteRowsEventData.java)/[UpdateRowsEventData](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/event/UpdateRowsEventData.java)/[DeleteRowsEventData](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/event/DeleteRowsEventData.java) event is preceded by [TableMapEventData](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/event/TableMapEventData.java) which contains schema & table name. If for some reason you need to know column names (types, etc). - the easiest way is to ```sql -select TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME, ORDINAL_POSITION, COLUMN_DEFAULT, IS_NULLABLE, -DATA_TYPE, CHARACTER_MAXIMUM_LENGTH, CHARACTER_OCTET_LENGTH, NUMERIC_PRECISION, NUMERIC_SCALE, +select TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME, ORDINAL_POSITION, COLUMN_DEFAULT, IS_NULLABLE, +DATA_TYPE, CHARACTER_MAXIMUM_LENGTH, CHARACTER_OCTET_LENGTH, NUMERIC_PRECISION, NUMERIC_SCALE, CHARACTER_SET_NAME, COLLATION_NAME from INFORMATION_SCHEMA.COLUMNS; # see https://dev.mysql.com/doc/refman/5.6/en/columns-table.html for more information ``` @@ -180,10 +181,10 @@ You can find JDBC snippet [here](https://github.com/shyiko/mysql-binlog-connecto #### API overview -There are two entry points - [BinaryLogClient](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java) (which you can use to read binary logs from a MySQL server) and -[BinaryLogFileReader](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogFileReader.java) (for offline log processing). Both of them rely on [EventDeserializer](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java) to deserialize -stream of events. Each [Event](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/event/Event.java) consists of [EventHeader](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/event/EventHeader.java) (containing among other things reference to [EventType](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java)) and -[EventData](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/event/EventData.java). The aforementioned EventDeserializer has one [EventHeaderDeserializer](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderDeserializer.java) ([EventHeaderV4Deserializer](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java) by default) +There are two entry points - [BinaryLogClient](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java) (which you can use to read binary logs from a MySQL server) and +[BinaryLogFileReader](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogFileReader.java) (for offline log processing). Both of them rely on [EventDeserializer](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java) to deserialize +stream of events. Each [Event](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/event/Event.java) consists of [EventHeader](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/event/EventHeader.java) (containing among other things reference to [EventType](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java)) and +[EventData](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/event/EventData.java). The aforementioned EventDeserializer has one [EventHeaderDeserializer](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderDeserializer.java) ([EventHeaderV4Deserializer](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java) by default) and [a collection of EventDataDeserializer|s](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java#L82). If there is no EventDataDeserializer registered for some particular type of Event - default EventDataDeserializer kicks in ([NullEventDataDeserializer](https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/NullEventDataDeserializer.java)). @@ -193,7 +194,7 @@ For the insight into the internals of MySQL look [here](https://dev.mysql.com/do ## Real-world applications -Some of the OSS using / built on top of mysql-binlog-conector-java: +Some of the OSS using / built on top of mysql-binlog-conector-java: * [apache/nifi](https://github.com/apache/nifi) An easy to use, powerful, and reliable system to process and distribute data. * [debezium](https://github.com/debezium/debezium) A low latency data streaming platform for change data capture (CDC). * [mavenlink/changestream](https://github.com/mavenlink/changestream) - A stream of changes for MySQL built on Akka. @@ -206,7 +207,7 @@ Some of the OSS using / built on top of mysql-binlog-conector-java: * [zendesk/maxwell](https://github.com/zendesk/maxwell) A MySQL-to-JSON Kafka producer. * [zzt93/syncer](https://github.com/zzt93/syncer) A tool sync & manipulate data from MySQL/MongoDB to ES/Kafka/MySQL, which make 'Eventual Consistency' promise. -It's also used [on a large scale](https://twitter.com/atwinmutt/status/626816601078300672) in MailChimp. You can read about it [here](http://devs.mailchimp.com/blog/powering-mailchimp-pro-reporting/). +It's also used [on a large scale](https://twitter.com/atwinmutt/status/626816601078300672) in MailChimp. You can read about it [here](http://devs.mailchimp.com/blog/powering-mailchimp-pro-reporting/). ## Development @@ -229,10 +230,10 @@ Secondly: ## Contributing -In lieu of a formal styleguide, please take care to maintain the existing coding style. -Executing `mvn checkstyle:check` within project directory should not produce any errors. +In lieu of a formal styleguide, please take care to maintain the existing coding style. +Executing `mvn checkstyle:check` within project directory should not produce any errors. If you are willing to install [vagrant](http://www.vagrantup.com/) (required by integration tests) it's highly recommended -to check (with `mvn clean verify`) that there are no test failures before sending a pull request. +to check (with `mvn clean verify`) that there are no test failures before sending a pull request. Additional tests for any new or changed functionality are also very welcomed. ## License From 7e95e4aa66148ffd77c5a4d6c05ccf9ad89199a8 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Sun, 26 Apr 2020 16:05:06 -0700 Subject: [PATCH 014/217] get rid of javax.xml dependency, target java 8 --- pom.xml | 4 ++-- .../event/deserialization/json/JsonStringFormatter.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 4a327429..3022e0a2 100644 --- a/pom.xml +++ b/pom.xml @@ -105,8 +105,8 @@ maven-compiler-plugin 3.5.1 - 1.6 - 1.6 + 8 + 8 diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonStringFormatter.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonStringFormatter.java index 5ae92c85..58a3b841 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonStringFormatter.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonStringFormatter.java @@ -19,6 +19,7 @@ import java.math.BigDecimal; import java.math.BigInteger; +import java.util.Base64; /** * A {@link JsonFormatter} implementation that creates a JSON string representation. @@ -197,7 +198,7 @@ public void valueTimestamp(long secondsPastEpoch, int microSeconds) { @Override public void valueOpaque(ColumnType type, byte[] value) { sb.append('"'); - sb.append(javax.xml.bind.DatatypeConverter.printBase64Binary(value)); + sb.append(Base64.getEncoder().encodeToString(value)); sb.append('"'); } From 95d80b2eb3c6fe94228c7ee8182516f8282e842b Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Sun, 26 Apr 2020 16:14:27 -0700 Subject: [PATCH 015/217] remove the other instance of javax.xml --- .../shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java index 7460f60e..e032b068 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java @@ -42,7 +42,6 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import javax.xml.bind.DatatypeConverter; import java.io.Closeable; import java.io.EOFException; import java.io.FilterInputStream; @@ -62,6 +61,7 @@ import java.sql.SQLSyntaxErrorException; import java.sql.Statement; import java.util.AbstractMap; +import java.util.Base64; import java.util.BitSet; import java.util.Calendar; import java.util.List; @@ -323,7 +323,7 @@ public void testDeserializationOfSTRING() throws Exception { assertEquals(writeAndCaptureRow("binary", "x'01'"), new Serializable[]{new byte[] {1}}); assertEquals(writeAndCaptureRow("binary", "x'FF'"), new Serializable[]{new byte[] {-1}}); assertEquals(writeAndCaptureRow("binary(16)", "unhex(md5(\"glob\"))"), - new Serializable[]{DatatypeConverter.parseHexBinary("8684147451a6cc3b92142c6f4b78e61c")}); + new Serializable[]{Base64.getDecoder().decode("hoQUdFGmzDuSFCxvS3jmHA==")}); } @Test From 32b457a5a9587bd9c77e2dd3fa9befe6b0a46ca3 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Sun, 26 Apr 2020 16:18:15 -0700 Subject: [PATCH 016/217] get rid of checkstyle & coverage --- pom.xml | 83 +-------------------------------------------------------- 1 file changed, 1 insertion(+), 82 deletions(-) diff --git a/pom.xml b/pom.xml index 3022e0a2..a598e8c6 100644 --- a/pom.xml +++ b/pom.xml @@ -158,33 +158,6 @@ - - org.apache.maven.plugins - maven-checkstyle-plugin - 2.9.1 - - true - supplement/codequality/checkstyle.xml - supplement/codequality/license.header - true - true - - - - verify - - checkstyle - - - - - - com.github.shyiko - checkstyle-nonstandard - 0.1.0 - - - com.github.shyiko.usage-maven-plugin usage-maven-plugin @@ -194,24 +167,10 @@ # build everything (append "-DskipTests=true" if you wish to skip tests) ./mvnw clean package - # run unit + integration tests, validate codebase using checkstyle + # run unit + integration tests ./mvnw -P coverage clean verify # use -Dvagrant.integration.box= to switch between MySQL sandboxes - # for aggregated coverage over different mysql releases use - ./mvnw clean - ./mvnw -P coverage verify \ - -Dvagrant.integration.box=supplement/vagrant/mysql-5.5.27-sandbox-prepackaged - ./mvnw -P coverage verify \ - -Dvagrant.integration.box=supplement/vagrant/mysql-5.6.12-sandbox-prepackaged - ./mvnw -P coverage verify \ - -Dvagrant.integration.box=supplement/vagrant/mysql-5.7.15-sandbox-prepackaged - ./mvnw -P coverage,mysql-8-compat verify \ - -Dvagrant.integration.box=supplement/vagrant/mysql-8.0.1-sandbox-prepackaged - - # submit coverage report to coveralls - ./mvnw -P coverage coveralls:jacoco -DrepoToken=<coveralls.io> - # publish a new version ./mvnw versions:set -DnewVersion=<version> ./mvnw -Ddeploy=maven-central @@ -241,46 +200,6 @@ - - coverage - - - - org.jacoco - jacoco-maven-plugin - 0.7.9 - - ${basedir}/target/coverage-reports/jacoco-unit.exec - ${basedir}/target/coverage-reports/jacoco-unit.exec - true - - **/ClientCapabilities.* - - - - - jacoco-initialize - - prepare-agent - - - - jacoco-site - post-integration-test - - report - - - - - - org.eluder.coveralls - coveralls-maven-plugin - 2.0.0 - - - - deploy-to-maven-central From 1811e4578c48ab06cae862fe936fef429a423fb4 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Mon, 27 Apr 2020 10:50:06 -0700 Subject: [PATCH 017/217] skip GTID tests on 5.5. better way of setting up GTID. --- .../BinaryLogClientGTIDIntegrationTest.java | 43 +++--------------- .../BinaryLogClientIntegrationTest.java | 12 ++++- .../mysql/binlog/MysqlOnetimeServer.java | 6 +++ .../shyiko/mysql/binlog/MysqlVersion.java | 45 +++++++++++++++++++ 4 files changed, 67 insertions(+), 39 deletions(-) create mode 100644 src/test/java/com/github/shyiko/mysql/binlog/MysqlVersion.java diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientGTIDIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientGTIDIntegrationTest.java index df033afa..ec86f377 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientGTIDIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientGTIDIntegrationTest.java @@ -18,6 +18,7 @@ import com.github.shyiko.mysql.binlog.event.QueryEventData; import com.github.shyiko.mysql.binlog.event.XidEventData; import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer; +import org.testng.SkipException; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -34,43 +35,11 @@ * @author Ben Osheroff */ public class BinaryLogClientGTIDIntegrationTest extends BinaryLogClientIntegrationTest { - - @BeforeClass - private void enableGTID() throws SQLException { - MySQLConnection[] servers = {slave, master}; - for (MySQLConnection m : servers) { - m.execute(new Callback() { - @Override - public void execute(Statement statement) throws SQLException { - ResultSet rs = statement.executeQuery("select @@GLOBAL.GTID_MODE as gtid_mode"); - rs.next(); - if ("ON".equals(rs.getString("gtid_mode"))) { - return; - } - statement.execute("SET @@GLOBAL.ENFORCE_GTID_CONSISTENCY = ON;"); - statement.execute("SET @@GLOBAL.GTID_MODE = OFF_PERMISSIVE;"); - statement.execute("SET @@GLOBAL.GTID_MODE = ON_PERMISSIVE;"); - statement.execute("SET @@GLOBAL.GTID_MODE = ON;"); - } - }, true); - } - } - - @AfterClass(alwaysRun = true) - private void disableGTID() throws SQLException { - MySQLConnection[] servers = {slave, master}; - for (MySQLConnection m : servers) { - m.execute(new Callback() { - @Override - public void execute(Statement statement) throws SQLException { - statement.execute("SET @@GLOBAL.GTID_MODE = ON_PERMISSIVE;"); - statement.execute("SET @@GLOBAL.GTID_MODE = OFF_PERMISSIVE;"); - statement.execute("SET @@GLOBAL.GTID_MODE = OFF;"); - statement.execute("SET @@GLOBAL.ENFORCE_GTID_CONSISTENCY = OFF;"); - } - }, true); - } - slave.execute("STOP SLAVE", "START SLAVE"); + @Override + protected MysqlOnetimeServerOptions getOptions() { + MysqlOnetimeServerOptions options = new MysqlOnetimeServerOptions(); + options.gtid = true; + return options; } @Test diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java index e032b068..5ad84d62 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java @@ -34,6 +34,7 @@ import com.github.shyiko.mysql.binlog.network.AuthenticationException; import com.github.shyiko.mysql.binlog.network.ServerException; import com.github.shyiko.mysql.binlog.network.SocketFactory; +import com.zendesk.maxwell.replication.MysqlVersion; import org.mockito.InOrder; import org.testng.SkipException; import org.testng.annotations.AfterClass; @@ -110,18 +111,25 @@ public class BinaryLogClientIntegrationTest { protected MySQLConnection master, slave; protected BinaryLogClient client; protected CountDownEventListener eventListener; + protected MysqlVersion mysqlVersion; + + protected MysqlOnetimeServerOptions getOptions() { + return null; + } @BeforeClass public void setUp() throws Exception { TimeZone.setDefault(TimeZone.getTimeZone("GMT")); - MysqlOnetimeServer masterServer = new MysqlOnetimeServer(); - MysqlOnetimeServer slaveServer = new MysqlOnetimeServer(); + MysqlOnetimeServer masterServer = new MysqlOnetimeServer(getOptions()); + MysqlOnetimeServer slaveServer = new MysqlOnetimeServer(getOptions()); + masterServer.boot(); slaveServer.boot(); slaveServer.setupSlave(masterServer.getPort()); master = new MySQLConnection("127.0.0.1", masterServer.getPort(), "root", ""); slave = new MySQLConnection("127.0.0.1", slaveServer.getPort(), "root", ""); + mysqlVersion = masterServer.getVersion(); client = new BinaryLogClient(slave.hostname, slave.port, slave.username, slave.password); EventDeserializer eventDeserializer = new EventDeserializer(); diff --git a/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java index 0810be7a..ea22faa4 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.zendesk.maxwell.replication.MysqlVersion; import java.io.BufferedReader; import java.io.IOException; @@ -245,6 +246,11 @@ public void shutDown() { } catch ( IOException e ) {} } + public MysqlVersion getVersion() { + String[] parts = getVersionString().split("\\."); + return new MysqlVersion(Integer.valueOf(parts[0]), Integer.valueOf(parts[1])); + } + private static String getVersionString() { String mysqlVersion = System.getenv("MYSQL_VERSION"); return mysqlVersion == null ? "5.7" : mysqlVersion; diff --git a/src/test/java/com/github/shyiko/mysql/binlog/MysqlVersion.java b/src/test/java/com/github/shyiko/mysql/binlog/MysqlVersion.java new file mode 100644 index 00000000..e46787b7 --- /dev/null +++ b/src/test/java/com/github/shyiko/mysql/binlog/MysqlVersion.java @@ -0,0 +1,45 @@ +package com.zendesk.maxwell.replication; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; + +public class MysqlVersion { + private final int major; + private final int minor; + + public MysqlVersion(int major, int minor) { + this.major = major; + this.minor = minor; + } + + public boolean atLeast(int major, int minor) { + return (this.major > major) || (this.major == major && this.minor >= minor); + } + + public boolean atLeast(MysqlVersion version) { + return atLeast(version.major, version.minor); + } + + public boolean lessThan(int major, int minor) { + return (this.major < major) || (this.major == major & this.minor < minor); + } + + public static MysqlVersion capture(Connection c) throws SQLException { + DatabaseMetaData meta = c.getMetaData(); + return new MysqlVersion(meta.getDatabaseMajorVersion(), meta.getDatabaseMinorVersion()); + } + + public int getMajor() { + return this.major; + } + + public int getMinor() { + return this.minor; + } + + @Override + public String toString() { + return "MysqlVersion[" + this.major + "," + this.minor + "]"; + } +} From bf2857810160370cad27bd01da65d0c2965abd98 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Mon, 27 Apr 2020 10:51:40 -0700 Subject: [PATCH 018/217] rename pipeline to 8.0 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 47806d6b..9d282aa7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -53,7 +53,7 @@ workflows: mysql: "5.7" requires: [ "build" ] - test: - name: "test-5.8" + name: "test-8.0" mysql: "8.0" requires: [ "build" ] From 1771e96979c27c4458fbb9411bdaefd75a2c33d1 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Mon, 27 Apr 2020 11:48:17 -0700 Subject: [PATCH 019/217] skip GTID on 5.5 --- .../mysql/binlog/BinaryLogClientGTIDIntegrationTest.java | 4 ++++ .../shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java | 2 +- .../com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientGTIDIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientGTIDIntegrationTest.java index ec86f377..b29c6d02 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientGTIDIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientGTIDIntegrationTest.java @@ -37,6 +37,10 @@ public class BinaryLogClientGTIDIntegrationTest extends BinaryLogClientIntegrationTest { @Override protected MysqlOnetimeServerOptions getOptions() { + if ( !this.mysqlVersion.atLeast(5,7) ) { + throw new SkipException("skipping gtid on 5.5"); + } + MysqlOnetimeServerOptions options = new MysqlOnetimeServerOptions(); options.gtid = true; return options; diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java index 5ad84d62..32d7e408 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java @@ -120,6 +120,7 @@ protected MysqlOnetimeServerOptions getOptions() { @BeforeClass public void setUp() throws Exception { TimeZone.setDefault(TimeZone.getTimeZone("GMT")); + mysqlVersion = MysqlOnetimeServer.getVersion(); MysqlOnetimeServer masterServer = new MysqlOnetimeServer(getOptions()); MysqlOnetimeServer slaveServer = new MysqlOnetimeServer(getOptions()); @@ -129,7 +130,6 @@ public void setUp() throws Exception { master = new MySQLConnection("127.0.0.1", masterServer.getPort(), "root", ""); slave = new MySQLConnection("127.0.0.1", slaveServer.getPort(), "root", ""); - mysqlVersion = masterServer.getVersion(); client = new BinaryLogClient(slave.hostname, slave.port, slave.username, slave.password); EventDeserializer eventDeserializer = new EventDeserializer(); diff --git a/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java index ea22faa4..a3e385b0 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java @@ -34,7 +34,7 @@ public MysqlOnetimeServer() { } public MysqlOnetimeServer(MysqlOnetimeServerOptions options) { - this.options = options; + this.options = options == null ? new MysqlOnetimeServerOptions() : options; } public void boot() throws Exception { @@ -246,7 +246,7 @@ public void shutDown() { } catch ( IOException e ) {} } - public MysqlVersion getVersion() { + public static MysqlVersion getVersion() { String[] parts = getVersionString().split("\\."); return new MysqlVersion(Integer.valueOf(parts[0]), Integer.valueOf(parts[1])); } From ecd1139261ae01dc113071405d5848444b380006 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Mon, 27 Apr 2020 11:57:12 -0700 Subject: [PATCH 020/217] re-enable 8.0 check --- .../mysql/binlog/MysqlOnetimeServer.java | 35 +++++++++---------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java index a3e385b0..9866e134 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java @@ -59,14 +59,14 @@ public void boot() throws Exception { serverID = "--server_id=" + options.serverID; String authPlugin = ""; - // if ( this.getVersion().atLeast(8, 0) ) { - if ( false ) { + + if ( getVersion().atLeast(8, 0) ) { authPlugin = "--default-authentication-plugin=mysql_native_password"; } ProcessBuilder pb = new ProcessBuilder( dir + "/src/test/onetimeserver", - "--mysql-version=" + this.getVersionString(), + "--mysql-version=" + getVersionString(), "--log-slave-updates", "--log-bin=master", "--binlog_format=row", @@ -91,24 +91,21 @@ public void boot() throws Exception { final BufferedReader errReader = new BufferedReader(new InputStreamReader(p.getErrorStream())); - new Thread() { - @Override - public void run() { - while (true) { - String l = null; - try { - l = errReader.readLine(); - } catch ( IOException e) {}; - - if (l == null) - break; - System.err.println(l); - } - } - }.start(); + new Thread(() -> { + while (true) { + String l = null; + try { + l = errReader.readLine(); + } catch ( IOException e) {}; + + if (l == null) + break; + System.err.println(l); + } + }).start(); String json = reader.readLine(); - String outputFile = null; + String outputFile; try { ObjectMapper mapper = new ObjectMapper(); Map output = mapper.readValue(json, MAP_STRING_OBJECT_REF); From 1452ab965b2fa327a9aa2a94bd7f2c45a11897bf Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Mon, 27 Apr 2020 12:20:15 -0700 Subject: [PATCH 021/217] fix merge --- .../json/JsonBinaryValueIntegrationTest.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java index 082ddb20..30925211 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java @@ -410,18 +410,18 @@ private String writeAndCaptureJSON(final String value) throws Exception { CapturingEventListener capturingEventListener = new CapturingEventListener(); client.registerEventListener(capturingEventListener); try { - master.execute(new BinaryLogClientIntegrationTest.Callback() { - @Override - public void execute(Statement statement) throws SQLException { - statement.execute("drop table if exists data_type_hell"); - statement.execute("create table data_type_hell (column_ " + "JSON" + ")"); - statement.execute("insert into data_type_hell values (" + value + ")"); - } + master.execute(statement -> { + statement.execute("drop table if exists data_type_hell"); + statement.execute("create table data_type_hell (column_ " + "JSON" + ")"); + statement.execute("insert into data_type_hell values (" + value + ")"); }); eventListener.waitFor(WriteRowsEventData.class, 1, DEFAULT_TIMEOUT); } finally { client.unregisterEventListener(capturingEventListener); } + if ( capturingEventListener.getEvents(WriteRowsEventData.class).size() == 0 ) { + assertTrue(false, "did not receive rows in json test for " + value); + } byte[] b = (byte[]) capturingEventListener.getEvents(WriteRowsEventData.class).get(0).getRows().get(0)[0]; return b == null ? null : JsonBinary.parseAsString(b); } @@ -430,12 +430,12 @@ public void execute(Statement statement) throws SQLException { public void afterEachTest() throws Exception { final CountDownLatch latch = new CountDownLatch(1); final String markerQuery = "drop table if exists _EOS_marker"; - BinaryLogClient.EventListener markerInterceptor = new BinaryLogClient.EventListener() { - @Override - public void onEvent(Event event) { - if (event.getHeader().getEventType() == EventType.QUERY) { - EventData data = event.getData(); - if (data != null && ((QueryEventData) data).getSql().toLowerCase().contains("_eos_marker")) { + BinaryLogClient.EventListener markerInterceptor = event -> { + if (event.getHeader().getEventType() == EventType.QUERY) { + EventData data = event.getData(); + if (data != null) { + String sql = ((QueryEventData) data).getSql().toLowerCase(); + if (sql.contains("_EOS_marker".toLowerCase())) { latch.countDown(); } } From 75b5de59daecfe9226294fd2ca0003caaa96c323 Mon Sep 17 00:00:00 2001 From: auntyellow Date: Tue, 28 Apr 2020 20:47:47 +0800 Subject: [PATCH 022/217] fix package name --- .../shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java | 1 - .../java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java | 1 - src/test/java/com/github/shyiko/mysql/binlog/MysqlVersion.java | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java index 32d7e408..43629133 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java @@ -34,7 +34,6 @@ import com.github.shyiko.mysql.binlog.network.AuthenticationException; import com.github.shyiko.mysql.binlog.network.ServerException; import com.github.shyiko.mysql.binlog.network.SocketFactory; -import com.zendesk.maxwell.replication.MysqlVersion; import org.mockito.InOrder; import org.testng.SkipException; import org.testng.annotations.AfterClass; diff --git a/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java index 9866e134..3dca9cb4 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java @@ -2,7 +2,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.zendesk.maxwell.replication.MysqlVersion; import java.io.BufferedReader; import java.io.IOException; diff --git a/src/test/java/com/github/shyiko/mysql/binlog/MysqlVersion.java b/src/test/java/com/github/shyiko/mysql/binlog/MysqlVersion.java index e46787b7..7213a724 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/MysqlVersion.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/MysqlVersion.java @@ -1,4 +1,4 @@ -package com.zendesk.maxwell.replication; +package com.github.shyiko.mysql.binlog; import java.sql.Connection; import java.sql.DatabaseMetaData; From 2c33620f62f98b31c57dd2ca547b1011b60b6c3f Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Tue, 28 Apr 2020 16:18:02 -0700 Subject: [PATCH 023/217] bump tests under 8.0.19 --- .../mysql/binlog/BinaryLogClientGTIDIntegrationTest.java | 4 ++-- src/test/onetimeserver | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientGTIDIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientGTIDIntegrationTest.java index b29c6d02..5246bd65 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientGTIDIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientGTIDIntegrationTest.java @@ -50,13 +50,13 @@ protected MysqlOnetimeServerOptions getOptions() { public void testGTIDAdvancesStatementBased() throws Exception { try { master.execute("set global binlog_format=statement"); - slave.execute("set global binlog_format=statement", "stop slave", "start slave"); + slave.execute("stop slave", "set global binlog_format=statement", "start slave"); master.reconnect(); master.execute("use test"); testGTIDAdvances(); } finally { master.execute("set global binlog_format=row"); - slave.execute("set global binlog_format=row", "stop slave", "start slave"); + slave.execute("stop slave", "set global binlog_format=row", "start slave"); master.reconnect(); master.execute("use test"); } diff --git a/src/test/onetimeserver b/src/test/onetimeserver index ef4196ce..a240a9c9 100755 --- a/src/test/onetimeserver +++ b/src/test/onetimeserver @@ -43,7 +43,7 @@ do --no-clean) NO_CLEAN="-no-clean" ;; - -d|--debug) + -d|--debug|-debug) DEBUG="-debug" ;; -h|--help) From 81ba86338b353015517a157a6f79b2c8bb6a9ce1 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Wed, 29 Apr 2020 14:48:09 -0700 Subject: [PATCH 024/217] update changelog --- CHANGELOG.md | 33 ++++++++++++++++++--------------- pom.xml | 2 +- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a2ca264..4ba26f3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,12 @@ # Changelog -All notable changes to this project will be documented in this file. -This project adheres to [Semantic Versioning](http://semver.org/). -## [0.22.0](https://github.com/shyiko/mysql-binlog-connector-java/compare/0.20.0...0.22.9) - 2020-04-24 +## [0.22.2](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.22.0...0.22.2) - 2020-04-29 +- Fix bugs in 0.22.0 involving nested JSON objects. + +## [0.22.0](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.20.1...0.22.0) - 2020-04-24 + +- *THIS RELEASE IS BUGGY. DO NOT USE.* - master server id is exposed in the library https://github.com/shyiko/mysql-binlog-connector-java/pull/319 - Fixes for JSON data in mysql 8.0.16+ https://github.com/shyiko/mysql-binlog-connector-java/pull/288 - more fixes for the bizarre azure platform https://github.com/shyiko/mysql-binlog-connector-java/pull/275 @@ -103,7 +106,7 @@ This project adheres to [Semantic Versioning](http://semver.org/). ## [0.10.1](https://github.com/shyiko/mysql-binlog-connector-java/compare/0.10.0...0.10.1) - 2017-02-28 ### Fixed -- HEARTBEAT tracking ([118](https://github.com/shyiko/mysql-binlog-connector-java/issues/118#issuecomment-283138143)). +- HEARTBEAT tracking ([118](https://github.com/shyiko/mysql-binlog-connector-java/issues/118#issuecomment-283138143)). ## [0.10.0](https://github.com/shyiko/mysql-binlog-connector-java/compare/0.9.2...0.10.0) - 2017-02-28 @@ -125,13 +128,13 @@ isn't reached within `BinaryLogClient::connectTimeout` from `BinaryLogClient::co ### Fixed - - NPE in case of EOF (BinaryLogClient) ([153](https://github.com/shyiko/mysql-binlog-connector-java/pull/153)). + - NPE in case of EOF (BinaryLogClient) ([153](https://github.com/shyiko/mysql-binlog-connector-java/pull/153)). ## [0.9.0](https://github.com/shyiko/mysql-binlog-connector-java/compare/0.8.1...0.9.0) - 2017-02-07 ### Added - - `BinaryLogClient::connectTimeout` (3 seconds by default). + - `BinaryLogClient::connectTimeout` (3 seconds by default). NOTE: `BinaryLogClient::keepAliveConnectTimeout` has been deprecated and is going to be removed in 1.0.0. ## [0.8.1](https://github.com/shyiko/mysql-binlog-connector-java/compare/0.8.0...0.8.1) - 2016-01-10 @@ -150,14 +153,14 @@ isn't reached within `BinaryLogClient::connectTimeout` from `BinaryLogClient::co ### Fixed - - `SSLMode.PREFERRED` handling (verification against the CA is no longer enforced) ([#142](https://github.com/shyiko/mysql-binlog-connector-java/pull/142)). + - `SSLMode.PREFERRED` handling (verification against the CA is no longer enforced) ([#142](https://github.com/shyiko/mysql-binlog-connector-java/pull/142)). NOTE: This change does NOT affect `SSLMode.VERIFY_CA` / `SSLMode.VERIFY_IDENTITY`. ## [0.7.3](https://github.com/shyiko/mysql-binlog-connector-java/compare/0.7.2...0.7.3) - 2016-12-26 ### Fixed - - Handling of DATE/DATETIME/TIMESTAMP "zero" value (e.g. '0000-00-00') when + - Handling of DATE/DATETIME/TIMESTAMP "zero" value (e.g. '0000-00-00') when `CompatibilityMode.DATE_AND_TIME_AS_LONG_MICRO` is set (false by default). ## [0.7.2](https://github.com/shyiko/mysql-binlog-connector-java/compare/0.7.1...0.7.2) - 2016-12-26 @@ -179,7 +182,7 @@ isn't reached within `BinaryLogClient::connectTimeout` from `BinaryLogClient::co ## [0.6.0](https://github.com/shyiko/mysql-binlog-connector-java/compare/0.5.2...0.6.0) - 2016-11-27 -### Added +### Added - `EventDeserializer` compatibility modes to mimic upcoming 1.0.0 event deserialization behavior ([#131](https://github.com/shyiko/mysql-binlog-connector-java/pull/131)). ## [0.5.2](https://github.com/shyiko/mysql-binlog-connector-java/compare/0.5.1...0.5.2) - 2016-11-19 @@ -208,9 +211,9 @@ isn't reached within `BinaryLogClient::connectTimeout` from `BinaryLogClient::co ### Fixed - GTID "rollover". - - binlog position tracking (`binaryLogClient.binlogPosition` is no longer updated on TABLE_MAP so that in case of - reconnect (using a different instance of client) table mapping (used by *RowsEventDataDeserializer|s) could be - reconstructed before hitting *RowsEvent. + - binlog position tracking (`binaryLogClient.binlogPosition` is no longer updated on TABLE_MAP so that in case of + reconnect (using a different instance of client) table mapping (used by *RowsEventDataDeserializer|s) could be + reconstructed before hitting *RowsEvent. ## [0.4.0](https://github.com/shyiko/mysql-binlog-connector-java/compare/0.3.3...0.4.0) - 2016-08-15 @@ -248,7 +251,7 @@ isn't reached within `BinaryLogClient::connectTimeout` from `BinaryLogClient::co ### Fixed - Possible infinite loop in case of EOF in the middle of `ByteArrayInputStream::fill`. - + ## [0.2.3](https://github.com/shyiko/mysql-binlog-connector-java/compare/0.2.2...0.2.3) - 2015-08-31 ### Fixed @@ -275,8 +278,8 @@ isn't reached within `BinaryLogClient::connectTimeout` from `BinaryLogClient::co - Support for authentication via empty password ([#39](https://github.com/shyiko/mysql-binlog-connector-java/issues/39)). ### Changed -- Server error reporting ([#37](https://github.com/shyiko/mysql-binlog-connector-java/issues/37)). - WARNING: If you are using exception message to identify specific server errors - you'll need to switch to +- Server error reporting ([#37](https://github.com/shyiko/mysql-binlog-connector-java/issues/37)). + WARNING: If you are using exception message to identify specific server errors - you'll need to switch to `ServerException`::[errorCode](https://github.com/shyiko/mysql-binlog-connector-java/commit/1817d0ff709c65c31af9236dcc4e50cc3ad1023b#diff-0dff747d57cb3f5f0548be89a81e29f8R37) (as message no longer includes error code). ### Fixed diff --git a/pom.xml b/pom.xml index a598e8c6..1609c3ad 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.zendesk mysql-binlog-connector-java - 0.22.1 + 0.22.2 mysql-binlog-connector-java MySQL Binary Log connector From e775757e55135844df748cc85486452cf994d7f3 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Wed, 29 Apr 2020 17:55:52 -0700 Subject: [PATCH 025/217] make releases automatic --- pom.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 1609c3ad..9399c5ec 100644 --- a/pom.xml +++ b/pom.xml @@ -264,8 +264,7 @@ https://oss.sonatype.org/ maven-central - true - + true From 620c10b175d8172c2e9b1f7152d03a19be165fc0 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Fri, 1 May 2020 09:29:18 -0700 Subject: [PATCH 026/217] add integration test for mysql 8.0 json partial update --- .../json/JsonBinaryValueIntegrationTest.java | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java index 30925211..dcc44f81 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java @@ -106,6 +106,72 @@ public void execute(Statement statement) throws SQLException { eventListener.reset(); } + @Test + public void testMysql8JsonSetPartialUpdateWithHoles() throws Exception { + CountDownEventListener eventListener = new CountDownEventListener(); + client.registerEventListener(eventListener); + CapturingEventListener capturingEventListener = new CapturingEventListener(); + client.registerEventListener(capturingEventListener); + String json = "{\"age\":22,\"addr\":{\"code\":100,\"detail\":{\"ab\":\"970785C8-C299\"}},\"name\":\"Alice\"}"; + master.execute("DROP TABLE IF EXISTS json_test", "create table json_test (j JSON)", + "INSERT INTO json_test VALUES ('" + json + "')", + "UPDATE json_test SET j = JSON_SET(j, '$.addr.detail.ab', '970785C8')"); + eventListener.waitFor(WriteRowsEventData.class, 1, DEFAULT_TIMEOUT); + eventListener.waitFor(UpdateRowsEventData.class, 1, DEFAULT_TIMEOUT); + List events = capturingEventListener.getEvents(WriteRowsEventData.class); + Serializable[] insertData = events.iterator().next().getRows().get(0); + assertEquals(JsonBinary.parseAsString((byte[]) insertData[0]), json); + + List updateEvents = capturingEventListener.getEvents(UpdateRowsEventData.class); + Serializable[] updateData = updateEvents.iterator().next().getRows().get(0).getValue(); + assertEquals(JsonBinary.parseAsString((byte[]) updateData[0]), json.replace("970785C8-C299", "970785C8")); + assertEquals(((byte[]) updateData[0]).length, ((byte[]) insertData[0]).length); + } + + @Test + public void testMysql8JsonRemovePartialUpdateWithHoles() throws Exception { + CountDownEventListener eventListener = new CountDownEventListener(); + client.registerEventListener(eventListener); + CapturingEventListener capturingEventListener = new CapturingEventListener(); + client.registerEventListener(capturingEventListener); + String json = "{\"age\":22,\"addr\":{\"code\":100,\"detail\":{\"ab\":\"970785C8-C299\"}},\"name\":\"Alice\"}"; + master.execute("DROP TABLE IF EXISTS json_test", "create table json_test (j JSON)", + "INSERT INTO json_test VALUES ('" + json + "')", + "UPDATE json_test SET j = JSON_REMOVE(j, '$.addr.detail.ab')"); + eventListener.waitFor(WriteRowsEventData.class, 1, DEFAULT_TIMEOUT); + eventListener.waitFor(UpdateRowsEventData.class, 1, DEFAULT_TIMEOUT); + List events = capturingEventListener.getEvents(WriteRowsEventData.class); + Serializable[] insertData = events.iterator().next().getRows().get(0); + assertEquals(JsonBinary.parseAsString((byte[]) insertData[0]), json); + + List updateEvents = capturingEventListener.getEvents(UpdateRowsEventData.class); + Serializable[] updateData = updateEvents.iterator().next().getRows().get(0).getValue(); + assertEquals(JsonBinary.parseAsString((byte[]) updateData[0]), json.replace("\"ab\":\"970785C8-C299\"", "")); + assertEquals(((byte[]) updateData[0]).length, ((byte[]) insertData[0]).length); + } + + @Test + public void testMysql8JsonReplacePartialUpdateWithHoles() throws Exception { + CountDownEventListener eventListener = new CountDownEventListener(); + client.registerEventListener(eventListener); + CapturingEventListener capturingEventListener = new CapturingEventListener(); + client.registerEventListener(capturingEventListener); + String json = "{\"age\":22,\"addr\":{\"code\":100,\"detail\":{\"ab\":\"970785C8-C299\"}},\"name\":\"Alice\"}"; + master.execute("DROP TABLE IF EXISTS json_test", "create table json_test (j JSON)", + "INSERT INTO json_test VALUES ('" + json + "')", + "UPDATE json_test SET j = JSON_REPLACE(j, '$.addr.detail.ab', '9707')"); + eventListener.waitFor(WriteRowsEventData.class, 1, DEFAULT_TIMEOUT); + eventListener.waitFor(UpdateRowsEventData.class, 1, DEFAULT_TIMEOUT); + List events = capturingEventListener.getEvents(WriteRowsEventData.class); + Serializable[] insertData = events.iterator().next().getRows().get(0); + assertEquals(JsonBinary.parseAsString((byte[]) insertData[0]), json); + + List updateEvents = capturingEventListener.getEvents(UpdateRowsEventData.class); + Serializable[] updateData = updateEvents.iterator().next().getRows().get(0).getValue(); + assertEquals(JsonBinary.parseAsString((byte[]) updateData[0]), json.replace("970785C8-C299", "9707")); + assertEquals(((byte[]) updateData[0]).length, ((byte[]) insertData[0]).length); + } + @Test public void testValueBoundariesAreHonored() throws Exception { CountDownEventListener eventListener = new CountDownEventListener(); From 16329a753cc4462c6540bebe813c1753dfebda51 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Fri, 1 May 2020 09:45:46 -0700 Subject: [PATCH 027/217] fix maven compiler args --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 9399c5ec..79a568f5 100644 --- a/pom.xml +++ b/pom.xml @@ -105,8 +105,8 @@ maven-compiler-plugin 3.5.1 - 8 - 8 + 1.8 + 1.8 From 8e83bd960891e1465726953238ec188abf0807f6 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Fri, 1 May 2020 09:45:48 -0700 Subject: [PATCH 028/217] add in some tests around the mysql 8.0.19 gaps --- .../json/JsonBinaryValueIntegrationTest.java | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java index dcc44f81..12552c31 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java @@ -26,6 +26,7 @@ import com.github.shyiko.mysql.binlog.event.EventData; import com.github.shyiko.mysql.binlog.event.EventType; import com.github.shyiko.mysql.binlog.event.QueryEventData; +import com.github.shyiko.mysql.binlog.event.UpdateRowsEventData; import com.github.shyiko.mysql.binlog.event.WriteRowsEventData; import org.skyscreamer.jsonassert.JSONAssert; import org.testng.annotations.AfterClass; @@ -125,7 +126,6 @@ public void testMysql8JsonSetPartialUpdateWithHoles() throws Exception { List updateEvents = capturingEventListener.getEvents(UpdateRowsEventData.class); Serializable[] updateData = updateEvents.iterator().next().getRows().get(0).getValue(); assertEquals(JsonBinary.parseAsString((byte[]) updateData[0]), json.replace("970785C8-C299", "970785C8")); - assertEquals(((byte[]) updateData[0]).length, ((byte[]) insertData[0]).length); } @Test @@ -147,7 +147,6 @@ public void testMysql8JsonRemovePartialUpdateWithHoles() throws Exception { List updateEvents = capturingEventListener.getEvents(UpdateRowsEventData.class); Serializable[] updateData = updateEvents.iterator().next().getRows().get(0).getValue(); assertEquals(JsonBinary.parseAsString((byte[]) updateData[0]), json.replace("\"ab\":\"970785C8-C299\"", "")); - assertEquals(((byte[]) updateData[0]).length, ((byte[]) insertData[0]).length); } @Test @@ -169,7 +168,29 @@ public void testMysql8JsonReplacePartialUpdateWithHoles() throws Exception { List updateEvents = capturingEventListener.getEvents(UpdateRowsEventData.class); Serializable[] updateData = updateEvents.iterator().next().getRows().get(0).getValue(); assertEquals(JsonBinary.parseAsString((byte[]) updateData[0]), json.replace("970785C8-C299", "9707")); - assertEquals(((byte[]) updateData[0]).length, ((byte[]) insertData[0]).length); + } + + @Test + public void testMysql8JsonRemoveArrayValue() throws Exception { + CountDownEventListener eventListener = new CountDownEventListener(); + client.registerEventListener(eventListener); + CapturingEventListener capturingEventListener = new CapturingEventListener(); + client.registerEventListener(capturingEventListener); + + String json = "[\"foo\",\"bar\",\"baz\"]"; + master.execute("DROP TABLE IF EXISTS json_test", "create table json_test (j JSON)", + "INSERT INTO json_test VALUES ('" + json + "')", + "UPDATE json_test SET j = JSON_REMOVE(j, '$[1]')"); + eventListener.waitFor(WriteRowsEventData.class, 1, DEFAULT_TIMEOUT); + eventListener.waitFor(UpdateRowsEventData.class, 1, DEFAULT_TIMEOUT); + + List events = capturingEventListener.getEvents(WriteRowsEventData.class); + Serializable[] insertData = events.iterator().next().getRows().get(0); + assertEquals(JsonBinary.parseAsString((byte[]) insertData[0]), json); + + List updateEvents = capturingEventListener.getEvents(UpdateRowsEventData.class); + Serializable[] updateData = updateEvents.iterator().next().getRows().get(0).getValue(); + assertEquals(JsonBinary.parseAsString((byte[]) updateData[0]), "[\"foo\",\"baz\"]"); } @Test From 430c4462f4aeeab1222a3227f5b8149d0b14530a Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Fri, 1 May 2020 09:46:31 -0700 Subject: [PATCH 029/217] gitignore vim stuff --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 38431278..88ff3803 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ target .project .settings .vagrant +.*.sw* From 13648dfa1aa514f2a0bdb95a938e673e7d18a33e Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Fri, 1 May 2020 09:47:01 -0700 Subject: [PATCH 030/217] remove vagrant stuff --- supplement/codequality/checkstyle.xml | 151 ------------------ supplement/codequality/license.header | 15 -- supplement/codequality/readme.md | 5 - .../vagrantfile | 6 - .../vagrant/mysql-5.5.27-sandbox/vagrantfile | 21 --- .../vagrantfile | 6 - .../vagrant/mysql-5.6.12-sandbox/vagrantfile | 21 --- .../vagrantfile | 6 - .../vagrant/mysql-5.7.15-sandbox/vagrantfile | 22 --- .../vagrantfile | 9 -- .../vagrant/mysql-8.0.1-sandbox/vagrantfile | 24 --- supplement/vagrant/readme.md | 7 - 12 files changed, 293 deletions(-) delete mode 100644 supplement/codequality/checkstyle.xml delete mode 100644 supplement/codequality/license.header delete mode 100644 supplement/codequality/readme.md delete mode 100644 supplement/vagrant/mysql-5.5.27-sandbox-prepackaged/vagrantfile delete mode 100644 supplement/vagrant/mysql-5.5.27-sandbox/vagrantfile delete mode 100644 supplement/vagrant/mysql-5.6.12-sandbox-prepackaged/vagrantfile delete mode 100644 supplement/vagrant/mysql-5.6.12-sandbox/vagrantfile delete mode 100644 supplement/vagrant/mysql-5.7.15-sandbox-prepackaged/vagrantfile delete mode 100644 supplement/vagrant/mysql-5.7.15-sandbox/vagrantfile delete mode 100644 supplement/vagrant/mysql-8.0.1-sandbox-prepackaged/vagrantfile delete mode 100644 supplement/vagrant/mysql-8.0.1-sandbox/vagrantfile delete mode 100644 supplement/vagrant/readme.md diff --git a/supplement/codequality/checkstyle.xml b/supplement/codequality/checkstyle.xml deleted file mode 100644 index 832357d4..00000000 --- a/supplement/codequality/checkstyle.xml +++ /dev/null @@ -1,151 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/supplement/codequality/license.header b/supplement/codequality/license.header deleted file mode 100644 index a0c33261..00000000 --- a/supplement/codequality/license.header +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright [yyyy] [name of copyright owner] - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ \ No newline at end of file diff --git a/supplement/codequality/readme.md b/supplement/codequality/readme.md deleted file mode 100644 index 99c9aebb..00000000 --- a/supplement/codequality/readme.md +++ /dev/null @@ -1,5 +0,0 @@ -### IDE integration - -1. import checkstyle.xml configuration file -2. set "checkstyle.header.file" property to the absolute path of license.header file -3. add [checkstyle-nonstandard-0.1.0.jar](http://search.maven.org/remotecontent?filepath=com/github/shyiko/checkstyle-nonstandard/0.1.0/checkstyle-nonstandard-0.1.0.jar) to the list of third-party check providers \ No newline at end of file diff --git a/supplement/vagrant/mysql-5.5.27-sandbox-prepackaged/vagrantfile b/supplement/vagrant/mysql-5.5.27-sandbox-prepackaged/vagrantfile deleted file mode 100644 index 5e349bfd..00000000 --- a/supplement/vagrant/mysql-5.5.27-sandbox-prepackaged/vagrantfile +++ /dev/null @@ -1,6 +0,0 @@ -Vagrant.configure("2") do |config| - config.vm.box = 'shyiko/mysql-sandbox-prepackaged' - config.vm.box_version = '5.5.27' - config.vm.network :forwarded_port, guest: 33061, host: 33061 - config.vm.network :forwarded_port, guest: 33062, host: 33062 -end diff --git a/supplement/vagrant/mysql-5.5.27-sandbox/vagrantfile b/supplement/vagrant/mysql-5.5.27-sandbox/vagrantfile deleted file mode 100644 index ddaf1c71..00000000 --- a/supplement/vagrant/mysql-5.5.27-sandbox/vagrantfile +++ /dev/null @@ -1,21 +0,0 @@ -Vagrant.configure("2") do |config| - config.vm.box = 'lucid32' - config.vm.box_url = 'http://files.vagrantup.com/lucid32.box' - config.vm.provision :shell, :inline => %Q( - apt-get update && apt-get install -y make libaio1 # libaio1 required by mysql - echo 'Downloading MySQL distribution ...' - wget --progress=dot:mega --content-disposition \ - http://downloads.mysql.com/archives/mysql-5.5/mysql-5.5.27-linux2.6-i686.tar.gz \ - 2>&1 | grep --line-buffered -o '[0-9]*%' - wget -O - https://launchpad.net/mysql-sandbox/mysql-sandbox-3/mysql-sandbox-3/+download/MySQL-Sandbox-3.0.33.tar.gz | tar xzv - (cd MySQL-Sandbox-3.0.33 && perl Makefile.PL && make && make install) - su -c "make_replication_sandbox ~/mysql-5.5.27-linux2.6-i686.tar.gz \ - --remote_access='%' --how_many_slaves=1 --sandbox_base_port=33061 \ - --master_options='-c binlog_format=ROW' \ - --slave_options='-c binlog_format=ROW -c log-slave-updates=TRUE'" vagrant - rm -f *.tar.gz - sed -i -e "s/exit\ 0/\\/home\\/vagrant\\/sandboxes\\/rsandbox_mysql-5_5_27\\/restart_all; exit 0/g" /etc/rc.local - ) - config.vm.network :forwarded_port, guest: 33061, host: 33061 - config.vm.network :forwarded_port, guest: 33062, host: 33062 -end diff --git a/supplement/vagrant/mysql-5.6.12-sandbox-prepackaged/vagrantfile b/supplement/vagrant/mysql-5.6.12-sandbox-prepackaged/vagrantfile deleted file mode 100644 index fc15f376..00000000 --- a/supplement/vagrant/mysql-5.6.12-sandbox-prepackaged/vagrantfile +++ /dev/null @@ -1,6 +0,0 @@ -Vagrant.configure("2") do |config| - config.vm.box = 'shyiko/mysql-sandbox-prepackaged' - config.vm.box_version = '5.6.12' - config.vm.network :forwarded_port, guest: 33061, host: 33061 - config.vm.network :forwarded_port, guest: 33062, host: 33062 -end diff --git a/supplement/vagrant/mysql-5.6.12-sandbox/vagrantfile b/supplement/vagrant/mysql-5.6.12-sandbox/vagrantfile deleted file mode 100644 index 4f36dc3a..00000000 --- a/supplement/vagrant/mysql-5.6.12-sandbox/vagrantfile +++ /dev/null @@ -1,21 +0,0 @@ -Vagrant.configure("2") do |config| - config.vm.box = 'lucid32' - config.vm.box_url = 'http://files.vagrantup.com/lucid32.box' - config.vm.provision :shell, :inline => %Q( - apt-get update && apt-get install -y make libaio1 # libaio1 required by mysql - echo 'Downloading MySQL distribution ...' - wget --progress=dot:mega --content-disposition \ - http://cdn.mysql.com/Downloads/MySQL-5.6/mysql-5.6.12-linux-glibc2.5-i686.tar.gz \ - 2>&1 | grep --line-buffered -o '[0-9]*%' - wget -O - https://launchpad.net/mysql-sandbox/mysql-sandbox-3/mysql-sandbox-3/+download/MySQL-Sandbox-3.0.33.tar.gz | tar xzv - (cd MySQL-Sandbox-3.0.33 && perl Makefile.PL && make && make install) - su -c "make_replication_sandbox ~/mysql-5.6.12-linux-glibc2.5-i686.tar.gz \ - --remote_access='%' --how_many_slaves=1 --sandbox_base_port=33061 \ - --master_options='-c binlog_format=ROW' \ - --slave_options='-c binlog_format=ROW -c log-slave-updates=TRUE'" vagrant - rm -f *.tar.gz - sed -i -e "s/exit\ 0/\\/home\\/vagrant\\/sandboxes\\/rsandbox_mysql-5_6_12\\/restart_all; exit 0/g" /etc/rc.local - ) - config.vm.network :forwarded_port, guest: 33061, host: 33061 - config.vm.network :forwarded_port, guest: 33062, host: 33062 -end diff --git a/supplement/vagrant/mysql-5.7.15-sandbox-prepackaged/vagrantfile b/supplement/vagrant/mysql-5.7.15-sandbox-prepackaged/vagrantfile deleted file mode 100644 index 7a644a5c..00000000 --- a/supplement/vagrant/mysql-5.7.15-sandbox-prepackaged/vagrantfile +++ /dev/null @@ -1,6 +0,0 @@ -Vagrant.configure("2") do |config| - config.vm.box = 'shyiko/mysql-sandbox-prepackaged' - config.vm.box_version = '5.7.15' - config.vm.network :forwarded_port, guest: 33061, host: 33061 - config.vm.network :forwarded_port, guest: 33062, host: 33062 -end diff --git a/supplement/vagrant/mysql-5.7.15-sandbox/vagrantfile b/supplement/vagrant/mysql-5.7.15-sandbox/vagrantfile deleted file mode 100644 index 07cfd309..00000000 --- a/supplement/vagrant/mysql-5.7.15-sandbox/vagrantfile +++ /dev/null @@ -1,22 +0,0 @@ -Vagrant.configure("2") do |config| - config.vm.box = 'lucid32' - config.vm.box_url = 'http://files.vagrantup.com/lucid32.box' - config.vm.provision :shell, :inline => %Q( - sed -i.bak -r 's/(us.)?(archive|security).ubuntu.com/old-releases.ubuntu.com/g' /etc/apt/sources.list - apt-get update && apt-get install -y make libaio1 # libaio1 required by mysql - echo 'Downloading MySQL distribution ...' - wget --progress=dot:mega --content-disposition \ - http://cdn.mysql.com/Downloads/MySQL-5.7/mysql-5.7.15-linux-glibc2.5-i686.tar.gz \ - 2>&1 | grep --line-buffered -o '[0-9]*%' - wget -O - https://github.com/datacharmer/mysql-sandbox/releases/download/3.1.13/MySQL-Sandbox-3.1.13.tar.gz | tar xzv - (cd MySQL-Sandbox-3.1.13 && perl Makefile.PL && make && make install) - su -c "make_replication_sandbox ~/mysql-5.7.15-linux-glibc2.5-i686.tar.gz \ - --remote_access='%' --how_many_slaves=1 --sandbox_base_port=33061 \ - --master_options='-c binlog_format=ROW' \ - --slave_options='-c binlog_format=ROW -c log-slave-updates=TRUE'" vagrant - rm -f *.tar.gz - sed -i -e "s/exit\ 0/\\/home\\/vagrant\\/sandboxes\\/rsandbox_mysql-5_7_15\\/restart_all; exit 0/g" /etc/rc.local - ) - config.vm.network :forwarded_port, guest: 33061, host: 33061 - config.vm.network :forwarded_port, guest: 33062, host: 33062 -end diff --git a/supplement/vagrant/mysql-8.0.1-sandbox-prepackaged/vagrantfile b/supplement/vagrant/mysql-8.0.1-sandbox-prepackaged/vagrantfile deleted file mode 100644 index 12b10d45..00000000 --- a/supplement/vagrant/mysql-8.0.1-sandbox-prepackaged/vagrantfile +++ /dev/null @@ -1,9 +0,0 @@ -Vagrant.configure("2") do |config| - config.vm.box = 'shyiko/mysql-sandbox-prepackaged' - config.vm.box_version = '8.0.1' - config.vm.network :forwarded_port, guest: 33061, host: 33061 - config.vm.network :forwarded_port, guest: 33062, host: 33062 - config.vm.provider "virtualbox" do |v| - v.memory = 1024 - end -end diff --git a/supplement/vagrant/mysql-8.0.1-sandbox/vagrantfile b/supplement/vagrant/mysql-8.0.1-sandbox/vagrantfile deleted file mode 100644 index ba712b6a..00000000 --- a/supplement/vagrant/mysql-8.0.1-sandbox/vagrantfile +++ /dev/null @@ -1,24 +0,0 @@ -Vagrant.configure("2") do |config| - config.vm.box = 'deb/jessie-i386' - config.vm.provision :shell, :inline => %Q( - sed -i.bak -r 's/(us.)?(archive|security).ubuntu.com/old-releases.ubuntu.com/g' /etc/apt/sources.list - apt-get update && apt-get install -y make libaio1 libnuma1 libtinfo5 # lib* required by mysql - echo 'Downloading MySQL distribution ...' - wget --no-check-certificate --progress=dot:mega --content-disposition \ - https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-8.0.1-dmr-linux-glibc2.12-i686.tar.gz \ - 2>&1 | grep --line-buffered -o '[0-9]*%' - wget -O - https://github.com/datacharmer/mysql-sandbox/releases/download/3.2.13/MySQL-Sandbox-3.2.13.tar.gz | tar xzv - (cd MySQL-Sandbox-3.2.13 && perl Makefile.PL && make && make install) - su -c "make_replication_sandbox ~/mysql-8.0.1-dmr-linux-glibc2.12-i686.tar.gz \ - --remote_access='%' --how_many_slaves=1 --sandbox_base_port=33061 \ - --master_options='-c binlog_format=ROW' \ - --slave_options='-c binlog_format=ROW -c log-slave-updates=TRUE'" vagrant - rm -f *.tar.gz - sed -i -e "s/exit\ 0/\\/home\\/vagrant\\/sandboxes\\/rsandbox_mysql-8_0_1\\/restart_all; exit 0/g" /etc/rc.local - ) - config.vm.network :forwarded_port, guest: 33061, host: 33061 - config.vm.network :forwarded_port, guest: 33062, host: 33062 - config.vm.provider "virtualbox" do |v| - v.memory = 1024 - end -end diff --git a/supplement/vagrant/readme.md b/supplement/vagrant/readme.md deleted file mode 100644 index 7099541f..00000000 --- a/supplement/vagrant/readme.md +++ /dev/null @@ -1,7 +0,0 @@ -### Sandbox repackaging - -1. cd mysql-X.X.XX-sandbox -2. vagrant up -3. vagrant package --output ../mysql-X.X.XX-sandbox.box -4. vagrant box add mysql-X.X.XX-sandbox ../mysql-X.X.XX-sandbox.box --force - From 01476e2452d0fd450c74b88fa6d5adb60bbe6cc1 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Fri, 1 May 2020 10:49:31 -0700 Subject: [PATCH 031/217] prevent TableMapEventData hash from growing unbounded. see https://github.com/zendesk/maxwell/issues/1468 for the full story. I'm not even sure how big this LRUCache really needs to be, given that we unconditionally put stuff in it before every transaction. I think the size might need to be 1? Unclear. --- .../shyiko/mysql/binlog/event/LRUCache.java | 19 +++++++++++++++++++ .../deserialization/EventDeserializer.java | 3 ++- 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/github/shyiko/mysql/binlog/event/LRUCache.java diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/LRUCache.java b/src/main/java/com/github/shyiko/mysql/binlog/event/LRUCache.java new file mode 100644 index 00000000..784cec0e --- /dev/null +++ b/src/main/java/com/github/shyiko/mysql/binlog/event/LRUCache.java @@ -0,0 +1,19 @@ +package com.github.shyiko.mysql.binlog.event; + +import java.util.LinkedHashMap; +import java.util.Map; + +public class LRUCache extends LinkedHashMap { + private int maxSize; + + // and other constructors for load factor and hashtable capacity + public LRUCache(int initialCapacity, float loadFactor, int maxSize) { + super(initialCapacity, loadFactor, true); + this.maxSize = maxSize; + } + + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > maxSize; + } +} diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java index 5ca3e692..d623d943 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java @@ -20,6 +20,7 @@ import com.github.shyiko.mysql.binlog.event.EventHeader; import com.github.shyiko.mysql.binlog.event.EventType; import com.github.shyiko.mysql.binlog.event.FormatDescriptionEventData; +import com.github.shyiko.mysql.binlog.event.LRUCache; import com.github.shyiko.mysql.binlog.event.TableMapEventData; import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream; @@ -65,7 +66,7 @@ public EventDeserializer( this.eventHeaderDeserializer = eventHeaderDeserializer; this.defaultEventDataDeserializer = defaultEventDataDeserializer; this.eventDataDeserializers = new IdentityHashMap(); - this.tableMapEventByTableId = new HashMap(); + this.tableMapEventByTableId = new LRUCache<>(100, 0.75f, 10000); registerDefaultEventDataDeserializers(); afterEventDataDeserializerSet(null); } From 04f8d4994d56805f6c97288391c27e1e5e3697a1 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Fri, 1 May 2020 10:58:38 -0700 Subject: [PATCH 032/217] fix bugs exposed by adding array test --- .../mysql/binlog/event/deserialization/json/JsonBinary.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java index 25ea2d7f..688cdc63 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java @@ -470,6 +470,8 @@ protected void parseObject(boolean small, JsonFormatter formatter) // checkstyle, please ignore MethodLength for the next line protected void parseArray(boolean small, JsonFormatter formatter) throws IOException { + int arrayOffset = this.reader.getPosition(); + // Read the header ... int numElements = readUnsignedIndex(Integer.MAX_VALUE, small, "number of elements in"); int numBytes = readUnsignedIndex(Integer.MAX_VALUE, small, "size of"); @@ -534,6 +536,9 @@ protected void parseArray(boolean small, JsonFormatter formatter) } } else { // Parse the value ... + this.reader.reset(); + this.reader.skip(arrayOffset + entry.index); + parse(entry.type, formatter); } } From 23d2c01a6ff1305d033fd4ee360d7b9b2e636d85 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Sat, 2 May 2020 13:17:40 -0700 Subject: [PATCH 033/217] add mysql 8 auth test --- .../binlog/BinaryLogClientIntegrationTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java index 43629133..7618fe40 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java @@ -1000,6 +1000,19 @@ public void execute(Statement statement) throws SQLException { } } + @Test + public void testMysql8Auth() throws Exception { + if ( !mysqlVersion.atLeast(8, 0) ) + throw new SkipException("skipping mysql8 auth test"); + + master.execute("create user 'mysql8' IDENTIFIED WITH caching_sha2_password BY 'testpass'"); + master.execute("grant replication slave on *.* to 'mysql8'"); + + final BinaryLogClient binaryLogClient = + new BinaryLogClient(master.hostname, master.port, "mysql8", "testPass"); + binaryLogClient.connect(5000); + } + @Test public void testMySQL8TableMetadata() throws Exception { master.execute("drop table if exists test_metameta"); From b70895c189ad56d2ce16f730a934ce7f49928073 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Sat, 2 May 2020 13:18:19 -0700 Subject: [PATCH 034/217] response with plugin-auth, deal with empty password on auth-switch --- .../command/AuthenticateSecurityPasswordCommand.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSecurityPasswordCommand.java b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSecurityPasswordCommand.java index d99d6317..d0741138 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSecurityPasswordCommand.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSecurityPasswordCommand.java @@ -56,7 +56,10 @@ public byte[] toByteArray() throws IOException { int clientCapabilities = this.clientCapabilities; if (clientCapabilities == 0) { clientCapabilities = ClientCapabilities.LONG_FLAG | - ClientCapabilities.PROTOCOL_41 | ClientCapabilities.SECURE_CONNECTION; + ClientCapabilities.PROTOCOL_41 | + ClientCapabilities.SECURE_CONNECTION | + ClientCapabilities.PLUGIN_AUTH; + if (schema != null) { clientCapabilities |= ClientCapabilities.CONNECT_WITH_DB; } @@ -68,7 +71,7 @@ public byte[] toByteArray() throws IOException { buffer.write(0); } buffer.writeZeroTerminatedString(username); - byte[] passwordSHA1 = "".equals(password) ? new byte[0] : passwordCompatibleWithMySQL411(password, salt); + byte[] passwordSHA1 = passwordCompatibleWithMySQL411(password, salt); buffer.writeInteger(passwordSHA1.length, 1); buffer.write(passwordSHA1); if (schema != null) { @@ -81,6 +84,9 @@ public byte[] toByteArray() throws IOException { * see mysql/sql/password.c scramble(...) */ public static byte[] passwordCompatibleWithMySQL411(String password, String salt) { + if ( "".equals(password) || password == null ) + return new byte[0]; + MessageDigest sha; try { sha = MessageDigest.getInstance("SHA-1"); From 6e52314318577516ad3a7c79f9460352989aa0e1 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Sat, 2 May 2020 13:19:41 -0700 Subject: [PATCH 035/217] WIP: authenticator refactor --- .../shyiko/mysql/binlog/BinaryLogClient.java | 117 +++++------------- .../mysql/binlog/network/Authenticator.java | 112 +++++++++++++++++ 2 files changed, 146 insertions(+), 83 deletions(-) create mode 100644 src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java index 477f3b08..b5fdc618 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java @@ -126,6 +126,7 @@ public X509Certificate[] getAcceptedIssuers() { private final String schema; private final String username; private final String password; + private int packetNumber = 1; private boolean blocking = true; private long serverId = 65535; @@ -140,6 +141,7 @@ public X509Certificate[] getAcceptedIssuers() { private boolean useBinlogFilenamePositionInGtidMode; private String gtid; private boolean tx; + private boolean isSSL; private EventDeserializer eventDeserializer = new EventDeserializer(); @@ -518,6 +520,8 @@ public void connect() throws IOException { ". Please make sure it's running.", e); } GreetingPacket greetingPacket = receiveGreeting(); + tryUpgradeToSSL(); + authenticate(greetingPacket); connectionId = greetingPacket.getThreadId(); if ("".equals(binlogFilename)) { @@ -652,7 +656,36 @@ private GreetingPacket receiveGreeting() throws IOException { return new GreetingPacket(initialHandshakePacket); } - private void enableHeartbeat() throws IOException { + private boolean tryUpgradeToSSL(GreetingPacket greetingPacket) throws IOException { + int collation = greetingPacket.getServerCollation(); + int packetNumber = 1; + + if (sslMode != SSLMode.DISABLED) { + boolean serverSupportsSSL = (greetingPacket.getServerCapabilities() & ClientCapabilities.SSL) != 0; + if (!serverSupportsSSL && (sslMode == SSLMode.REQUIRED || sslMode == SSLMode.VERIFY_CA || + sslMode == SSLMode.VERIFY_IDENTITY)) { + throw new IOException("MySQL server does not support SSL"); + } + if (serverSupportsSSL) { + SSLRequestCommand sslRequestCommand = new SSLRequestCommand(); + sslRequestCommand.setCollation(collation); + channel.write(sslRequestCommand, packetNumber++); + SSLSocketFactory sslSocketFactory = + this.sslSocketFactory != null ? + this.sslSocketFactory : + sslMode == SSLMode.REQUIRED || sslMode == SSLMode.PREFERRED ? + DEFAULT_REQUIRED_SSL_MODE_SOCKET_FACTORY : + DEFAULT_VERIFY_CA_SSL_MODE_SOCKET_FACTORY; + channel.upgradeToSSL(sslSocketFactory, + sslMode == SSLMode.VERIFY_IDENTITY ? new TLSHostnameVerifier() : null); + return true; + } + } + return false; + } + + + private void enableHeartbeat() throws IOException { channel.write(new QueryCommand("set @master_heartbeat_period=" + heartbeatInterval * 1000000)); byte[] statementResult = channel.read(); if (statementResult[0] == (byte) 0xFF /* error */) { @@ -704,88 +737,6 @@ private void ensureEventDataDeserializer(EventType eventType, } } - private void authenticate(GreetingPacket greetingPacket) throws IOException { - int collation = greetingPacket.getServerCollation(); - int packetNumber = 1; - - boolean usingSSLSocket = false; - if (sslMode != SSLMode.DISABLED) { - boolean serverSupportsSSL = (greetingPacket.getServerCapabilities() & ClientCapabilities.SSL) != 0; - if (!serverSupportsSSL && (sslMode == SSLMode.REQUIRED || sslMode == SSLMode.VERIFY_CA || - sslMode == SSLMode.VERIFY_IDENTITY)) { - throw new IOException("MySQL server does not support SSL"); - } - if (serverSupportsSSL) { - SSLRequestCommand sslRequestCommand = new SSLRequestCommand(); - sslRequestCommand.setCollation(collation); - channel.write(sslRequestCommand, packetNumber++); - SSLSocketFactory sslSocketFactory = - this.sslSocketFactory != null ? - this.sslSocketFactory : - sslMode == SSLMode.REQUIRED || sslMode == SSLMode.PREFERRED ? - DEFAULT_REQUIRED_SSL_MODE_SOCKET_FACTORY : - DEFAULT_VERIFY_CA_SSL_MODE_SOCKET_FACTORY; - channel.upgradeToSSL(sslSocketFactory, - sslMode == SSLMode.VERIFY_IDENTITY ? new TLSHostnameVerifier() : null); - usingSSLSocket = true; - } - } - - Command authenticateCommand = "caching_sha2_password".equals(greetingPacket.getPluginProvidedData()) ? - new AuthenticateSHA2Command(schema, username, password, greetingPacket.getScramble(), collation) : - new AuthenticateSecurityPasswordCommand(schema, username, password, greetingPacket.getScramble(), collation); - - channel.write(authenticateCommand, packetNumber); - byte[] authenticationResult = channel.read(); - if (authenticationResult[0] != (byte) 0x00 /* ok */) { - if (authenticationResult[0] == (byte) 0xFF /* error */) { - byte[] bytes = Arrays.copyOfRange(authenticationResult, 1, authenticationResult.length); - ErrorPacket errorPacket = new ErrorPacket(bytes); - throw new AuthenticationException(errorPacket.getErrorMessage(), errorPacket.getErrorCode(), - errorPacket.getSqlState()); - } else if (authenticationResult[0] == (byte) 0xFE) { - switchAuthentication(authenticationResult, usingSSLSocket); - } else if (authenticationResult[0] == (byte) 0x01) { - if (authenticationResult.length >= 2 && (authenticationResult[1] == 3) || (authenticationResult[1] == 4)) { - // 8.0 auth ok - byte[] authenticationResultSha2 = channel.read(); - logger.log(Level.FINEST, "SHA2 auth result {0}", authenticationResultSha2); - } else { - throw new AuthenticationException("Unexpected authentication result (" + authenticationResult[0] + "&" + authenticationResult[1] + ")"); - } - } else { - throw new AuthenticationException("Unexpected authentication result (" + authenticationResult[0] + ")"); - } - } - } - - private void switchAuthentication(byte[] authenticationResult, boolean usingSSLSocket) throws IOException { - /* - Azure-MySQL likes to tell us to switch authentication methods, even though - we haven't advertised that we support any. It uses this for some-odd - reason to send the real password scramble. - */ - ByteArrayInputStream buffer = new ByteArrayInputStream(authenticationResult); - buffer.read(1); - - String authName = buffer.readZeroTerminatedString(); - if ("mysql_native_password".equals(authName)) { - String scramble = buffer.readZeroTerminatedString(); - - Command switchCommand = new AuthenticateNativePasswordCommand(scramble, password); - channel.write(switchCommand, (usingSSLSocket ? 4 : 3)); - byte[] authResult = channel.read(); - - if (authResult[0] != (byte) 0x00) { - byte[] bytes = Arrays.copyOfRange(authResult, 1, authResult.length); - ErrorPacket errorPacket = new ErrorPacket(bytes); - throw new AuthenticationException(errorPacket.getErrorMessage(), errorPacket.getErrorCode(), - errorPacket.getSqlState()); - } - } else { - throw new AuthenticationException("Unsupported authentication type: " + authName); - } - } private void spawnKeepAliveThread() { final ExecutorService threadExecutor = diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java b/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java new file mode 100644 index 00000000..8d5f2800 --- /dev/null +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java @@ -0,0 +1,112 @@ +package com.github.shyiko.mysql.binlog.network; + +import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream; +import com.github.shyiko.mysql.binlog.network.protocol.ErrorPacket; +import com.github.shyiko.mysql.binlog.network.protocol.GreetingPacket; +import com.github.shyiko.mysql.binlog.network.protocol.PacketChannel; +import com.github.shyiko.mysql.binlog.network.protocol.command.AuthenticateNativePasswordCommand; +import com.github.shyiko.mysql.binlog.network.protocol.command.AuthenticateSHA2Command; +import com.github.shyiko.mysql.binlog.network.protocol.command.AuthenticateSecurityPasswordCommand; +import com.github.shyiko.mysql.binlog.network.protocol.command.Command; +import com.github.shyiko.mysql.binlog.network.protocol.command.SSLRequestCommand; + +import java.io.IOException; +import java.util.Arrays; +import java.util.logging.Level; + +public class Authenticator { + private final GreetingPacket greetingPacket; + private final PacketChannel channel; + + public Authenticator(GreetingPacket greetingPacket, PacketChannel channel) { + this.greetingPacket = greetingPacket; + this.channel = channel; + } + + private void authenticate(GreetingPacket greetingPacket) throws IOException { + int collation = greetingPacket.getServerCollation(); + int packetNumber = 1; + + boolean usingSSLSocket = false; + if (sslMode != SSLMode.DISABLED) { + boolean serverSupportsSSL = (greetingPacket.getServerCapabilities() & ClientCapabilities.SSL) != 0; + if (!serverSupportsSSL && (sslMode == SSLMode.REQUIRED || sslMode == SSLMode.VERIFY_CA || + sslMode == SSLMode.VERIFY_IDENTITY)) { + throw new IOException("MySQL server does not support SSL"); + } + if (serverSupportsSSL) { + SSLRequestCommand sslRequestCommand = new SSLRequestCommand(); + sslRequestCommand.setCollation(collation); + channel.write(sslRequestCommand, packetNumber++); + SSLSocketFactory sslSocketFactory = + this.sslSocketFactory != null ? + this.sslSocketFactory : + sslMode == SSLMode.REQUIRED || sslMode == SSLMode.PREFERRED ? + DEFAULT_REQUIRED_SSL_MODE_SOCKET_FACTORY : + DEFAULT_VERIFY_CA_SSL_MODE_SOCKET_FACTORY; + channel.upgradeToSSL(sslSocketFactory, + sslMode == SSLMode.VERIFY_IDENTITY ? new TLSHostnameVerifier() : null); + usingSSLSocket = true; + } + } + + logger.log(Level.INFO, greetingPacket.getPluginProvidedData()); + + Command authenticateCommand = "caching_sha2_password".equals(greetingPacket.getPluginProvidedData()) ? + new AuthenticateSHA2Command(schema, username, password, greetingPacket.getScramble(), collation) : + new AuthenticateSecurityPasswordCommand(schema, username, password, greetingPacket.getScramble(), collation); + + channel.write(authenticateCommand, packetNumber); + byte[] authenticationResult = channel.read(); + if (authenticationResult[0] != (byte) 0x00 /* ok */) { + if (authenticationResult[0] == (byte) 0xFF /* error */) { + byte[] bytes = Arrays.copyOfRange(authenticationResult, 1, authenticationResult.length); + ErrorPacket errorPacket = new ErrorPacket(bytes); + throw new AuthenticationException(errorPacket.getErrorMessage(), errorPacket.getErrorCode(), + errorPacket.getSqlState()); + } else if (authenticationResult[0] == (byte) 0xFE) { + switchAuthentication(authenticationResult, usingSSLSocket); + } else if (authenticationResult[0] == (byte) 0x01) { + if (authenticationResult.length >= 2 && (authenticationResult[1] == 3) || (authenticationResult[1] == 4)) { + // 8.0 auth ok + byte[] authenticationResultSha2 = channel.read(); + logger.log(Level.FINEST, "SHA2 auth result {0}", authenticationResultSha2); + } else { + throw new AuthenticationException("Unexpected authentication result (" + authenticationResult[0] + "&" + authenticationResult[1] + ")"); + } + } else { + throw new AuthenticationException("Unexpected authentication result (" + authenticationResult[0] + ")"); + } + } + } + + private void switchAuthentication(byte[] authenticationResult, boolean usingSSLSocket) throws IOException { + /* + Azure-MySQL likes to tell us to switch authentication methods, even though + we haven't advertised that we support any. It uses this for some-odd + reason to send the real password scramble. + */ + ByteArrayInputStream buffer = new ByteArrayInputStream(authenticationResult); + buffer.read(1); + + String authName = buffer.readZeroTerminatedString(); + if ("mysql_native_password".equals(authName)) { + String scramble = buffer.readZeroTerminatedString(); + + Command switchCommand = new AuthenticateNativePasswordCommand(scramble, password); + channel.write(switchCommand, (usingSSLSocket ? 4 : 3)); + byte[] authResult = channel.read(); + + if (authResult[0] != (byte) 0x00) { + byte[] bytes = Arrays.copyOfRange(authResult, 1, authResult.length); + ErrorPacket errorPacket = new ErrorPacket(bytes); + throw new AuthenticationException(errorPacket.getErrorMessage(), errorPacket.getErrorCode(), + errorPacket.getSqlState()); + } + } else { + throw new AuthenticationException("Unsupported authentication type: " + authName); + } + } + + +} From 4a08f0d447d7b20ec3b45cbe98cbf3c3c78ea8d1 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Sat, 2 May 2020 13:46:18 -0700 Subject: [PATCH 036/217] move packet number ownership into PacketChannel --- .../shyiko/mysql/binlog/BinaryLogClient.java | 8 +++-- .../network/protocol/PacketChannel.java | 31 ++++++++++--------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java index 0a860374..cda961cd 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java @@ -717,7 +717,7 @@ private void authenticate(GreetingPacket greetingPacket) throws IOException { if (serverSupportsSSL) { SSLRequestCommand sslRequestCommand = new SSLRequestCommand(); sslRequestCommand.setCollation(collation); - channel.write(sslRequestCommand, packetNumber++); + channel.write(sslRequestCommand); SSLSocketFactory sslSocketFactory = this.sslSocketFactory != null ? this.sslSocketFactory : @@ -732,7 +732,7 @@ private void authenticate(GreetingPacket greetingPacket) throws IOException { AuthenticateCommand authenticateCommand = new AuthenticateCommand(schema, username, password, greetingPacket.getScramble()); authenticateCommand.setCollation(collation); - channel.write(authenticateCommand, packetNumber); + channel.write(authenticateCommand); byte[] authenticationResult = channel.read(); if (authenticationResult[0] != (byte) 0x00 /* ok */) { if (authenticationResult[0] == (byte) 0xFF /* error */) { @@ -746,6 +746,8 @@ private void authenticate(GreetingPacket greetingPacket) throws IOException { throw new AuthenticationException("Unexpected authentication result (" + authenticationResult[0] + ")"); } } + + channel.authenticationComplete(); } private void switchAuthentication(byte[] authenticationResult, boolean usingSSLSocket) throws IOException { @@ -762,7 +764,7 @@ private void switchAuthentication(byte[] authenticationResult, boolean usingSSLS String scramble = buffer.readZeroTerminatedString(); Command switchCommand = new AuthenticateNativePasswordCommand(scramble, password); - channel.write(switchCommand, (usingSSLSocket ? 4 : 3)); + channel.write(switchCommand); byte[] authResult = channel.read(); if (authResult[0] != (byte) 0x00) { diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/PacketChannel.java b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/PacketChannel.java index fbbe950f..6262f3f6 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/PacketChannel.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/PacketChannel.java @@ -32,7 +32,8 @@ * @author Stanley Shyiko */ public class PacketChannel implements Channel { - + private int packetNumber = 1; + private boolean authenticationComplete; private Socket socket; private ByteArrayInputStream inputStream; private ByteArrayOutputStream outputStream; @@ -55,17 +56,29 @@ public ByteArrayOutputStream getOutputStream() { return outputStream; } + public void authenticationComplete() { + authenticationComplete = true; + } + public byte[] read() throws IOException { int length = inputStream.readInteger(3); inputStream.skip(1); //sequence return inputStream.read(length); } - public void write(Command command, int packetNumber) throws IOException { + public void write(Command command) throws IOException { byte[] body = command.toByteArray(); ByteArrayOutputStream buffer = new ByteArrayOutputStream(); buffer.writeInteger(body.length, 3); // packet length - buffer.writeInteger(packetNumber, 1); + + // see https://dev.mysql.com/doc/dev/mysql-server/8.0.11/page_protocol_basic_packets.html#sect_protocol_basic_packets_sequence_id + // we only have to maintain a sequence number in the authentication phase. + // what the point is, I do not know + if ( authenticationComplete ) + buffer.writeInteger(0, 1); + else + buffer.writeInteger(packetNumber++, 1); + buffer.write(body, 0, body.length); outputStream.write(buffer.toByteArray()); // though it has no effect in case of default (underlying) output stream (SocketOutputStream), @@ -73,18 +86,6 @@ public void write(Command command, int packetNumber) throws IOException { outputStream.flush(); } - /** - * @deprecated use {@link #write(Command, int)} instead - */ - @Deprecated - public void writeBuffered(Command command, int packetNumber) throws IOException { - write(command, packetNumber); - } - - public void write(Command command) throws IOException { - write(command, 0); - } - public void upgradeToSSL(SSLSocketFactory sslSocketFactory, HostnameVerifier hostnameVerifier) throws IOException { SSLSocket sslSocket = sslSocketFactory.createSocket(this.socket); sslSocket.startHandshake(); From 49ed9ab7976262e6e8929c11ba98bc690f1a7f11 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Sat, 2 May 2020 16:00:55 -0700 Subject: [PATCH 037/217] wait for tests longer --- .../deserialization/json/JsonBinaryValueIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java index 12552c31..96750afd 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java @@ -53,7 +53,7 @@ */ public class JsonBinaryValueIntegrationTest { - private static final long DEFAULT_TIMEOUT = TimeUnit.SECONDS.toMillis(3); + private static final long DEFAULT_TIMEOUT = TimeUnit.SECONDS.toMillis(6); private final Logger logger = Logger.getLogger(getClass().getSimpleName()); From 2cfcc59b2bbcac8b0a261990aec7c05fc77e4ce6 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Sat, 2 May 2020 17:15:40 -0700 Subject: [PATCH 038/217] try some debug logging --- .../github/shyiko/mysql/binlog/CountDownEventListener.java | 4 ++++ .../deserialization/json/JsonBinaryValueIntegrationTest.java | 1 + 2 files changed, 5 insertions(+) diff --git a/src/test/java/com/github/shyiko/mysql/binlog/CountDownEventListener.java b/src/test/java/com/github/shyiko/mysql/binlog/CountDownEventListener.java index 73a6f5b9..38c5cfb2 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/CountDownEventListener.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/CountDownEventListener.java @@ -35,9 +35,12 @@ public class CountDownEventListener implements BinaryLogClient.EventListener { @Override public void onEvent(Event event) { + System.out.println("increment Counter for " + event.getHeader().getEventType() + " " + getCounter(countersByType, event.getHeader().getEventType())); + incrementCounter(getCounter(countersByType, event.getHeader().getEventType())); EventData data = event.getData(); if (data != null) { + System.out.println("increment Counter for " + data.getClass() + " " + getCounter(countersByDataClass, data.getClass())); incrementCounter(getCounter(countersByDataClass, data.getClass())); } } @@ -76,6 +79,7 @@ private void waitForCounterToGetZero(String counterName, AtomicInteger counter, long timeoutInMilliseconds) throws TimeoutException, InterruptedException { synchronized (counter) { counter.set(counter.get() - numberOfExpectedEvents); + System.out.println("counter " + counterName + " set to " + counter.get()); if (counter.get() != 0) { counter.wait(timeoutInMilliseconds); if (counter.get() != 0) { diff --git a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java index 96750afd..05ffc10e 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java @@ -507,6 +507,7 @@ private String writeAndCaptureJSON(final String value) throws Exception { client.unregisterEventListener(capturingEventListener); } if ( capturingEventListener.getEvents(WriteRowsEventData.class).size() == 0 ) { + System.out.println("I am about to fail an expectation..."); assertTrue(false, "did not receive rows in json test for " + value); } byte[] b = (byte[]) capturingEventListener.getEvents(WriteRowsEventData.class).get(0).getRows().get(0)[0]; From 35fa10f91ee203ef8d093521b5b903f7d3e49b64 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Sat, 2 May 2020 17:26:47 -0700 Subject: [PATCH 039/217] remove superfluous extra event listeners --- .../json/JsonBinaryValueIntegrationTest.java | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java index 05ffc10e..72dc26d8 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java @@ -109,8 +109,6 @@ public void execute(Statement statement) throws SQLException { @Test public void testMysql8JsonSetPartialUpdateWithHoles() throws Exception { - CountDownEventListener eventListener = new CountDownEventListener(); - client.registerEventListener(eventListener); CapturingEventListener capturingEventListener = new CapturingEventListener(); client.registerEventListener(capturingEventListener); String json = "{\"age\":22,\"addr\":{\"code\":100,\"detail\":{\"ab\":\"970785C8-C299\"}},\"name\":\"Alice\"}"; @@ -130,8 +128,6 @@ public void testMysql8JsonSetPartialUpdateWithHoles() throws Exception { @Test public void testMysql8JsonRemovePartialUpdateWithHoles() throws Exception { - CountDownEventListener eventListener = new CountDownEventListener(); - client.registerEventListener(eventListener); CapturingEventListener capturingEventListener = new CapturingEventListener(); client.registerEventListener(capturingEventListener); String json = "{\"age\":22,\"addr\":{\"code\":100,\"detail\":{\"ab\":\"970785C8-C299\"}},\"name\":\"Alice\"}"; @@ -147,12 +143,12 @@ public void testMysql8JsonRemovePartialUpdateWithHoles() throws Exception { List updateEvents = capturingEventListener.getEvents(UpdateRowsEventData.class); Serializable[] updateData = updateEvents.iterator().next().getRows().get(0).getValue(); assertEquals(JsonBinary.parseAsString((byte[]) updateData[0]), json.replace("\"ab\":\"970785C8-C299\"", "")); + + client.unregisterEventListener(capturingEventListener); } @Test public void testMysql8JsonReplacePartialUpdateWithHoles() throws Exception { - CountDownEventListener eventListener = new CountDownEventListener(); - client.registerEventListener(eventListener); CapturingEventListener capturingEventListener = new CapturingEventListener(); client.registerEventListener(capturingEventListener); String json = "{\"age\":22,\"addr\":{\"code\":100,\"detail\":{\"ab\":\"970785C8-C299\"}},\"name\":\"Alice\"}"; @@ -168,12 +164,12 @@ public void testMysql8JsonReplacePartialUpdateWithHoles() throws Exception { List updateEvents = capturingEventListener.getEvents(UpdateRowsEventData.class); Serializable[] updateData = updateEvents.iterator().next().getRows().get(0).getValue(); assertEquals(JsonBinary.parseAsString((byte[]) updateData[0]), json.replace("970785C8-C299", "9707")); + + client.unregisterEventListener(capturingEventListener); } @Test public void testMysql8JsonRemoveArrayValue() throws Exception { - CountDownEventListener eventListener = new CountDownEventListener(); - client.registerEventListener(eventListener); CapturingEventListener capturingEventListener = new CapturingEventListener(); client.registerEventListener(capturingEventListener); @@ -191,12 +187,12 @@ public void testMysql8JsonRemoveArrayValue() throws Exception { List updateEvents = capturingEventListener.getEvents(UpdateRowsEventData.class); Serializable[] updateData = updateEvents.iterator().next().getRows().get(0).getValue(); assertEquals(JsonBinary.parseAsString((byte[]) updateData[0]), "[\"foo\",\"baz\"]"); + + client.unregisterEventListener(capturingEventListener); } @Test public void testValueBoundariesAreHonored() throws Exception { - CountDownEventListener eventListener = new CountDownEventListener(); - client.registerEventListener(eventListener); CapturingEventListener capturingEventListener = new CapturingEventListener(); client.registerEventListener(capturingEventListener); master.execute("create table json_b (h varchar(255), j JSON, k varchar(255))", @@ -207,6 +203,8 @@ public void testValueBoundariesAreHonored() throws Exception { assertEquals(data[0], "sponge"); assertEquals(JsonBinary.parseAsString((byte[]) data[1]), "{}"); assertEquals(data[2], "bob"); + + client.unregisterEventListener(capturingEventListener); } @Test From 9c999c7a4679abea75f1a75994073ff879841ebd Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Sat, 2 May 2020 17:50:04 -0700 Subject: [PATCH 040/217] fix test race condition. It was previously possible that CountDownEventListener woke up the test before CapturingEventListener had received rows. Unifying the latch and the capture together in one class fixes this. --- .../mysql/binlog/BinaryLogClientTest.java | 2 +- .../mysql/binlog/CapturingEventListener.java | 3 ++- .../mysql/binlog/CountDownEventListener.java | 4 ---- .../json/JsonBinaryValueIntegrationTest.java | 20 +++++++++---------- 4 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientTest.java index fece36b4..542c2452 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientTest.java @@ -49,7 +49,7 @@ public void testEventListenersManagement() { assertEquals(binaryLogClient.getEventListeners().size(), 3); binaryLogClient.unregisterEventListener(traceEventListener); assertEquals(binaryLogClient.getEventListeners().size(), 2); - binaryLogClient.unregisterEventListener(CountDownEventListener.class); + binaryLogClient.unregisterEventListener(CapturingEventListener.class); assertEquals(binaryLogClient.getEventListeners().size(), 1); } diff --git a/src/test/java/com/github/shyiko/mysql/binlog/CapturingEventListener.java b/src/test/java/com/github/shyiko/mysql/binlog/CapturingEventListener.java index f6b7c6b0..0249f6e1 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/CapturingEventListener.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/CapturingEventListener.java @@ -25,7 +25,7 @@ /** * @author Stanley Shyiko */ -public class CapturingEventListener implements BinaryLogClient.EventListener { +public class CapturingEventListener extends CountDownEventListener { private final List events = new LinkedList(); @@ -33,6 +33,7 @@ public class CapturingEventListener implements BinaryLogClient.EventListener { public void onEvent(Event event) { synchronized (events) { events.add(event); + super.onEvent(event); } } diff --git a/src/test/java/com/github/shyiko/mysql/binlog/CountDownEventListener.java b/src/test/java/com/github/shyiko/mysql/binlog/CountDownEventListener.java index 38c5cfb2..73a6f5b9 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/CountDownEventListener.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/CountDownEventListener.java @@ -35,12 +35,9 @@ public class CountDownEventListener implements BinaryLogClient.EventListener { @Override public void onEvent(Event event) { - System.out.println("increment Counter for " + event.getHeader().getEventType() + " " + getCounter(countersByType, event.getHeader().getEventType())); - incrementCounter(getCounter(countersByType, event.getHeader().getEventType())); EventData data = event.getData(); if (data != null) { - System.out.println("increment Counter for " + data.getClass() + " " + getCounter(countersByDataClass, data.getClass())); incrementCounter(getCounter(countersByDataClass, data.getClass())); } } @@ -79,7 +76,6 @@ private void waitForCounterToGetZero(String counterName, AtomicInteger counter, long timeoutInMilliseconds) throws TimeoutException, InterruptedException { synchronized (counter) { counter.set(counter.get() - numberOfExpectedEvents); - System.out.println("counter " + counterName + " set to " + counter.get()); if (counter.get() != 0) { counter.wait(timeoutInMilliseconds); if (counter.get() != 0) { diff --git a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java index 72dc26d8..e6eb2221 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java @@ -115,8 +115,8 @@ public void testMysql8JsonSetPartialUpdateWithHoles() throws Exception { master.execute("DROP TABLE IF EXISTS json_test", "create table json_test (j JSON)", "INSERT INTO json_test VALUES ('" + json + "')", "UPDATE json_test SET j = JSON_SET(j, '$.addr.detail.ab', '970785C8')"); - eventListener.waitFor(WriteRowsEventData.class, 1, DEFAULT_TIMEOUT); - eventListener.waitFor(UpdateRowsEventData.class, 1, DEFAULT_TIMEOUT); + capturingEventListener.waitFor(WriteRowsEventData.class, 1, DEFAULT_TIMEOUT); + capturingEventListener.waitFor(UpdateRowsEventData.class, 1, DEFAULT_TIMEOUT); List events = capturingEventListener.getEvents(WriteRowsEventData.class); Serializable[] insertData = events.iterator().next().getRows().get(0); assertEquals(JsonBinary.parseAsString((byte[]) insertData[0]), json); @@ -134,8 +134,8 @@ public void testMysql8JsonRemovePartialUpdateWithHoles() throws Exception { master.execute("DROP TABLE IF EXISTS json_test", "create table json_test (j JSON)", "INSERT INTO json_test VALUES ('" + json + "')", "UPDATE json_test SET j = JSON_REMOVE(j, '$.addr.detail.ab')"); - eventListener.waitFor(WriteRowsEventData.class, 1, DEFAULT_TIMEOUT); - eventListener.waitFor(UpdateRowsEventData.class, 1, DEFAULT_TIMEOUT); + capturingEventListener.waitFor(WriteRowsEventData.class, 1, DEFAULT_TIMEOUT); + capturingEventListener.waitFor(UpdateRowsEventData.class, 1, DEFAULT_TIMEOUT); List events = capturingEventListener.getEvents(WriteRowsEventData.class); Serializable[] insertData = events.iterator().next().getRows().get(0); assertEquals(JsonBinary.parseAsString((byte[]) insertData[0]), json); @@ -155,8 +155,8 @@ public void testMysql8JsonReplacePartialUpdateWithHoles() throws Exception { master.execute("DROP TABLE IF EXISTS json_test", "create table json_test (j JSON)", "INSERT INTO json_test VALUES ('" + json + "')", "UPDATE json_test SET j = JSON_REPLACE(j, '$.addr.detail.ab', '9707')"); - eventListener.waitFor(WriteRowsEventData.class, 1, DEFAULT_TIMEOUT); - eventListener.waitFor(UpdateRowsEventData.class, 1, DEFAULT_TIMEOUT); + capturingEventListener.waitFor(WriteRowsEventData.class, 1, DEFAULT_TIMEOUT); + capturingEventListener.waitFor(UpdateRowsEventData.class, 1, DEFAULT_TIMEOUT); List events = capturingEventListener.getEvents(WriteRowsEventData.class); Serializable[] insertData = events.iterator().next().getRows().get(0); assertEquals(JsonBinary.parseAsString((byte[]) insertData[0]), json); @@ -177,8 +177,8 @@ public void testMysql8JsonRemoveArrayValue() throws Exception { master.execute("DROP TABLE IF EXISTS json_test", "create table json_test (j JSON)", "INSERT INTO json_test VALUES ('" + json + "')", "UPDATE json_test SET j = JSON_REMOVE(j, '$[1]')"); - eventListener.waitFor(WriteRowsEventData.class, 1, DEFAULT_TIMEOUT); - eventListener.waitFor(UpdateRowsEventData.class, 1, DEFAULT_TIMEOUT); + capturingEventListener.waitFor(WriteRowsEventData.class, 1, DEFAULT_TIMEOUT); + capturingEventListener.waitFor(UpdateRowsEventData.class, 1, DEFAULT_TIMEOUT); List events = capturingEventListener.getEvents(WriteRowsEventData.class); Serializable[] insertData = events.iterator().next().getRows().get(0); @@ -197,7 +197,7 @@ public void testValueBoundariesAreHonored() throws Exception { client.registerEventListener(capturingEventListener); master.execute("create table json_b (h varchar(255), j JSON, k varchar(255))", "INSERT INTO json_b VALUES ('sponge', '{}', 'bob');"); - eventListener.waitFor(WriteRowsEventData.class, 1, DEFAULT_TIMEOUT); + capturingEventListener.waitFor(WriteRowsEventData.class, 1, DEFAULT_TIMEOUT); List events = capturingEventListener.getEvents(WriteRowsEventData.class); Serializable[] data = events.iterator().next().getRows().get(0); assertEquals(data[0], "sponge"); @@ -500,7 +500,7 @@ private String writeAndCaptureJSON(final String value) throws Exception { statement.execute("create table data_type_hell (column_ " + "JSON" + ")"); statement.execute("insert into data_type_hell values (" + value + ")"); }); - eventListener.waitFor(WriteRowsEventData.class, 1, DEFAULT_TIMEOUT); + capturingEventListener.waitFor(WriteRowsEventData.class, 1, DEFAULT_TIMEOUT); } finally { client.unregisterEventListener(capturingEventListener); } From 148ce23eb806d45158502784e8fff23db10ba09f Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Sun, 3 May 2020 02:25:03 -0700 Subject: [PATCH 041/217] finish refactor into Authenticator class, more sequence-number fixes The old implementation was pretty janky, although it worked. sequence numbers are actually incremented and checked by both client and server. --- .../shyiko/mysql/binlog/BinaryLogClient.java | 14 +++-- .../mysql/binlog/network/Authenticator.java | 58 ++++++++----------- .../network/protocol/PacketChannel.java | 16 +++-- 3 files changed, 41 insertions(+), 47 deletions(-) diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java index b5fdc618..b6690126 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java @@ -33,6 +33,7 @@ import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream; import com.github.shyiko.mysql.binlog.jmx.BinaryLogClientMXBean; import com.github.shyiko.mysql.binlog.network.AuthenticationException; +import com.github.shyiko.mysql.binlog.network.Authenticator; import com.github.shyiko.mysql.binlog.network.ClientCapabilities; import com.github.shyiko.mysql.binlog.network.DefaultSSLSocketFactory; import com.github.shyiko.mysql.binlog.network.SSLMode; @@ -126,7 +127,6 @@ public X509Certificate[] getAcceptedIssuers() { private final String schema; private final String username; private final String password; - private int packetNumber = 1; private boolean blocking = true; private long serverId = 65535; @@ -520,9 +520,12 @@ public void connect() throws IOException { ". Please make sure it's running.", e); } GreetingPacket greetingPacket = receiveGreeting(); - tryUpgradeToSSL(); - authenticate(greetingPacket); + tryUpgradeToSSL(greetingPacket); + + new Authenticator(greetingPacket, channel, schema, username, password).authenticate(); + channel.authenticationComplete(); + connectionId = greetingPacket.getThreadId(); if ("".equals(binlogFilename)) { synchronized (gtidSetAccessLock) { @@ -658,7 +661,6 @@ private GreetingPacket receiveGreeting() throws IOException { private boolean tryUpgradeToSSL(GreetingPacket greetingPacket) throws IOException { int collation = greetingPacket.getServerCollation(); - int packetNumber = 1; if (sslMode != SSLMode.DISABLED) { boolean serverSupportsSSL = (greetingPacket.getServerCapabilities() & ClientCapabilities.SSL) != 0; @@ -669,7 +671,7 @@ private boolean tryUpgradeToSSL(GreetingPacket greetingPacket) throws IOExceptio if (serverSupportsSSL) { SSLRequestCommand sslRequestCommand = new SSLRequestCommand(); sslRequestCommand.setCollation(collation); - channel.write(sslRequestCommand, packetNumber++); + channel.write(sslRequestCommand); SSLSocketFactory sslSocketFactory = this.sslSocketFactory != null ? this.sslSocketFactory : @@ -685,7 +687,7 @@ private boolean tryUpgradeToSSL(GreetingPacket greetingPacket) throws IOExceptio } - private void enableHeartbeat() throws IOException { + private void enableHeartbeat() throws IOException { channel.write(new QueryCommand("set @master_heartbeat_period=" + heartbeatInterval * 1000000)); byte[] statementResult = channel.read(); if (statementResult[0] == (byte) 0xFF /* error */) { diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java b/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java index 8d5f2800..e75ee455 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java @@ -13,50 +13,40 @@ import java.io.IOException; import java.util.Arrays; import java.util.logging.Level; +import java.util.logging.Logger; public class Authenticator { private final GreetingPacket greetingPacket; private final PacketChannel channel; + private final String schema; + private final String username; + private final String password; - public Authenticator(GreetingPacket greetingPacket, PacketChannel channel) { + private final Logger logger = Logger.getLogger(getClass().getName()); + + public Authenticator( + GreetingPacket greetingPacket, + PacketChannel channel, + String schema, + String username, + String password + ) { this.greetingPacket = greetingPacket; this.channel = channel; + this.schema = schema; + this.username = username; + this.password = password; } - private void authenticate(GreetingPacket greetingPacket) throws IOException { + public void authenticate() throws IOException { + logger.log(Level.INFO, "Begin auth for " + username); int collation = greetingPacket.getServerCollation(); - int packetNumber = 1; - - boolean usingSSLSocket = false; - if (sslMode != SSLMode.DISABLED) { - boolean serverSupportsSSL = (greetingPacket.getServerCapabilities() & ClientCapabilities.SSL) != 0; - if (!serverSupportsSSL && (sslMode == SSLMode.REQUIRED || sslMode == SSLMode.VERIFY_CA || - sslMode == SSLMode.VERIFY_IDENTITY)) { - throw new IOException("MySQL server does not support SSL"); - } - if (serverSupportsSSL) { - SSLRequestCommand sslRequestCommand = new SSLRequestCommand(); - sslRequestCommand.setCollation(collation); - channel.write(sslRequestCommand, packetNumber++); - SSLSocketFactory sslSocketFactory = - this.sslSocketFactory != null ? - this.sslSocketFactory : - sslMode == SSLMode.REQUIRED || sslMode == SSLMode.PREFERRED ? - DEFAULT_REQUIRED_SSL_MODE_SOCKET_FACTORY : - DEFAULT_VERIFY_CA_SSL_MODE_SOCKET_FACTORY; - channel.upgradeToSSL(sslSocketFactory, - sslMode == SSLMode.VERIFY_IDENTITY ? new TLSHostnameVerifier() : null); - usingSSLSocket = true; - } - } - - logger.log(Level.INFO, greetingPacket.getPluginProvidedData()); Command authenticateCommand = "caching_sha2_password".equals(greetingPacket.getPluginProvidedData()) ? new AuthenticateSHA2Command(schema, username, password, greetingPacket.getScramble(), collation) : new AuthenticateSecurityPasswordCommand(schema, username, password, greetingPacket.getScramble(), collation); - channel.write(authenticateCommand, packetNumber); + channel.write(authenticateCommand); byte[] authenticationResult = channel.read(); if (authenticationResult[0] != (byte) 0x00 /* ok */) { if (authenticationResult[0] == (byte) 0xFF /* error */) { @@ -65,12 +55,11 @@ private void authenticate(GreetingPacket greetingPacket) throws IOException { throw new AuthenticationException(errorPacket.getErrorMessage(), errorPacket.getErrorCode(), errorPacket.getSqlState()); } else if (authenticationResult[0] == (byte) 0xFE) { - switchAuthentication(authenticationResult, usingSSLSocket); + switchAuthentication(authenticationResult); } else if (authenticationResult[0] == (byte) 0x01) { if (authenticationResult.length >= 2 && (authenticationResult[1] == 3) || (authenticationResult[1] == 4)) { // 8.0 auth ok byte[] authenticationResultSha2 = channel.read(); - logger.log(Level.FINEST, "SHA2 auth result {0}", authenticationResultSha2); } else { throw new AuthenticationException("Unexpected authentication result (" + authenticationResult[0] + "&" + authenticationResult[1] + ")"); } @@ -78,9 +67,10 @@ private void authenticate(GreetingPacket greetingPacket) throws IOException { throw new AuthenticationException("Unexpected authentication result (" + authenticationResult[0] + ")"); } } + logger.log(Level.INFO, "Auth complete " + username); } - private void switchAuthentication(byte[] authenticationResult, boolean usingSSLSocket) throws IOException { + private void switchAuthentication(byte[] authenticationResult) throws IOException { /* Azure-MySQL likes to tell us to switch authentication methods, even though we haven't advertised that we support any. It uses this for some-odd @@ -94,7 +84,7 @@ private void switchAuthentication(byte[] authenticationResult, boolean usingSSLS String scramble = buffer.readZeroTerminatedString(); Command switchCommand = new AuthenticateNativePasswordCommand(scramble, password); - channel.write(switchCommand, (usingSSLSocket ? 4 : 3)); + channel.write(switchCommand); byte[] authResult = channel.read(); if (authResult[0] != (byte) 0x00) { @@ -107,6 +97,4 @@ private void switchAuthentication(byte[] authenticationResult, boolean usingSSLS throw new AuthenticationException("Unsupported authentication type: " + authName); } } - - } diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/PacketChannel.java b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/PacketChannel.java index 6262f3f6..34208a10 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/PacketChannel.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/PacketChannel.java @@ -32,7 +32,7 @@ * @author Stanley Shyiko */ public class PacketChannel implements Channel { - private int packetNumber = 1; + private int packetNumber = 0; private boolean authenticationComplete; private Socket socket; private ByteArrayInputStream inputStream; @@ -62,7 +62,10 @@ public void authenticationComplete() { public byte[] read() throws IOException { int length = inputStream.readInteger(3); - inputStream.skip(1); //sequence + int sequence = inputStream.read(); // sequence + if ( sequence != packetNumber++ ) { + throw new IOException("unexpected sequence #" + sequence); + } return inputStream.read(length); } @@ -74,10 +77,11 @@ public void write(Command command) throws IOException { // see https://dev.mysql.com/doc/dev/mysql-server/8.0.11/page_protocol_basic_packets.html#sect_protocol_basic_packets_sequence_id // we only have to maintain a sequence number in the authentication phase. // what the point is, I do not know - if ( authenticationComplete ) - buffer.writeInteger(0, 1); - else - buffer.writeInteger(packetNumber++, 1); + if ( authenticationComplete ) { + packetNumber = 0; + } + + buffer.writeInteger(packetNumber++, 1); buffer.write(body, 0, body.length); outputStream.write(buffer.toByteArray()); From 9bc770592590bad8b440034ee60f114c44e93fb2 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Sun, 3 May 2020 15:27:46 -0700 Subject: [PATCH 042/217] mysql 8 authentication: first passing test. We code and test the normal-authentication (non-cached) path, in which we negotiate an SSL connection with the server, miss the cache for SHA2 passwords, and have to send a plaintext password. --- .../shyiko/mysql/binlog/BinaryLogClient.java | 1 + .../mysql/binlog/network/Authenticator.java | 99 ++++++++++++++----- .../network/protocol/PacketChannel.java | 6 ++ .../command/AuthenticateSHA2Command.java | 14 +++ .../protocol/command/ByteArrayCommand.java | 15 +++ .../protocol/command/SSLRequestCommand.java | 4 +- .../BinaryLogClientIntegrationTest.java | 8 +- 7 files changed, 118 insertions(+), 29 deletions(-) create mode 100644 src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/ByteArrayCommand.java diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java index b6690126..16d47cd5 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java @@ -680,6 +680,7 @@ private boolean tryUpgradeToSSL(GreetingPacket greetingPacket) throws IOExceptio DEFAULT_VERIFY_CA_SSL_MODE_SOCKET_FACTORY; channel.upgradeToSSL(sslSocketFactory, sslMode == SSLMode.VERIFY_IDENTITY ? new TLSHostnameVerifier() : null); + logger.info("SSL enabled"); return true; } } diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java b/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java index e75ee455..8abb920d 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java @@ -1,12 +1,14 @@ package com.github.shyiko.mysql.binlog.network; import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream; +import com.github.shyiko.mysql.binlog.io.ByteArrayOutputStream; import com.github.shyiko.mysql.binlog.network.protocol.ErrorPacket; import com.github.shyiko.mysql.binlog.network.protocol.GreetingPacket; import com.github.shyiko.mysql.binlog.network.protocol.PacketChannel; import com.github.shyiko.mysql.binlog.network.protocol.command.AuthenticateNativePasswordCommand; import com.github.shyiko.mysql.binlog.network.protocol.command.AuthenticateSHA2Command; import com.github.shyiko.mysql.binlog.network.protocol.command.AuthenticateSecurityPasswordCommand; +import com.github.shyiko.mysql.binlog.network.protocol.command.ByteArrayCommand; import com.github.shyiko.mysql.binlog.network.protocol.command.Command; import com.github.shyiko.mysql.binlog.network.protocol.command.SSLRequestCommand; @@ -16,6 +18,11 @@ import java.util.logging.Logger; public class Authenticator { + private enum AuthMethod { + NATIVE, + CACHING_SHA2 + }; + private final GreetingPacket greetingPacket; private final PacketChannel channel; private final String schema; @@ -24,6 +31,11 @@ public class Authenticator { private final Logger logger = Logger.getLogger(getClass().getName()); + private final String SHA2_PASSWORD = "caching_sha2_password"; + private final String MYSQL_NATIVE = "mysql_native_password"; + + private AuthMethod authMethod = AuthMethod.NATIVE; + public Authenticator( GreetingPacket greetingPacket, PacketChannel channel, @@ -42,32 +54,69 @@ public void authenticate() throws IOException { logger.log(Level.INFO, "Begin auth for " + username); int collation = greetingPacket.getServerCollation(); - Command authenticateCommand = "caching_sha2_password".equals(greetingPacket.getPluginProvidedData()) ? - new AuthenticateSHA2Command(schema, username, password, greetingPacket.getScramble(), collation) : - new AuthenticateSecurityPasswordCommand(schema, username, password, greetingPacket.getScramble(), collation); + Command authenticateCommand; + if ( SHA2_PASSWORD.equals(greetingPacket.getPluginProvidedData()) ) { + authMethod = AuthMethod.CACHING_SHA2; + authenticateCommand = new AuthenticateSHA2Command(schema, username, password, greetingPacket.getScramble(), collation); + } else { + authMethod = AuthMethod.NATIVE; + authenticateCommand = new AuthenticateSecurityPasswordCommand(schema, username, password, greetingPacket.getScramble(), collation); + } channel.write(authenticateCommand); + readResult(); + logger.log(Level.INFO, "Auth complete " + username); + } + + private void readResult() throws IOException { byte[] authenticationResult = channel.read(); - if (authenticationResult[0] != (byte) 0x00 /* ok */) { - if (authenticationResult[0] == (byte) 0xFF /* error */) { + switch(authenticationResult[0]) { + case (byte) 0x00: + // success + return; + case (byte) 0xFF: + // error byte[] bytes = Arrays.copyOfRange(authenticationResult, 1, authenticationResult.length); ErrorPacket errorPacket = new ErrorPacket(bytes); throw new AuthenticationException(errorPacket.getErrorMessage(), errorPacket.getErrorCode(), errorPacket.getSqlState()); - } else if (authenticationResult[0] == (byte) 0xFE) { + case (byte) 0xFE: switchAuthentication(authenticationResult); - } else if (authenticationResult[0] == (byte) 0x01) { - if (authenticationResult.length >= 2 && (authenticationResult[1] == 3) || (authenticationResult[1] == 4)) { - // 8.0 auth ok - byte[] authenticationResultSha2 = channel.read(); + return; + default: + if ( authMethod == AuthMethod.NATIVE ) + throw new AuthenticationException("Unexpected authentication result (" + authenticationResult[0] + ")"); + else + processCachingSHA2Result(authenticationResult); + } + } + + private void processCachingSHA2Result(byte[] authenticationResult) throws IOException { + if (authenticationResult.length < 2) + throw new AuthenticationException("caching_sha2_password response too short!"); + + ByteArrayInputStream stream = new ByteArrayInputStream(authenticationResult); + stream.readPackedInteger(); // throw away length, always 1 + + switch(stream.read()) { + case 0x03: + // successful fast authentication + return; + case 0x04: + // need to send continue auth. + if ( channel.isSSL() ) { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + + buffer.write(password.getBytes()); + buffer.write(0); + + Command c = new ByteArrayCommand(buffer.toByteArray()); + channel.write(c); + readResult(); } else { - throw new AuthenticationException("Unexpected authentication result (" + authenticationResult[0] + "&" + authenticationResult[1] + ")"); + throw new AuthenticationException("Please enable SSL in order to support caching_sha2_password auth"); } - } else { - throw new AuthenticationException("Unexpected authentication result (" + authenticationResult[0] + ")"); - } } - logger.log(Level.INFO, "Auth complete " + username); } private void switchAuthentication(byte[] authenticationResult) throws IOException { @@ -80,21 +129,21 @@ private void switchAuthentication(byte[] authenticationResult) throws IOExceptio buffer.read(1); String authName = buffer.readZeroTerminatedString(); - if ("mysql_native_password".equals(authName)) { + if (MYSQL_NATIVE.equals(authName)) { + authMethod = AuthMethod.NATIVE; + String scramble = buffer.readZeroTerminatedString(); Command switchCommand = new AuthenticateNativePasswordCommand(scramble, password); channel.write(switchCommand); - byte[] authResult = channel.read(); + } else if ( SHA2_PASSWORD.equals(authName) ) { + authMethod = AuthMethod.CACHING_SHA2; - if (authResult[0] != (byte) 0x00) { - byte[] bytes = Arrays.copyOfRange(authResult, 1, authResult.length); - ErrorPacket errorPacket = new ErrorPacket(bytes); - throw new AuthenticationException(errorPacket.getErrorMessage(), errorPacket.getErrorCode(), - errorPacket.getSqlState()); - } - } else { - throw new AuthenticationException("Unsupported authentication type: " + authName); + String scramble = buffer.readZeroTerminatedString(); + Command authCommand = new AuthenticateSHA2Command(password, scramble); + channel.write(authCommand); } + + readResult(); } } diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/PacketChannel.java b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/PacketChannel.java index 34208a10..95a4b2be 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/PacketChannel.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/PacketChannel.java @@ -34,6 +34,7 @@ public class PacketChannel implements Channel { private int packetNumber = 0; private boolean authenticationComplete; + private boolean isSSL = false; private Socket socket; private ByteArrayInputStream inputStream; private ByteArrayOutputStream outputStream; @@ -101,6 +102,11 @@ public void upgradeToSSL(SSLSocketFactory sslSocketFactory, HostnameVerifier hos throw new IdentityVerificationException("\"" + sslSocket.getInetAddress().getHostName() + "\" identity was not confirmed"); } + isSSL = true; + } + + public boolean isSSL() { + return isSSL; } @Override diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2Command.java b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2Command.java index c8afca7e..d5e7579c 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2Command.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2Command.java @@ -34,6 +34,7 @@ public class AuthenticateSHA2Command implements Command { private String salt; private int clientCapabilities; private int collation; + private boolean rawPassword = false; public AuthenticateSHA2Command(String schema, String username, String password, String salt, int collation) { this.schema = schema; @@ -43,6 +44,12 @@ public AuthenticateSHA2Command(String schema, String username, String password, this.collation = collation; } + public AuthenticateSHA2Command(String password, String salt) { + this.rawPassword = true; + this.password = password; + this.salt = salt; + } + public void setClientCapabilities(int clientCapabilities) { this.clientCapabilities = clientCapabilities; } @@ -50,6 +57,13 @@ public void setClientCapabilities(int clientCapabilities) { @Override public byte[] toByteArray() throws IOException { ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + + if ( rawPassword ) { + byte[] passwordSHA1 = encodePassword(); + buffer.write(passwordSHA1); + return buffer.toByteArray(); + } + int clientCapabilities = this.clientCapabilities; if (clientCapabilities == 0) { clientCapabilities |= ClientCapabilities.LONG_FLAG; diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/ByteArrayCommand.java b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/ByteArrayCommand.java new file mode 100644 index 00000000..94ee6998 --- /dev/null +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/ByteArrayCommand.java @@ -0,0 +1,15 @@ +package com.github.shyiko.mysql.binlog.network.protocol.command; + +import java.io.IOException; + +public class ByteArrayCommand implements Command { + private final byte[] command; + + public ByteArrayCommand(byte[] command) { + this.command = command; + } + @Override + public byte[] toByteArray() throws IOException { + return command; + } +} diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/SSLRequestCommand.java b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/SSLRequestCommand.java index ea748104..959cf5f9 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/SSLRequestCommand.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/SSLRequestCommand.java @@ -42,7 +42,9 @@ public byte[] toByteArray() throws IOException { int clientCapabilities = this.clientCapabilities; if (clientCapabilities == 0) { clientCapabilities = ClientCapabilities.LONG_FLAG | - ClientCapabilities.PROTOCOL_41 | ClientCapabilities.SECURE_CONNECTION; + ClientCapabilities.PROTOCOL_41 | + ClientCapabilities.SECURE_CONNECTION | + ClientCapabilities.PLUGIN_AUTH; } clientCapabilities |= ClientCapabilities.SSL; buffer.writeInteger(clientCapabilities, 4); diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java index 7618fe40..2909b2b6 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java @@ -32,6 +32,7 @@ import com.github.shyiko.mysql.binlog.io.BufferedSocketInputStream; import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream; import com.github.shyiko.mysql.binlog.network.AuthenticationException; +import com.github.shyiko.mysql.binlog.network.SSLMode; import com.github.shyiko.mysql.binlog.network.ServerException; import com.github.shyiko.mysql.binlog.network.SocketFactory; import org.mockito.InOrder; @@ -1006,11 +1007,12 @@ public void testMysql8Auth() throws Exception { throw new SkipException("skipping mysql8 auth test"); master.execute("create user 'mysql8' IDENTIFIED WITH caching_sha2_password BY 'testpass'"); - master.execute("grant replication slave on *.* to 'mysql8'"); + master.execute("grant replication slave, replication client on *.* to 'mysql8'"); final BinaryLogClient binaryLogClient = - new BinaryLogClient(master.hostname, master.port, "mysql8", "testPass"); - binaryLogClient.connect(5000); + new BinaryLogClient(master.hostname, master.port, "mysql8", "testpass"); + binaryLogClient.setSSLMode(SSLMode.PREFERRED); + binaryLogClient.connect(500000); } @Test From ce680a801316eb009b1158ba05e00bb8c3275a23 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Mon, 4 May 2020 11:08:41 -0700 Subject: [PATCH 043/217] finish off slow path for caching_sha2_authentication this involves either sending the password over a TLS channel or grabbing an RSA key from the server and encrypting the password with that. whew! --- .../mysql/binlog/network/Authenticator.java | 53 ++++++++++---- .../AuthenticateSHA2RSAPasswordCommand.java | 73 +++++++++++++++++++ .../BinaryLogClientIntegrationTest.java | 60 +++++++++++++-- .../mysql/binlog/MysqlOnetimeServer.java | 2 +- 4 files changed, 168 insertions(+), 20 deletions(-) create mode 100644 src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2RSAPasswordCommand.java diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java b/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java index 8abb920d..fa891d2a 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java @@ -7,6 +7,7 @@ import com.github.shyiko.mysql.binlog.network.protocol.PacketChannel; import com.github.shyiko.mysql.binlog.network.protocol.command.AuthenticateNativePasswordCommand; import com.github.shyiko.mysql.binlog.network.protocol.command.AuthenticateSHA2Command; +import com.github.shyiko.mysql.binlog.network.protocol.command.AuthenticateSHA2RSAPasswordCommand; import com.github.shyiko.mysql.binlog.network.protocol.command.AuthenticateSecurityPasswordCommand; import com.github.shyiko.mysql.binlog.network.protocol.command.ByteArrayCommand; import com.github.shyiko.mysql.binlog.network.protocol.command.Command; @@ -24,6 +25,7 @@ private enum AuthMethod { }; private final GreetingPacket greetingPacket; + private String scramble; private final PacketChannel channel; private final String schema; private final String username; @@ -44,6 +46,7 @@ public Authenticator( String password ) { this.greetingPacket = greetingPacket; + this.scramble = greetingPacket.getScramble(); this.channel = channel; this.schema = schema; this.username = username; @@ -57,10 +60,10 @@ public void authenticate() throws IOException { Command authenticateCommand; if ( SHA2_PASSWORD.equals(greetingPacket.getPluginProvidedData()) ) { authMethod = AuthMethod.CACHING_SHA2; - authenticateCommand = new AuthenticateSHA2Command(schema, username, password, greetingPacket.getScramble(), collation); + authenticateCommand = new AuthenticateSHA2Command(schema, username, password, scramble, collation); } else { authMethod = AuthMethod.NATIVE; - authenticateCommand = new AuthenticateSecurityPasswordCommand(schema, username, password, greetingPacket.getScramble(), collation); + authenticateCommand = new AuthenticateSecurityPasswordCommand(schema, username, password, scramble, collation); } channel.write(authenticateCommand); @@ -100,22 +103,46 @@ private void processCachingSHA2Result(byte[] authenticationResult) throws IOExce switch(stream.read()) { case 0x03: + logger.log(Level.INFO, "caching auth successful"); // successful fast authentication + readResult(); return; case 0x04: - // need to send continue auth. - if ( channel.isSSL() ) { - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + continueCachingSHA2Authentication(); + } + } + + private void continueCachingSHA2Authentication() throws IOException { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + if ( channel.isSSL() ) { + // over SSL we simply send the password in cleartext. - buffer.write(password.getBytes()); - buffer.write(0); + buffer.write(password.getBytes()); + buffer.write(0); - Command c = new ByteArrayCommand(buffer.toByteArray()); + Command c = new ByteArrayCommand(buffer.toByteArray()); + channel.write(c); + readResult(); + } else { + // try to download an RSA key + buffer.write(0x02); + channel.write(new ByteArrayCommand(buffer.toByteArray())); + + ByteArrayInputStream stream = new ByteArrayInputStream(channel.read()); + int result = stream.read(); + switch(result) { + case 0x01: + byte[] rsaKey = new byte[stream.available()]; + stream.read(rsaKey); + + Command c = new AuthenticateSHA2RSAPasswordCommand(new String(rsaKey), password, scramble); channel.write(c); + readResult(); - } else { - throw new AuthenticationException("Please enable SSL in order to support caching_sha2_password auth"); - } + return; + default: + throw new AuthenticationException("Unkown response fetching RSA key in caching_sha2_pasword auth: " + result); + } } } @@ -132,14 +159,14 @@ private void switchAuthentication(byte[] authenticationResult) throws IOExceptio if (MYSQL_NATIVE.equals(authName)) { authMethod = AuthMethod.NATIVE; - String scramble = buffer.readZeroTerminatedString(); + this.scramble = buffer.readZeroTerminatedString(); Command switchCommand = new AuthenticateNativePasswordCommand(scramble, password); channel.write(switchCommand); } else if ( SHA2_PASSWORD.equals(authName) ) { authMethod = AuthMethod.CACHING_SHA2; - String scramble = buffer.readZeroTerminatedString(); + this.scramble = buffer.readZeroTerminatedString(); Command authCommand = new AuthenticateSHA2Command(password, scramble); channel.write(authCommand); } diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2RSAPasswordCommand.java b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2RSAPasswordCommand.java new file mode 100644 index 00000000..3799c6dd --- /dev/null +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2RSAPasswordCommand.java @@ -0,0 +1,73 @@ +package com.github.shyiko.mysql.binlog.network.protocol.command; + +import com.github.shyiko.mysql.binlog.network.AuthenticationException; + +import javax.crypto.Cipher; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.security.KeyFactory; +import java.security.interfaces.RSAPublicKey; +import java.security.spec.X509EncodedKeySpec; +import java.util.Base64; + +public class AuthenticateSHA2RSAPasswordCommand implements Command { + private static final String RSA_METHOD = "RSA/ECB/OAEPWithSHA-1AndMGF1Padding"; + private final String rsaKey; + private final String password; + private final String scramble; + + public AuthenticateSHA2RSAPasswordCommand(String rsaKey, String password, String scramble) { + this.rsaKey = rsaKey; + this.password = password; + this.scramble = scramble; + } + + @Override + public byte[] toByteArray() throws IOException { + RSAPublicKey key = decodeKey(rsaKey); + + ByteArrayOutputStream nullTerminatedPassword = new ByteArrayOutputStream(); + if ( password != null ) + nullTerminatedPassword.write(password.getBytes()); + nullTerminatedPassword.write(0); + + byte[] passBytes = nullTerminatedPassword.toByteArray(); + byte[] scrambleBytes = scramble.getBytes(); + byte[] xorBuffer = new byte[passBytes.length]; + + for(int pos = 0; pos < passBytes.length; pos++) { + xorBuffer[pos] = (byte) (passBytes[pos] ^ scrambleBytes[pos % scrambleBytes.length]); + } + + byte[] encrypted = encrypt(xorBuffer, key, RSA_METHOD); + return encrypted; + } + + private RSAPublicKey decodeKey(String key) throws AuthenticationException { + int beginIndex = key.indexOf("\n") + 1; + int endIndex = key.indexOf("-----END PUBLIC KEY-----"); + String innerKey = key.substring(beginIndex, endIndex).replaceAll("\\n", ""); + + Base64.Decoder decoder = Base64.getDecoder(); + byte[] certificateData = decoder.decode(innerKey.getBytes()); + + X509EncodedKeySpec spec = new X509EncodedKeySpec(certificateData); + try { + KeyFactory kf = KeyFactory.getInstance("RSA"); + return (RSAPublicKey) kf.generatePublic(spec); + } catch (Exception e) { + throw new AuthenticationException("Unable to decode public key: " + key); + } + } + + private byte[] encrypt(byte[] source, RSAPublicKey key, String transformation) throws AuthenticationException { + try { + Cipher cipher = Cipher.getInstance(transformation); + cipher.init(Cipher.ENCRYPT_MODE, key); + return cipher.doFinal(source); + } catch (Exception e) { + throw new AuthenticationException("couldn't encrypt password: " + e.getMessage()); + } + } + +} diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java index 2909b2b6..22d92a9f 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java @@ -35,6 +35,7 @@ import com.github.shyiko.mysql.binlog.network.SSLMode; import com.github.shyiko.mysql.binlog.network.ServerException; import com.github.shyiko.mysql.binlog.network.SocketFactory; +import com.mysql.cj.MysqlConnection; import org.mockito.InOrder; import org.testng.SkipException; import org.testng.annotations.AfterClass; @@ -1001,18 +1002,65 @@ public void execute(Statement statement) throws SQLException { } } + private BinaryLogClient setupMysql8Login(MySQLConnection server) throws Exception { + server.execute("create user 'mysql8' IDENTIFIED WITH caching_sha2_password BY 'testpass'"); + server.execute("grant replication slave, replication client on *.* to 'mysql8'"); + + return new BinaryLogClient(server.hostname, server.port, "mysql8", "testpass"); + } + @Test public void testMysql8Auth() throws Exception { if ( !mysqlVersion.atLeast(8, 0) ) throw new SkipException("skipping mysql8 auth test"); - master.execute("create user 'mysql8' IDENTIFIED WITH caching_sha2_password BY 'testpass'"); - master.execute("grant replication slave, replication client on *.* to 'mysql8'"); + BinaryLogClient client = setupMysql8Login(master); + client.setSSLMode(SSLMode.PREFERRED); + client.connect(DEFAULT_TIMEOUT); + } + + @Test + public void testMysql8FastAuth() throws Exception { + if ( !mysqlVersion.atLeast(8, 0) ) + throw new SkipException("skipping mysql8 auth test"); + + BinaryLogClient client = setupMysql8Login(master); + client.setSSLMode(SSLMode.PREFERRED); + client.connect(DEFAULT_TIMEOUT); + + client.disconnect(); + + // this call should hit the sha2 cache + client.connect(DEFAULT_TIMEOUT); + } + + + @Test + public void testSHA2CachingAuthAsDefault() throws Exception { + if ( !mysqlVersion.atLeast(8, 0) ) + throw new SkipException("skipping mysql8 auth test"); + + MysqlOnetimeServerOptions opts = new MysqlOnetimeServerOptions(); + opts.extraParams = "--default-authentication-plugin=caching_sha2_password"; + MysqlOnetimeServer server = new MysqlOnetimeServer(opts); + server.boot(); + + MySQLConnection cx = new MySQLConnection("127.0.0.1", server.getPort(), "root", ""); + + BinaryLogClient client = setupMysql8Login(cx); + client.setSSLMode(SSLMode.PREFERRED); + client.connect(DEFAULT_TIMEOUT); + + server.shutDown(); + } + + @Test + public void testSHA2CachingWithoutSSL() throws Exception { + if ( !mysqlVersion.atLeast(8, 0) ) + throw new SkipException("skipping mysql8 auth test"); - final BinaryLogClient binaryLogClient = - new BinaryLogClient(master.hostname, master.port, "mysql8", "testpass"); - binaryLogClient.setSSLMode(SSLMode.PREFERRED); - binaryLogClient.connect(500000); + BinaryLogClient client = setupMysql8Login(master); + client.connect(DEFAULT_TIMEOUT); } @Test diff --git a/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java index 3dca9cb4..a74e159e 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java @@ -59,7 +59,7 @@ public void boot() throws Exception { String authPlugin = ""; - if ( getVersion().atLeast(8, 0) ) { + if ( getVersion().atLeast(8, 0) && !xtraParams.contains("--default-authentication-plugin")) { authPlugin = "--default-authentication-plugin=mysql_native_password"; } From f84cfdebeab9c38fa707b7286f29f80a8c7ee18d Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Tue, 5 May 2020 06:59:33 -0700 Subject: [PATCH 044/217] make the tests play nice with each other --- .../BinaryLogClientIntegrationTest.java | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java index 22d92a9f..892bdc5d 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java @@ -152,6 +152,11 @@ public void execute(Statement statement) throws SQLException { } }); eventListener.waitFor(EventType.QUERY, 2, DEFAULT_TIMEOUT); + + if ( mysqlVersion.atLeast(8, 0) ) { + setupMysql8Login(master); + eventListener.waitFor(EventType.QUERY, 2, DEFAULT_TIMEOUT); + } } @BeforeMethod @@ -1002,11 +1007,9 @@ public void execute(Statement statement) throws SQLException { } } - private BinaryLogClient setupMysql8Login(MySQLConnection server) throws Exception { + private void setupMysql8Login(MySQLConnection server) throws Exception { server.execute("create user 'mysql8' IDENTIFIED WITH caching_sha2_password BY 'testpass'"); server.execute("grant replication slave, replication client on *.* to 'mysql8'"); - - return new BinaryLogClient(server.hostname, server.port, "mysql8", "testpass"); } @Test @@ -1014,7 +1017,7 @@ public void testMysql8Auth() throws Exception { if ( !mysqlVersion.atLeast(8, 0) ) throw new SkipException("skipping mysql8 auth test"); - BinaryLogClient client = setupMysql8Login(master); + BinaryLogClient client = new BinaryLogClient(master.hostname, master.port, "mysql8", "testpass"); client.setSSLMode(SSLMode.PREFERRED); client.connect(DEFAULT_TIMEOUT); } @@ -1024,7 +1027,7 @@ public void testMysql8FastAuth() throws Exception { if ( !mysqlVersion.atLeast(8, 0) ) throw new SkipException("skipping mysql8 auth test"); - BinaryLogClient client = setupMysql8Login(master); + BinaryLogClient client = new BinaryLogClient(master.hostname, master.port, "mysql8", "testpass"); client.setSSLMode(SSLMode.PREFERRED); client.connect(DEFAULT_TIMEOUT); @@ -1047,9 +1050,10 @@ public void testSHA2CachingAuthAsDefault() throws Exception { MySQLConnection cx = new MySQLConnection("127.0.0.1", server.getPort(), "root", ""); - BinaryLogClient client = setupMysql8Login(cx); - client.setSSLMode(SSLMode.PREFERRED); - client.connect(DEFAULT_TIMEOUT); + setupMysql8Login(cx); + BinaryLogClient c = new BinaryLogClient(cx.hostname, cx.port, "mysql8", "testpass"); + c.setSSLMode(SSLMode.PREFERRED); + c.connect(DEFAULT_TIMEOUT); server.shutDown(); } @@ -1059,7 +1063,7 @@ public void testSHA2CachingWithoutSSL() throws Exception { if ( !mysqlVersion.atLeast(8, 0) ) throw new SkipException("skipping mysql8 auth test"); - BinaryLogClient client = setupMysql8Login(master); + BinaryLogClient client = new BinaryLogClient(master.hostname, master.port, "mysql8", "testpass"); client.connect(DEFAULT_TIMEOUT); } From 02bd10ba42a40c93003e7017813331ae4ed2b0c7 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Tue, 5 May 2020 16:03:59 -0700 Subject: [PATCH 045/217] code clean-up; unify XOR code, switch some arguments around --- .../mysql/binlog/network/Authenticator.java | 2 +- .../command/AuthenticateSHA2Command.java | 28 +++++-------------- .../AuthenticateSHA2RSAPasswordCommand.java | 9 ++---- .../AuthenticateSecurityPasswordCommand.java | 11 +------- .../protocol/command/CommandUtils.java | 12 ++++++++ 5 files changed, 23 insertions(+), 39 deletions(-) create mode 100644 src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/CommandUtils.java diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java b/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java index fa891d2a..37728d12 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java @@ -167,7 +167,7 @@ private void switchAuthentication(byte[] authenticationResult) throws IOExceptio authMethod = AuthMethod.CACHING_SHA2; this.scramble = buffer.readZeroTerminatedString(); - Command authCommand = new AuthenticateSHA2Command(password, scramble); + Command authCommand = new AuthenticateSHA2Command(scramble, password); channel.write(authCommand); } diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2Command.java b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2Command.java index d5e7579c..b776df58 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2Command.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2Command.java @@ -31,23 +31,23 @@ public class AuthenticateSHA2Command implements Command { private String schema; private String username; private String password; - private String salt; + private String scramble; private int clientCapabilities; private int collation; private boolean rawPassword = false; - public AuthenticateSHA2Command(String schema, String username, String password, String salt, int collation) { + public AuthenticateSHA2Command(String schema, String username, String password, String scramble, int collation) { this.schema = schema; this.username = username; this.password = password; - this.salt = salt; + this.scramble = scramble; this.collation = collation; } - public AuthenticateSHA2Command(String password, String salt) { + public AuthenticateSHA2Command(String scramble, String password) { this.rawPassword = true; this.password = password; - this.salt = salt; + this.scramble = scramble; } public void setClientCapabilities(int clientCapabilities) { @@ -128,29 +128,15 @@ private byte[] encodePassword() { // SHA2(digest_stage2, m_rnd) => scramble_stage1 md.update(dig2, 0, dig1.length); - md.update(salt.getBytes(), 0, salt.getBytes().length); + md.update(scramble.getBytes(), 0, scramble.getBytes().length); md.digest(scramble1, 0, CACHING_SHA2_DIGEST_LENGTH); // XOR(digest_stage1, scramble_stage1) => scramble - byte[] mysqlScrambleBuff = new byte[CACHING_SHA2_DIGEST_LENGTH]; - xor(dig1, mysqlScrambleBuff, scramble1, CACHING_SHA2_DIGEST_LENGTH); - - return mysqlScrambleBuff; + return CommandUtils.xor(dig1, scramble1); } catch (NoSuchAlgorithmException ex) { throw new RuntimeException(ex); } catch (DigestException e) { throw new RuntimeException(e); } } - - private void xor(byte[] from, byte[] to, byte[] scramble, int length) { - int pos = 0; - int scrambleLength = scramble.length; - - while (pos < length) { - to[pos] = (byte) (from[pos] ^ scramble[pos % scrambleLength]); - pos++; - } - } - } diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2RSAPasswordCommand.java b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2RSAPasswordCommand.java index 3799c6dd..2a7913b0 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2RSAPasswordCommand.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2RSAPasswordCommand.java @@ -32,14 +32,9 @@ public byte[] toByteArray() throws IOException { nullTerminatedPassword.write(0); byte[] passBytes = nullTerminatedPassword.toByteArray(); - byte[] scrambleBytes = scramble.getBytes(); - byte[] xorBuffer = new byte[passBytes.length]; - - for(int pos = 0; pos < passBytes.length; pos++) { - xorBuffer[pos] = (byte) (passBytes[pos] ^ scrambleBytes[pos % scrambleBytes.length]); - } - + byte[] xorBuffer = CommandUtils.xor(passBytes, scramble.getBytes()); byte[] encrypted = encrypt(xorBuffer, key, RSA_METHOD); + return encrypted; } diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSecurityPasswordCommand.java b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSecurityPasswordCommand.java index d0741138..ea3dfc53 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSecurityPasswordCommand.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSecurityPasswordCommand.java @@ -94,7 +94,7 @@ public static byte[] passwordCompatibleWithMySQL411(String password, String salt throw new RuntimeException(e); } byte[] passwordHash = sha.digest(password.getBytes()); - return xor(passwordHash, sha.digest(union(salt.getBytes(), sha.digest(passwordHash)))); + return CommandUtils.xor(passwordHash, sha.digest(union(salt.getBytes(), sha.digest(passwordHash)))); } private static byte[] union(byte[] a, byte[] b) { @@ -103,13 +103,4 @@ private static byte[] union(byte[] a, byte[] b) { System.arraycopy(b, 0, r, a.length, b.length); return r; } - - private static byte[] xor(byte[] a, byte[] b) { - byte[] r = new byte[a.length]; - for (int i = 0; i < r.length; i++) { - r[i] = (byte) (a[i] ^ b[i]); - } - return r; - } - } diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/CommandUtils.java b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/CommandUtils.java new file mode 100644 index 00000000..68c09095 --- /dev/null +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/CommandUtils.java @@ -0,0 +1,12 @@ +package com.github.shyiko.mysql.binlog.network.protocol.command; + +public class CommandUtils { + public static byte[] xor(byte[] input, byte[] against) { + byte[] to = new byte[input.length]; + + for( int i = 0; i < input.length; i++ ) { + to[i] = (byte) (input[i] ^ against[i % against.length]); + } + return to; + } +} From 48fe68669c18b5fa8ae68d0c0392f7ab3aa6fd54 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Tue, 5 May 2020 16:09:35 -0700 Subject: [PATCH 046/217] use the already existing `writeZeroTerminatedString` --- .../mysql/binlog/io/ByteArrayOutputStream.java | 4 +++- .../mysql/binlog/network/Authenticator.java | 3 +-- .../AuthenticateSHA2RSAPasswordCommand.java | 15 +++++---------- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayOutputStream.java b/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayOutputStream.java index 7154ba22..0bbdf3aa 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayOutputStream.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayOutputStream.java @@ -59,7 +59,9 @@ public void writeString(String value) throws IOException { * @see ByteArrayInputStream#readZeroTerminatedString() */ public void writeZeroTerminatedString(String value) throws IOException { - write(value.getBytes()); + if ( value != null ) + write(value.getBytes()); + write(0); } diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java b/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java index 37728d12..7e597d3e 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java @@ -117,8 +117,7 @@ private void continueCachingSHA2Authentication() throws IOException { if ( channel.isSSL() ) { // over SSL we simply send the password in cleartext. - buffer.write(password.getBytes()); - buffer.write(0); + buffer.writeZeroTerminatedString(password); Command c = new ByteArrayCommand(buffer.toByteArray()); channel.write(c); diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2RSAPasswordCommand.java b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2RSAPasswordCommand.java index 2a7913b0..e8de8d4f 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2RSAPasswordCommand.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSHA2RSAPasswordCommand.java @@ -1,9 +1,9 @@ package com.github.shyiko.mysql.binlog.network.protocol.command; import com.github.shyiko.mysql.binlog.network.AuthenticationException; +import com.github.shyiko.mysql.binlog.io.ByteArrayOutputStream; import javax.crypto.Cipher; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.security.KeyFactory; import java.security.interfaces.RSAPublicKey; @@ -26,16 +26,11 @@ public AuthenticateSHA2RSAPasswordCommand(String rsaKey, String password, String public byte[] toByteArray() throws IOException { RSAPublicKey key = decodeKey(rsaKey); - ByteArrayOutputStream nullTerminatedPassword = new ByteArrayOutputStream(); - if ( password != null ) - nullTerminatedPassword.write(password.getBytes()); - nullTerminatedPassword.write(0); + ByteArrayOutputStream passBuffer = new ByteArrayOutputStream(); + passBuffer.writeZeroTerminatedString(password); - byte[] passBytes = nullTerminatedPassword.toByteArray(); - byte[] xorBuffer = CommandUtils.xor(passBytes, scramble.getBytes()); - byte[] encrypted = encrypt(xorBuffer, key, RSA_METHOD); - - return encrypted; + byte[] xorBuffer = CommandUtils.xor(passBuffer.toByteArray(), scramble.getBytes()); + return encrypt(xorBuffer, key, RSA_METHOD); } private RSAPublicKey decodeKey(String key) throws AuthenticationException { From 3810fcd12dcecefeec50a618c645e2ab9e18cea7 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Tue, 5 May 2020 16:12:07 -0700 Subject: [PATCH 047/217] downgrade logging to debug level --- .../github/shyiko/mysql/binlog/network/Authenticator.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java b/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java index 7e597d3e..bc8c684b 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java @@ -54,7 +54,7 @@ public Authenticator( } public void authenticate() throws IOException { - logger.log(Level.INFO, "Begin auth for " + username); + logger.log(Level.FINE, "Begin auth for " + username); int collation = greetingPacket.getServerCollation(); Command authenticateCommand; @@ -68,7 +68,7 @@ public void authenticate() throws IOException { channel.write(authenticateCommand); readResult(); - logger.log(Level.INFO, "Auth complete " + username); + logger.log(Level.FINE, "Auth complete " + username); } private void readResult() throws IOException { @@ -103,11 +103,12 @@ private void processCachingSHA2Result(byte[] authenticationResult) throws IOExce switch(stream.read()) { case 0x03: - logger.log(Level.INFO, "caching auth successful"); + logger.log(Level.FINE, "cached sha2 auth successful"); // successful fast authentication readResult(); return; case 0x04: + logger.log(Level.FINE, "cached sha2 auth not successful, moving to full auth path"); continueCachingSHA2Authentication(); } } @@ -134,6 +135,7 @@ private void continueCachingSHA2Authentication() throws IOException { byte[] rsaKey = new byte[stream.available()]; stream.read(rsaKey); + logger.log(Level.FINE, "received RSA key: " + rsaKey); Command c = new AuthenticateSHA2RSAPasswordCommand(new String(rsaKey), password, scramble); channel.write(c); From b33e8abd99c54c14eda4e39e1106d884f0043032 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Tue, 5 May 2020 17:35:58 -0700 Subject: [PATCH 048/217] fix EOFError crash in test suite mysql8 is, for some oddball reason, occasionally asking us to try sha256_password (non-caching) authentication. We should crash, but not with an EOFError --- .../com/github/shyiko/mysql/binlog/network/Authenticator.java | 2 ++ .../shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java | 1 + 2 files changed, 3 insertions(+) diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java b/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java index bc8c684b..0faf012d 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/Authenticator.java @@ -170,6 +170,8 @@ private void switchAuthentication(byte[] authenticationResult) throws IOExceptio this.scramble = buffer.readZeroTerminatedString(); Command authCommand = new AuthenticateSHA2Command(scramble, password); channel.write(authCommand); + } else { + throw new AuthenticationException("unsupported authentication method: " + authName); } readResult(); diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java index 892bdc5d..7b7efa31 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java @@ -851,6 +851,7 @@ public void testExceptionIsThrownWhenInsufficientPermissionsToDetectPosition() t String prefix = "jdbc.mysql.replication."; String slaveUsername = bundle.getString(prefix + "slave.slaveUsername"); String slavePassword = bundle.getString(prefix + "slave.slavePassword"); + new BinaryLogClient(slave.hostname, slave.port, slaveUsername, slavePassword).connect(); } From 5f8e7b58f549740d13c13977d55420436210ad6d Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Mon, 11 May 2020 21:59:39 -0700 Subject: [PATCH 049/217] bump version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 79a568f5..1d8253ff 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.zendesk mysql-binlog-connector-java - 0.22.2 + 0.23.0-rc1 mysql-binlog-connector-java MySQL Binary Log connector From 1a00ac240a59d6504744c3f75e5f3f0256897be4 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Mon, 11 May 2020 21:59:49 -0700 Subject: [PATCH 050/217] remove unused var --- .../java/com/github/shyiko/mysql/binlog/BinaryLogClient.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java index 16d47cd5..ce8a1abd 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java @@ -141,7 +141,6 @@ public X509Certificate[] getAcceptedIssuers() { private boolean useBinlogFilenamePositionInGtidMode; private String gtid; private boolean tx; - private boolean isSSL; private EventDeserializer eventDeserializer = new EventDeserializer(); From d1395ed1a534d2bdc7f984589c28fd26b35b92de Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Mon, 11 May 2020 22:10:31 -0700 Subject: [PATCH 051/217] rework resultset error handling do an obvious hoist into a function also add code targeting https://github.com/zendesk/maxwell/issues/1374, where a row itself can have an ErrPacket --- .../shyiko/mysql/binlog/BinaryLogClient.java | 40 ++++++++----------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java index ce8a1abd..2fe57f53 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java @@ -647,14 +647,19 @@ public Object call() throws Exception { }; } - private GreetingPacket receiveGreeting() throws IOException { - byte[] initialHandshakePacket = channel.read(); - if (initialHandshakePacket[0] == (byte) 0xFF /* error */) { - byte[] bytes = Arrays.copyOfRange(initialHandshakePacket, 1, initialHandshakePacket.length); + private void checkError(byte[] packet) throws IOException { + if (packet[0] == (byte) 0xFF /* error */) { + byte[] bytes = Arrays.copyOfRange(packet, 1, packet.length); ErrorPacket errorPacket = new ErrorPacket(bytes); throw new ServerException(errorPacket.getErrorMessage(), errorPacket.getErrorCode(), - errorPacket.getSqlState()); + errorPacket.getSqlState()); } + } + + private GreetingPacket receiveGreeting() throws IOException { + byte[] initialHandshakePacket = channel.read(); + checkError(initialHandshakePacket); + return new GreetingPacket(initialHandshakePacket); } @@ -690,12 +695,7 @@ private boolean tryUpgradeToSSL(GreetingPacket greetingPacket) throws IOExceptio private void enableHeartbeat() throws IOException { channel.write(new QueryCommand("set @master_heartbeat_period=" + heartbeatInterval * 1000000)); byte[] statementResult = channel.read(); - if (statementResult[0] == (byte) 0xFF /* error */) { - byte[] bytes = Arrays.copyOfRange(statementResult, 1, statementResult.length); - ErrorPacket errorPacket = new ErrorPacket(bytes); - throw new ServerException(errorPacket.getErrorMessage(), errorPacket.getErrorCode(), - errorPacket.getSqlState()); - } + checkError(statementResult); } private void setMasterServerId() throws IOException { @@ -904,12 +904,7 @@ private ChecksumType fetchBinlogChecksum() throws IOException { private void confirmSupportOfChecksum(ChecksumType checksumType) throws IOException { channel.write(new QueryCommand("set @master_binlog_checksum= @@global.binlog_checksum")); byte[] statementResult = channel.read(); - if (statementResult[0] == (byte) 0xFF /* error */) { - byte[] bytes = Arrays.copyOfRange(statementResult, 1, statementResult.length); - ErrorPacket errorPacket = new ErrorPacket(bytes); - throw new ServerException(errorPacket.getErrorMessage(), errorPacket.getErrorCode(), - errorPacket.getSqlState()); - } + checkError(statementResult); eventDeserializer.setChecksumType(checksumType); } @@ -1051,16 +1046,13 @@ private void commitGtid() { } private ResultSetRowPacket[] readResultSet() throws IOException { - List resultSet = new LinkedList(); + List resultSet = new LinkedList<>(); byte[] statementResult = channel.read(); - if (statementResult[0] == (byte) 0xFF /* error */) { - byte[] bytes = Arrays.copyOfRange(statementResult, 1, statementResult.length); - ErrorPacket errorPacket = new ErrorPacket(bytes); - throw new ServerException(errorPacket.getErrorMessage(), errorPacket.getErrorCode(), - errorPacket.getSqlState()); - } + checkError(statementResult); + while ((channel.read())[0] != (byte) 0xFE /* eof */) { /* skip */ } for (byte[] bytes; (bytes = channel.read())[0] != (byte) 0xFE /* eof */; ) { + checkError(bytes); resultSet.add(new ResultSetRowPacket(bytes)); } return resultSet.toArray(new ResultSetRowPacket[resultSet.size()]); From 073532e55d029f69444830c332c0b47f089641cc Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Mon, 25 May 2020 22:22:42 -0700 Subject: [PATCH 052/217] v0.23.1 --- pom.xml | 2 +- .../deserialization/AbstractRowsEventDataDeserializer.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 1d8253ff..200b0c3f 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.zendesk mysql-binlog-connector-java - 0.23.0-rc1 + 0.23.1 mysql-binlog-connector-java MySQL Binary Log connector diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java index e7cf6d53..0b2ac624 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java @@ -411,7 +411,6 @@ protected byte[] deserializeJson(int meta, ByteArrayInputStream inputStream) thr return inputStream.read(blobLength); } - // checkstyle, please ignore ParameterNumber for the next line protected Long asUnixTime(int year, int month, int day, int hour, int minute, int second, int millis) { // https://dev.mysql.com/doc/refman/5.0/en/datetime.html if (year == 0 || month == 0 || day == 0) { From f546e2e88b2f67db77422fd22169d01a49b898e1 Mon Sep 17 00:00:00 2001 From: Mark Crisp Date: Thu, 23 Jul 2020 19:50:12 -0400 Subject: [PATCH 053/217] propagate all exceptions that occur in binary log client --- .../com/github/shyiko/mysql/binlog/BinaryLogClient.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java index 2fe57f53..e4b8b4df 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java @@ -495,8 +495,9 @@ public void setThreadFactory(ThreadFactory threadFactory) { * @throws AuthenticationException if authentication fails * @throws ServerException if MySQL server responds with an error * @throws IOException if anything goes wrong while trying to connect + * @throws IllegalStateException if binary log client is already connected */ - public void connect() throws IOException { + public void connect() throws IOException, IllegalStateException { if (!connectLock.tryLock()) { throw new IllegalStateException("BinaryLogClient is already connected"); } @@ -836,8 +837,8 @@ public void run() { try { setConnectTimeout(timeout); connect(); - } catch (IOException e) { - exceptionReference.set(e); + } catch (Exception e) { + exceptionReference.set(new IOException(e)); // method is asynchronous, catch all exceptions so that they are not lost countDownLatch.countDown(); // making sure we don't end up waiting whole "timeout" } } From d3c019e77546e1813dfd98266d98a56a24e0fbf0 Mon Sep 17 00:00:00 2001 From: Mark Crisp Date: Thu, 23 Jul 2020 20:03:58 -0400 Subject: [PATCH 054/217] added integration test --- .../shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java index 7b7efa31..aa81b588 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java @@ -833,6 +833,12 @@ public void testExceptionIsThrownWhenTryingToConnectAlreadyConnectedClient() thr client.connect(); } + @Test(expectedExceptions = IOException.class) + public void testExceptionIsThrownWhenTryingToConnectAlreadyConnectedClientWithTimeout() throws Exception { + assertTrue(client.isConnected()); + client.connect(1000); + } + @Test public void testExceptionIsThrownWhenProvidedWithWrongCredentials() throws Exception { BinaryLogClient binaryLogClient = From 48623192a02f02826a39c89b9356ae5dd4490ba1 Mon Sep 17 00:00:00 2001 From: Mark Crisp Date: Sat, 25 Jul 2020 17:37:51 -0400 Subject: [PATCH 055/217] fixed auth exception test --- .../java/com/github/shyiko/mysql/binlog/BinaryLogClient.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java index e4b8b4df..e3f1857c 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java @@ -837,6 +837,9 @@ public void run() { try { setConnectTimeout(timeout); connect(); + } catch (IOException e) { + exceptionReference.set(e); + countDownLatch.countDown(); // making sure we don't end up waiting whole "timeout" } catch (Exception e) { exceptionReference.set(new IOException(e)); // method is asynchronous, catch all exceptions so that they are not lost countDownLatch.countDown(); // making sure we don't end up waiting whole "timeout" From 2e6147bc81ac458a7325e220840be285388a0145 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Sat, 25 Jul 2020 21:49:15 -0700 Subject: [PATCH 056/217] v0.23.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 200b0c3f..c8769558 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.zendesk mysql-binlog-connector-java - 0.23.1 + 0.23.2 mysql-binlog-connector-java MySQL Binary Log connector From 9764d2ee03b56d0c122068309a65a5a713d1b3ca Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Tue, 4 Aug 2020 19:39:43 -0700 Subject: [PATCH 057/217] update new maven slug --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0c11b688..adf41cb4 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,9 @@ Get the latest JAR(s) from [here](http://search.maven.org/#search%7Cga%7C1%7Cg%3 ```xml - com.github.shyiko + com.zendesk mysql-binlog-connector-java - 0.18.1 + 0.23.2 ``` From 8ca21a43f1de9b5ce9abe8597acb79f18ac63056 Mon Sep 17 00:00:00 2001 From: auntyellow Date: Sun, 9 Aug 2020 22:23:11 +0800 Subject: [PATCH 058/217] update maven url --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index adf41cb4..05d65a8c 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ but ended up as a complete rewrite. Key differences/features: ## Usage -Get the latest JAR(s) from [here](http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22com.github.shyiko%22%20AND%20a%3A%22mysql-binlog-connector-java%22). Alternatively you can include following Maven dependency (available through Maven Central): +Get the latest JAR(s) from [here](https://search.maven.org/search?q=g:com.zendesk%20AND%20a:mysql-binlog-connector-java). Alternatively you can include following Maven dependency (available through Maven Central): ```xml From 7d6d12349fd647733a0a727905ffe9c78595d402 Mon Sep 17 00:00:00 2001 From: zzt Date: Sat, 24 Oct 2020 21:38:26 +0800 Subject: [PATCH 059/217] add EventDeserializer.CompatibilityMode.INTEGER_AS_BYTE_ARRAY impl. --- .../AbstractRowsEventDataDeserializer.java | 20 +++++++++++++++++++ .../deserialization/EventDeserializer.java | 9 ++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java index 0b2ac624..7b910da4 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java @@ -75,6 +75,7 @@ public abstract class AbstractRowsEventDataDeserializer imp private Long invalidDateAndTimeRepresentation; private boolean microsecondsPrecision; private boolean deserializeCharAndBinaryAsByteArray; + private boolean deserializeIntegerAsByteArray; public AbstractRowsEventDataDeserializer(Map tableMapEventByTableId) { this.tableMapEventByTableId = tableMapEventByTableId; @@ -97,6 +98,10 @@ void setDeserializeCharAndBinaryAsByteArray(boolean value) { this.deserializeCharAndBinaryAsByteArray = value; } + void setDeserializeIntegerAsByteArray(boolean deserializeIntegerAsByteArray) { + this.deserializeIntegerAsByteArray = deserializeIntegerAsByteArray; + } + protected Serializable[] deserializeRow(long tableId, BitSet includedColumns, ByteArrayInputStream inputStream) throws IOException { TableMapEventData tableMapEvent = tableMapEventByTableId.get(tableId); @@ -203,22 +208,37 @@ protected Serializable deserializeBit(int meta, ByteArrayInputStream inputStream } protected Serializable deserializeTiny(ByteArrayInputStream inputStream) throws IOException { + if (deserializeIntegerAsByteArray) { + return inputStream.read(1); + } return (int) ((byte) inputStream.readInteger(1)); } protected Serializable deserializeShort(ByteArrayInputStream inputStream) throws IOException { + if (deserializeIntegerAsByteArray) { + return inputStream.read(2); + } return (int) ((short) inputStream.readInteger(2)); } protected Serializable deserializeInt24(ByteArrayInputStream inputStream) throws IOException { + if (deserializeIntegerAsByteArray) { + return inputStream.read(3); + } return (inputStream.readInteger(3) << 8) >> 8; } protected Serializable deserializeLong(ByteArrayInputStream inputStream) throws IOException { + if (deserializeIntegerAsByteArray) { + return inputStream.read(4); + } return inputStream.readInteger(4); } protected Serializable deserializeLongLong(ByteArrayInputStream inputStream) throws IOException { + if (deserializeIntegerAsByteArray) { + return inputStream.read(8); + } return inputStream.readLong(8); } diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java index d623d943..fbec7c38 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java @@ -200,6 +200,9 @@ private void ensureCompatibility(EventDataDeserializer eventDataDeserializer) { deserializer.setDeserializeCharAndBinaryAsByteArray( compatibilitySet.contains(CompatibilityMode.CHAR_AND_BINARY_AS_BYTE_ARRAY) ); + deserializer.setDeserializeIntegerAsByteArray( + compatibilitySet.contains(CompatibilityMode.INTEGER_AS_BYTE_ARRAY) + ); } } @@ -350,7 +353,11 @@ public enum CompatibilityMode { * *

This option is going to be enabled by default starting from mysql-binlog-connector-java@1.0.0. */ - CHAR_AND_BINARY_AS_BYTE_ARRAY + CHAR_AND_BINARY_AS_BYTE_ARRAY, + /** + * Return TINY/SHORT/INT24/LONG/LONGLONG values as byte[]|s (instead of int|s). + */ + INTEGER_AS_BYTE_ARRAY } /** From 09e9e64cb29c41dd8dca1148a37e06e9e45b4d10 Mon Sep 17 00:00:00 2001 From: zzt Date: Sun, 25 Oct 2020 15:05:17 +0800 Subject: [PATCH 060/217] add testDeserializationOfIntegerAsByteArray in BinaryLogClientIntegrationTest. --- .../BinaryLogClientIntegrationTest.java | 38 +++++++++++++++++++ src/test/onetimeserver | 6 +-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java index aa81b588..44bb2fa5 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java @@ -421,6 +421,44 @@ public void testDeserializationOfDateAndTimeAsLong() throws Exception { } } + @Test + public void testDeserializationOfIntegerAsByteArray() throws Exception { + final BinaryLogClient client = new BinaryLogClient(slave.hostname, slave.port, + slave.username, slave.password); + EventDeserializer eventDeserializer = new EventDeserializer(); + eventDeserializer.setCompatibilityMode(CompatibilityMode.INTEGER_AS_BYTE_ARRAY); + client.setEventDeserializer(eventDeserializer); + client.connect(DEFAULT_TIMEOUT); + try { + assertEquals(writeAndCaptureRow("tinyint unsigned", "0", "1", "255"), + new Serializable[]{new byte[]{0}, new byte[]{1}, new byte[]{-1}}); + assertEquals(writeAndCaptureRow("tinyint", "-128", "-1", "0", "1", "127"), + new Serializable[]{new byte[]{-0x80}, new byte[]{-1}, new byte[]{0}, new byte[]{1}, new byte[]{0x7f}}); + + assertEquals(writeAndCaptureRow("smallint unsigned", "0", "1", "65535"), + new Serializable[]{new byte[]{0, 0}, new byte[]{0, 1}, new byte[]{-1, -1}}); + assertEquals(writeAndCaptureRow("smallint", "-32768", "-1", "0", "1", "32767"), + new Serializable[]{new byte[]{-0x80, 0}, new byte[]{-1, -1}, new byte[]{0, 0}, new byte[]{0, 1}, new byte[]{0x7f, -1}}); + + assertEquals(writeAndCaptureRow("mediumint unsigned", "0", "1", "16777215"), + new Serializable[]{new byte[]{0, 0, 0}, new byte[]{0, 0, 1}, new byte[]{-1, -1, -1}}); + assertEquals(writeAndCaptureRow("mediumint", "-8388608", "-1", "0", "1", "8388607"), + new Serializable[]{new byte[]{-0x80, 0, 0}, new byte[]{-1, -1, -1}, new byte[]{0, 0, 0}, new byte[]{0, 0, 1}, new byte[]{0x7f, -1, -1}}); + + assertEquals(writeAndCaptureRow("int unsigned", "0", "1", "4294967295"), + new Serializable[]{new byte[]{0, 0, 0, 0}, new byte[]{0, 0, 0, 1}, new byte[]{-1, -1, -1, -1}}); + assertEquals(writeAndCaptureRow("int", "-2147483648", "-1", "0", "1", "2147483647"), + new Serializable[]{new byte[]{-0x80, 0, 0, 0}, new byte[]{-1, -1, -1, -1}, new byte[]{0, 0, 0, 0}, new byte[]{0, 0, 0, 1}, new byte[]{0x7f, -1, -1, -1}}); + + assertEquals(writeAndCaptureRow("bigint unsigned", "0", "1", "18446744073709551615"), + new Serializable[]{new byte[]{0, 0, 0, 0, 0, 0, 0, 0}, new byte[]{0, 0, 0, 0, 0, 0, 0, 1}, new byte[]{-1, -1, -1, -1, -1, -1, -1, -1}}); + assertEquals(writeAndCaptureRow("bigint", "-9223372036854775808", "-1", "0", "1", "9223372036854775807"), + new Serializable[]{new byte[]{-0x80, 0, 0, 0, 0, 0, 0, 0}, new byte[]{-1, -1, -1, -1, -1, -1, -1, -1}, new byte[]{0, 0, 0, 0, 0, 0, 0, 0}, new byte[]{0, 0, 0, 0, 0, 0, 0, 1}, new byte[]{0x7f, -1, -1, -1, -1, -1, -1, -1}}); + } finally { + client.disconnect(); + } + } + @Test public void testDeserializationOfDateAndTimeAsLongMicrosecondsPrecision() throws Exception { final BinaryLogClient client = new BinaryLogClient(slave.hostname, slave.port, diff --git a/src/test/onetimeserver b/src/test/onetimeserver index a240a9c9..cbf8a2f2 100755 --- a/src/test/onetimeserver +++ b/src/test/onetimeserver @@ -5,9 +5,9 @@ PLATFORM=${_PLATFORM/ /-} OS=`uname -s | tr '[:upper:]' '[:lower:]'` -THIS_URL="https://raw.githubusercontent.com/osheroff/onetimeserver/master/onetimeserver" -WRAPPER_URL="https://raw.githubusercontent.com/osheroff/onetimeserver/master/wrapper/wrapper.c" -ONETIMESERVER_URL="https://raw.githubusercontent.com/osheroff/onetimeserver-binaries/master/onetimeserver-go/$OS/onetimeserver-go" +THIS_URL="https://gitee.com/zengzetang/onetimeserver/raw/master/onetimeserver" +WRAPPER_URL="https://gitee.com/zengzetang/onetimeserver/raw/master/wrapper/wrapper.c" +ONETIMESERVER_URL="http://localhost:8888/Desktop/onetimeserver-go.txt" CACHE_DIR=$HOME/.onetimeserver/$PLATFORM function usage() { From 45586e8a8183dc0d95b645e65f09fb2c4b2ade3a Mon Sep 17 00:00:00 2001 From: zzt Date: Sun, 25 Oct 2020 15:06:51 +0800 Subject: [PATCH 061/217] recover onetimeserver script. --- src/test/onetimeserver | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/onetimeserver b/src/test/onetimeserver index cbf8a2f2..a240a9c9 100755 --- a/src/test/onetimeserver +++ b/src/test/onetimeserver @@ -5,9 +5,9 @@ PLATFORM=${_PLATFORM/ /-} OS=`uname -s | tr '[:upper:]' '[:lower:]'` -THIS_URL="https://gitee.com/zengzetang/onetimeserver/raw/master/onetimeserver" -WRAPPER_URL="https://gitee.com/zengzetang/onetimeserver/raw/master/wrapper/wrapper.c" -ONETIMESERVER_URL="http://localhost:8888/Desktop/onetimeserver-go.txt" +THIS_URL="https://raw.githubusercontent.com/osheroff/onetimeserver/master/onetimeserver" +WRAPPER_URL="https://raw.githubusercontent.com/osheroff/onetimeserver/master/wrapper/wrapper.c" +ONETIMESERVER_URL="https://raw.githubusercontent.com/osheroff/onetimeserver-binaries/master/onetimeserver-go/$OS/onetimeserver-go" CACHE_DIR=$HOME/.onetimeserver/$PLATFORM function usage() { From 9e2fb39d2d7c1b4a4833beea5aa351894cdc84cf Mon Sep 17 00:00:00 2001 From: Sergei Morozov Date: Tue, 27 Oct 2020 13:30:24 -0700 Subject: [PATCH 062/217] DBZ-2499: Debezium Connectors are failing while reading binlog: Unknown event type 100 --- .../EventHeaderV4Deserializer.java | 4 +-- .../BinaryLogFileReaderIntegrationTest.java | 31 ++++++++++++++++++ src/test/resources/mysql-bin.aurora-padding | Bin 0 -> 1294 bytes 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 src/test/resources/mysql-bin.aurora-padding diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java index 2d8bcdd8..c0569f09 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java @@ -40,9 +40,9 @@ public EventHeaderV4 deserialize(ByteArrayInputStream inputStream) throws IOExce return header; } - private static EventType getEventType(int ordinal) throws IOException { + private static EventType getEventType(int ordinal) { if (ordinal >= EVENT_TYPES.length) { - throw new IOException("Unknown event type " + ordinal); + return EventType.UNKNOWN; } return EVENT_TYPES[ordinal]; } diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogFileReaderIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogFileReaderIntegrationTest.java index cbf523a6..ef9eca57 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogFileReaderIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogFileReaderIntegrationTest.java @@ -30,6 +30,7 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; @@ -70,6 +71,36 @@ public void testChecksumCRC32WithCustomEventDataDeserializer() throws Exception readAll(reader, 303); } + @Test + public void testUnsupportedEventType() throws Exception { + EventDeserializer eventDeserializer = new EventDeserializer(); + + // mysql> SHOW BINLOG EVENTS IN 'mysql-bin.aurora-padding'; + // +--------------------------+------+----------------+-------------+---------------------------------------+ + // | Log_name | Pos | Event_type | End_log_pos | Info | + // +--------------------------+------+----------------+-------------+---------------------------------------+ + // | mysql-bin.aurora-padding | 4 | Format_desc | 185 | Server ver: 5.7.12-log, Binlog ver: 4 | + // | mysql-bin.aurora-padding | 185 | Previous_gtids | 216 | | + // | mysql-bin.aurora-padding | 216 | Anonymous_Gtid | 281 | SET @@SESSION.GTID_NEXT= 'ANONYMOUS' | + // | mysql-bin.aurora-padding | 281 | Aurora_padding | 1209 | Ignorable | + // | mysql-bin.aurora-padding | 1209 | Query | 1294 | BEGIN | + BinaryLogFileReader reader = new BinaryLogFileReader( + new FileInputStream("src/test/resources/mysql-bin.aurora-padding"), eventDeserializer); + try { + for (int i = 0; i < 3; i++) { + assertNotNull(reader.readEvent()); + } + try { + reader.readEvent(); + } catch (IOException e) { + // this simulates the Debezium's event.processing.failure.handling.mode = warn + } + assertEquals(reader.readEvent().getHeader().getEventType(), EventType.QUERY); + } finally { + reader.close(); + } + } + private void readAll(BinaryLogFileReader reader, int expect) throws IOException { try { int numberOfEvents = 0; diff --git a/src/test/resources/mysql-bin.aurora-padding b/src/test/resources/mysql-bin.aurora-padding new file mode 100644 index 0000000000000000000000000000000000000000..688b51b2c9b31f157f40c768366da58da03e4dd9 GIT binary patch literal 1294 zcmeyDl$p0eeNsHX0CybMRt5%!oj}aMz`|gvXRc>xq??nU4i+Gi5VqiD;9wA9U;zRl z28Ou|EK(dmtsER2OhAf@i%Uz3LC6Fof@UBKlpmj&l$IUOz+eRBv!ba3aT)*U{}TW? zS{dYMd7#J*AZ`F+2sl3Z$4ij35=hz+C??4Wk^mDB6{sZB%WR0i7bt!ES!DpofE17c z3z&g{0(4#jgVrb*4S~@R82AwQJ?~I3D1S16@@FV8@A3hQ2@Zyh_aA`sI41*x5+p-| uxDFsD8*_0<3X3ko1E4Ziw&2L%5Le%nr1-qll7hra1}9f{Pe0issn!6j|H|k9 literal 0 HcmV?d00001 From 712ce17320e1b4d4556d511289fecbe7ad2ec3e6 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Thu, 29 Oct 2020 15:32:36 -0700 Subject: [PATCH 063/217] fix tests --- .../BinaryLogClientIntegrationTest.java | 55 +++++++++++-------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java index 44bb2fa5..8d813793 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java @@ -430,30 +430,37 @@ public void testDeserializationOfIntegerAsByteArray() throws Exception { client.setEventDeserializer(eventDeserializer); client.connect(DEFAULT_TIMEOUT); try { - assertEquals(writeAndCaptureRow("tinyint unsigned", "0", "1", "255"), - new Serializable[]{new byte[]{0}, new byte[]{1}, new byte[]{-1}}); - assertEquals(writeAndCaptureRow("tinyint", "-128", "-1", "0", "1", "127"), - new Serializable[]{new byte[]{-0x80}, new byte[]{-1}, new byte[]{0}, new byte[]{1}, new byte[]{0x7f}}); - - assertEquals(writeAndCaptureRow("smallint unsigned", "0", "1", "65535"), - new Serializable[]{new byte[]{0, 0}, new byte[]{0, 1}, new byte[]{-1, -1}}); - assertEquals(writeAndCaptureRow("smallint", "-32768", "-1", "0", "1", "32767"), - new Serializable[]{new byte[]{-0x80, 0}, new byte[]{-1, -1}, new byte[]{0, 0}, new byte[]{0, 1}, new byte[]{0x7f, -1}}); - - assertEquals(writeAndCaptureRow("mediumint unsigned", "0", "1", "16777215"), - new Serializable[]{new byte[]{0, 0, 0}, new byte[]{0, 0, 1}, new byte[]{-1, -1, -1}}); - assertEquals(writeAndCaptureRow("mediumint", "-8388608", "-1", "0", "1", "8388607"), - new Serializable[]{new byte[]{-0x80, 0, 0}, new byte[]{-1, -1, -1}, new byte[]{0, 0, 0}, new byte[]{0, 0, 1}, new byte[]{0x7f, -1, -1}}); - - assertEquals(writeAndCaptureRow("int unsigned", "0", "1", "4294967295"), - new Serializable[]{new byte[]{0, 0, 0, 0}, new byte[]{0, 0, 0, 1}, new byte[]{-1, -1, -1, -1}}); - assertEquals(writeAndCaptureRow("int", "-2147483648", "-1", "0", "1", "2147483647"), - new Serializable[]{new byte[]{-0x80, 0, 0, 0}, new byte[]{-1, -1, -1, -1}, new byte[]{0, 0, 0, 0}, new byte[]{0, 0, 0, 1}, new byte[]{0x7f, -1, -1, -1}}); - - assertEquals(writeAndCaptureRow("bigint unsigned", "0", "1", "18446744073709551615"), - new Serializable[]{new byte[]{0, 0, 0, 0, 0, 0, 0, 0}, new byte[]{0, 0, 0, 0, 0, 0, 0, 1}, new byte[]{-1, -1, -1, -1, -1, -1, -1, -1}}); - assertEquals(writeAndCaptureRow("bigint", "-9223372036854775808", "-1", "0", "1", "9223372036854775807"), - new Serializable[]{new byte[]{-0x80, 0, 0, 0, 0, 0, 0, 0}, new byte[]{-1, -1, -1, -1, -1, -1, -1, -1}, new byte[]{0, 0, 0, 0, 0, 0, 0, 0}, new byte[]{0, 0, 0, 0, 0, 0, 0, 1}, new byte[]{0x7f, -1, -1, -1, -1, -1, -1, -1}}); + Serializable[] result; + + result = writeAndCaptureRow("tinyint unsigned", "0", "1", "255"); + assertEquals(result[0], 0); + assertEquals(result[1], 1); + assertEquals(result[2], -1); + + + result = writeAndCaptureRow("tinyint", "-128", "-1", "0", "1", "127"); + assertEquals(result[0], -128); + assertEquals(result[1], -1); + assertEquals(result[2], 0); + assertEquals(result[3], 1); + assertEquals(result[4], 127); + + result = writeAndCaptureRow("smallint unsigned", "0", "1", "65535"); + assertEquals(result[0], 0); + assertEquals(result[1], 1); + assertEquals(result[2], -1); + + result = writeAndCaptureRow("smallint", "-32768", "-1", "0", "1", "32767"); + assertEquals(result[0], -32768); + assertEquals(result[1], -1); + assertEquals(result[2], 0); + assertEquals(result[3], 1); + assertEquals(result[4], 32767); + + result = writeAndCaptureRow("mediumint unsigned", "0", "1", "16777215"); + assertEquals(result[0], 0); + assertEquals(result[1], 1); + assertEquals(result[2], -1); } finally { client.disconnect(); } From b4aaed26c927f2a866e9b7d13badb9f3ab01c41b Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Thu, 29 Oct 2020 15:48:32 -0700 Subject: [PATCH 064/217] v0.23.3 --- CHANGELOG.md | 13 +++++++++++++ pom.xml | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ba26f3c..68943c40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## [0.23.3](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.23.2...0.23.3) - 2020-10-29 + +- add EventDeserializer.CompatibilityMode.INTEGER_AS_BYTE_ARRAY if you want raw integer data +- don't crash on AWS Aurora's unknown event types + +## [0.23.2](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.23.1...0.23.2) - 2020-07-25 + +- `connect` now throws `IllegalStateException` when already connected + +## [0.23.1](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.22.2...0.23.1) - 2020-05-25 + +- this releases adds support for mysql 8's `caching_sha2_password` authentication method + ## [0.22.2](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.22.0...0.22.2) - 2020-04-29 - Fix bugs in 0.22.0 involving nested JSON objects. diff --git a/pom.xml b/pom.xml index c8769558..2d6d2bea 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.zendesk mysql-binlog-connector-java - 0.23.2 + 0.23.3 mysql-binlog-connector-java MySQL Binary Log connector From a0a65cdd86a2ee0d6dee015a6062e9607b1cd3ae Mon Sep 17 00:00:00 2001 From: Gil Cottle Date: Fri, 15 Jan 2021 22:29:05 -0800 Subject: [PATCH 065/217] Add Auth Plugin Name to for native password auth to fix Azure connections PLUGIN_AUTH is listed as one of the capabilities and connections to Azure MySQL require the Auth Plugin Name to be specified when that capability is present. If it is not there, Azure MySQL instances respond with "The connection string may not be right. Please visit portal for references.". See https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::HandshakeResponse and https://github.com/osheroff/mysql-binlog-connector-java/issues/26 for more details about the investigation. --- .../protocol/command/AuthenticateSecurityPasswordCommand.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSecurityPasswordCommand.java b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSecurityPasswordCommand.java index ea3dfc53..de93da45 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSecurityPasswordCommand.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSecurityPasswordCommand.java @@ -77,6 +77,7 @@ public byte[] toByteArray() throws IOException { if (schema != null) { buffer.writeZeroTerminatedString(schema); } + buffer.writeZeroTerminatedString("mysql_native_password"); return buffer.toByteArray(); } From ed9336a49ee96862a6ab923dbf84b0b1015df7e2 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Sun, 17 Jan 2021 10:06:17 -0800 Subject: [PATCH 066/217] v0.23.4 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2d6d2bea..916fc616 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.zendesk mysql-binlog-connector-java - 0.23.3 + 0.23.4 mysql-binlog-connector-java MySQL Binary Log connector From fc2da16ec801def74680d63d780b1da7edef050d Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Thu, 21 Jan 2021 22:09:41 -0800 Subject: [PATCH 067/217] wip: new tls verifier so we can upgrade to jvm 11 --- pom.xml | 13 +- .../mysql/binlog/network/HostnameChecker.java | 568 ++++++++++++++++++ .../binlog/network/TLSHostnameVerifier.java | 10 +- .../mysql/binlog/BinaryLogClientTest.java | 4 +- 4 files changed, 581 insertions(+), 14 deletions(-) create mode 100644 src/main/java/com/github/shyiko/mysql/binlog/network/HostnameChecker.java diff --git a/pom.xml b/pom.xml index 916fc616..a21feffa 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.zendesk mysql-binlog-connector-java - 0.23.4 + 0.23.4-SNAPSHOT mysql-binlog-connector-java MySQL Binary Log connector @@ -228,17 +228,18 @@ org.apache.maven.plugins maven-javadoc-plugin - 2.10.4 + 3.2.0 + + buar + -J--add-exports + -Jjava.base/sun.security.util=ALL-UNNAMED + attach-javadocs jar - - -Xdoclint:none - true - diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/HostnameChecker.java b/src/main/java/com/github/shyiko/mysql/binlog/network/HostnameChecker.java new file mode 100644 index 00000000..86b73aed --- /dev/null +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/HostnameChecker.java @@ -0,0 +1,568 @@ +/* + * $HeadURL: file:///opt/dev/not-yet-commons-ssl-SVN-repo/tags/commons-ssl-0.3.17/src/java/org/apache/commons/ssl/HostnameVerifier.java $ + * $Revision: 121 $ + * $Date: 2007-11-13 21:26:57 -0800 (Tue, 13 Nov 2007) $ + * + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ + +/* + * after looking around the landscape of java verifying a certificate hostname, the best I found + * was to copy and paste from https://github.com/narupley/not-going-to-be-commons-ssl/blob/master/src/main/java/org/apache/commons/ssl/Certificates.java + * + * given that it seems like it's never going to be maintained upstream... + * is it bad? time will tell. + */ + +package com.github.shyiko.mysql.binlog.network; + +import javax.naming.InvalidNameException; +import javax.naming.NamingException; +import javax.naming.directory.Attribute; +import javax.naming.directory.Attributes; +import javax.naming.ldap.LdapName; +import javax.naming.ldap.Rdn; +import javax.net.ssl.SSLException; +import javax.net.ssl.SSLPeerUnverifiedException; +import javax.net.ssl.SSLSession; +import javax.net.ssl.SSLSocket; +import javax.security.auth.x500.X500Principal; +import java.io.IOException; +import java.io.InputStream; +import java.security.cert.Certificate; +import java.security.cert.X509Certificate; +import java.util.Arrays; +import java.util.Collection; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.TreeSet; + +/** + * Interface for checking if a hostname matches the names stored inside the + * server's X.509 certificate. Correctly implements + * javax.net.ssl.HostnameVerifier, but that interface is not recommended. + * Instead we added several check() methods that take SSLSocket, + * or X509Certificate, or ultimately (they all end up calling this one), + * String. (It's easier to supply JUnit with Strings instead of mock + * SSLSession objects!) + *

Our check() methods throw exceptions if the name is + * invalid, whereas javax.net.ssl.HostnameVerifier just returns true/false. + *

+ * We provide the HostnameVerifier.DEFAULT, HostnameVerifier.STRICT, and + * HostnameVerifier.ALLOW_ALL implementations. We also provide the more + * specialized HostnameVerifier.DEFAULT_AND_LOCALHOST, as well as + * HostnameVerifier.STRICT_IE6. But feel free to define your own + * implementations! + *

+ * Inspired by Sebastian Hauer's original StrictSSLProtocolSocketFactory in the + * HttpClient "contrib" repository. + * + * @author Julius Davies + * @author Sebastian Hauer + * @since 8-Dec-2006 + */ +public interface HostnameChecker extends javax.net.ssl.HostnameVerifier { + + boolean verify(String host, SSLSession session); + + void check(String host, SSLSocket ssl) throws IOException; + + void check(String host, X509Certificate cert) throws SSLException; + + void check(String host, String[] cns, String[] subjectAlts) + throws SSLException; + + void check(String[] hosts, SSLSocket ssl) throws IOException; + + void check(String[] hosts, X509Certificate cert) throws SSLException; + + + /** + * Checks to see if the supplied hostname matches any of the supplied CNs + * or "DNS" Subject-Alts. Most implementations only look at the first CN, + * and ignore any additional CNs. Most implementations do look at all of + * the "DNS" Subject-Alts. The CNs or Subject-Alts may contain wildcards + * according to RFC 2818. + * + * @param cns CN fields, in order, as extracted from the X.509 + * certificate. + * @param subjectAlts Subject-Alt fields of type 2 ("DNS"), as extracted + * from the X.509 certificate. + * @param hosts The array of hostnames to verify. + * @throws SSLException If verification failed. + */ + void check(String[] hosts, String[] cns, String[] subjectAlts) + throws SSLException; + + + /** + * The DEFAULT HostnameVerifier works the same way as Curl and Firefox. + *

+ * The hostname must match either the first CN, or any of the subject-alts. + * A wildcard can occur in the CN, and in any of the subject-alts. + *

+ * The only difference between DEFAULT and STRICT is that a wildcard (such + * as "*.foo.com") with DEFAULT matches all subdomains, including + * "a.b.foo.com". + */ + public final static HostnameChecker DEFAULT = + new AbstractChecker() { + public final void check(final String[] hosts, final String[] cns, + final String[] subjectAlts) + throws SSLException { + check(hosts, cns, subjectAlts, false, false); + } + + public final String toString() { return "DEFAULT"; } + }; + + + /** + * The DEFAULT_AND_LOCALHOST HostnameVerifier works like the DEFAULT + * one with one additional relaxation: a host of "localhost", + * "localhost.localdomain", "127.0.0.1", "::1" will always pass, no matter + * what is in the server's certificate. + */ + public final static HostnameChecker DEFAULT_AND_LOCALHOST = + new AbstractChecker() { + public final void check(final String[] hosts, final String[] cns, + final String[] subjectAlts) + throws SSLException { + if (isLocalhost(hosts[0])) { + return; + } + check(hosts, cns, subjectAlts, false, false); + } + + public final String toString() { return "DEFAULT_AND_LOCALHOST"; } + }; + + /** + * The STRICT HostnameVerifier works the same way as java.net.URL in Sun + * Java 1.4, Sun Java 5, Sun Java 6. It's also pretty close to IE6. + * This implementation appears to be compliant with RFC 2818 for dealing + * with wildcards. + *

+ * The hostname must match either the first CN, or any of the subject-alts. + * A wildcard can occur in the CN, and in any of the subject-alts. The + * one divergence from IE6 is how we only check the first CN. IE6 allows + * a match against any of the CNs present. We decided to follow in + * Sun Java 1.4's footsteps and only check the first CN. + *

+ * A wildcard such as "*.foo.com" matches only subdomains in the same + * level, for example "a.foo.com". It does not match deeper subdomains + * such as "a.b.foo.com". + */ + public final static HostnameChecker STRICT = + new AbstractChecker() { + public final void check(final String[] host, final String[] cns, + final String[] subjectAlts) + throws SSLException { + check(host, cns, subjectAlts, false, true); + } + + public final String toString() { return "STRICT"; } + }; + + /** + * The STRICT_IE6 HostnameVerifier works just like the STRICT one with one + * minor variation: the hostname can match against any of the CN's in the + * server's certificate, not just the first one. This behaviour is + * identical to IE6's behaviour. + */ + public final static HostnameChecker STRICT_IE6 = + new AbstractChecker() { + public final void check(final String[] host, final String[] cns, + final String[] subjectAlts) + throws SSLException { + check(host, cns, subjectAlts, true, true); + } + + public final String toString() { return "STRICT_IE6"; } + }; + + /** + * The ALLOW_ALL HostnameVerifier essentially turns hostname verification + * off. This implementation is a no-op, and never throws the SSLException. + */ + public final static HostnameChecker ALLOW_ALL = + new AbstractChecker() { + public final void check(final String[] host, final String[] cns, + final String[] subjectAlts) { + // Allow everything - so never blowup. + } + + public final String toString() { return "ALLOW_ALL"; } + }; + + abstract class AbstractChecker implements HostnameChecker { + public static String[] getCNs(X509Certificate cert) { + try { + final String subjectPrincipal = cert.getSubjectX500Principal().getName(X500Principal.RFC2253); + final LinkedList cnList = new LinkedList(); + final LdapName subjectDN = new LdapName(subjectPrincipal); + for (final Rdn rds : subjectDN.getRdns()) { + final Attributes attributes = rds.toAttributes(); + final Attribute cn = attributes.get("cn"); + if (cn != null) { + try { + final Object value = cn.get(); + if (value != null) { + cnList.add(value.toString()); + } + } catch (NoSuchElementException ignore) { + } catch (NamingException ignore) { + } + } + } + if (!cnList.isEmpty()) { + return cnList.toArray(new String[cnList.size()]); + } + } catch (InvalidNameException ignore) { + } + return null; + } + + /** + * Extracts the array of SubjectAlt DNS names from an X509Certificate. + * Returns null if there aren't any. + *

+ * Note: Java doesn't appear able to extract international characters + * from the SubjectAlts. It can only extract international characters + * from the CN field. + *

+ * (Or maybe the version of OpenSSL I'm using to test isn't storing the + * international characters correctly in the SubjectAlts?). + * + * @param cert X509Certificate + * @return Array of SubjectALT DNS names stored in the certificate. + */ + public static String[] getDNSSubjectAlts(X509Certificate cert) { + LinkedList subjectAltList = new LinkedList(); + Collection c = null; + try { + c = cert.getSubjectAlternativeNames(); + } + catch (Exception cpe) { } + if (c != null) { + Iterator it = c.iterator(); + while (it.hasNext()) { + List list = (List) it.next(); + int type = ((Integer) list.get(0)).intValue(); + // If type is 2, then we've got a dNSName + if (type == 2) { + String s = (String) list.get(1); + subjectAltList.add(s); + } + } + } + if (!subjectAltList.isEmpty()) { + String[] subjectAlts = new String[subjectAltList.size()]; + subjectAltList.toArray(subjectAlts); + return subjectAlts; + } else { + return null; + } + } + + /** + * This contains a list of 2nd-level domains that aren't allowed to + * have wildcards when combined with country-codes. + * For example: [*.co.uk]. + *

+ * The [*.co.uk] problem is an interesting one. Should we just hope + * that CA's would never foolishly allow such a certificate to happen? + * Looks like we're the only implementation guarding against this. + * Firefox, Curl, Sun Java 1.4, 5, 6 don't bother with this check. + */ + private final static String[] BAD_COUNTRY_2LDS = + {"ac", "co", "com", "ed", "edu", "go", "gouv", "gov", "info", + "lg", "ne", "net", "or", "org"}; + + private final static String[] LOCALHOSTS = {"::1", "127.0.0.1", + "localhost", + "localhost.localdomain"}; + + + static { + // Just in case developer forgot to manually sort the array. :-) + Arrays.sort(BAD_COUNTRY_2LDS); + Arrays.sort(LOCALHOSTS); + } + + protected AbstractChecker() {} + + /** + * The javax.net.ssl.HostnameVerifier contract. + * + * @param host 'hostname' we used to create our socket + * @param session SSLSession with the remote server + * @return true if the host matched the one in the certificate. + */ + public boolean verify(String host, SSLSession session) { + try { + Certificate[] certs = session.getPeerCertificates(); + X509Certificate x509 = (X509Certificate) certs[0]; + check(new String[]{host}, x509); + return true; + } + catch (SSLException e) { + return false; + } + } + + public void check(String host, SSLSocket ssl) throws IOException { + check(new String[]{host}, ssl); + } + + public void check(String host, X509Certificate cert) + throws SSLException { + check(new String[]{host}, cert); + } + + public void check(String host, String[] cns, String[] subjectAlts) + throws SSLException { + check(new String[]{host}, cns, subjectAlts); + } + + public void check(String host[], SSLSocket ssl) + throws IOException { + if (host == null) { + throw new NullPointerException("host to verify is null"); + } + + SSLSession session = ssl.getSession(); + if (session == null) { + // In our experience this only happens under IBM 1.4.x when + // spurious (unrelated) certificates show up in the server' + // chain. Hopefully this will unearth the real problem: + InputStream in = ssl.getInputStream(); + in.available(); + /* + If you're looking at the 2 lines of code above because + you're running into a problem, you probably have two + options: + + #1. Clean up the certificate chain that your server + is presenting (e.g. edit "/etc/apache2/server.crt" + or wherever it is your server's certificate chain + is defined). + + OR + + #2. Upgrade to an IBM 1.5.x or greater JVM, or switch + to a non-IBM JVM. + */ + + // If ssl.getInputStream().available() didn't cause an + // exception, maybe at least now the session is available? + session = ssl.getSession(); + if (session == null) { + // If it's still null, probably a startHandshake() will + // unearth the real problem. + ssl.startHandshake(); + + // Okay, if we still haven't managed to cause an exception, + // might as well go for the NPE. Or maybe we're okay now? + session = ssl.getSession(); + } + } + Certificate[] certs; + try { + certs = session.getPeerCertificates(); + } catch (SSLPeerUnverifiedException spue) { + InputStream in = ssl.getInputStream(); + in.available(); + // Didn't trigger anything interesting? Okay, just throw + // original. + throw spue; + } + X509Certificate x509 = (X509Certificate) certs[0]; + check(host, x509); + } + + public void check(String[] host, X509Certificate cert) + throws SSLException { + String[] cns = AbstractChecker.getCNs(cert); + String[] subjectAlts = AbstractChecker.getDNSSubjectAlts(cert); + check(host, cns, subjectAlts); + } + + public void check(final String[] hosts, final String[] cns, + final String[] subjectAlts, final boolean ie6, + final boolean strictWithSubDomains) + throws SSLException { + // Build up lists of allowed hosts For logging/debugging purposes. + StringBuffer buf = new StringBuffer(32); + buf.append('<'); + for (int i = 0; i < hosts.length; i++) { + String h = hosts[i]; + h = h != null ? h.trim().toLowerCase() : ""; + hosts[i] = h; + if (i > 0) { + buf.append('/'); + } + buf.append(h); + } + buf.append('>'); + String hostnames = buf.toString(); + // Build the list of names we're going to check. Our DEFAULT and + // STRICT implementations of the HostnameVerifier only use the + // first CN provided. All other CNs are ignored. + // (Firefox, wget, curl, Sun Java 1.4, 5, 6 all work this way). + TreeSet names = new TreeSet(); + if (cns != null && cns.length > 0 && cns[0] != null) { + names.add(cns[0]); + if (ie6) { + for (int i = 1; i < cns.length; i++) { + names.add(cns[i]); + } + } + } + if (subjectAlts != null) { + for (int i = 0; i < subjectAlts.length; i++) { + if (subjectAlts[i] != null) { + names.add(subjectAlts[i]); + } + } + } + if (names.isEmpty()) { + String msg = "Certificate for " + hosts[0] + " doesn't contain CN or DNS subjectAlt"; + throw new SSLException(msg); + } + + // StringBuffer for building the error message. + buf = new StringBuffer(); + + boolean match = false; + out: + for (Iterator it = names.iterator(); it.hasNext();) { + // Don't trim the CN, though! + String cn = (String) it.next(); + cn = cn.toLowerCase(); + // Store CN in StringBuffer in case we need to report an error. + buf.append(" <"); + buf.append(cn); + buf.append('>'); + if (it.hasNext()) { + buf.append(" OR"); + } + + // The CN better have at least two dots if it wants wildcard + // action. It also can't be [*.co.uk] or [*.co.jp] or + // [*.org.uk], etc... + boolean doWildcard = cn.startsWith("*.") && + cn.lastIndexOf('.') >= 0 && + !isIP4Address(cn) && + acceptableCountryWildcard(cn); + + for (int i = 0; i < hosts.length; i++) { + final String hostName = hosts[i].trim().toLowerCase(); + if (doWildcard) { + match = hostName.endsWith(cn.substring(1)); + if (match && strictWithSubDomains) { + // If we're in strict mode, then [*.foo.com] is not + // allowed to match [a.b.foo.com] + match = countDots(hostName) == countDots(cn); + } + } else { + match = hostName.equals(cn); + } + if (match) { + break out; + } + } + } + if (!match) { + throw new SSLException("hostname in certificate didn't match: " + hostnames + " !=" + buf); + } + } + + public static boolean isIP4Address(final String cn) { + boolean isIP4 = true; + String tld = cn; + int x = cn.lastIndexOf('.'); + // We only bother analyzing the characters after the final dot + // in the name. + if (x >= 0 && x + 1 < cn.length()) { + tld = cn.substring(x + 1); + } + for (int i = 0; i < tld.length(); i++) { + if (!Character.isDigit(tld.charAt(0))) { + isIP4 = false; + break; + } + } + return isIP4; + } + + public static boolean acceptableCountryWildcard(final String cn) { + int cnLen = cn.length(); + if (cnLen >= 7 && cnLen <= 9) { + // Look for the '.' in the 3rd-last position: + if (cn.charAt(cnLen - 3) == '.') { + // Trim off the [*.] and the [.XX]. + String s = cn.substring(2, cnLen - 3); + // And test against the sorted array of bad 2lds: + int x = Arrays.binarySearch(BAD_COUNTRY_2LDS, s); + return x < 0; + } + } + return true; + } + + public static boolean isLocalhost(String host) { + host = host != null ? host.trim().toLowerCase() : ""; + if (host.startsWith("::1")) { + int x = host.lastIndexOf('%'); + if (x >= 0) { + host = host.substring(0, x); + } + } + int x = Arrays.binarySearch(LOCALHOSTS, host); + return x >= 0; + } + + /** + * Counts the number of dots "." in a string. + * + * @param s string to count dots from + * @return number of dots + */ + public static int countDots(final String s) { + int count = 0; + for (int i = 0; i < s.length(); i++) { + if (s.charAt(i) == '.') { + count++; + } + } + return count; + } + } + +} diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/TLSHostnameVerifier.java b/src/main/java/com/github/shyiko/mysql/binlog/network/TLSHostnameVerifier.java index 28b106d6..6dd3aa43 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/network/TLSHostnameVerifier.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/TLSHostnameVerifier.java @@ -15,13 +15,11 @@ */ package com.github.shyiko.mysql.binlog.network; -import sun.security.util.HostnameChecker; - import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLException; import javax.net.ssl.SSLPeerUnverifiedException; import javax.net.ssl.SSLSession; import java.security.cert.Certificate; -import java.security.cert.CertificateException; import java.security.cert.X509Certificate; /** @@ -30,15 +28,15 @@ public class TLSHostnameVerifier implements HostnameVerifier { public boolean verify(String hostname, SSLSession session) { - HostnameChecker checker = HostnameChecker.getInstance(HostnameChecker.TYPE_TLS); + HostnameChecker checker = HostnameChecker.DEFAULT; try { Certificate[] peerCertificates = session.getPeerCertificates(); if (peerCertificates.length > 0 && peerCertificates[0] instanceof X509Certificate) { X509Certificate peerCertificate = (X509Certificate) peerCertificates[0]; try { - checker.match(hostname, peerCertificate); + checker.check(hostname, peerCertificate); return true; - } catch (CertificateException ignored) { + } catch (SSLException ignored) { } } } catch (SSLPeerUnverifiedException ignored) { diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientTest.java index 542c2452..7f13b7c1 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientTest.java @@ -115,7 +115,7 @@ public void testNullEventDeserializerIsNotAllowed() throws Exception { @Test(timeOut = 15000) public void testDisconnectWhileBlockedByFBRead() throws Exception { - final BinaryLogClient binaryLogClient = new BinaryLogClient("localhost", 33060, "root", "mysql"); + final BinaryLogClient binaryLogClient = new BinaryLogClient("localhost", 33061, "root", "mysql"); final CountDownLatch readAttempted = new CountDownLatch(1); binaryLogClient.setSocketFactory(new SocketFactory() { @Override @@ -144,7 +144,7 @@ public void run() { try { final ServerSocket serverSocket = new ServerSocket(); try { - serverSocket.bind(new InetSocketAddress("localhost", 33060)); + serverSocket.bind(new InetSocketAddress("localhost", 33061)); socketBound.countDown(); serverSocket.accept(); // accept socket but do NOT send anything assertTrue(readAttempted.await(3000, TimeUnit.MILLISECONDS)); From 174798b0e2bab5e9cf9ceb4bc8a346718cc631a3 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Fri, 22 Jan 2021 11:00:27 -0800 Subject: [PATCH 068/217] move test off mysqlx port --- .../com/github/shyiko/mysql/binlog/BinaryLogClientTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientTest.java index 542c2452..7f13b7c1 100644 --- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientTest.java +++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientTest.java @@ -115,7 +115,7 @@ public void testNullEventDeserializerIsNotAllowed() throws Exception { @Test(timeOut = 15000) public void testDisconnectWhileBlockedByFBRead() throws Exception { - final BinaryLogClient binaryLogClient = new BinaryLogClient("localhost", 33060, "root", "mysql"); + final BinaryLogClient binaryLogClient = new BinaryLogClient("localhost", 33061, "root", "mysql"); final CountDownLatch readAttempted = new CountDownLatch(1); binaryLogClient.setSocketFactory(new SocketFactory() { @Override @@ -144,7 +144,7 @@ public void run() { try { final ServerSocket serverSocket = new ServerSocket(); try { - serverSocket.bind(new InetSocketAddress("localhost", 33060)); + serverSocket.bind(new InetSocketAddress("localhost", 33061)); socketBound.countDown(); serverSocket.accept(); // accept socket but do NOT send anything assertTrue(readAttempted.await(3000, TimeUnit.MILLISECONDS)); From b912bbb74acd45a97791896fe4b3fb26b9536633 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Fri, 22 Jan 2021 19:39:27 -0800 Subject: [PATCH 069/217] add some logging, build for jdk 11 --- pom.xml | 3 +-- .../mysql/binlog/network/HostnameChecker.java | 13 +++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index a21feffa..46724235 100644 --- a/pom.xml +++ b/pom.xml @@ -105,8 +105,7 @@ maven-compiler-plugin 3.5.1 - 1.8 - 1.8 + 11 diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/HostnameChecker.java b/src/main/java/com/github/shyiko/mysql/binlog/network/HostnameChecker.java index 86b73aed..080c4590 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/network/HostnameChecker.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/network/HostnameChecker.java @@ -61,6 +61,8 @@ import java.util.List; import java.util.NoSuchElementException; import java.util.TreeSet; +import java.util.logging.Level; +import java.util.logging.Logger; /** * Interface for checking if a hostname matches the names stored inside the @@ -221,6 +223,8 @@ public final void check(final String[] host, final String[] cns, }; abstract class AbstractChecker implements HostnameChecker { + private final Logger logger = Logger.getLogger(getClass().getName()); + public static String[] getCNs(X509Certificate cert) { try { final String subjectPrincipal = cert.getSubjectX500Principal().getName(X500Principal.RFC2253); @@ -406,10 +410,19 @@ is presenting (e.g. edit "/etc/apache2/server.crt" check(host, x509); } + private String commaJoin(String [] input) { + if ( input == null ) return ""; + return String.join(",", Arrays.asList(input)); + } + public void check(String[] host, X509Certificate cert) throws SSLException { String[] cns = AbstractChecker.getCNs(cert); String[] subjectAlts = AbstractChecker.getDNSSubjectAlts(cert); + logger.log(Level.INFO, + "attempting to verify SSL identity '" + commaJoin(host) + "' " + + "against cns: [" + commaJoin(cns) + "], " + + "subject-alts: [" + commaJoin(subjectAlts) + "]"); check(host, cns, subjectAlts); } From e2c05f60733b563c2692287b4df395f725883cc7 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Fri, 22 Jan 2021 20:55:06 -0800 Subject: [PATCH 070/217] remove dead config --- pom.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pom.xml b/pom.xml index 46724235..15feaccd 100644 --- a/pom.xml +++ b/pom.xml @@ -228,11 +228,6 @@ org.apache.maven.plugins maven-javadoc-plugin 3.2.0 - - buar - -J--add-exports - -Jjava.base/sun.security.util=ALL-UNNAMED - attach-javadocs From c72e3562239112e4dc1ac4f84a8257c582220667 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Fri, 22 Jan 2021 21:14:41 -0800 Subject: [PATCH 071/217] bump maven-compiler, maven-source plugin --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 15feaccd..2c5eb964 100644 --- a/pom.xml +++ b/pom.xml @@ -103,7 +103,7 @@ org.apache.maven.plugins maven-compiler-plugin - 3.5.1 + 3.8.1 11 @@ -213,7 +213,7 @@ org.apache.maven.plugins maven-source-plugin - 3.0.1 + 3.8.1 attach-sources From cd618db00d84cdb846201f89092e7ec5b106d53d Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Mon, 25 Jan 2021 10:47:26 -0800 Subject: [PATCH 072/217] update version in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 05d65a8c..f665442c 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Get the latest JAR(s) from [here](https://search.maven.org/search?q=g:com.zendes com.zendesk mysql-binlog-connector-java - 0.23.2 + 0.23.4 ``` From 47466e712684e1b3551367768b94eb6b68f18d83 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Thu, 4 Feb 2021 15:14:21 -0800 Subject: [PATCH 073/217] version 0.24.0 --- CHANGELOG.md | 8 ++++++++ pom.xml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68943c40..af30c11b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.24.0](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.23.3...0.24.0) - 2021-02-04 + +- Move up to JDK 11, drop support for JDK 8 + +## [0.23.4](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.23.3...0.23.4) - 2021-01-17 + +- correct authentication error that was causing a problem with Azure + ## [0.23.3](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.23.2...0.23.3) - 2020-10-29 - add EventDeserializer.CompatibilityMode.INTEGER_AS_BYTE_ARRAY if you want raw integer data diff --git a/pom.xml b/pom.xml index 2c5eb964..1410905f 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.zendesk mysql-binlog-connector-java - 0.23.4-SNAPSHOT + 0.24.0 mysql-binlog-connector-java MySQL Binary Log connector From 9519b3ce36e9a2525f69082751512333a1adefc1 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Thu, 4 Feb 2021 16:39:32 -0800 Subject: [PATCH 074/217] correct maven-source-plugin version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1410905f..a8c8fc9c 100644 --- a/pom.xml +++ b/pom.xml @@ -213,7 +213,7 @@ org.apache.maven.plugins maven-source-plugin - 3.8.1 + 3.2.1 attach-sources From 0953a3fb447371d6b3363ce9bfc90ca82df216a1 Mon Sep 17 00:00:00 2001 From: Ben Osheroff Date: Thu, 4 Feb 2021 16:39:48 -0800 Subject: [PATCH 075/217] slog through and cleanup javadoc warnings --- .../shyiko/mysql/binlog/BinaryLogClient.java | 23 ++++++++++ .../mysql/binlog/BinaryLogFileReader.java | 1 + .../github/shyiko/mysql/binlog/GtidSet.java | 2 +- .../AbstractRowsEventDataDeserializer.java | 3 -- .../deserialization/EventDeserializer.java | 5 +++ .../deserialization/json/JsonBinary.java | 4 +- .../mysql/binlog/io/ByteArrayInputStream.java | 27 +++++++++--- .../binlog/io/ByteArrayOutputStream.java | 8 ++++ .../mysql/binlog/network/ErrorCode.java | 44 +++++++++---------- .../mysql/binlog/network/HostnameChecker.java | 20 ++++----- .../shyiko/mysql/binlog/network/SSLMode.java | 3 +- .../mysql/binlog/network/ServerException.java | 1 + .../AuthenticateSecurityPasswordCommand.java | 3 ++ 13 files changed, 98 insertions(+), 46 deletions(-) diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java index e3f1857c..e15f68c6 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java @@ -172,6 +172,8 @@ public X509Certificate[] getAcceptedIssuers() { /** * Alias for BinaryLogClient("localhost", 3306, <no schema> = null, username, password). * @see BinaryLogClient#BinaryLogClient(String, int, String, String, String) + * @param username login name + * @param password password */ public BinaryLogClient(String username, String password) { this("localhost", 3306, null, username, password); @@ -180,6 +182,9 @@ public BinaryLogClient(String username, String password) { /** * Alias for BinaryLogClient("localhost", 3306, schema, username, password). * @see BinaryLogClient#BinaryLogClient(String, int, String, String, String) + * @param schema database name, nullable + * @param username login name + * @param password password */ public BinaryLogClient(String schema, String username, String password) { this("localhost", 3306, schema, username, password); @@ -188,6 +193,10 @@ public BinaryLogClient(String schema, String username, String password) { /** * Alias for BinaryLogClient(hostname, port, <no schema> = null, username, password). * @see BinaryLogClient#BinaryLogClient(String, int, String, String, String) + * @param hostname mysql server hostname + * @param port mysql server port + * @param username login name + * @param password password */ public BinaryLogClient(String hostname, int port, String username, String password) { this(hostname, port, null, username, password); @@ -332,6 +341,7 @@ public void setGtidSet(String gtidSet) { /** * @see #setGtidSetFallbackToPurged(boolean) + * @return whether gtid_purged is used as a fallback */ public boolean isGtidSetFallbackToPurged() { return gtidSetFallbackToPurged; @@ -347,6 +357,7 @@ public void setGtidSetFallbackToPurged(boolean gtidSetFallbackToPurged) { /** * @see #setUseBinlogFilenamePositionInGtidMode(boolean) + * @return value of useBinlogFilenamePostionInGtidMode */ public boolean isUseBinlogFilenamePositionInGtidMode() { return useBinlogFilenamePositionInGtidMode; @@ -1072,6 +1083,7 @@ public List getEventListeners() { /** * Register event listener. Note that multiple event listeners will be called in order they * where registered. + * @param eventListener event listener */ public void registerEventListener(EventListener eventListener) { eventListeners.add(eventListener); @@ -1079,6 +1091,7 @@ public void registerEventListener(EventListener eventListener) { /** * Unregister all event listener of specific type. + * @param listenerClass event listener class to unregister */ public void unregisterEventListener(Class listenerClass) { for (EventListener eventListener: eventListeners) { @@ -1090,6 +1103,7 @@ public void unregisterEventListener(Class listenerClass /** * Unregister single event listener. + * @param eventListener event listener to unregister */ public void unregisterEventListener(EventListener eventListener) { eventListeners.remove(eventListener); @@ -1120,6 +1134,7 @@ public List getLifecycleListeners() { /** * Register lifecycle listener. Note that multiple lifecycle listeners will be called in order they * where registered. + * @param lifecycleListener lifecycle listener to register */ public void registerLifecycleListener(LifecycleListener lifecycleListener) { lifecycleListeners.add(lifecycleListener); @@ -1127,6 +1142,7 @@ public void registerLifecycleListener(LifecycleListener lifecycleListener) { /** * Unregister all lifecycle listener of specific type. + * @param listenerClass lifecycle listener class to unregister */ public void unregisterLifecycleListener(Class listenerClass) { for (LifecycleListener lifecycleListener : lifecycleListeners) { @@ -1138,6 +1154,7 @@ public void unregisterLifecycleListener(Class liste /** * Unregister single lifecycle listener. + * @param eventListener lifecycle listener to unregister */ public void unregisterLifecycleListener(LifecycleListener eventListener) { lifecycleListeners.remove(eventListener); @@ -1215,23 +1232,29 @@ public interface LifecycleListener { /** * Called once client has successfully logged in but before started to receive binlog events. + * @param client the client that logged in */ void onConnect(BinaryLogClient client); /** * It's guarantied to be called before {@link #onDisconnect(BinaryLogClient)}) in case of * communication failure. + * @param client the client that triggered the communication failure + * @param ex The exception that triggered the communication failutre */ void onCommunicationFailure(BinaryLogClient client, Exception ex); /** * Called in case of failed event deserialization. Note this type of error does NOT cause client to * disconnect. If you wish to stop receiving events you'll need to fire client.disconnect() manually. + * @param client the client that failed event deserialization + * @param ex The exception that triggered the failutre */ void onEventDeserializationFailure(BinaryLogClient client, Exception ex); /** * Called upon disconnect (regardless of the reason). + * @param client the client that disconnected */ void onDisconnect(BinaryLogClient client); } diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogFileReader.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogFileReader.java index 96b79fc6..d06a2d2d 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogFileReader.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogFileReader.java @@ -77,6 +77,7 @@ public BinaryLogFileReader(InputStream inputStream, EventDeserializer eventDeser /** * @return deserialized event or null in case of end-of-stream + * @throws IOException if reading the event fails */ public Event readEvent() throws IOException { return eventDeserializer.nextEvent(inputStream); diff --git a/src/main/java/com/github/shyiko/mysql/binlog/GtidSet.java b/src/main/java/com/github/shyiko/mysql/binlog/GtidSet.java index 6899eafe..55d619e1 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/GtidSet.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/GtidSet.java @@ -31,7 +31,7 @@ * gtid_set: uuid_set[,uuid_set]... * uuid_set: uuid:interval[:interval]... * uuid: hhhhhhhh-hhhh-hhhh-hhhh-hhhhhhhhhhhh, h: [0-9|A-F] - * interval: n[-n], (n >= 1) + * interval: n[-n], (n >= 1) * * * @author Stanley Shyiko diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java index 7b910da4..89971365 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java @@ -471,9 +471,6 @@ private static int[] split(long value, int divider, int length) { return result; } - /** - * see mysql/strings/decimal.c - */ public static BigDecimal asBigDecimal(int precision, int scale, byte[] value) { boolean positive = (value[0] & 0x80) == 0x80; value[0] ^= 0x80; diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java index fbec7c38..cf936852 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java @@ -153,6 +153,7 @@ private void afterEventDataDeserializerSet(EventType eventType) { /** * @deprecated resolved based on FORMAT_DESCRIPTION + * @param checksumType don't use this function. */ @Deprecated public void setChecksumType(ChecksumType checksumType) { @@ -161,6 +162,8 @@ public void setChecksumType(ChecksumType checksumType) { /** * @see CompatibilityMode + * @param first at least one CompatabilityMode + * @param rest many modes */ public void setCompatibilityMode(CompatibilityMode first, CompatibilityMode... rest) { this.compatibilitySet = EnumSet.of(first, rest); @@ -208,6 +211,8 @@ private void ensureCompatibility(EventDataDeserializer eventDataDeserializer) { /** * @return deserialized event or null in case of end-of-stream + * @param inputStream input stream to fetch event from + * @throws IOException if connection gets closed */ public Event nextEvent(ByteArrayInputStream inputStream) throws IOException { if (inputStream.peek() == -1) { diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java index 688cdc63..153e317b 100644 --- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java +++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java @@ -80,8 +80,6 @@ *

Grammar

* The grammar of the binary representation of JSON objects are defined in the MySQL codebase in the * json_binary.h file: - *

- * * 

  *   doc ::= type value
  *   type ::=
@@ -662,7 +660,6 @@ protected void parseString(JsonFormatter formatter) throws IOException {
      * See the 
      * MySQL source code for the logic used in this method.
-     * 

      * 
Grammar

      *
      * 
@@ -958,6 +955,7 @@ protected BigInteger readUInt64() throws IOException {
      * to 16383, and so on...
      *
      * @return the integer value
+	 * @throws IOException if we don't encounter an end-of-int marker
      */
     protected int readVariableInt() throws IOException {
         int length = 0;
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java b/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java
index ff71cbee..c6fd4c3e 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java
@@ -41,6 +41,9 @@ public ByteArrayInputStream(byte[] bytes) {
 
     /**
      * Read int written in little-endian format.
+	 * @param length length of the integer to read
+	 * @throws IOException in case of EOF
+	 * @return the integer from the binlog
      */
     public int readInteger(int length) throws IOException {
         int result = 0;
@@ -52,6 +55,9 @@ public int readInteger(int length) throws IOException {
 
     /**
      * Read long written in little-endian format.
+	 * @param length length of the long to read
+	 * @throws IOException in case of EOF
+	 * @return the long from the binlog
      */
     public long readLong(int length) throws IOException {
         long result = 0;
@@ -63,6 +69,9 @@ public long readLong(int length) throws IOException {
 
     /**
      * Read fixed length string.
+	 * @param length length of string to read
+	 * @throws IOException in case of EOF
+	 * @return string
      */
     public String readString(int length) throws IOException {
         return new String(read(length));
@@ -70,6 +79,8 @@ public String readString(int length) throws IOException {
 
     /**
      * Read variable-length string. Preceding packed integer indicates the length of the string.
+	 * @throws IOException in case of EOF
+	 * @return string
      */
     public String readLengthEncodedString() throws IOException {
         return readString(readPackedInteger());
@@ -77,6 +88,8 @@ public String readLengthEncodedString() throws IOException {
 
     /**
      * Read variable-length string. End is indicated by 0x00 byte.
+	 * @throws IOException in case of EOF
+	 * @return string
      */
     public String readZeroTerminatedString() throws IOException {
         ByteArrayOutputStream s = new ByteArrayOutputStream();
@@ -128,6 +141,8 @@ private byte[] reverse(byte[] bytes) {
 
     /**
      * @see #readPackedNumber()
+	 * @throws IOException in case of malformed number, eof, null, or long
+	 * @return integer
      */
     public int readPackedInteger() throws IOException {
         Number number = readPackedNumber();
@@ -141,12 +156,14 @@ public int readPackedInteger() throws IOException {
     }
 
     /**
-     * Format (first-byte-based):

-     * 0-250 - The first byte is the number (in the range 0-250). No additional bytes are used.

-     * 251 - SQL NULL value

-     * 252 - Two more bytes are used. The number is in the range 251-0xffff.

-     * 253 - Three more bytes are used. The number is in the range 0xffff-0xffffff.

+     * Format (first-byte-based):

+     * 0-250 - The first byte is the number (in the range 0-250). No additional bytes are used.

+     * 251 - SQL NULL value

+     * 252 - Two more bytes are used. The number is in the range 251-0xffff.

+     * 253 - Three more bytes are used. The number is in the range 0xffff-0xffffff.

      * 254 - Eight more bytes are used. The number is in the range 0xffffff-0xffffffffffffffff.
+	 * @throws IOException in case of malformed number or EOF
+	 * @return long or null
      */
     public Number readPackedNumber() throws IOException {
         int b = this.read();
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayOutputStream.java b/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayOutputStream.java
index 0bbdf3aa..17840c6f 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayOutputStream.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayOutputStream.java
@@ -35,6 +35,9 @@ public ByteArrayOutputStream(OutputStream outputStream) {
 
     /**
      * Write int in little-endian format.
+	 * @throws IOException on underlying stream error
+	 * @param value integer to write
+	 * @param length length in bytes of the integer
      */
     public void writeInteger(int value, int length) throws IOException {
         for (int i = 0; i < length; i++) {
@@ -44,6 +47,9 @@ public void writeInteger(int value, int length) throws IOException {
 
     /**
      * Write long in little-endian format.
+	 * @throws IOException on underlying stream error
+	 * @param value long to write
+	 * @param length length in bytes of the long
      */
     public void writeLong(long value, int length) throws IOException {
         for (int i = 0; i < length; i++) {
@@ -57,6 +63,8 @@ public void writeString(String value) throws IOException {
 
     /**
      * @see ByteArrayInputStream#readZeroTerminatedString()
+	 * @param value string to write
+	 * @throws IOException on underlying stream error
      */
     public void writeZeroTerminatedString(String value) throws IOException {
         if ( value != null )
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/ErrorCode.java b/src/main/java/com/github/shyiko/mysql/binlog/network/ErrorCode.java
index e7e46752..2cd4484e 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/network/ErrorCode.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/network/ErrorCode.java
@@ -2178,7 +2178,7 @@ public final class ErrorCode {
     public static final int ER_TOO_BIG_PRECISION = 1426;
 
     /**
-     * For float(M,D), double(M,D) or decimal(M,D), M must be >= D (column '%-.192s').
+     * For float(M,D), double(M,D) or decimal(M,D), M must be >= D (column '%-.192s').
      */
     public static final int ER_M_BIGGER_THAN_D = 1427;
 
@@ -3146,7 +3146,7 @@ public final class ErrorCode {
     public static final int WARN_NO_MASTER_INFO = 1617;
 
     /**
-     * <%-.64s> option ignored
+     * %-.64s option ignored
      */
     public static final int WARN_OPTION_IGNORED = 1618;
 
@@ -3988,56 +3988,56 @@ public final class ErrorCode {
     public static final int ER_CANT_DO_IMPLICIT_COMMIT_IN_TRX_WHEN_GTID_NEXT_IS_SET = 1778;
 
     /**
-     * @@GLOBAL.GTID_MODE = ON or UPGRADE_STEP_2 requires @@GLOBAL.ENFORCE_GTID_CONSISTENCY = 1.
+       @@GLOBAL.GTID_MODE = ON or UPGRADE_STEP_2 requires @@GLOBAL.ENFORCE_GTID_CONSISTENCY = 1.
      */
     public static final int ER_GTID_MODE_2_OR_3_REQUIRES_ENFORCE_GTID_CONSISTENCY_ON = 1779;
 
     /**
-     * @@GLOBAL.GTID_MODE = ON or UPGRADE_STEP_1 or UPGRADE_STEP_2 requires --log-bin and --log-slave-updates.
+     * @@GLOBAL.GTID_MODE = ON or UPGRADE_STEP_1 or UPGRADE_STEP_2 requires --log-bin and --log-slave-updates.
      */
     public static final int ER_GTID_MODE_REQUIRES_BINLOG = 1780;
 
     /**
-     * @@SESSION.GTID_NEXT cannot be set to UUID:NUMBER when @@GLOBAL.GTID_MODE = OFF.
+     * @@SESSION.GTID_NEXT cannot be set to UUID:NUMBER when @@GLOBAL.GTID_MODE = OFF.
      */
     public static final int ER_CANT_SET_GTID_NEXT_TO_GTID_WHEN_GTID_MODE_IS_OFF = 1781;
 
     /**
-     * @@SESSION.GTID_NEXT cannot be set to ANONYMOUS when @@GLOBAL.GTID_MODE = ON.
+     * @@SESSION.GTID_NEXT cannot be set to ANONYMOUS when @@GLOBAL.GTID_MODE = ON.
      */
     public static final int ER_CANT_SET_GTID_NEXT_TO_ANONYMOUS_WHEN_GTID_MODE_IS_ON = 1782;
 
     /**
-     * @@SESSION.GTID_NEXT_LIST cannot be set to a non-NULL value when @@GLOBAL.GTID_MODE = OFF.
+     * @@SESSION.GTID_NEXT_LIST cannot be set to a non-NULL value when @@GLOBAL.GTID_MODE = OFF.
      */
     public static final int ER_CANT_SET_GTID_NEXT_LIST_TO_NON_NULL_WHEN_GTID_MODE_IS_OFF = 1783;
 
     /**
-     * Found a Gtid_log_event or Previous_gtids_log_event when @@GLOBAL.GTID_MODE = OFF.
+     * Found a Gtid_log_event or Previous_gtids_log_event when @@GLOBAL.GTID_MODE = OFF.
      */
     public static final int ER_FOUND_GTID_EVENT_WHEN_GTID_MODE_IS_OFF = 1784;
 
     /**
-     * When @@GLOBAL.ENFORCE_GTID_CONSISTENCY = 1, updates to non-transactional tables can only be done in either
+     * When @@GLOBAL.ENFORCE_GTID_CONSISTENCY = 1, updates to non-transactional tables can only be done in either
      * autocommitted statements or single-statement transactions, and never in the same statement as updates to
      * transactional tables.
      */
     public static final int ER_GTID_UNSAFE_NON_TRANSACTIONAL_TABLE = 1785;
 
     /**
-     * CREATE TABLE ... SELECT is forbidden when @@GLOBAL.ENFORCE_GTID_CONSISTENCY = 1.
+     * CREATE TABLE ... SELECT is forbidden when @@GLOBAL.ENFORCE_GTID_CONSISTENCY = 1.
      */
     public static final int ER_GTID_UNSAFE_CREATE_SELECT = 1786;
 
     /**
-     * When @@GLOBAL.ENFORCE_GTID_CONSISTENCY = 1, the statements CREATE TEMPORARY TABLE and DROP TEMPORARY TABLE can
+     * When @@GLOBAL.ENFORCE_GTID_CONSISTENCY = 1, the statements CREATE TEMPORARY TABLE and DROP TEMPORARY TABLE can
      * be executed in a non-transactional context only, and require that AUTOCOMMIT = 1.
      */
     public static final int ER_GTID_UNSAFE_CREATE_DROP_TEMPORARY_TABLE_IN_TRANSACTION = 1787;
 
     /**
-     * The value of @@GLOBAL.GTID_MODE can only change one step at a time: OFF <-> UPGRADE_STEP_1 <-> UPGRADE_STEP_2
-     * <-> ON. Also note that this value must be stepped up or down simultaneously on all servers; see the Manual for
+     * The value of @@GLOBAL.GTID_MODE can only change one step at a time: OFF <-> UPGRADE_STEP_1 <-> UPGRADE_STEP_2
+     * <-> ON. Also note that this value must be stepped up or down simultaneously on all servers; see the Manual for
      * instructions.
      */
     public static final int ER_GTID_MODE_CAN_ONLY_CHANGE_ONE_STEP_AT_A_TIME = 1788;
@@ -4049,7 +4049,7 @@ public final class ErrorCode {
     public static final int ER_MASTER_HAS_PURGED_REQUIRED_GTIDS = 1789;
 
     /**
-     * @@SESSION.GTID_NEXT cannot be changed by a client that owns a GTID. The client owns %s. Ownership is released
+     * @@SESSION.GTID_NEXT cannot be changed by a client that owns a GTID. The client owns %s. Ownership is released
      * on COMMIT or ROLLBACK.
      */
     public static final int ER_CANT_SET_GTID_NEXT_WHEN_OWNING_GTID = 1790;
@@ -4291,8 +4291,8 @@ public final class ErrorCode {
     public static final int ER_READ_ONLY_MODE = 1836;
 
     /**
-     * When @@SESSION.GTID_NEXT is set to a GTID, you must explicitly set it to a different value after a COMMIT or
-     * ROLLBACK. Please check GTID_NEXT variable manual page for detailed explanation. Current @@SESSION.GTID_NEXT is
+     * When @@SESSION.GTID_NEXT is set to a GTID, you must explicitly set it to a different value after a COMMIT or
+     * ROLLBACK. Please check GTID_NEXT variable manual page for detailed explanation. Current @@SESSION.GTID_NEXT is
      * '%s'.
      */
     public static final int ER_GTID_NEXT_TYPE_UNDEFINED_GROUP = 1837;
@@ -4303,27 +4303,27 @@ public final class ErrorCode {
     public static final int ER_VARIABLE_NOT_SETTABLE_IN_SP = 1838;
 
     /**
-     * @@GLOBAL.GTID_PURGED can only be set when @@GLOBAL.GTID_MODE = ON.
+     * @@GLOBAL.GTID_PURGED can only be set when @@GLOBAL.GTID_MODE = ON.
      */
     public static final int ER_CANT_SET_GTID_PURGED_WHEN_GTID_MODE_IS_OFF = 1839;
 
     /**
-     * @@GLOBAL.GTID_PURGED can only be set when @@GLOBAL.GTID_EXECUTED is empty.
+     * @@GLOBAL.GTID_PURGED can only be set when @@GLOBAL.GTID_EXECUTED is empty.
      */
     public static final int ER_CANT_SET_GTID_PURGED_WHEN_GTID_EXECUTED_IS_NOT_EMPTY = 1840;
 
     /**
-     * @@GLOBAL.GTID_PURGED can only be set when there are no ongoing transactions (not even in other clients).
+     * @@GLOBAL.GTID_PURGED can only be set when there are no ongoing transactions (not even in other clients).
      */
     public static final int ER_CANT_SET_GTID_PURGED_WHEN_OWNED_GTIDS_IS_NOT_EMPTY = 1841;
 
     /**
-     * @@GLOBAL.GTID_PURGED was changed from '%s' to '%s'.
+     * @@GLOBAL.GTID_PURGED was changed from '%s' to '%s'.
      */
     public static final int ER_GTID_PURGED_WAS_CHANGED = 1842;
 
     /**
-     * @@GLOBAL.GTID_EXECUTED was changed from '%s' to '%s'.
+     * @@GLOBAL.GTID_EXECUTED was changed from '%s' to '%s'.
      */
     public static final int ER_GTID_EXECUTED_WAS_CHANGED = 1843;
 
@@ -4518,7 +4518,7 @@ public final class ErrorCode {
     public static final int ER_OLD_TEMPORALS_UPGRADED = 1880;
 
     /**
-     * Operation not allowed when innodb_forced_recovery > 0.
+     * Operation not allowed when innodb_forced_recovery > 0.
      */
     public static final int ER_INNODB_FORCED_RECOVERY = 1881;
 
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/HostnameChecker.java b/src/main/java/com/github/shyiko/mysql/binlog/network/HostnameChecker.java
index 080c4590..059a5f33 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/network/HostnameChecker.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/network/HostnameChecker.java
@@ -72,15 +72,15 @@
  * or X509Certificate, or ultimately (they all end up calling this one),
  * String.  (It's easier to supply JUnit with Strings instead of mock
  * SSLSession objects!)
- * 
Our check() methods throw exceptions if the name is
+ * 
Our check() methods throw exceptions if the name is
  * invalid, whereas javax.net.ssl.HostnameVerifier just returns true/false.
- * 

+ * 

  * We provide the HostnameVerifier.DEFAULT, HostnameVerifier.STRICT, and
  * HostnameVerifier.ALLOW_ALL implementations.  We also provide the more
  * specialized HostnameVerifier.DEFAULT_AND_LOCALHOST, as well as
  * HostnameVerifier.STRICT_IE6.  But feel free to define your own
  * implementations!
- * 

+ * 

  * Inspired by Sebastian Hauer's original StrictSSLProtocolSocketFactory in the
  * HttpClient "contrib" repository.
  *
@@ -124,10 +124,10 @@ void check(String[] hosts, String[] cns, String[] subjectAlts)
 
     /**
      * The DEFAULT HostnameVerifier works the same way as Curl and Firefox.
-     * 

+     * 

      * The hostname must match either the first CN, or any of the subject-alts.
      * A wildcard can occur in the CN, and in any of the subject-alts.
-     * 

+     * 

      * The only difference between DEFAULT and STRICT is that a wildcard (such
      * as "*.foo.com") with DEFAULT matches all subdomains, including
      * "a.b.foo.com".
@@ -169,13 +169,13 @@ public final void check(final String[] hosts, final String[] cns,
      * Java 1.4, Sun Java 5, Sun Java 6.  It's also pretty close to IE6.
      * This implementation appears to be compliant with RFC 2818 for dealing
      * with wildcards.
-     * 

+     * 

      * The hostname must match either the first CN, or any of the subject-alts.
      * A wildcard can occur in the CN, and in any of the subject-alts.  The
      * one divergence from IE6 is how we only check the first CN.  IE6 allows
      * a match against any of the CNs present.  We decided to follow in
      * Sun Java 1.4's footsteps and only check the first CN.
-     * 

+     * 

      * A wildcard such as "*.foo.com" matches only subdomains in the same
      * level, for example "a.foo.com".  It does not match deeper subdomains
      * such as "a.b.foo.com".
@@ -255,11 +255,11 @@ public static String[] getCNs(X509Certificate cert) {
         /**
          * Extracts the array of SubjectAlt DNS names from an X509Certificate.
          * Returns null if there aren't any.
-         * 

+         * 

          * Note:  Java doesn't appear able to extract international characters
          * from the SubjectAlts.  It can only extract international characters
          * from the CN field.
-         * 

+         * 

          * (Or maybe the version of OpenSSL I'm using to test isn't storing the
          * international characters correctly in the SubjectAlts?).
          *
@@ -298,7 +298,7 @@ public static String[] getDNSSubjectAlts(X509Certificate cert) {
          * This contains a list of 2nd-level domains that aren't allowed to
          * have wildcards when combined with country-codes.
          * For example: [*.co.uk].
-         * 

+         * 

          * The [*.co.uk] problem is an interesting one.  Should we just hope
          * that CA's would never foolishly allow such a certificate to happen?
          * Looks like we're the only implementation guarding against this.
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/SSLMode.java b/src/main/java/com/github/shyiko/mysql/binlog/network/SSLMode.java
index a5ce7f42..041dd6ea 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/network/SSLMode.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/network/SSLMode.java
@@ -16,8 +16,7 @@
 package com.github.shyiko.mysql.binlog.network;
 
 /**
- * @see * ssl-mode for the original documentation.
  * @author Stanley Shyiko
  */
 public enum SSLMode {
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/ServerException.java b/src/main/java/com/github/shyiko/mysql/binlog/network/ServerException.java
index cfd026d2..03ce3b72 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/network/ServerException.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/network/ServerException.java
@@ -33,6 +33,7 @@ public ServerException(String message, int errorCode, String sqlState) {
 
     /**
      * @see ErrorCode
+	 * @return error code
      */
     public int getErrorCode() {
         return errorCode;
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSecurityPasswordCommand.java b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSecurityPasswordCommand.java
index de93da45..744e85fe 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSecurityPasswordCommand.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/AuthenticateSecurityPasswordCommand.java
@@ -83,6 +83,9 @@ public byte[] toByteArray() throws IOException {
 
     /**
      * see mysql/sql/password.c scramble(...)
+	 * @param password the password
+	 * @param salt salt received from server
+	 * @return hashed password
      */
     public static byte[] passwordCompatibleWithMySQL411(String password, String salt) {
         if ( "".equals(password) || password == null )

From d475cb453f80f379f42cfcd63ae96453c82f0ed8 Mon Sep 17 00:00:00 2001
From: Jiri Pechanec 
Date: Thu, 18 Feb 2021 08:09:16 +0100
Subject: [PATCH 076/217] Provide fast skip method implementation

---
 .../mysql/binlog/event/deserialization/json/JsonBinary.java  | 4 ++--
 .../github/shyiko/mysql/binlog/io/ByteArrayInputStream.java  | 5 +++++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
index 688cdc63..98212c72 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
@@ -403,7 +403,7 @@ protected void parseObject(boolean small, JsonFormatter formatter)
             } else {
                 // Parse the value ...
                 this.reader.reset();
-                this.reader.skip(objectOffset + entry.index);
+                this.reader.fastSkip(objectOffset + entry.index);
                 parse(entry.type, formatter);
             }
         }
@@ -537,7 +537,7 @@ protected void parseArray(boolean small, JsonFormatter formatter)
             } else {
                 // Parse the value ...
                 this.reader.reset();
-                this.reader.skip(arrayOffset + entry.index);
+                this.reader.fastSkip(arrayOffset + entry.index);
 
                 parse(entry.type, formatter);
             }
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java b/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java
index ff71cbee..b089ac03 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java
@@ -241,4 +241,9 @@ public synchronized void reset() throws IOException {
         pos = markPosition;
         inputStream.reset();
     }
+
+    public synchronized long fastSkip(long n) throws IOException {
+        pos += (int) n;
+        return inputStream.skip(n);
+    }
 }

From 02f1eda704d7785f12ecaaec50ef3337c1b810e8 Mon Sep 17 00:00:00 2001
From: Jiri Pechanec 
Date: Mon, 1 Mar 2021 11:33:05 +0100
Subject: [PATCH 077/217] Add support for blocks

---
 .../mysql/binlog/io/ByteArrayInputStream.java | 23 ++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java b/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java
index b089ac03..d7d6f282 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java
@@ -242,8 +242,25 @@ public synchronized void reset() throws IOException {
         inputStream.reset();
     }
 
+    /**
+     * This method implements fast-forward skipping in the stream.
+     * It can be used if and only if the underlying stream is fully available till its end.
+     * In other cases the regular {@link #skip(long)} method must be used.
+     *
+     * @param n - number of bytes to skip
+     * @return number of bytes skipped
+     * @throws IOException
+     */
     public synchronized long fastSkip(long n) throws IOException {
-        pos += (int) n;
-        return inputStream.skip(n);
-    }
+        long skipOf = n;
+        if (blockLength != -1) {
+            skipOf = Math.min(blockLength, skipOf);
+            blockLength -= skipOf;
+            if (blockLength == 0) {
+                blockLength = -1;
+            }
+        }
+        pos += (int) skipOf;
+        return inputStream.skip(skipOf);
+     }
 }

From 584b80090ccf740fd2ad318f29c683cbf84356e1 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Wed, 3 Mar 2021 09:36:44 -0800
Subject: [PATCH 078/217] v0.24.1

---
 CHANGELOG.md | 4 ++++
 pom.xml      | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index af30c11b..3e98127a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Changelog
 
+## [0.24.1](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.24.0...0.24.1) - 2021-03-03
+
+- Fix for performance issues read JSON columns
+
 ## [0.24.0](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.23.3...0.24.0) - 2021-02-04
 
 - Move up to JDK 11, drop support for JDK 8
diff --git a/pom.xml b/pom.xml
index a8c8fc9c..183e02c9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.24.0
+    0.24.1
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From 891ffee0752b75632af2a57d0964ca98661aab99 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Thu, 4 Mar 2021 10:17:58 -0800
Subject: [PATCH 079/217] 0.25.0

---
 CHANGELOG.md | 4 ++++
 pom.xml      | 4 ++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3e98127a..805767f6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Changelog
 
+## [0.25.0](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.24.1...0.25.0) - 2021-03-04
+
+- bring back jdk 8 support, this caused... ahem.  Issues.
+
 ## [0.24.1](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.24.0...0.24.1) - 2021-03-03
 
 - Fix for performance issues read JSON columns
diff --git a/pom.xml b/pom.xml
index 183e02c9..8c788d4c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.24.1
+    0.25.0
 
     mysql-binlog-connector-java
     MySQL Binary Log connector
@@ -105,7 +105,7 @@
                 maven-compiler-plugin
                 3.8.1
                 
-                    11
+                    8
                 
             
             

From 4d1bf9cbc0d017030c3e3324b9946ef4ef0021f0 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Mon, 29 Mar 2021 17:02:45 -0700
Subject: [PATCH 080/217] update README.md

---
 README.md                                              | 10 ++++++----
 .../binlog/event/deserialization/json/JsonBinary.java  |  2 --
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index f665442c..73ab8158 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,8 @@
 # mysql-binlog-connector-java [![Build Status](https://travis-ci.org/shyiko/mysql-binlog-connector-java.svg?branch=master)](https://travis-ci.org/shyiko/mysql-binlog-connector-java) [![Coverage Status](https://coveralls.io/repos/shyiko/mysql-binlog-connector-java/badge.svg?branch=master)](https://coveralls.io/r/shyiko/mysql-binlog-connector-java?branch=master) [![Maven Central](https://img.shields.io/maven-central/v/com.github.shyiko/mysql-binlog-connector-java.svg)](http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22com.github.shyiko%22%20AND%20a%3A%22mysql-binlog-connector-java%22)
 
 
-MySQL Binary Log connector.
+MySQL Binary Log connector.  @osheroff's fork of @shiyko's project, probably
+the "official" version of this.  With help from the Debezium devs.
 
 Initially project was started as a fork of [open-replicator](https://code.google.com/p/open-replicator),
 but ended up as a complete rewrite. Key differences/features:
@@ -23,16 +24,17 @@ but ended up as a complete rewrite. Key differences/features:
 
 ## Usage
 
-Get the latest JAR(s) from [here](https://search.maven.org/search?q=g:com.zendesk%20AND%20a:mysql-binlog-connector-java). Alternatively you can include following Maven dependency (available through Maven Central):
 
 ```xml
 
     com.zendesk
     mysql-binlog-connector-java
-    0.23.4
+    0.25.0
 
 ```
 
+Or get the latest JAR(s) from [here](https://search.maven.org/search?q=g:com.zendesk%20AND%20a:mysql-binlog-connector-java).
+
 #### Reading binary log file
 
 ```java
@@ -197,6 +199,7 @@ For the insight into the internals of MySQL look [here](https://dev.mysql.com/do
 Some of the OSS using / built on top of mysql-binlog-conector-java:
 * [apache/nifi](https://github.com/apache/nifi) An easy to use, powerful, and reliable system to process and distribute data.
 * [debezium](https://github.com/debezium/debezium) A low latency data streaming platform for change data capture (CDC).
+* [zendesk/maxwell](https://github.com/zendesk/maxwell) A MySQL-to-JSON Kafka producer.
 * [mavenlink/changestream](https://github.com/mavenlink/changestream) - A stream of changes for MySQL built on Akka.
 * [mardambey/mypipe](https://github.com/mardambey/mypipe) MySQL binary log consumer with the ability to act on changed rows and publish changes to different systems with emphasis on Apache Kafka.
 * [ngocdaothanh/mydit](https://github.com/ngocdaothanh/mydit) MySQL to MongoDB data replicator.
@@ -204,7 +207,6 @@ Some of the OSS using / built on top of mysql-binlog-conector-java:
 * [shyiko/rook](https://github.com/shyiko/rook) Generic Change Data Capture (CDC) toolkit.
 * [streamsets/datacollector](https://github.com/streamsets/datacollector) Continuous big data ingestion infrastructure.
 * [twingly/ecco](https://github.com/twingly/ecco) MySQL replication binlog parser in JRuby.
-* [zendesk/maxwell](https://github.com/zendesk/maxwell) A MySQL-to-JSON Kafka producer.
 * [zzt93/syncer](https://github.com/zzt93/syncer) A tool sync & manipulate data from MySQL/MongoDB to ES/Kafka/MySQL, which make 'Eventual Consistency' promise.
 
 It's also used [on a large scale](https://twitter.com/atwinmutt/status/626816601078300672) in MailChimp. You can read about it [here](http://devs.mailchimp.com/blog/powering-mailchimp-pro-reporting/).
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
index 81e7f1ae..d8b824eb 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
@@ -261,8 +261,6 @@ protected void parse(ValueType type, JsonFormatter formatter) throws IOException
      * json_binary.h file:
      * 
Grammar

      *
-     * 
Grammar

-     *
      * 
      *   value ::=
      *       object  |

From 67e6a95a9e7f031987e3ad45733cc9b9b8f913d8 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Mon, 29 Mar 2021 17:03:37 -0700
Subject: [PATCH 081/217] move usage up

---
 README.md | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 73ab8158..11d04419 100644
--- a/README.md
+++ b/README.md
@@ -4,6 +4,16 @@
 MySQL Binary Log connector.  @osheroff's fork of @shiyko's project, probably
 the "official" version of this.  With help from the Debezium devs.
 
+## Usage
+
+```xml
+
+    com.zendesk
+    mysql-binlog-connector-java
+    0.25.0
+
+```
+
 Initially project was started as a fork of [open-replicator](https://code.google.com/p/open-replicator),
 but ended up as a complete rewrite. Key differences/features:
 
@@ -22,16 +32,6 @@ but ended up as a complete rewrite. Key differences/features:
 [siddontang/go-mysql](https://github.com/siddontang/go-mysql) (Go),
 [noplay/python-mysql-replication](https://github.com/noplay/python-mysql-replication) (Python).
 
-## Usage
-
-
-```xml
-
-    com.zendesk
-    mysql-binlog-connector-java
-    0.25.0
-
-```
 
 Or get the latest JAR(s) from [here](https://search.maven.org/search?q=g:com.zendesk%20AND%20a:mysql-binlog-connector-java).
 

From 252dead8a8cc1ee6854184320a506f3c3e057dfe Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Mon, 29 Mar 2021 17:04:30 -0700
Subject: [PATCH 082/217] fix badgey thingy

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 11d04419..407e27a3 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-# mysql-binlog-connector-java [![Build Status](https://travis-ci.org/shyiko/mysql-binlog-connector-java.svg?branch=master)](https://travis-ci.org/shyiko/mysql-binlog-connector-java) [![Coverage Status](https://coveralls.io/repos/shyiko/mysql-binlog-connector-java/badge.svg?branch=master)](https://coveralls.io/r/shyiko/mysql-binlog-connector-java?branch=master) [![Maven Central](https://img.shields.io/maven-central/v/com.github.shyiko/mysql-binlog-connector-java.svg)](http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22com.github.shyiko%22%20AND%20a%3A%22mysql-binlog-connector-java%22)
+# mysql-binlog-connector-java [![Build Status](https://travis-ci.org/shyiko/mysql-binlog-connector-java.svg?branch=master)](https://travis-ci.org/shyiko/mysql-binlog-connector-java) [![Coverage Status](https://coveralls.io/repos/shyiko/mysql-binlog-connector-java/badge.svg?branch=master)](https://coveralls.io/r/shyiko/mysql-binlog-connector-java?branch=master) [![Maven Central](https://img.shields.io/maven-central/v/com.zendesk/mysql-binlog-connector-java.svg)](http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22com.github.shyiko%22%20AND%20a%3A%22mysql-binlog-connector-java%22)
 
 
 MySQL Binary Log connector.  @osheroff's fork of @shiyko's project, probably

From ff9a79139127dea6fd3e9112e3d0213c108989c6 Mon Sep 17 00:00:00 2001
From: Vadzim Ramanenka 
Date: Fri, 26 Mar 2021 22:09:50 +0300
Subject: [PATCH 083/217] Optimize read operation on ByteArrayInputStream

---
 .../mysql/binlog/io/ByteArrayInputStream.java |  44 ++++++++
 .../binlog/io/ByteArrayInputStreamTest.java   | 102 ++++++++++++++++++
 2 files changed, 146 insertions(+)
 create mode 100644 src/test/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStreamTest.java

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java b/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java
index b883394f..3cfaf979 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java
@@ -222,6 +222,50 @@ private int readWithinBlockBoundaries() throws IOException {
         return inputStream.read();
     }
 
+    @Override
+    public int read(byte[] b, int off, int len) throws IOException {
+        if (b == null) {
+            throw new NullPointerException();
+        } else if (off < 0 || len < 0 || len > b.length - off) {
+            throw new IndexOutOfBoundsException();
+        } else if (len == 0) {
+            return 0;
+        }
+
+        if (peek != null) {
+            b[off] = (byte)(int)peek;
+            off += 1;
+            len -= 1;
+        }
+
+        int read = readWithinBlockBoundaries(b, off, len);
+
+        if (read > 0) {
+            this.pos += read;
+        }
+
+        if (peek != null) {
+            peek = null;
+            read = read <= 0 ? 1 : read + 1;
+        }
+
+        return read;
+    }
+
+    private int readWithinBlockBoundaries(byte[] b, int off, int len) throws IOException {
+        if (blockLength == -1) {
+            return inputStream.read(b, off, len);
+        } else if (blockLength == 0) {
+            return -1;
+        }
+
+        int read = inputStream.read(b, off, Math.min(len, blockLength));
+        if (read > 0) {
+            blockLength -= read;
+        }
+        return read;
+    }
+
     @Override
     public void close() throws IOException {
         inputStream.close();
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStreamTest.java b/src/test/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStreamTest.java
new file mode 100644
index 00000000..8f0eb112
--- /dev/null
+++ b/src/test/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStreamTest.java
@@ -0,0 +1,102 @@
+package com.github.shyiko.mysql.binlog.io;
+
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+
+public class ByteArrayInputStreamTest {
+    @Test
+    public void testReadToArray() throws Exception {
+        byte[] buff = new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
+        ByteArrayInputStream in = new ByteArrayInputStream(buff);
+        assertEquals(in.getPosition(), 0);
+
+        byte[] b = new byte[20];
+
+        int read = in.read(b, 0, 0);
+        assertEquals(read, 0);
+        assertEquals(in.getPosition(), 0);
+
+        read = in.read(b, 0, 4);
+        assertEquals(read, 4);
+        assertEquals(b[3], 3);
+        assertEquals(in.getPosition(), 4);
+
+        read = in.read(b, 4, 4);
+        assertEquals(read, 4);
+        assertEquals(b[7], 7);
+        assertEquals(in.getPosition(), 8);
+
+        read = in.read(b, 8, 4);
+        assertEquals(read, 4);
+        assertEquals(b[11], 11);
+        assertEquals(in.getPosition(), 12);
+
+        read = in.read(b, 12, 4);
+        assertEquals(read, 4);
+        assertEquals(b[15], 15);
+        assertEquals(in.getPosition(), 16);
+
+        read = in.read(b, 16, 4);
+        assertEquals(read, -1);
+        assertEquals(in.getPosition(), 16);
+    }
+
+    @Test
+    public void testReadToArrayWithinBlockBoundaries() throws Exception {
+        byte[] buff = new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8};
+        ByteArrayInputStream in = new ByteArrayInputStream(buff);
+        byte[] b = new byte[8];
+
+        in.enterBlock(4);
+
+        int read = in.read(b, 0, 3);
+        assertEquals(read, 3);
+        assertEquals(b[2], 2);
+
+        read = in.read(b, 3, 3);
+        assertEquals(read, 1);
+        assertEquals(b[3], 3);
+
+        read = in.read(b, 4, 3);
+        assertEquals(read, -1);
+    }
+
+    @Test(expectedExceptions = NullPointerException.class)
+    public void testReadToArrayWithNullBuff() throws Exception {
+        ByteArrayInputStream in = new ByteArrayInputStream(new byte[]{});
+        in.read(null, 0, 4);
+    }
+
+    @Test(expectedExceptions = IndexOutOfBoundsException.class)
+    public void testReadToArrayWhenLenExceedsBuffSize() throws Exception {
+        ByteArrayInputStream in = new ByteArrayInputStream(new byte[]{0, 1, 2});
+        byte[] b = new byte[1];
+        in.read(b, 0, 4);
+    }
+
+    @Test(expectedExceptions = IndexOutOfBoundsException.class)
+    public void testReadToArrayWhenOffsetNegative() throws Exception {
+        ByteArrayInputStream in = new ByteArrayInputStream(new byte[]{0, 1, 2});
+        byte[] b = new byte[1];
+        in.read(b, -1, 1);
+    }
+
+    @Test(expectedExceptions = IndexOutOfBoundsException.class)
+    public void testReadToArrayWhenLengthNegative() throws Exception {
+        ByteArrayInputStream in = new ByteArrayInputStream(new byte[]{0, 1, 2});
+        byte[] b = new byte[1];
+        in.read(b, 0, -1);
+    }
+
+    @Test
+    public void testPeekAndReadToArray() throws Exception {
+        ByteArrayInputStream in = new ByteArrayInputStream(new byte[]{5, 6, 7});
+        byte[] b = new byte[3];
+        assertEquals(in.peek(), 5);
+        int read = in.read(b, 0, 3);
+        assertEquals(read, 3);
+        assertEquals(b[0], 5);
+        assertEquals(b[2], 7);
+    }
+}

From 6fc750f56fffbea6c3e6d6602a1808020fa646a0 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Tue, 20 Apr 2021 16:12:46 -0700
Subject: [PATCH 084/217] v0.25.1

---
 CHANGELOG.md | 4 ++++
 pom.xml      | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 805767f6..92babdb3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Changelog
 
+## [0.25.1](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.25.0...0.25.1) - 2021-04-20
+
+- performance improves in ByteArrayInputStream#read
+
 ## [0.25.0](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.24.1...0.25.0) - 2021-03-04
 
 - bring back jdk 8 support, this caused... ahem.  Issues.
diff --git a/pom.xml b/pom.xml
index 8c788d4c..8df99701 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.25.0
+    0.25.1
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From 532f474585029e3188ad0eb0d31d4905add54daf Mon Sep 17 00:00:00 2001
From: Igor Kostromin 
Date: Wed, 9 Jun 2021 11:06:41 +0300
Subject: [PATCH 085/217] Introduce overridable setupConnection() method

---
 .../shyiko/mysql/binlog/BinaryLogClient.java  | 23 ++++++++++++-------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index e15f68c6..d8b8299f 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -554,14 +554,7 @@ public void connect() throws IOException, IllegalStateException {
                     }
                     binlogPosition = 4;
                 }
-                ChecksumType checksumType = fetchBinlogChecksum();
-                if (checksumType != ChecksumType.NONE) {
-                    confirmSupportOfChecksum(checksumType);
-                }
-                setMasterServerId();
-                if (heartbeatInterval > 0) {
-                    enableHeartbeat();
-                }
+                setupConnection();
                 gtid = null;
                 tx = false;
                 requestBinaryLogStream();
@@ -614,6 +607,20 @@ public void connect() throws IOException, IllegalStateException {
         }
     }
 
+    /**
+     * Apply additional options for connection before requesting binlog stream.
+     */
+    protected void setupConnection() throws IOException {
+        ChecksumType checksumType = fetchBinlogChecksum();
+        if (checksumType != ChecksumType.NONE) {
+            confirmSupportOfChecksum(checksumType);
+        }
+        setMasterServerId();
+        if (heartbeatInterval > 0) {
+            enableHeartbeat();
+        }
+    }
+
     private PacketChannel openChannel() throws IOException {
         Socket socket = socketFactory != null ? socketFactory.createSocket() : new Socket();
         socket.connect(new InetSocketAddress(hostname, port), (int) connectTimeout);

From efcb63504b54bceaaee3531cdf15ba76106824e7 Mon Sep 17 00:00:00 2001
From: Jack Low 
Date: Tue, 15 Jun 2021 17:16:25 +1000
Subject: [PATCH 086/217] Set DefaultSSLSocketFactory to TLSv1.2 in line with
 JDK 11.0.11:
 https://www.oracle.com/java/technologies/javase/11-0-11-relnotes.html#JDK-8202343

---
 .../shyiko/mysql/binlog/network/DefaultSSLSocketFactory.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/DefaultSSLSocketFactory.java b/src/main/java/com/github/shyiko/mysql/binlog/network/DefaultSSLSocketFactory.java
index 0fabaa10..388e95d2 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/network/DefaultSSLSocketFactory.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/network/DefaultSSLSocketFactory.java
@@ -30,11 +30,11 @@ public class DefaultSSLSocketFactory implements SSLSocketFactory {
     private final String protocol;
 
     public DefaultSSLSocketFactory() {
-        this("TLSv1");
+        this("TLSv1.2");
     }
 
     /**
-     * @param protocol TLSv1, TLSv1.1 or TLSv1.2 (the last two require JDK 7+)
+     * @param protocol TLSv1, TLSv1.1 or TLSv1.2. Since JDK 11.0.11, TLSv1 and TLSv1.1 are no longer supported.
      */
     public DefaultSSLSocketFactory(String protocol) {
         this.protocol = protocol;

From 84eaf6dbfbb70583327b8cd59187575c97cdd6aa Mon Sep 17 00:00:00 2001
From: Jack Low 
Date: Wed, 16 Jun 2021 10:00:25 +1000
Subject: [PATCH 087/217] Update version from 0.25.1 to 0.26.0

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 8df99701..42fb2c1c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.25.1
+    0.26.0
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From 4771a48fe8dc3ac89aeeafc09ce5cd5a72239f9a Mon Sep 17 00:00:00 2001
From: Jack Low 
Date: Thu, 17 Jun 2021 15:32:43 +1000
Subject: [PATCH 088/217] Revert "Update version from 0.25.1 to 0.26.0"

This reverts commit 84eaf6dbfbb70583327b8cd59187575c97cdd6aa.
---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 42fb2c1c..8df99701 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.26.0
+    0.25.1
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From 273fc1502be0bf3c7bbe5cd9f680b1cc6f0a3e02 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Fri, 25 Jun 2021 08:02:24 -0700
Subject: [PATCH 089/217] v0.25.2

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 8df99701..b6b62f90 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.25.1
+    0.25.2
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From 9cc270ee804442945ac6ed984be9e8adea460177 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Fri, 25 Jun 2021 08:03:32 -0700
Subject: [PATCH 090/217] update CHANGELOG

---
 CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 92babdb3..98cb70e4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,10 @@
 # Changelog
 
+## [0.25.2](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.25.1...0.25.2) - 2021-06-25
+
+- allow `setupConnection()` to be overridden
+- upgrade to TLS v1.2
+
 ## [0.25.1](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.25.0...0.25.1) - 2021-04-20
 
 - performance improves in ByteArrayInputStream#read

From 3598de77bf327ac4a662a77c83d71cb1ce10bd39 Mon Sep 17 00:00:00 2001
From: "jackey.zhang" 
Date: Tue, 13 Jul 2021 11:39:58 +0800
Subject: [PATCH 091/217] Adding support Mariadb events, such as GTID and
 annotate rows event(sql)

---
 .../shyiko/mysql/binlog/BinaryLogClient.java  | 101 +++++++----
 .../shyiko/mysql/binlog/MariadbGtidSet.java   | 158 ++++++++++++++++++
 .../binlog/event/AnnotateRowsEventData.java   |  31 ++++
 .../shyiko/mysql/binlog/event/EventType.java  | 119 +++++++------
 .../binlog/event/MariadbGtidEventData.java    |  43 +++++
 .../event/MariadbGtidListEventData.java       |  29 ++++
 .../AnnotateRowsEventDataDeserializer.java    |  24 +++
 .../deserialization/EventDeserializer.java    |   6 +
 .../EventHeaderV4Deserializer.java            |   8 +-
 .../MariadbGtidEventDataDeserializer.java     |  33 ++++
 .../MariadbGtidListEventDataDeserializer.java |  39 +++++
 .../command/DumpBinaryLogCommand.java         |  13 +-
 ...BinaryLogClientMariadbIntegrationTest.java |  87 ++++++++++
 13 files changed, 605 insertions(+), 86 deletions(-)
 create mode 100644 src/main/java/com/github/shyiko/mysql/binlog/MariadbGtidSet.java
 create mode 100644 src/main/java/com/github/shyiko/mysql/binlog/event/AnnotateRowsEventData.java
 create mode 100644 src/main/java/com/github/shyiko/mysql/binlog/event/MariadbGtidEventData.java
 create mode 100644 src/main/java/com/github/shyiko/mysql/binlog/event/MariadbGtidListEventData.java
 create mode 100644 src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AnnotateRowsEventDataDeserializer.java
 create mode 100644 src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidEventDataDeserializer.java
 create mode 100644 src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidListEventDataDeserializer.java
 create mode 100644 src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientMariadbIntegrationTest.java

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index d8b8299f..5ba4cb2d 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -15,21 +15,9 @@
  */
 package com.github.shyiko.mysql.binlog;
 
-import com.github.shyiko.mysql.binlog.event.Event;
-import com.github.shyiko.mysql.binlog.event.EventHeader;
-import com.github.shyiko.mysql.binlog.event.EventHeaderV4;
-import com.github.shyiko.mysql.binlog.event.EventType;
-import com.github.shyiko.mysql.binlog.event.GtidEventData;
-import com.github.shyiko.mysql.binlog.event.QueryEventData;
-import com.github.shyiko.mysql.binlog.event.RotateEventData;
-import com.github.shyiko.mysql.binlog.event.deserialization.ChecksumType;
-import com.github.shyiko.mysql.binlog.event.deserialization.EventDataDeserializationException;
-import com.github.shyiko.mysql.binlog.event.deserialization.EventDataDeserializer;
-import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer;
+import com.github.shyiko.mysql.binlog.event.*;
+import com.github.shyiko.mysql.binlog.event.deserialization.*;
 import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer.EventDataWrapper;
-import com.github.shyiko.mysql.binlog.event.deserialization.GtidEventDataDeserializer;
-import com.github.shyiko.mysql.binlog.event.deserialization.QueryEventDataDeserializer;
-import com.github.shyiko.mysql.binlog.event.deserialization.RotateEventDataDeserializer;
 import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream;
 import com.github.shyiko.mysql.binlog.jmx.BinaryLogClientMXBean;
 import com.github.shyiko.mysql.binlog.network.AuthenticationException;
@@ -46,15 +34,7 @@
 import com.github.shyiko.mysql.binlog.network.protocol.Packet;
 import com.github.shyiko.mysql.binlog.network.protocol.PacketChannel;
 import com.github.shyiko.mysql.binlog.network.protocol.ResultSetRowPacket;
-import com.github.shyiko.mysql.binlog.network.protocol.command.AuthenticateNativePasswordCommand;
-import com.github.shyiko.mysql.binlog.network.protocol.command.AuthenticateSHA2Command;
-import com.github.shyiko.mysql.binlog.network.protocol.command.AuthenticateSecurityPasswordCommand;
-import com.github.shyiko.mysql.binlog.network.protocol.command.Command;
-import com.github.shyiko.mysql.binlog.network.protocol.command.DumpBinaryLogCommand;
-import com.github.shyiko.mysql.binlog.network.protocol.command.DumpBinaryLogGtidCommand;
-import com.github.shyiko.mysql.binlog.network.protocol.command.PingCommand;
-import com.github.shyiko.mysql.binlog.network.protocol.command.QueryCommand;
-import com.github.shyiko.mysql.binlog.network.protocol.command.SSLRequestCommand;
+import com.github.shyiko.mysql.binlog.network.protocol.command.*;
 
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.TrustManager;
@@ -141,6 +121,8 @@ public X509Certificate[] getAcceptedIssuers() {
     private boolean useBinlogFilenamePositionInGtidMode;
     private String gtid;
     private boolean tx;
+    private boolean isMariadb = false;
+    private boolean mariadbSendAnnotateRowsEvent = false;
 
     private EventDeserializer eventDeserializer = new EventDeserializer();
 
@@ -335,7 +317,12 @@ public void setGtidSet(String gtidSet) {
             this.binlogFilename = "";
         }
         synchronized (gtidSetAccessLock) {
-            this.gtidSet = gtidSet != null ? new GtidSet(gtidSet) : null;
+            // mariadb GtidSet format will be domainId-serverId-sequence
+            if (gtidSet != null && !gtidSet.contains(":")) {
+                this.gtidSet = new MariadbGtidSet(gtidSet);
+            } else {
+                this.gtidSet = gtidSet != null ? new GtidSet(gtidSet) : null;
+            }
         }
     }
 
@@ -501,6 +488,19 @@ public void setThreadFactory(ThreadFactory threadFactory) {
         this.threadFactory = threadFactory;
     }
 
+    public boolean isMariadbSendAnnotateRowsEvent() {
+        return mariadbSendAnnotateRowsEvent;
+    }
+
+    /**
+     * Only in Mariadb, if set true, the Slave server connects with the BINLOG_SEND_ANNOTATE_ROWS_EVENT flag (value is 2)
+     * in the COM_BINLOG_DUMP Slave Registration phase
+     * @param mariadbSendAnnotateRowsEvent
+     */
+    public void setMariadbSendAnnotateRowsEvent(boolean mariadbSendAnnotateRowsEvent) {
+        this.mariadbSendAnnotateRowsEvent = mariadbSendAnnotateRowsEvent;
+    }
+
     /**
      * Connect to the replication stream. Note that this method blocks until disconnected.
      * @throws AuthenticationException if authentication fails
@@ -538,10 +538,15 @@ public void connect() throws IOException, IllegalStateException {
                 channel.authenticationComplete();
 
                 connectionId = greetingPacket.getThreadId();
+                isMariadb = greetingPacket.getServerVersion().toLowerCase().contains("mariadb");
                 if ("".equals(binlogFilename)) {
                     synchronized (gtidSetAccessLock) {
                         if (gtidSet != null && "".equals(gtidSet.toString()) && gtidSetFallbackToPurged) {
-                            gtidSet = new GtidSet(fetchGtidPurged());
+                            if (isMariadb) {
+                                gtidSet = new MariadbGtidSet(fetchGtidPurged());
+                            } else {
+                                gtidSet = new GtidSet(fetchGtidPurged());
+                            }
                         }
                     }
                 }
@@ -594,6 +599,11 @@ public void connect() throws IOException, IllegalStateException {
                 if (gtidSet != null) {
                     ensureEventDataDeserializer(EventType.GTID, GtidEventDataDeserializer.class);
                     ensureEventDataDeserializer(EventType.QUERY, QueryEventDataDeserializer.class);
+                    if (isMariadb) {
+                        ensureEventDataDeserializer(EventType.ANNOTATE_ROWS, AnnotateRowsEventDataDeserializer.class);
+                        ensureEventDataDeserializer(EventType.MARIADB_GTID, MariadbGtidEventDataDeserializer.class);
+                        ensureEventDataDeserializer(EventType.MARIADB_GTID_LIST, MariadbGtidListEventDataDeserializer.class);
+                    }
                 }
             }
             listenForEventPackets();
@@ -730,10 +740,22 @@ private void requestBinaryLogStream() throws IOException {
         Command dumpBinaryLogCommand;
         synchronized (gtidSetAccessLock) {
             if (gtidSet != null) {
-                dumpBinaryLogCommand = new DumpBinaryLogGtidCommand(serverId,
-                    useBinlogFilenamePositionInGtidMode ? binlogFilename : "",
-                    useBinlogFilenamePositionInGtidMode ? binlogPosition : 4,
-                    gtidSet);
+                if (isMariadb) {
+                    channel.write(new QueryCommand("SET @mariadb_slave_capability=4"));
+                    checkError(channel.read());
+                    channel.write(new QueryCommand("SET @slave_connect_state = '" + gtidSet.toString() + "'"));
+                    checkError(channel.read());
+                    channel.write(new QueryCommand("SET @slave_gtid_strict_mode = 0"));
+                    checkError(channel.read());
+                    channel.write(new QueryCommand("SET @slave_gtid_ignore_duplicates = 0"));
+                    checkError(channel.read());
+                    dumpBinaryLogCommand = new DumpBinaryLogCommand(serverId, "", 0L, isMariadbSendAnnotateRowsEvent());
+                } else {
+                    dumpBinaryLogCommand = new DumpBinaryLogGtidCommand(serverId,
+                        useBinlogFilenamePositionInGtidMode ? binlogFilename : "",
+                        useBinlogFilenamePositionInGtidMode ? binlogPosition : 4,
+                        gtidSet);
+                }
             } else {
                 dumpBinaryLogCommand = new DumpBinaryLogCommand(serverId, binlogFilename, binlogPosition);
             }
@@ -1034,13 +1056,30 @@ private void updateGtidSet(Event event) {
                 GtidEventData gtidEventData = (GtidEventData) EventDataWrapper.internal(event.getData());
                 gtid = gtidEventData.getGtid();
                 break;
+            case MARIADB_GTID:
+                MariadbGtidEventData mariadbGtidEventData =  (MariadbGtidEventData) EventDataWrapper.internal(event.getData());
+                mariadbGtidEventData.setServerId(eventHeader.getServerId());
+                gtid = mariadbGtidEventData.toString();
+                break;
+            case MARIADB_GTID_LIST:
+                MariadbGtidListEventData mariadbGtidListEventData =  (MariadbGtidListEventData) EventDataWrapper.internal(event.getData());
+                gtid = mariadbGtidListEventData.getMariaGTIDSet().toString();
+                break;
             case XID:
                 commitGtid();
                 tx = false;
                 break;
             case QUERY:
-                QueryEventData queryEventData = (QueryEventData) EventDataWrapper.internal(event.getData());
-                String sql = queryEventData.getSql();
+            case ANNOTATE_ROWS:
+                String sql;
+                if (eventHeader.getEventType() == EventType.QUERY) {
+                    QueryEventData queryEventData = (QueryEventData) EventDataWrapper.internal(event.getData());
+                    sql = queryEventData.getSql();
+                } else {
+                    AnnotateRowsEventData annotateRowsEventData = (AnnotateRowsEventData) EventDataWrapper.internal(event.getData());
+                    sql = annotateRowsEventData.getRowsQuery();
+                }
+
                 if (sql == null) {
                     break;
                 }
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/MariadbGtidSet.java b/src/main/java/com/github/shyiko/mysql/binlog/MariadbGtidSet.java
new file mode 100644
index 00000000..c86ee54a
--- /dev/null
+++ b/src/main/java/com/github/shyiko/mysql/binlog/MariadbGtidSet.java
@@ -0,0 +1,158 @@
+package com.github.shyiko.mysql.binlog;
+
+import java.util.*;
+
+/**
+ * Mariadb Global Transaction ID
+ *
+ * @author Winger
+ * @see GTID for the original doc
+ */
+public class MariadbGtidSet extends GtidSet {
+
+    private Map map = new HashMap<>();
+
+    public MariadbGtidSet() {
+        super(null); //
+    }
+
+    public MariadbGtidSet(String gtidSet) {
+        super(null);
+        if (gtidSet != null && gtidSet.length() > 0) {
+            String[] gtids = gtidSet.replaceAll("\n", "").split(",");
+            for (String gtid : gtids) {
+                MariaGtid mariaGtid = MariaGtid.parse(gtid);
+                map.put(mariaGtid.getDomainId(), mariaGtid);
+            }
+        }
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder();
+        for (MariaGtid gtid : map.values()) {
+            if (sb.length() > 0) {
+                sb.append(",");
+            }
+            sb.append(gtid.toString());
+        }
+        return sb.toString();
+    }
+
+    @Override
+    public Collection getUUIDSets() {
+        throw new UnsupportedOperationException("Mariadb gtid not support this method");
+    }
+
+    @Override
+    public UUIDSet getUUIDSet(String uuid) {
+        throw new UnsupportedOperationException("Mariadb gtid not support this method");
+    }
+
+    @Override
+    public UUIDSet putUUIDSet(UUIDSet uuidSet) {
+        throw new UnsupportedOperationException("Mariadb gtid not support this method");
+    }
+
+    @Override
+    public boolean add(String gtid) {
+        MariaGtid mariaGtid = MariaGtid.parse(gtid);
+        map.put(mariaGtid.getDomainId(), mariaGtid);
+        return true;
+    }
+
+    public void add(MariaGtid gtid) {
+        map.put(gtid.getDomainId(), gtid);
+    }
+
+    @Override
+    public boolean isContainedWithin(GtidSet other) {
+        throw new UnsupportedOperationException("Mariadb gtid not support this method");
+    }
+
+    @Override
+    public int hashCode() {
+        return map.keySet().hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == this) {
+            return true;
+        }
+        if (obj instanceof MariadbGtidSet) {
+            MariadbGtidSet that = (MariadbGtidSet) obj;
+            return this.map.equals(that.map);
+        }
+        return false;
+    }
+
+    public static class MariaGtid {
+
+        // {domainId}-{serverId}-{sequence}
+        private long domainId;
+        private long serverId;
+        private long sequence;
+
+        public MariaGtid(long domainId, long serverId, long sequence) {
+            this.domainId = domainId;
+            this.serverId = serverId;
+            this.sequence = sequence;
+        }
+
+        public MariaGtid(String gtid) {
+            String[] gtidArr = gtid.split("-");
+            this.domainId = Long.parseLong(gtidArr[0]);
+            this.serverId = Long.parseLong(gtidArr[1]);
+            this.sequence = Long.parseLong(gtidArr[2]);
+        }
+
+        public static MariaGtid parse(String gtid) {
+            return new MariaGtid(gtid);
+        }
+
+        public long getDomainId() {
+            return domainId;
+        }
+
+        public void setDomainId(long domainId) {
+            this.domainId = domainId;
+        }
+
+        public long getServerId() {
+            return serverId;
+        }
+
+        public void setServerId(long serverId) {
+            this.serverId = serverId;
+        }
+
+        public long getSequence() {
+            return sequence;
+        }
+
+        public void setSequence(long sequence) {
+            this.sequence = sequence;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+            MariaGtid mariaGtid = (MariaGtid) o;
+            return domainId == mariaGtid.domainId &&
+                serverId == mariaGtid.serverId &&
+                sequence == mariaGtid.sequence;
+        }
+
+        @Override
+        public String toString() {
+            return String.format("%s-%s-%s", domainId, serverId, sequence);
+        }
+    }
+}
+
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/AnnotateRowsEventData.java b/src/main/java/com/github/shyiko/mysql/binlog/event/AnnotateRowsEventData.java
new file mode 100644
index 00000000..85fa79e0
--- /dev/null
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/AnnotateRowsEventData.java
@@ -0,0 +1,31 @@
+package com.github.shyiko.mysql.binlog.event;
+
+/**
+ * Mariadb ANNOTATE_ROWS_EVENT events accompany row events and describe the query which caused the row event
+ * Enable this with --binlog-annotate-row-events (default on from MariaDB 10.2.4).
+ * In the binary log, each Annotate_rows event precedes the corresponding Table map event.
+ * Note the master server sends ANNOTATE_ROWS_EVENT events only if the Slave server connects
+ * with the BINLOG_SEND_ANNOTATE_ROWS_EVENT flag (value is 2) in the COM_BINLOG_DUMP Slave Registration phase.
+ *
+ * @author Winger
+ * @see ANNOTATE_ROWS_EVENT for the original doc
+ */
+public class AnnotateRowsEventData implements EventData {
+
+    private String rowsQuery;
+
+    public String getRowsQuery() {
+        return rowsQuery;
+    }
+
+    public void setRowsQuery(String rowsQuery) {
+        this.rowsQuery = rowsQuery;
+    }
+
+    @Override
+    public String toString() {
+        return "AnnotateRowsEventData{" +
+            "rowsQuery='" + rowsQuery + '\'' +
+            '}';
+    }
+}
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
index 7f94ccfd..fb1fcaa8 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
@@ -16,99 +16,99 @@
 package com.github.shyiko.mysql.binlog.event;
 
 /**
+ * @author Stanley Shyiko
  * @see Event Meanings for the original
  * documentation.
- * @author Stanley Shyiko
  */
 public enum EventType {
 
     /**
      * Events of this event type should never occur. Not written to a binary log.
      */
-    UNKNOWN,
+    UNKNOWN(0),
     /**
      * A descriptor event that is written to the beginning of the each binary log file. (In MySQL 4.0 and 4.1,
      * this event is written only to the first binary log file that the server creates after startup.) This event is
      * used in MySQL 3.23 through 4.1 and superseded in MySQL 5.0 by {@link #FORMAT_DESCRIPTION}.
      */
-    START_V3,
+    START_V3(1),
     /**
      * Written when an updating statement is done.
      */
-    QUERY,
+    QUERY(2),
     /**
      * Written when mysqld stops.
      */
-    STOP,
+    STOP(3),
     /**
      * Written when mysqld switches to a new binary log file. This occurs when someone issues a FLUSH LOGS statement or
      * the current binary log file becomes larger than max_binlog_size.
      */
-    ROTATE,
+    ROTATE(4),
     /**
      * Written every time a statement uses an AUTO_INCREMENT column or the LAST_INSERT_ID() function; precedes other
      * events for the statement. This is written only before a {@link #QUERY} and is not used in case of RBR.
      */
-    INTVAR,
+    INTVAR(5),
     /**
      * Used for LOAD DATA INFILE statements in MySQL 3.23.
      */
-    LOAD,
+    LOAD(6),
     /**
      * Not used.
      */
-    SLAVE,
+    SLAVE(7),
     /**
      * Used for LOAD DATA INFILE statements in MySQL 4.0 and 4.1.
      */
-    CREATE_FILE,
+    CREATE_FILE(8),
     /**
      * Used for LOAD DATA INFILE statements as of MySQL 4.0.
      */
-    APPEND_BLOCK,
+    APPEND_BLOCK(9),
     /**
      * Used for LOAD DATA INFILE statements in 4.0 and 4.1.
      */
-    EXEC_LOAD,
+    EXEC_LOAD(10),
     /**
      * Used for LOAD DATA INFILE statements as of MySQL 4.0.
      */
-    DELETE_FILE,
+    DELETE_FILE(11),
     /**
      * Used for LOAD DATA INFILE statements in MySQL 4.0 and 4.1.
      */
-    NEW_LOAD,
+    NEW_LOAD(12),
     /**
      * Written every time a statement uses the RAND() function; precedes other events for the statement. Indicates the
      * seed values to use for generating a random number with RAND() in the next statement. This is written only
      * before a {@link #QUERY} and is not used in case of RBR.
      */
-    RAND,
+    RAND(13),
     /**
      * Written every time a statement uses a user variable; precedes other events for the statement. Indicates the
      * value to use for the user variable in the next statement. This is written only before a {@link #QUERY} and
      * is not used in case of RBR.
      */
-    USER_VAR,
+    USER_VAR(14),
     /**
      * A descriptor event that is written to the beginning of the each binary log file.
      * This event is used as of MySQL 5.0; it supersedes {@link #START_V3}.
      */
-    FORMAT_DESCRIPTION,
+    FORMAT_DESCRIPTION(15),
     /**
      * Generated for a commit of a transaction that modifies one or more tables of an XA-capable storage engine.
      * Normal transactions are implemented by sending a {@link #QUERY} containing a BEGIN statement and a {@link #QUERY}
      * containing a COMMIT statement (or a ROLLBACK statement if the transaction is rolled back).
      */
-    XID,
+    XID(16),
     /**
      * Used for LOAD DATA INFILE statements as of MySQL 5.0.
      */
-    BEGIN_LOAD_QUERY,
+    BEGIN_LOAD_QUERY(17),
     /**
      * Used for LOAD DATA INFILE statements as of MySQL 5.0.
      */
-    EXECUTE_LOAD_QUERY,
+    EXECUTE_LOAD_QUERY(18),
     /**
      * This event precedes each row operation event. It maps a table definition to a number, where the table definition
      * consists of database and table names and column definitions. The purpose of this event is to enable replication
@@ -117,105 +117,128 @@ public enum EventType {
      * of TABLE_MAP events: one per table used by events in the sequence.
      * Used in case of RBR.
      */
-    TABLE_MAP,
+    TABLE_MAP(19),
     /**
      * Describes inserted rows (within a single table).
      * Used in case of RBR (5.1.0 - 5.1.15).
      */
-    PRE_GA_WRITE_ROWS,
+    PRE_GA_WRITE_ROWS(20),
     /**
      * Describes updated rows (within a single table).
      * Used in case of RBR (5.1.0 - 5.1.15).
      */
-    PRE_GA_UPDATE_ROWS,
+    PRE_GA_UPDATE_ROWS(21),
     /**
      * Describes deleted rows (within a single table).
      * Used in case of RBR (5.1.0 - 5.1.15).
      */
-    PRE_GA_DELETE_ROWS,
+    PRE_GA_DELETE_ROWS(22),
     /**
      * Describes inserted rows (within a single table).
      * Used in case of RBR (5.1.16 - mysql-trunk).
      */
-    WRITE_ROWS,
+    WRITE_ROWS(23),
     /**
      * Describes updated rows (within a single table).
      * Used in case of RBR (5.1.16 - mysql-trunk).
      */
-    UPDATE_ROWS,
+    UPDATE_ROWS(24),
     /**
      * Describes deleted rows (within a single table).
      * Used in case of RBR (5.1.16 - mysql-trunk).
      */
-    DELETE_ROWS,
+    DELETE_ROWS(25),
     /**
      * Used to log an out of the ordinary event that occurred on the master. It notifies the slave that something
      * happened on the master that might cause data to be in an inconsistent state.
      */
-    INCIDENT,
+    INCIDENT(26),
     /**
      * Sent by a master to a slave to let the slave know that the master is still alive. Not written to a binary log.
      */
-    HEARTBEAT,
+    HEARTBEAT(27),
     /**
      * In some situations, it is necessary to send over ignorable data to the slave: data that a slave can handle in
      * case there is code for handling it, but which can be ignored if it is not recognized.
      */
-    IGNORABLE,
+    IGNORABLE(28),
     /**
      * Introduced to record the original query for rows events in RBR.
      */
-    ROWS_QUERY,
+    ROWS_QUERY(29),
     /**
      * Describes inserted rows (within a single table).
      * Used in case of RBR (5.1.18+).
      */
-    EXT_WRITE_ROWS,
+    EXT_WRITE_ROWS(30),
     /**
      * Describes updated rows (within a single table).
      * Used in case of RBR (5.1.18+).
      */
-    EXT_UPDATE_ROWS,
+    EXT_UPDATE_ROWS(31),
     /**
      * Describes deleted rows (within a single table).
      * Used in case of RBR (5.1.18+).
      */
-    EXT_DELETE_ROWS,
+    EXT_DELETE_ROWS(32),
     /**
      * Global Transaction Identifier.
      */
-    GTID,
-    ANONYMOUS_GTID,
-    PREVIOUS_GTIDS,
-    TRANSACTION_CONTEXT,
-    VIEW_CHANGE,
+    GTID(33),
+    ANONYMOUS_GTID(34),
+    PREVIOUS_GTIDS(35),
+    TRANSACTION_CONTEXT(36),
+    VIEW_CHANGE(37),
     /**
      * Prepared XA transaction terminal event similar to XID except that it is specific to XA transaction.
      */
-    XA_PREPARE;
+    XA_PREPARE(38),
+
+    /**
+     * MariaDB Support Events
+     *
+     * @see Replication Protocol for the original doc.
+     */
+    ANNOTATE_ROWS(160), //
+    MARIADB_GTID(162),
+    MARIADB_GTID_LIST(163);
+
+    private final int eventNumber;
+
+    EventType(int eventNumber) {
+        this.eventNumber = eventNumber;
+    }
 
     public static boolean isRowMutation(EventType eventType) {
         return EventType.isWrite(eventType) ||
-               EventType.isUpdate(eventType) ||
-               EventType.isDelete(eventType);
+            EventType.isUpdate(eventType) ||
+            EventType.isDelete(eventType);
     }
 
     public static boolean isWrite(EventType eventType) {
         return eventType == PRE_GA_WRITE_ROWS ||
-               eventType == WRITE_ROWS ||
-               eventType == EXT_WRITE_ROWS;
+            eventType == WRITE_ROWS ||
+            eventType == EXT_WRITE_ROWS;
     }
 
     public static boolean isUpdate(EventType eventType) {
         return eventType == PRE_GA_UPDATE_ROWS ||
-               eventType == UPDATE_ROWS ||
-               eventType == EXT_UPDATE_ROWS;
+            eventType == UPDATE_ROWS ||
+            eventType == EXT_UPDATE_ROWS;
     }
 
     public static boolean isDelete(EventType eventType) {
         return eventType == PRE_GA_DELETE_ROWS ||
-               eventType == DELETE_ROWS ||
-               eventType == EXT_DELETE_ROWS;
+            eventType == DELETE_ROWS ||
+            eventType == EXT_DELETE_ROWS;
     }
 
+    public static EventType byEventNumber(int num) {
+        for (EventType type : EventType.values()) {
+            if (type.eventNumber == num) {
+                return type;
+            }
+        }
+        return null;
+    }
 }
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/MariadbGtidEventData.java b/src/main/java/com/github/shyiko/mysql/binlog/event/MariadbGtidEventData.java
new file mode 100644
index 00000000..54838d03
--- /dev/null
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/MariadbGtidEventData.java
@@ -0,0 +1,43 @@
+package com.github.shyiko.mysql.binlog.event;
+
+/**
+ * MariaDB and MySQL have different GTID implementations, and that these are not compatible with each other.
+ *
+ * @author Winger
+ * @see GTID_EVENT for the original doc
+ */
+public class MariadbGtidEventData implements EventData {
+
+    private long sequence;
+    private long domainId;
+    private long serverId;
+
+    public long getSequence() {
+        return sequence;
+    }
+
+    public void setSequence(long sequence) {
+        this.sequence = sequence;
+    }
+
+    public long getDomainId() {
+        return domainId;
+    }
+
+    public void setDomainId(long domainId) {
+        this.domainId = domainId;
+    }
+
+    public long getServerId() {
+        return serverId;
+    }
+
+    public void setServerId(long serverId) {
+        this.serverId = serverId;
+    }
+
+    @Override
+    public String toString() {
+        return domainId + "-" + serverId + "-" + sequence;
+    }
+}
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/MariadbGtidListEventData.java b/src/main/java/com/github/shyiko/mysql/binlog/event/MariadbGtidListEventData.java
new file mode 100644
index 00000000..ae6e91f6
--- /dev/null
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/MariadbGtidListEventData.java
@@ -0,0 +1,29 @@
+package com.github.shyiko.mysql.binlog.event;
+
+import com.github.shyiko.mysql.binlog.MariadbGtidSet;
+
+/**
+ * Logged in every binlog to record the current replication state
+ *
+ * @author Winger
+ * @see GTID_LIST_EVENT for the original doc
+ */
+public class MariadbGtidListEventData implements EventData {
+
+    private MariadbGtidSet mariaGTIDSet;
+
+    public MariadbGtidSet getMariaGTIDSet() {
+        return mariaGTIDSet;
+    }
+
+    public void setMariaGTIDSet(MariadbGtidSet mariaGTIDSet) {
+        this.mariaGTIDSet = mariaGTIDSet;
+    }
+
+    @Override
+    public String toString() {
+        return "MariadbGtidListEventData{" +
+            "mariaGTIDSet=" + mariaGTIDSet +
+            '}';
+    }
+}
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AnnotateRowsEventDataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AnnotateRowsEventDataDeserializer.java
new file mode 100644
index 00000000..688a9b29
--- /dev/null
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AnnotateRowsEventDataDeserializer.java
@@ -0,0 +1,24 @@
+package com.github.shyiko.mysql.binlog.event.deserialization;
+
+import com.github.shyiko.mysql.binlog.event.AnnotateRowsEventData;
+import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream;
+
+import java.io.IOException;
+
+/**
+ * Mariadb ANNOTATE_ROWS_EVENT Fields
+ * 
+ *  string The SQL statement (not null-terminated)
+ * 

+ *
+ * @author Winger
+ */
+public class AnnotateRowsEventDataDeserializer implements EventDataDeserializer {
+
+    @Override
+    public AnnotateRowsEventData deserialize(ByteArrayInputStream inputStream) throws IOException {
+        AnnotateRowsEventData event = new AnnotateRowsEventData();
+        event.setRowsQuery(inputStream.readString(inputStream.available()));
+        return event;
+    }
+}
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java
index cf936852..ffb37880 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java
@@ -120,6 +120,12 @@ private void registerDefaultEventDataDeserializers() {
                new PreviousGtidSetDeserializer());
         eventDataDeserializers.put(EventType.XA_PREPARE,
                 new XAPrepareEventDataDeserializer());
+        eventDataDeserializers.put(EventType.ANNOTATE_ROWS,
+            new AnnotateRowsEventDataDeserializer());
+        eventDataDeserializers.put(EventType.MARIADB_GTID,
+            new MariadbGtidEventDataDeserializer());
+        eventDataDeserializers.put(EventType.MARIADB_GTID_LIST,
+            new MariadbGtidListEventDataDeserializer());
     }
 
     public void setEventDataDeserializer(EventType eventType, EventDataDeserializer eventDataDeserializer) {
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
index c0569f09..e9533629 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
@@ -26,8 +26,6 @@
  */
 public class EventHeaderV4Deserializer implements EventHeaderDeserializer {
 
-    private static final EventType[] EVENT_TYPES = EventType.values();
-
     @Override
     public EventHeaderV4 deserialize(ByteArrayInputStream inputStream) throws IOException {
         EventHeaderV4 header = new EventHeaderV4();
@@ -41,10 +39,8 @@ public EventHeaderV4 deserialize(ByteArrayInputStream inputStream) throws IOExce
     }
 
     private static EventType getEventType(int ordinal) {
-        if (ordinal >= EVENT_TYPES.length) {
-            return EventType.UNKNOWN;
-        }
-        return EVENT_TYPES[ordinal];
+        EventType eventType = EventType.byEventNumber(ordinal);
+        return eventType == null ? EventType.UNKNOWN : eventType;
     }
 
 }
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidEventDataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidEventDataDeserializer.java
new file mode 100644
index 00000000..c57e2558
--- /dev/null
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidEventDataDeserializer.java
@@ -0,0 +1,33 @@
+package com.github.shyiko.mysql.binlog.event.deserialization;
+
+import com.github.shyiko.mysql.binlog.event.MariadbGtidEventData;
+import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream;
+
+import java.io.IOException;
+
+/**
+ * Mariadb GTID_EVENT Fields
+ * 
+ *     uint<8> GTID sequence
+ *     uint<4> Replication Domain ID
+ *     uint<1> Flags
+ *
+ * 	if flag & FL_GROUP_COMMIT_ID
+ * 	    uint<8> commit_id
+ * 	else
+ * 	    uint<6> 0
+ * 

+ *
+ * @author Winger
+ * @see GTID_EVENT for the original doc
+ */
+public class MariadbGtidEventDataDeserializer implements EventDataDeserializer {
+    @Override
+    public MariadbGtidEventData deserialize(ByteArrayInputStream inputStream) throws IOException {
+        MariadbGtidEventData event = new MariadbGtidEventData();
+        event.setSequence(inputStream.readLong(8));
+        event.setDomainId(inputStream.readInteger(4));
+        // Flags ignore
+        return event;
+    }
+}
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidListEventDataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidListEventDataDeserializer.java
new file mode 100644
index 00000000..8a429532
--- /dev/null
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidListEventDataDeserializer.java
@@ -0,0 +1,39 @@
+package com.github.shyiko.mysql.binlog.event.deserialization;
+
+
+import com.github.shyiko.mysql.binlog.MariadbGtidSet;
+import com.github.shyiko.mysql.binlog.event.MariadbGtidListEventData;
+import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream;
+
+import java.io.IOException;
+
+/**
+ * Mariadb GTID_LIST_EVENT Fields
+ * 
+ *  uint<4> Number of GTIDs
+ *  GTID[0]
+ *      uint<4> Replication Domain ID
+ *      uint<4> Server_ID
+ *      uint<8> GTID sequence ...
+ * GTID[n]
+ * 

+ *
+ * @author Winger
+ * @see GTID_EVENT for the original doc
+ */
+public class MariadbGtidListEventDataDeserializer implements EventDataDeserializer {
+    @Override
+    public MariadbGtidListEventData deserialize(ByteArrayInputStream inputStream) throws IOException {
+        MariadbGtidListEventData eventData = new MariadbGtidListEventData();
+        long gtidLength = inputStream.readInteger(4);
+        MariadbGtidSet mariaGTIDSet = new MariadbGtidSet();
+        for (int i = 0; i < gtidLength; i++) {
+            long domainId = inputStream.readInteger(4);
+            long serverId = inputStream.readInteger(4);
+            long sequence = inputStream.readLong(8);
+            mariaGTIDSet.add(new MariadbGtidSet.MariaGtid(domainId, serverId, sequence));
+        }
+        eventData.setMariaGTIDSet(mariaGTIDSet);
+        return eventData;
+    }
+}
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/DumpBinaryLogCommand.java b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/DumpBinaryLogCommand.java
index 36216c76..b7436aab 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/DumpBinaryLogCommand.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/command/DumpBinaryLogCommand.java
@@ -24,9 +24,11 @@
  */
 public class DumpBinaryLogCommand implements Command {
 
+    public static final int BINLOG_SEND_ANNOTATE_ROWS_EVENT = 2;
     private long serverId;
     private String binlogFilename;
     private long binlogPosition;
+    private boolean sendAnnotateRowsEvent;
 
     public DumpBinaryLogCommand(long serverId, String binlogFilename, long binlogPosition) {
         this.serverId = serverId;
@@ -34,12 +36,21 @@ public DumpBinaryLogCommand(long serverId, String binlogFilename, long binlogPos
         this.binlogPosition = binlogPosition;
     }
 
+    public DumpBinaryLogCommand(long serverId, String binlogFilename, long binlogPosition, boolean sendAnnotateRowsEvent) {
+        this(serverId, binlogFilename, binlogPosition);
+        this.sendAnnotateRowsEvent = sendAnnotateRowsEvent;
+    }
+
     @Override
     public byte[] toByteArray() throws IOException {
         ByteArrayOutputStream buffer = new ByteArrayOutputStream();
         buffer.writeInteger(CommandType.BINLOG_DUMP.ordinal(), 1);
         buffer.writeLong(this.binlogPosition, 4);
-        buffer.writeInteger(0, 2); // flag
+        int binlogFlags = 0;
+        if (sendAnnotateRowsEvent) {
+            binlogFlags |= BINLOG_SEND_ANNOTATE_ROWS_EVENT;
+        }
+        buffer.writeInteger(binlogFlags, 2); // flag
         buffer.writeLong(this.serverId, 4);
         buffer.writeString(this.binlogFilename);
         return buffer.toByteArray();
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientMariadbIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientMariadbIntegrationTest.java
new file mode 100644
index 00000000..4b7ea9ca
--- /dev/null
+++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientMariadbIntegrationTest.java
@@ -0,0 +1,87 @@
+package com.github.shyiko.mysql.binlog;
+
+import com.github.shyiko.mysql.binlog.event.AnnotateRowsEventData;
+import com.github.shyiko.mysql.binlog.event.MariadbGtidEventData;
+import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer;
+import org.testng.annotations.Test;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.concurrent.TimeUnit;
+
+import static org.testng.Assert.assertNotEquals;
+import static org.testng.AssertJUnit.assertNotNull;
+
+/**
+ * BinaryLogClientMariadbGTIDIntegrationTest.java
+ *
+ * @author winger[jackey.zhang@woqutech.com]
+ * @create 2021-07-13 10:52 上午
+ */
+public class BinaryLogClientMariadbIntegrationTest {
+
+    protected BinaryLogClientIntegrationTest.MySQLConnection master;
+
+    @Test
+    public void testMariadbUseGTIDAndAnnotateRowsEvent() throws Exception {
+        master = new BinaryLogClientIntegrationTest.MySQLConnection("127.0.0.1", 3306, "root", "123456");
+        master.execute(new BinaryLogClientIntegrationTest.Callback() {
+            @Override
+            public void execute(Statement statement) throws SQLException {
+                statement.execute("drop database if exists mbcj_test");
+                statement.execute("create database mbcj_test");
+                statement.execute("use mbcj_test");
+                statement.execute("CREATE TABLE if not exists foo (i int)");
+                statement.execute("CREATE TABLE if not exists bar (i int)");
+            }
+        });
+        // get current gtid
+        final String[] currentGtidPos = new String[1];
+        master.query("show global variables like 'gtid_current_pos%'", new BinaryLogClientIntegrationTest.Callback() {
+
+            @Override
+            public void execute(ResultSet rs) throws SQLException {
+                rs.next();
+                currentGtidPos[0] = rs.getString(2);
+            }
+        });
+
+        CountDownEventListener eventListener;
+        BinaryLogClient client = new BinaryLogClient("127.0.0.1", 3306, "root", "123456");
+        client.setGtidSet(currentGtidPos[0]);
+        client.setMariadbSendAnnotateRowsEvent(true);
+
+        EventDeserializer eventDeserializer = new EventDeserializer();
+        eventDeserializer.setCompatibilityMode(EventDeserializer.CompatibilityMode.CHAR_AND_BINARY_AS_BYTE_ARRAY,
+            EventDeserializer.CompatibilityMode.DATE_AND_TIME_AS_LONG);
+        client.setEventDeserializer(eventDeserializer);
+        client.registerEventListener(new TraceEventListener());
+        client.registerLifecycleListener(new TraceLifecycleListener());
+        client.registerEventListener(eventListener = new CountDownEventListener());
+
+        master.execute(new BinaryLogClientIntegrationTest.Callback() {
+            @Override
+            public void execute(Statement statement) throws SQLException {
+                statement.execute("INSERT INTO foo set i = 2");
+                statement.execute("DROP TABLE IF EXISTS bar");
+            }
+        });
+
+        try {
+            eventListener.reset();
+            client.connect();
+
+            eventListener.waitFor(MariadbGtidEventData.class, 1, TimeUnit.SECONDS.toMillis(4));
+            String gtidSet = client.getGtidSet();
+            assertNotNull(gtidSet);
+
+            eventListener.reset();
+            eventListener.waitFor(AnnotateRowsEventData.class, 1, TimeUnit.SECONDS.toMillis(4));
+            gtidSet = client.getGtidSet();
+            assertNotEquals(currentGtidPos[0], gtidSet);
+        } finally {
+            client.disconnect();
+        }
+    }
+}

From 1f2779137fa202624aed00c97564b0f1995ba721 Mon Sep 17 00:00:00 2001
From: winger 
Date: Thu, 15 Jul 2021 13:46:57 +0800
Subject: [PATCH 092/217] refact to support mariadb types

---
 .../shyiko/mysql/binlog/BinaryLogClient.java  | 167 +++++++++---------
 .../mysql/binlog/MariadbBinaryLogClient.java  |  88 +++++++++
 ...ariadbBinaryLogClientIntegrationTest.java} |  13 +-
 .../mysql/binlog/MariadbGtidSetTest.java      |  36 ++++
 ...AnnotateRowsEventDataDeserializerTest.java |  27 +++
 .../MariadbGtidEventDataDeserializerTest.java |  26 +++
 ...iadbGtidListEventDataDeserializerTest.java |  26 +++
 7 files changed, 288 insertions(+), 95 deletions(-)
 create mode 100644 src/main/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClient.java
 rename src/test/java/com/github/shyiko/mysql/binlog/{BinaryLogClientMariadbIntegrationTest.java => MariadbBinaryLogClientIntegrationTest.java} (89%)
 create mode 100644 src/test/java/com/github/shyiko/mysql/binlog/MariadbGtidSetTest.java
 create mode 100644 src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/AnnotateRowsEventDataDeserializerTest.java
 create mode 100644 src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidEventDataDeserializerTest.java
 create mode 100644 src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidListEventDataDeserializerTest.java

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index 5ba4cb2d..62fc0ae0 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -15,9 +15,24 @@
  */
 package com.github.shyiko.mysql.binlog;
 
-import com.github.shyiko.mysql.binlog.event.*;
-import com.github.shyiko.mysql.binlog.event.deserialization.*;
+import com.github.shyiko.mysql.binlog.event.AnnotateRowsEventData;
+import com.github.shyiko.mysql.binlog.event.Event;
+import com.github.shyiko.mysql.binlog.event.EventHeader;
+import com.github.shyiko.mysql.binlog.event.EventHeaderV4;
+import com.github.shyiko.mysql.binlog.event.EventType;
+import com.github.shyiko.mysql.binlog.event.GtidEventData;
+import com.github.shyiko.mysql.binlog.event.MariadbGtidEventData;
+import com.github.shyiko.mysql.binlog.event.MariadbGtidListEventData;
+import com.github.shyiko.mysql.binlog.event.QueryEventData;
+import com.github.shyiko.mysql.binlog.event.RotateEventData;
+import com.github.shyiko.mysql.binlog.event.deserialization.ChecksumType;
+import com.github.shyiko.mysql.binlog.event.deserialization.EventDataDeserializationException;
+import com.github.shyiko.mysql.binlog.event.deserialization.EventDataDeserializer;
+import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer;
 import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer.EventDataWrapper;
+import com.github.shyiko.mysql.binlog.event.deserialization.GtidEventDataDeserializer;
+import com.github.shyiko.mysql.binlog.event.deserialization.QueryEventDataDeserializer;
+import com.github.shyiko.mysql.binlog.event.deserialization.RotateEventDataDeserializer;
 import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream;
 import com.github.shyiko.mysql.binlog.jmx.BinaryLogClientMXBean;
 import com.github.shyiko.mysql.binlog.network.AuthenticationException;
@@ -34,7 +49,12 @@
 import com.github.shyiko.mysql.binlog.network.protocol.Packet;
 import com.github.shyiko.mysql.binlog.network.protocol.PacketChannel;
 import com.github.shyiko.mysql.binlog.network.protocol.ResultSetRowPacket;
-import com.github.shyiko.mysql.binlog.network.protocol.command.*;
+import com.github.shyiko.mysql.binlog.network.protocol.command.Command;
+import com.github.shyiko.mysql.binlog.network.protocol.command.DumpBinaryLogCommand;
+import com.github.shyiko.mysql.binlog.network.protocol.command.DumpBinaryLogGtidCommand;
+import com.github.shyiko.mysql.binlog.network.protocol.command.PingCommand;
+import com.github.shyiko.mysql.binlog.network.protocol.command.QueryCommand;
+import com.github.shyiko.mysql.binlog.network.protocol.command.SSLRequestCommand;
 
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.TrustManager;
@@ -115,14 +135,12 @@ public X509Certificate[] getAcceptedIssuers() {
     private volatile long connectionId;
     private SSLMode sslMode = SSLMode.DISABLED;
 
-    private GtidSet gtidSet;
-    private final Object gtidSetAccessLock = new Object();
+    protected GtidSet gtidSet;
+    protected final Object gtidSetAccessLock = new Object();
     private boolean gtidSetFallbackToPurged;
     private boolean useBinlogFilenamePositionInGtidMode;
     private String gtid;
     private boolean tx;
-    private boolean isMariadb = false;
-    private boolean mariadbSendAnnotateRowsEvent = false;
 
     private EventDeserializer eventDeserializer = new EventDeserializer();
 
@@ -132,7 +150,7 @@ public X509Certificate[] getAcceptedIssuers() {
     private SocketFactory socketFactory;
     private SSLSocketFactory sslSocketFactory;
 
-    private volatile PacketChannel channel;
+    protected volatile PacketChannel channel;
     private volatile boolean connected;
     private volatile long masterServerId = -1;
 
@@ -317,15 +335,14 @@ public void setGtidSet(String gtidSet) {
             this.binlogFilename = "";
         }
         synchronized (gtidSetAccessLock) {
-            // mariadb GtidSet format will be domainId-serverId-sequence
-            if (gtidSet != null && !gtidSet.contains(":")) {
-                this.gtidSet = new MariadbGtidSet(gtidSet);
-            } else {
-                this.gtidSet = gtidSet != null ? new GtidSet(gtidSet) : null;
-            }
+            this.gtidSet = gtidSet != null ? buildGtidSet(gtidSet) : null;
         }
     }
 
+    protected GtidSet buildGtidSet(String gtidSet) {
+        return new GtidSet(gtidSet);
+    }
+
     /**
      * @see #setGtidSetFallbackToPurged(boolean)
 	 * @return whether gtid_purged is used as a fallback
@@ -488,19 +505,6 @@ public void setThreadFactory(ThreadFactory threadFactory) {
         this.threadFactory = threadFactory;
     }
 
-    public boolean isMariadbSendAnnotateRowsEvent() {
-        return mariadbSendAnnotateRowsEvent;
-    }
-
-    /**
-     * Only in Mariadb, if set true, the Slave server connects with the BINLOG_SEND_ANNOTATE_ROWS_EVENT flag (value is 2)
-     * in the COM_BINLOG_DUMP Slave Registration phase
-     * @param mariadbSendAnnotateRowsEvent
-     */
-    public void setMariadbSendAnnotateRowsEvent(boolean mariadbSendAnnotateRowsEvent) {
-        this.mariadbSendAnnotateRowsEvent = mariadbSendAnnotateRowsEvent;
-    }
-
     /**
      * Connect to the replication stream. Note that this method blocks until disconnected.
      * @throws AuthenticationException if authentication fails
@@ -538,17 +542,8 @@ public void connect() throws IOException, IllegalStateException {
                 channel.authenticationComplete();
 
                 connectionId = greetingPacket.getThreadId();
-                isMariadb = greetingPacket.getServerVersion().toLowerCase().contains("mariadb");
                 if ("".equals(binlogFilename)) {
-                    synchronized (gtidSetAccessLock) {
-                        if (gtidSet != null && "".equals(gtidSet.toString()) && gtidSetFallbackToPurged) {
-                            if (isMariadb) {
-                                gtidSet = new MariadbGtidSet(fetchGtidPurged());
-                            } else {
-                                gtidSet = new GtidSet(fetchGtidPurged());
-                            }
-                        }
-                    }
+                    setupGtidSet();
                 }
                 if (binlogFilename == null) {
                     fetchBinlogFilenameAndPosition();
@@ -597,13 +592,7 @@ public void connect() throws IOException, IllegalStateException {
             ensureEventDataDeserializer(EventType.ROTATE, RotateEventDataDeserializer.class);
             synchronized (gtidSetAccessLock) {
                 if (gtidSet != null) {
-                    ensureEventDataDeserializer(EventType.GTID, GtidEventDataDeserializer.class);
-                    ensureEventDataDeserializer(EventType.QUERY, QueryEventDataDeserializer.class);
-                    if (isMariadb) {
-                        ensureEventDataDeserializer(EventType.ANNOTATE_ROWS, AnnotateRowsEventDataDeserializer.class);
-                        ensureEventDataDeserializer(EventType.MARIADB_GTID, MariadbGtidEventDataDeserializer.class);
-                        ensureEventDataDeserializer(EventType.MARIADB_GTID_LIST, MariadbGtidListEventDataDeserializer.class);
-                    }
+                    ensureGtidEventDataDeserializer();
                 }
             }
             listenForEventPackets();
@@ -676,7 +665,7 @@ public Object call() throws Exception {
         };
     }
 
-    private void checkError(byte[] packet) throws IOException {
+    protected void checkError(byte[] packet) throws IOException {
         if (packet[0] == (byte) 0xFF /* error */) {
             byte[] bytes = Arrays.copyOfRange(packet, 1, packet.length);
             ErrorPacket errorPacket = new ErrorPacket(bytes);
@@ -720,7 +709,6 @@ private boolean tryUpgradeToSSL(GreetingPacket greetingPacket) throws IOExceptio
         return false;
     }
 
-
     private void enableHeartbeat() throws IOException {
         channel.write(new QueryCommand("set @master_heartbeat_period=" + heartbeatInterval * 1000000));
         byte[] statementResult = channel.read();
@@ -735,27 +723,15 @@ private void setMasterServerId() throws IOException {
         }
     }
 
-    private void requestBinaryLogStream() throws IOException {
+    protected void requestBinaryLogStream() throws IOException {
         long serverId = blocking ? this.serverId : 0; // http://bugs.mysql.com/bug.php?id=71178
         Command dumpBinaryLogCommand;
         synchronized (gtidSetAccessLock) {
             if (gtidSet != null) {
-                if (isMariadb) {
-                    channel.write(new QueryCommand("SET @mariadb_slave_capability=4"));
-                    checkError(channel.read());
-                    channel.write(new QueryCommand("SET @slave_connect_state = '" + gtidSet.toString() + "'"));
-                    checkError(channel.read());
-                    channel.write(new QueryCommand("SET @slave_gtid_strict_mode = 0"));
-                    checkError(channel.read());
-                    channel.write(new QueryCommand("SET @slave_gtid_ignore_duplicates = 0"));
-                    checkError(channel.read());
-                    dumpBinaryLogCommand = new DumpBinaryLogCommand(serverId, "", 0L, isMariadbSendAnnotateRowsEvent());
-                } else {
-                    dumpBinaryLogCommand = new DumpBinaryLogGtidCommand(serverId,
-                        useBinlogFilenamePositionInGtidMode ? binlogFilename : "",
-                        useBinlogFilenamePositionInGtidMode ? binlogPosition : 4,
-                        gtidSet);
-                }
+                dumpBinaryLogCommand = new DumpBinaryLogGtidCommand(serverId,
+                    useBinlogFilenamePositionInGtidMode ? binlogFilename : "",
+                    useBinlogFilenamePositionInGtidMode ? binlogPosition : 4,
+                    gtidSet);
             } else {
                 dumpBinaryLogCommand = new DumpBinaryLogCommand(serverId, binlogFilename, binlogPosition);
             }
@@ -763,7 +739,7 @@ private void requestBinaryLogStream() throws IOException {
         channel.write(dumpBinaryLogCommand);
     }
 
-    private void ensureEventDataDeserializer(EventType eventType,
+    protected void ensureEventDataDeserializer(EventType eventType,
              Class eventDataDeserializerClass) {
         EventDataDeserializer eventDataDeserializer = eventDeserializer.getEventDataDeserializer(eventType);
         if (eventDataDeserializer.getClass() != eventDataDeserializerClass &&
@@ -780,6 +756,10 @@ private void ensureEventDataDeserializer(EventType eventType,
         }
     }
 
+    protected void ensureGtidEventDataDeserializer() {
+        ensureEventDataDeserializer(EventType.GTID, GtidEventDataDeserializer.class);
+        ensureEventDataDeserializer(EventType.QUERY, QueryEventDataDeserializer.class);
+    }
 
     private void spawnKeepAliveThread() {
         final ExecutorService threadExecutor =
@@ -924,6 +904,14 @@ private String fetchGtidPurged() throws IOException {
         return "";
     }
 
+    protected void setupGtidSet() throws IOException{
+        synchronized (gtidSetAccessLock) {
+            if (gtidSet != null && "".equals(gtidSet.toString()) && gtidSetFallbackToPurged) {
+                gtidSet = new GtidSet(fetchGtidPurged());
+            }
+        }
+    }
+
     private void fetchBinlogFilenameAndPosition() throws IOException {
         ResultSetRowPacket[] resultSet;
         channel.write(new QueryCommand("show master status"));
@@ -1025,7 +1013,7 @@ private byte[] readPacketSplitInChunks(ByteArrayInputStream inputStream, int pac
         return result;
     }
 
-    private void updateClientBinlogFilenameAndPosition(Event event) {
+    protected void updateClientBinlogFilenameAndPosition(Event event) {
         EventHeader eventHeader = event.getHeader();
         EventType eventType = eventHeader.getEventType();
         if (eventType == EventType.ROTATE) {
@@ -1044,7 +1032,7 @@ private void updateClientBinlogFilenameAndPosition(Event event) {
         }
     }
 
-    private void updateGtidSet(Event event) {
+    protected void updateGtidSet(Event event) {
         synchronized (gtidSetAccessLock) {
             if (gtidSet == null) {
                 return;
@@ -1070,34 +1058,39 @@ private void updateGtidSet(Event event) {
                 tx = false;
                 break;
             case QUERY:
-            case ANNOTATE_ROWS:
-                String sql;
-                if (eventHeader.getEventType() == EventType.QUERY) {
-                    QueryEventData queryEventData = (QueryEventData) EventDataWrapper.internal(event.getData());
-                    sql = queryEventData.getSql();
-                } else {
-                    AnnotateRowsEventData annotateRowsEventData = (AnnotateRowsEventData) EventDataWrapper.internal(event.getData());
-                    sql = annotateRowsEventData.getRowsQuery();
-                }
-
+                QueryEventData queryEventData = (QueryEventData) EventDataWrapper.internal(event.getData());
+                String sql = queryEventData.getSql();
                 if (sql == null) {
                     break;
                 }
-                if ("BEGIN".equals(sql)) {
-                    tx = true;
-                } else
-                if ("COMMIT".equals(sql) || "ROLLBACK".equals(sql)) {
-                    commitGtid();
-                    tx = false;
-                } else
-                if (!tx) {
-                    // auto-commit query, likely DDL
-                    commitGtid();
+                commitGtid(sql);
+                break;
+            case ANNOTATE_ROWS:
+                AnnotateRowsEventData annotateRowsEventData = (AnnotateRowsEventData) EventDeserializer.EventDataWrapper.internal(event.getData());
+                sql = annotateRowsEventData.getRowsQuery();
+                if (sql == null) {
+                    break;
                 }
+                commitGtid(sql);
+                break;
             default:
         }
     }
 
+    protected void commitGtid(String sql) {
+        if ("BEGIN".equals(sql)) {
+            tx = true;
+        } else
+        if ("COMMIT".equals(sql) || "ROLLBACK".equals(sql)) {
+            commitGtid();
+            tx = false;
+        } else
+        if (!tx) {
+            // auto-commit query, likely DDL
+            commitGtid();
+        }
+    }
+
     private void commitGtid() {
         if (gtid != null) {
             synchronized (gtidSetAccessLock) {
@@ -1308,7 +1301,7 @@ public interface LifecycleListener {
     /**
      * Default (no-op) implementation of {@link LifecycleListener}.
      */
-    public static abstract class AbstractLifecycleListener implements LifecycleListener {
+        public static abstract class AbstractLifecycleListener implements LifecycleListener {
 
         public void onConnect(BinaryLogClient client) { }
 
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClient.java
new file mode 100644
index 00000000..3dc67fa9
--- /dev/null
+++ b/src/main/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClient.java
@@ -0,0 +1,88 @@
+package com.github.shyiko.mysql.binlog;
+
+import com.github.shyiko.mysql.binlog.event.AnnotateRowsEventData;
+import com.github.shyiko.mysql.binlog.event.Event;
+import com.github.shyiko.mysql.binlog.event.EventType;
+import com.github.shyiko.mysql.binlog.event.deserialization.AnnotateRowsEventDataDeserializer;
+import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer;
+import com.github.shyiko.mysql.binlog.event.deserialization.MariadbGtidEventDataDeserializer;
+import com.github.shyiko.mysql.binlog.event.deserialization.MariadbGtidListEventDataDeserializer;
+import com.github.shyiko.mysql.binlog.network.protocol.command.Command;
+import com.github.shyiko.mysql.binlog.network.protocol.command.DumpBinaryLogCommand;
+import com.github.shyiko.mysql.binlog.network.protocol.command.QueryCommand;
+
+import java.io.IOException;
+
+/**
+ * Mariadb replication stream client.
+ *
+ * @author Winger
+ */
+public class MariadbBinaryLogClient extends BinaryLogClient {
+
+    private boolean useSendAnnotateRowsEvent;
+
+    public MariadbBinaryLogClient(String username, String password) {
+        super(username, password);
+    }
+
+    public MariadbBinaryLogClient(String schema, String username, String password) {
+        super(schema, username, password);
+    }
+
+    public MariadbBinaryLogClient(String hostname, int port, String username, String password) {
+        super(hostname, port, username, password);
+    }
+
+    public MariadbBinaryLogClient(String hostname, int port, String schema, String username, String password) {
+        super(hostname, port, schema, username, password);
+    }
+
+    @Override
+    protected GtidSet buildGtidSet(String gtidSet) {
+        return new MariadbGtidSet(gtidSet);
+    }
+
+    @Override
+    protected void setupGtidSet() throws IOException {
+        //Mariadb ignore
+    }
+
+    @Override
+    protected void requestBinaryLogStream() throws IOException {
+        long serverId = isBlocking() ? this.getServerId() : 0; // http://bugs.mysql.com/bug.php?id=71178
+        Command dumpBinaryLogCommand;
+        synchronized (gtidSetAccessLock) {
+            if (gtidSet != null) {
+                channel.write(new QueryCommand("SET @mariadb_slave_capability=4"));
+                checkError(channel.read());
+                channel.write(new QueryCommand("SET @slave_connect_state = '" + gtidSet.toString() + "'"));
+                checkError(channel.read());
+                channel.write(new QueryCommand("SET @slave_gtid_strict_mode = 0"));
+                checkError(channel.read());
+                channel.write(new QueryCommand("SET @slave_gtid_ignore_duplicates = 0"));
+                checkError(channel.read());
+                dumpBinaryLogCommand = new DumpBinaryLogCommand(serverId, "", 0L, isUseSendAnnotateRowsEvent());
+
+            } else {
+                dumpBinaryLogCommand = new DumpBinaryLogCommand(serverId, getBinlogFilename(), getBinlogPosition());
+            }
+        }
+        channel.write(dumpBinaryLogCommand);
+    }
+
+    @Override
+    protected void ensureGtidEventDataDeserializer() {
+        ensureEventDataDeserializer(EventType.ANNOTATE_ROWS, AnnotateRowsEventDataDeserializer.class);
+        ensureEventDataDeserializer(EventType.MARIADB_GTID, MariadbGtidEventDataDeserializer.class);
+        ensureEventDataDeserializer(EventType.MARIADB_GTID_LIST, MariadbGtidListEventDataDeserializer.class);
+    }
+
+    public boolean isUseSendAnnotateRowsEvent() {
+        return useSendAnnotateRowsEvent;
+    }
+
+    public void setUseSendAnnotateRowsEvent(boolean useSendAnnotateRowsEvent) {
+        this.useSendAnnotateRowsEvent = useSendAnnotateRowsEvent;
+    }
+}
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientMariadbIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClientIntegrationTest.java
similarity index 89%
rename from src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientMariadbIntegrationTest.java
rename to src/test/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClientIntegrationTest.java
index 4b7ea9ca..fac8b579 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientMariadbIntegrationTest.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClientIntegrationTest.java
@@ -14,18 +14,15 @@
 import static org.testng.AssertJUnit.assertNotNull;
 
 /**
- * BinaryLogClientMariadbGTIDIntegrationTest.java
- *
- * @author winger[jackey.zhang@woqutech.com]
- * @create 2021-07-13 10:52 上午
+ * @author Winger
  */
-public class BinaryLogClientMariadbIntegrationTest {
+public class MariadbBinaryLogClientIntegrationTest {
 
     protected BinaryLogClientIntegrationTest.MySQLConnection master;
 
     @Test
     public void testMariadbUseGTIDAndAnnotateRowsEvent() throws Exception {
-        master = new BinaryLogClientIntegrationTest.MySQLConnection("127.0.0.1", 3306, "root", "123456");
+        master = new BinaryLogClientIntegrationTest.MySQLConnection("127.0.0.1", 3306, "root", "");
         master.execute(new BinaryLogClientIntegrationTest.Callback() {
             @Override
             public void execute(Statement statement) throws SQLException {
@@ -48,9 +45,9 @@ public void execute(ResultSet rs) throws SQLException {
         });
 
         CountDownEventListener eventListener;
-        BinaryLogClient client = new BinaryLogClient("127.0.0.1", 3306, "root", "123456");
+        MariadbBinaryLogClient client = new MariadbBinaryLogClient("127.0.0.1", 3306, "root", "123456");
         client.setGtidSet(currentGtidPos[0]);
-        client.setMariadbSendAnnotateRowsEvent(true);
+        client.setUseSendAnnotateRowsEvent(true);
 
         EventDeserializer eventDeserializer = new EventDeserializer();
         eventDeserializer.setCompatibilityMode(EventDeserializer.CompatibilityMode.CHAR_AND_BINARY_AS_BYTE_ARRAY,
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/MariadbGtidSetTest.java b/src/test/java/com/github/shyiko/mysql/binlog/MariadbGtidSetTest.java
new file mode 100644
index 00000000..71d5753a
--- /dev/null
+++ b/src/test/java/com/github/shyiko/mysql/binlog/MariadbGtidSetTest.java
@@ -0,0 +1,36 @@
+package com.github.shyiko.mysql.binlog;
+
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotEquals;
+
+/**
+ * @author Winger
+ */
+public class MariadbGtidSetTest {
+
+    @Test
+    public void testAdd() {
+        MariadbGtidSet gtidSet = new MariadbGtidSet("0-102-7255");
+        gtidSet.add("0-102-7256");
+        gtidSet.add("0-102-7257");
+        gtidSet.add("0-102-7259");
+        gtidSet.add("1-102-7300");
+        assertNotEquals(gtidSet.toString(), "1-102-7300");
+        assertNotEquals(gtidSet.toString(), "0-102-7259");
+        assertEquals(gtidSet.toString(), "0-102-7259,1-102-7300");
+    }
+
+    @Test
+    public void testEmptySet() {
+        assertEquals(new MariadbGtidSet("").toString(), "");
+    }
+
+    @Test
+    public void testEquals() {
+        assertEquals(new MariadbGtidSet(""), new MariadbGtidSet(null));
+        assertEquals(new MariadbGtidSet(""), new MariadbGtidSet(""));
+        assertEquals(new MariadbGtidSet("0-0-7404"), new MariadbGtidSet("0-0-7404"));
+    }
+}
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/AnnotateRowsEventDataDeserializerTest.java b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/AnnotateRowsEventDataDeserializerTest.java
new file mode 100644
index 00000000..e117d32f
--- /dev/null
+++ b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/AnnotateRowsEventDataDeserializerTest.java
@@ -0,0 +1,27 @@
+package com.github.shyiko.mysql.binlog.event.deserialization;
+
+import com.github.shyiko.mysql.binlog.event.AnnotateRowsEventData;
+import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static junit.framework.Assert.assertEquals;
+
+/**
+ * @author Winger
+ */
+public class AnnotateRowsEventDataDeserializerTest {
+
+    private static final byte[] DATA = {73, 78, 83, 69, 82, 84, 32, 73, 78, 84, 79, 32, 102, 111, 111, 32, 115, 101, 116, 32, 105, 32, 61, 32, 50};
+
+    private static final String sql = "INSERT INTO foo set i = 2";
+
+    @Test
+    public void deserialize() throws IOException {
+        AnnotateRowsEventDataDeserializer deserializer = new AnnotateRowsEventDataDeserializer();
+        AnnotateRowsEventData eventData = deserializer.deserialize(new ByteArrayInputStream(DATA));
+
+        assertEquals(sql, eventData.getRowsQuery());
+    }
+}
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidEventDataDeserializerTest.java b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidEventDataDeserializerTest.java
new file mode 100644
index 00000000..f6e703bf
--- /dev/null
+++ b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidEventDataDeserializerTest.java
@@ -0,0 +1,26 @@
+package com.github.shyiko.mysql.binlog.event.deserialization;
+
+import com.github.shyiko.mysql.binlog.event.MariadbGtidEventData;
+import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static junit.framework.Assert.assertEquals;
+
+/**
+ * @author Winger
+ */
+public class MariadbGtidEventDataDeserializerTest {
+
+    private static final byte[] DATA = {-20, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 121};
+
+    private static final String GTID_SET = "0-0-7404";
+
+    @Test
+    public void deserialize() throws IOException {
+        MariadbGtidEventDataDeserializer deserializer = new MariadbGtidEventDataDeserializer();
+        MariadbGtidEventData eventData = deserializer.deserialize(new ByteArrayInputStream(DATA));
+        assertEquals(GTID_SET, eventData.toString());
+    }
+}
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidListEventDataDeserializerTest.java b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidListEventDataDeserializerTest.java
new file mode 100644
index 00000000..3a6f33fc
--- /dev/null
+++ b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidListEventDataDeserializerTest.java
@@ -0,0 +1,26 @@
+package com.github.shyiko.mysql.binlog.event.deserialization;
+
+import com.github.shyiko.mysql.binlog.event.MariadbGtidListEventData;
+import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static junit.framework.Assert.assertEquals;
+
+/**
+ * @author Winger
+ */
+public class MariadbGtidListEventDataDeserializerTest {
+
+    private static final byte[] DATA = {1, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 87, 28, 0, 0, 0, 0, 0, 0, 77};
+
+    private static final String GTID_SET_LIST = "MariadbGtidListEventData{mariaGTIDSet=0-102-7255}";
+
+    @Test
+    public void deserialize() throws IOException {
+        MariadbGtidListEventDataDeserializer deserializer = new MariadbGtidListEventDataDeserializer();
+        MariadbGtidListEventData eventData = deserializer.deserialize(new ByteArrayInputStream(DATA));
+        assertEquals(GTID_SET_LIST, eventData.toString());
+    }
+}

From 486a447a10d378ebf34a132ce31327cdc743318d Mon Sep 17 00:00:00 2001
From: winger 
Date: Fri, 16 Jul 2021 11:34:08 +0800
Subject: [PATCH 093/217] add some doc informations about mariadb

---
 README.md                                                 | 8 ++++++++
 .../com/github/shyiko/mysql/binlog/BinaryLogClient.java   | 2 +-
 .../shyiko/mysql/binlog/MariadbBinaryLogClient.java       | 3 ---
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 407e27a3..d17ce527 100644
--- a/README.md
+++ b/README.md
@@ -84,6 +84,14 @@ kick off from a specific filename or position, use `client.setBinlogFilename(fil
 > `client.connect()` is blocking (meaning that client will listen for events in the current thread).
 `client.connect(timeout)`, on the other hand, spawns a separate thread.
 
+> Note difference between MariaDB and MySQL
+```
+BinaryLogClient client = new MariadbBinaryLogClient("hostname", 3306, "username", "password");
+// ... as same as BinaryLogClient
+```
+> `client.setGtidSet(gtid)` meaning that client kick off from a specific gtid, MariaDB also support.
+> `client.setUseSendAnnotateRowsEvent(true)` meaning that client will send annotate rows events(describe the query which caused the row event), and 'false' by default
+
 #### Controlling event deserialization
 
 > You might need it for several reasons:
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index 62fc0ae0..b7b3b230 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -1301,7 +1301,7 @@ public interface LifecycleListener {
     /**
      * Default (no-op) implementation of {@link LifecycleListener}.
      */
-        public static abstract class AbstractLifecycleListener implements LifecycleListener {
+    public static abstract class AbstractLifecycleListener implements LifecycleListener {
 
         public void onConnect(BinaryLogClient client) { }
 
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClient.java
index 3dc67fa9..0c6740d6 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClient.java
@@ -1,10 +1,7 @@
 package com.github.shyiko.mysql.binlog;
 
-import com.github.shyiko.mysql.binlog.event.AnnotateRowsEventData;
-import com.github.shyiko.mysql.binlog.event.Event;
 import com.github.shyiko.mysql.binlog.event.EventType;
 import com.github.shyiko.mysql.binlog.event.deserialization.AnnotateRowsEventDataDeserializer;
-import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer;
 import com.github.shyiko.mysql.binlog.event.deserialization.MariadbGtidEventDataDeserializer;
 import com.github.shyiko.mysql.binlog.event.deserialization.MariadbGtidListEventDataDeserializer;
 import com.github.shyiko.mysql.binlog.network.protocol.command.Command;

From ae3c591a4beef66315e59785c3fa2db7edd0d769 Mon Sep 17 00:00:00 2001
From: Gunnar Morling 
Date: Thu, 17 Jun 2021 18:42:48 +0200
Subject: [PATCH 094/217] #40 Adding support for column visibility as supported
 by MySQL 8.0.23

---
 .../binlog/event/TableMapEventMetadata.java   | 11 +++++++
 .../TableMapEventDataDeserializer.java        |  1 +
 .../TableMapEventMetadataDeserializer.java    | 31 +++++++++++++++----
 .../BinaryLogClientIntegrationTest.java       | 19 +++++++++++-
 .../mysql/binlog/MysqlOnetimeServer.java      |  6 ++++
 .../binlog/MysqlOnetimeServerOptions.java     |  1 +
 6 files changed, 62 insertions(+), 7 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/TableMapEventMetadata.java b/src/main/java/com/github/shyiko/mysql/binlog/event/TableMapEventMetadata.java
index 66f030ec..8b4a48ec 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/TableMapEventMetadata.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/TableMapEventMetadata.java
@@ -36,6 +36,7 @@ public class TableMapEventMetadata implements EventData {
     private Map primaryKeysWithPrefix;
     private DefaultCharset enumAndSetDefaultCharset;
     private List enumAndSetColumnCharsets;
+    private BitSet visibility;
 
     public BitSet getSignedness() {
         return signedness;
@@ -125,6 +126,14 @@ public void setEnumAndSetColumnCharsets(List enumAndSetColumnCharsets)
         this.enumAndSetColumnCharsets = enumAndSetColumnCharsets;
     }
 
+    public BitSet getVisibility() {
+        return visibility;
+    }
+
+    public void setVisibility(BitSet visibility) {
+        this.visibility = visibility;
+    }
+
     @Override
     public String toString() {
         final StringBuilder sb = new StringBuilder();
@@ -163,6 +172,8 @@ public String toString() {
         sb.append(", enumAndSetColumnCharsets=").append(enumAndSetColumnCharsets == null ? "null" : "");
         appendList(sb, enumAndSetColumnCharsets);
 
+        sb.append(",visibility=").append(visibility);
+
         sb.append('}');
         return sb.toString();
     }
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventDataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventDataDeserializer.java
index 35ca1edf..6cb814cb 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventDataDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventDataDeserializer.java
@@ -46,6 +46,7 @@ public TableMapEventData deserialize(ByteArrayInputStream inputStream) throws IO
         if (metadataLength > 0) {
             metadata = metadataDeserializer.deserialize(
                 new ByteArrayInputStream(inputStream.read(metadataLength)),
+                eventData.getColumnTypes().length,
                 numericColumnCount(eventData.getColumnTypes())
             );
         }
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
index e175f444..2a0705d0 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
@@ -26,13 +26,17 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 /**
  * @author Ahmed Abdul Hamid
  */
 public class TableMapEventMetadataDeserializer {
 
-    public TableMapEventMetadata deserialize(ByteArrayInputStream inputStream, int nIntColumns) throws IOException {
+    private final Logger logger = Logger.getLogger(getClass().getName());
+
+    public TableMapEventMetadata deserialize(ByteArrayInputStream inputStream, int nColumns, int nNumericColumns) throws IOException {
         int remainingBytes = inputStream.available();
         if (remainingBytes <= 0) {
             return null;
@@ -41,7 +45,13 @@ public TableMapEventMetadata deserialize(ByteArrayInputStream inputStream, int n
         TableMapEventMetadata result = new TableMapEventMetadata();
 
         for (; remainingBytes > 0; inputStream.enterBlock(remainingBytes)) {
-            MetadataFieldType fieldType = MetadataFieldType.byCode(inputStream.readInteger(1));
+            int code = inputStream.readInteger(1);
+
+            MetadataFieldType fieldType = MetadataFieldType.byCode(code);
+            if (fieldType == null) {
+                throw new IOException("Unsupported table metadata field type " + code);
+            }
+
             int fieldLength = inputStream.readPackedInteger();
 
             remainingBytes = inputStream.available();
@@ -49,7 +59,7 @@ public TableMapEventMetadata deserialize(ByteArrayInputStream inputStream, int n
 
             switch (fieldType) {
                 case SIGNEDNESS:
-                    result.setSignedness(readSignedness(inputStream, nIntColumns));
+                    result.setSignedness(readBooleanList(inputStream, nNumericColumns));
                     break;
                 case DEFAULT_CHARSET:
                     result.setDefaultCharset(readDefaultCharset(inputStream));
@@ -80,17 +90,24 @@ public TableMapEventMetadata deserialize(ByteArrayInputStream inputStream, int n
                     break;
                 case ENUM_AND_SET_COLUMN_CHARSET:
                     result.setEnumAndSetColumnCharsets(readIntegers(inputStream));
+                case VISIBILITY:
+                    result.setVisibility(readBooleanList(inputStream, nColumns));
+                    break;
+                case UNKNOWN_METADATA_FIELD_TYPE:
+                    if (logger.isLoggable(Level.FINE)) {
+                        logger.fine("Received metadata field of unknown type");
+                    }
                     break;
                 default:
                     inputStream.enterBlock(remainingBytes);
-                    throw new IOException("Unsupported table metadata field type " + fieldType);
+                    throw new IOException("Unsupported table metadata field type " + code);
             }
             remainingBytes -= fieldLength;
         }
         return result;
     }
 
-    private static BitSet readSignedness(ByteArrayInputStream inputStream, int length) throws IOException {
+    private static BitSet readBooleanList(ByteArrayInputStream inputStream, int length) throws IOException {
         BitSet result = new BitSet();
         // according to MySQL internals the amount of storage required for N columns is INT((N+7)/8) bytes
         byte[] bytes = inputStream.read((length + 7) >> 3);
@@ -162,7 +179,9 @@ private enum MetadataFieldType {
         SIMPLE_PRIMARY_KEY(8),              // The primary key without any prefix
         PRIMARY_KEY_WITH_PREFIX(9),         // The primary key with some prefix
         ENUM_AND_SET_DEFAULT_CHARSET(10),   // Charsets of ENUM and SET columns
-        ENUM_AND_SET_COLUMN_CHARSET(11);    // Charsets of ENUM and SET columns
+        ENUM_AND_SET_COLUMN_CHARSET(11),    // Charsets of ENUM and SET columns
+        VISIBILITY(12),                     // Column visibility (8.0.23 and newer)
+        UNKNOWN_METADATA_FIELD_TYPE(128);   // Returned with binlog-row-metadata=FULL from MySQL 8.0 in some cases
 
         private final int code;
 
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java
index 8d813793..7e04312c 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java
@@ -115,7 +115,9 @@ public class BinaryLogClientIntegrationTest {
     protected MysqlVersion mysqlVersion;
 
     protected MysqlOnetimeServerOptions getOptions() {
-        return null;
+        MysqlOnetimeServerOptions options = new MysqlOnetimeServerOptions();
+        options.fullRowMetaData = true;
+        return options;
     }
 
     @BeforeClass
@@ -1140,6 +1142,21 @@ public void execute(final ResultSet rs) throws SQLException {
         });
     }
 
+    @Test
+    public void testMySQL8InvisibleColumn() throws Exception {
+        if ( !mysqlVersion.atLeast(8, 0) )
+            throw new SkipException("skipping mysql8 invisible column test");
+
+        master.execute("drop table if exists test_invisible_column");
+        master.execute("create table test_invisible_column (\n"
+                + "id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,\n"
+                + "name varchar(100) not null,\n"
+                + "created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP INVISIBLE\n"
+                + ");");
+        master.execute("insert into test_invisible_column (name) values ('User 1')");
+        eventListener.waitFor(WriteRowsEventData.class, 1, DEFAULT_TIMEOUT);
+    }
+
     @AfterMethod
     public void afterEachTest() throws Exception {
         final CountDownLatch latch = new CountDownLatch(1);
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java
index a74e159e..a3c5f51f 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java
@@ -63,6 +63,11 @@ public void boot() throws Exception {
 			authPlugin = "--default-authentication-plugin=mysql_native_password";
 		}
 
+		String fullRowMetaData = "";
+		if ( getVersion().atLeast(8, 0) && options.fullRowMetaData ) {
+			fullRowMetaData = "--binlog-row-metadata=FULL";
+		}
+
 		ProcessBuilder pb = new ProcessBuilder(
 			dir + "/src/test/onetimeserver",
 			"--mysql-version=" + getVersionString(),
@@ -74,6 +79,7 @@ public void boot() throws Exception {
 			"--character-set-server=utf8",
 			"--sync_binlog=0",
 			"--default-time-zone=+00:00",
+			fullRowMetaData,
 			isRoot ? "--user=root" : "",
 			authPlugin,
 			gtidParams
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServerOptions.java b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServerOptions.java
index 2e0ca936..ae677cae 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServerOptions.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServerOptions.java
@@ -5,4 +5,5 @@ public class MysqlOnetimeServerOptions {
     public boolean gtid = false;
     public MysqlOnetimeServer masterServer;
     public String extraParams;
+    public boolean fullRowMetaData;
 }

From 12f1585b5625223f2f5ab97912d213a14f6bbb52 Mon Sep 17 00:00:00 2001
From: Gunnar Morling 
Date: Sat, 17 Jul 2021 20:07:10 +0200
Subject: [PATCH 095/217] #47 Adding GitHub Actions CI

---
 .github/workflows/mysql-57.yml | 45 ++++++++++++++++++++++++++++++++++
 .github/workflows/mysql-80.yml | 45 ++++++++++++++++++++++++++++++++++
 2 files changed, 90 insertions(+)
 create mode 100644 .github/workflows/mysql-57.yml
 create mode 100644 .github/workflows/mysql-80.yml

diff --git a/.github/workflows/mysql-57.yml b/.github/workflows/mysql-57.yml
new file mode 100644
index 00000000..46b090a9
--- /dev/null
+++ b/.github/workflows/mysql-57.yml
@@ -0,0 +1,45 @@
+#
+# Copyright Gunnar Morling
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This workflow will build a Java project with Maven
+# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+
+name: Build and run tests against MySQL 5.7
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up JDK 11
+      uses: actions/setup-java@v1
+      with:
+        java-version: 11
+    - name: Cache Maven packages
+      uses: actions/cache@v1
+      with:
+        path: ~/.m2
+        key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+        restore-keys: ${{ runner.os }}-m2
+    - name: Build with Maven
+      run: mvn -B install --file pom.xml
diff --git a/.github/workflows/mysql-80.yml b/.github/workflows/mysql-80.yml
new file mode 100644
index 00000000..2787a97c
--- /dev/null
+++ b/.github/workflows/mysql-80.yml
@@ -0,0 +1,45 @@
+#
+# Copyright Gunnar Morling
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This workflow will build a Java project with Maven
+# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+
+name: Build and run tests against MySQL 8.0
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up JDK 11
+      uses: actions/setup-java@v1
+      with:
+        java-version: 11
+    - name: Cache Maven packages
+      uses: actions/cache@v1
+      with:
+        path: ~/.m2
+        key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+        restore-keys: ${{ runner.os }}-m2
+    - name: Build with Maven
+      run: MYSQL_VERSION=8.0 mvn -B install --file pom.xml

From 96edcd5feaf208559cae4a4b44a49d9ab96df5d4 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Thu, 29 Jul 2021 12:50:16 -0700
Subject: [PATCH 096/217] v0.25.3

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index b6b62f90..8b2635fe 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.25.2
+    0.25.3
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From 9bf861ab242f98e021e65e07034a585ba7a7ab2f Mon Sep 17 00:00:00 2001
From: jolivares 
Date: Thu, 12 Aug 2021 15:04:41 +0200
Subject: [PATCH 097/217] avoid computing lag for events not having timestamp
 (HEARTBEAT)

---
 .../shyiko/mysql/binlog/jmx/BinaryLogClientStatistics.java    | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/jmx/BinaryLogClientStatistics.java b/src/main/java/com/github/shyiko/mysql/binlog/jmx/BinaryLogClientStatistics.java
index b1910901..dd13946f 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/jmx/BinaryLogClientStatistics.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/jmx/BinaryLogClientStatistics.java
@@ -18,6 +18,7 @@
 import com.github.shyiko.mysql.binlog.BinaryLogClient;
 import com.github.shyiko.mysql.binlog.event.Event;
 import com.github.shyiko.mysql.binlog.event.EventHeader;
+import com.github.shyiko.mysql.binlog.event.EventType;
 
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
@@ -65,6 +66,9 @@ public long getSecondsBehindMaster() {
         if (timestamp == 0 || eventHeader == null) {
             return -1;
         }
+        if (eventHeader.getEventType() == EventType.HEARTBEAT && eventHeader.getTimestamp() == 0) {
+            return 0;
+        }
         return (timestamp - eventHeader.getTimestamp()) / 1000;
     }
 

From bd5959d1656f5dbef253890f56e442f51d3b703d Mon Sep 17 00:00:00 2001
From: Sergei Morozov 
Date: Fri, 10 Sep 2021 11:00:58 -0700
Subject: [PATCH 098/217] Add details to the EOF exceptions

Co-authored-by: Vadzim Ramanenka 
---
 .../shyiko/mysql/binlog/io/ByteArrayInputStream.java     | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java b/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java
index 3cfaf979..66dbe13a 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/io/ByteArrayInputStream.java
@@ -29,6 +29,7 @@ public class ByteArrayInputStream extends InputStream {
     private Integer peek;
     private Integer pos, markPosition;
     private int blockLength = -1;
+    private int initialBlockLength = -1;
 
     public ByteArrayInputStream(InputStream inputStream) {
         this.inputStream = inputStream;
@@ -110,7 +111,10 @@ public void fill(byte[] bytes, int offset, int length) throws IOException {
         while (remaining != 0) {
             int read = read(bytes, offset + length - remaining, remaining);
             if (read == -1) {
-                throw new EOFException();
+                throw new EOFException(
+                    String.format("Failed to read remaining %d of %d bytes from position %d. Block length: %d. Initial block length: %d.",
+                        remaining, length, pos, blockLength, initialBlockLength)
+                );
             }
             remaining -= read;
         }
@@ -206,7 +210,7 @@ public int read() throws IOException {
             peek = null;
         }
         if (result == -1) {
-            throw new EOFException();
+            throw new EOFException(String.format("Failed to read next byte from position %d", this.pos));
         }
         this.pos += 1;
         return result;
@@ -273,6 +277,7 @@ public void close() throws IOException {
 
     public void enterBlock(int length) {
         this.blockLength = length < -1 ? -1 : length;
+        this.initialBlockLength = length;
     }
 
     public void skipToTheEndOfTheBlock() throws IOException {

From bbf53d79c37ca6bd7dc5e95c60ca03e1c8d12fce Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Wed, 13 Oct 2021 15:11:04 -0700
Subject: [PATCH 099/217] bump to 0.25.4

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 8b2635fe..5920031c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.25.3
+    0.25.4
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From e50366259d6e38e9368db5b00f6e8c82510faa70 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Wed, 13 Oct 2021 15:12:30 -0700
Subject: [PATCH 100/217] update changelog

---
 CHANGELOG.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 98cb70e4..feadff22 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,13 @@
 # Changelog
 
+## [0.25.4](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.25.3...0.25.4) - 2021-10-13
+
+- add debugging info to eof exception
+
+## [0.25.3](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.25.2...0.25.3) - 2021-07-29
+
+- support mysql 8's invisible columns
+
 ## [0.25.2](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.25.1...0.25.2) - 2021-06-25
 
 - allow `setupConnection()` to be overridden

From 02926960a0c6d3a963be05e103066473d9f1d643 Mon Sep 17 00:00:00 2001
From: Jiri Pechanec 
Date: Fri, 28 Jan 2022 09:40:44 +0100
Subject: [PATCH 101/217] Process JSON field names using offsets

---
 .../deserialization/json/JsonBinary.java      |  38 +++++-
 .../json/JsonBinaryValueIntegrationTest.java  |  23 +++-
 .../json/JsonPartialUpdateParseTest.java      | 109 ++++++++++++++++++
 3 files changed, 163 insertions(+), 7 deletions(-)
 create mode 100644 src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonPartialUpdateParseTest.java

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
index d8b824eb..b167ba75 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
@@ -326,10 +326,11 @@ protected void parseObject(boolean small, JsonFormatter formatter)
         int valueSize = small ? 2 : 4;
 
         // Read each key-entry, consisting of the offset and length of each key ...
-        int[] keyLengths = new int[numElements];
+        KeyEntry[] keys = new KeyEntry[numElements];
         for (int i = 0; i != numElements; ++i) {
-            readUnsignedIndex(numBytes, small, "key offset in"); // unused
-            keyLengths[i] = readUInt16();
+            keys[i] = new KeyEntry(
+                    readUnsignedIndex(numBytes, small, "key offset in"),
+                    readUInt16());
         }
 
         // Read each key value value-entry
@@ -374,9 +375,14 @@ protected void parseObject(boolean small, JsonFormatter formatter)
         }
 
         // Read each key ...
-        String[] keys = new String[numElements];
         for (int i = 0; i != numElements; ++i) {
-            keys[i] = reader.readString(keyLengths[i]);
+            final int skipBytes = keys[i].index + objectOffset - reader.getPosition();
+            // Skip to a start of a field name if the current position does not point to it
+            // This can happen for MySQL 8
+            if (skipBytes != 0) {
+                reader.fastSkip(skipBytes);
+            }
+            keys[i].name = reader.readString(keys[i].length);
         }
 
         // Now parse the values ...
@@ -385,7 +391,7 @@ protected void parseObject(boolean small, JsonFormatter formatter)
             if (i != 0) {
                 formatter.nextEntry();
             }
-            formatter.name(keys[i]);
+            formatter.name(keys[i].name);
             ValueEntry entry = entries[i];
             if (entry.resolved) {
                 Object value = entry.value;
@@ -996,6 +1002,26 @@ protected static String asHex(int value) {
         return Integer.toHexString(value);
     }
 
+    /**
+     * Class used internally to hold key entry information.
+     */
+    protected static final class KeyEntry {
+
+        protected final int index;
+        protected final int length;
+        protected String name;
+
+        public KeyEntry(int index, int length) {
+            this.index = index;
+            this.length = length;
+        }
+
+        public KeyEntry setKey(String key) {
+            this.name = key;
+            return this;
+        }
+    }
+
     /**
      * Class used internally to hold value entry information.
      */
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java
index e6eb2221..8fb3917c 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java
@@ -22,7 +22,6 @@
 import com.github.shyiko.mysql.binlog.MysqlOnetimeServer;
 import com.github.shyiko.mysql.binlog.TraceEventListener;
 import com.github.shyiko.mysql.binlog.TraceLifecycleListener;
-import com.github.shyiko.mysql.binlog.event.Event;
 import com.github.shyiko.mysql.binlog.event.EventData;
 import com.github.shyiko.mysql.binlog.event.EventType;
 import com.github.shyiko.mysql.binlog.event.QueryEventData;
@@ -147,6 +146,28 @@ public void testMysql8JsonRemovePartialUpdateWithHoles() throws Exception {
         client.unregisterEventListener(capturingEventListener);
     }
 
+    @Test
+    public void testMysql8JsonRemovePartialUpdateWithHolesAndSparseKeys() throws Exception {
+        CapturingEventListener capturingEventListener = new CapturingEventListener();
+        client.registerEventListener(capturingEventListener);
+        String json = "{\"17fc9889474028063990914001f6854f6b8b5784\":\"test_field_for_remove_fields_behaviour_2\",\"1f3a2ea5bc1f60258df20521bee9ac636df69a3a\":{\"currency\":\"USD\"},\"4f4d99a438f334d7dbf83a1816015b361b848b3b\":{\"currency\":\"USD\"},\"9021162291be72f5a8025480f44bf44d5d81d07c\":\"test_field_for_remove_fields_behaviour_3_will_be_removed\",\"9b0ed11532efea688fdf12b28f142b9eb08a80c5\":{\"currency\":\"USD\"},\"e65ad0762c259b05b4866f7249eabecabadbe577\":\"test_field_for_remove_fields_behaviour_1_updated\",\"ff2c07edcaa3e987c23fb5cc4fe860bb52becf00\":{\"currency\":\"USD\"}}";
+        master.execute("DROP TABLE IF EXISTS json_test", "create table json_test (j JSON)",
+                "INSERT INTO json_test VALUES ('" + json + "')",
+                "UPDATE json_test SET j = JSON_REMOVE(j, '$.\"17fc9889474028063990914001f6854f6b8b5784\"')");
+        capturingEventListener.waitFor(WriteRowsEventData.class, 1, DEFAULT_TIMEOUT);
+        capturingEventListener.waitFor(UpdateRowsEventData.class, 1, DEFAULT_TIMEOUT);
+        List events = capturingEventListener.getEvents(WriteRowsEventData.class);
+        Serializable[] insertData = events.iterator().next().getRows().get(0);
+        assertEquals(JsonBinary.parseAsString((byte[]) insertData[0]), json);
+
+        List updateEvents = capturingEventListener.getEvents(UpdateRowsEventData.class);
+        Serializable[] updateData = updateEvents.iterator().next().getRows().get(0).getValue();
+        assertEquals(JsonBinary.parseAsString((byte[]) updateData[0]), json.replace(
+                "\"17fc9889474028063990914001f6854f6b8b5784\":\"test_field_for_remove_fields_behaviour_2\",", ""));
+
+        client.unregisterEventListener(capturingEventListener);
+    }
+
     @Test
     public void testMysql8JsonReplacePartialUpdateWithHoles() throws Exception {
         CapturingEventListener capturingEventListener = new CapturingEventListener();
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonPartialUpdateParseTest.java b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonPartialUpdateParseTest.java
new file mode 100644
index 00000000..8d347dba
--- /dev/null
+++ b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonPartialUpdateParseTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.github.shyiko.mysql.binlog.event.deserialization.json;
+
+import java.io.IOException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * The column value
+ * 
+ * {
+ *    "17fc9889474028063990914001f6854f6b8b5784":"test_field_for_remove_fields_behaviour_2",
+ *    "1f3a2ea5bc1f60258df20521bee9ac636df69a3a":{
+ *       "currency":"USD"
+ *    },
+ *    "4f4d99a438f334d7dbf83a1816015b361b848b3b":{
+ *       "currency":"USD"
+ *    },
+ *    "9021162291be72f5a8025480f44bf44d5d81d07c":"test_field_for_remove_fields_behaviour_3_will_be_removed",
+ *    "9b0ed11532efea688fdf12b28f142b9eb08a80c5":{
+ *       "currency":"USD"
+ *    },
+ *    "e65ad0762c259b05b4866f7249eabecabadbe577":"test_field_for_remove_fields_behaviour_1_updated",
+ *    "ff2c07edcaa3e987c23fb5cc4fe860bb52becf00":{
+ *       "currency":"USD"
+ *    }
+ * }
+ * 

+ * is partially updated using
+ * 
+ * JSON_REMOVE(custom_fields, '$.\"17fc9889474028063990914001f6854f6b8b5784\"')
+ * 

+ * MySQL 5.7 and MySQL 8.0 emits different values in binlog (MySQL 8 value is sparse and requires strict using of offsets) 
+ */
+public class JsonPartialUpdateParseTest {
+
+    private static final byte[] BINLOG_UPDATE_57 = { 0, 6, 0, -28, 1, 46, 0, 40, 0, 86, 0, 40, 0, 126, 0, 40, 0, -90, 0,
+            40, 0, -50, 0, 40, 0, -10, 0, 40, 0, 0, 30, 1, 0, 53, 1, 12, 76, 1, 0, -123, 1, 12, -100, 1, 0, -51, 1, 49,
+            102, 51, 97, 50, 101, 97, 53, 98, 99, 49, 102, 54, 48, 50, 53, 56, 100, 102, 50, 48, 53, 50, 49, 98, 101,
+            101, 57, 97, 99, 54, 51, 54, 100, 102, 54, 57, 97, 51, 97, 52, 102, 52, 100, 57, 57, 97, 52, 51, 56, 102,
+            51, 51, 52, 100, 55, 100, 98, 102, 56, 51, 97, 49, 56, 49, 54, 48, 49, 53, 98, 51, 54, 49, 98, 56, 52, 56,
+            98, 51, 98, 57, 48, 50, 49, 49, 54, 50, 50, 57, 49, 98, 101, 55, 50, 102, 53, 97, 56, 48, 50, 53, 52, 56,
+            48, 102, 52, 52, 98, 102, 52, 52, 100, 53, 100, 56, 49, 100, 48, 55, 99, 57, 98, 48, 101, 100, 49, 49, 53,
+            51, 50, 101, 102, 101, 97, 54, 56, 56, 102, 100, 102, 49, 50, 98, 50, 56, 102, 49, 52, 50, 98, 57, 101, 98,
+            48, 56, 97, 56, 48, 99, 53, 101, 54, 53, 97, 100, 48, 55, 54, 50, 99, 50, 53, 57, 98, 48, 53, 98, 52, 56,
+            54, 54, 102, 55, 50, 52, 57, 101, 97, 98, 101, 99, 97, 98, 97, 100, 98, 101, 53, 55, 55, 102, 102, 50, 99,
+            48, 55, 101, 100, 99, 97, 97, 51, 101, 57, 56, 55, 99, 50, 51, 102, 98, 53, 99, 99, 52, 102, 101, 56, 54,
+            48, 98, 98, 53, 50, 98, 101, 99, 102, 48, 48, 1, 0, 23, 0, 11, 0, 8, 0, 12, 19, 0, 99, 117, 114, 114, 101,
+            110, 99, 121, 3, 85, 83, 68, 1, 0, 23, 0, 11, 0, 8, 0, 12, 19, 0, 99, 117, 114, 114, 101, 110, 99, 121, 3,
+            85, 83, 68, 56, 116, 101, 115, 116, 95, 102, 105, 101, 108, 100, 95, 102, 111, 114, 95, 114, 101, 109, 111,
+            118, 101, 95, 102, 105, 101, 108, 100, 115, 95, 98, 101, 104, 97, 118, 105, 111, 117, 114, 95, 51, 95, 119,
+            105, 108, 108, 95, 98, 101, 95, 114, 101, 109, 111, 118, 101, 100, 1, 0, 23, 0, 11, 0, 8, 0, 12, 19, 0, 99,
+            117, 114, 114, 101, 110, 99, 121, 3, 85, 83, 68, 48, 116, 101, 115, 116, 95, 102, 105, 101, 108, 100, 95,
+            102, 111, 114, 95, 114, 101, 109, 111, 118, 101, 95, 102, 105, 101, 108, 100, 115, 95, 98, 101, 104, 97,
+            118, 105, 111, 117, 114, 95, 49, 95, 117, 112, 100, 97, 116, 101, 100, 1, 0, 23, 0, 11, 0, 8, 0, 12, 19, 0,
+            99, 117, 114, 114, 101, 110, 99, 121, 3, 85, 83, 68 };
+
+    private static final byte[] BINLOG_UPDATE_80 = { 0, 6, 0, 60, 2, 93, 0, 40, 0, -123, 0, 40, 0, -83, 0, 40, 0, -43,
+            0, 40, 0, -3, 0, 40, 0, 37, 1, 40, 0, 0, 118, 1, 0, -115, 1, 12, -92, 1, 0, -35, 1, 12, -12, 1, 0, 37, 2, 1,
+            12, -12, 1, 0, 37, 2, 49, 55, 102, 99, 57, 56, 56, 57, 52, 55, 52, 48, 50, 56, 48, 54, 51, 57, 57, 48, 57,
+            49, 52, 48, 48, 49, 102, 54, 56, 53, 52, 102, 54, 98, 56, 98, 53, 55, 56, 52, 49, 102, 51, 97, 50, 101, 97,
+            53, 98, 99, 49, 102, 54, 48, 50, 53, 56, 100, 102, 50, 48, 53, 50, 49, 98, 101, 101, 57, 97, 99, 54, 51, 54,
+            100, 102, 54, 57, 97, 51, 97, 52, 102, 52, 100, 57, 57, 97, 52, 51, 56, 102, 51, 51, 52, 100, 55, 100, 98,
+            102, 56, 51, 97, 49, 56, 49, 54, 48, 49, 53, 98, 51, 54, 49, 98, 56, 52, 56, 98, 51, 98, 57, 48, 50, 49, 49,
+            54, 50, 50, 57, 49, 98, 101, 55, 50, 102, 53, 97, 56, 48, 50, 53, 52, 56, 48, 102, 52, 52, 98, 102, 52, 52,
+            100, 53, 100, 56, 49, 100, 48, 55, 99, 57, 98, 48, 101, 100, 49, 49, 53, 51, 50, 101, 102, 101, 97, 54, 56,
+            56, 102, 100, 102, 49, 50, 98, 50, 56, 102, 49, 52, 50, 98, 57, 101, 98, 48, 56, 97, 56, 48, 99, 53, 101,
+            54, 53, 97, 100, 48, 55, 54, 50, 99, 50, 53, 57, 98, 48, 53, 98, 52, 56, 54, 54, 102, 55, 50, 52, 57, 101,
+            97, 98, 101, 99, 97, 98, 97, 100, 98, 101, 53, 55, 55, 102, 102, 50, 99, 48, 55, 101, 100, 99, 97, 97, 51,
+            101, 57, 56, 55, 99, 50, 51, 102, 98, 53, 99, 99, 52, 102, 101, 56, 54, 48, 98, 98, 53, 50, 98, 101, 99,
+            102, 48, 48, 40, 116, 101, 115, 116, 95, 102, 105, 101, 108, 100, 95, 102, 111, 114, 95, 114, 101, 109, 111,
+            118, 101, 95, 102, 105, 101, 108, 100, 115, 95, 98, 101, 104, 97, 118, 105, 111, 117, 114, 95, 50, 1, 0, 23,
+            0, 11, 0, 8, 0, 12, 19, 0, 99, 117, 114, 114, 101, 110, 99, 121, 3, 85, 83, 68, 1, 0, 23, 0, 11, 0, 8, 0,
+            12, 19, 0, 99, 117, 114, 114, 101, 110, 99, 121, 3, 85, 83, 68, 56, 116, 101, 115, 116, 95, 102, 105, 101,
+            108, 100, 95, 102, 111, 114, 95, 114, 101, 109, 111, 118, 101, 95, 102, 105, 101, 108, 100, 115, 95, 98,
+            101, 104, 97, 118, 105, 111, 117, 114, 95, 51, 95, 119, 105, 108, 108, 95, 98, 101, 95, 114, 101, 109, 111,
+            118, 101, 100, 1, 0, 23, 0, 11, 0, 8, 0, 12, 19, 0, 99, 117, 114, 114, 101, 110, 99, 121, 3, 85, 83, 68, 48,
+            116, 101, 115, 116, 95, 102, 105, 101, 108, 100, 95, 102, 111, 114, 95, 114, 101, 109, 111, 118, 101, 95,
+            102, 105, 101, 108, 100, 115, 95, 98, 101, 104, 97, 118, 105, 111, 117, 114, 95, 49, 95, 117, 112, 100, 97,
+            116, 101, 100, 1, 0, 23, 0, 11, 0, 8, 0, 12, 19, 0, 99, 117, 114, 114, 101, 110, 99, 121, 3, 85, 83, 68 };
+
+    @Test
+    public void test57PartialUpdateBinlog() throws IOException {
+        final String json = JsonBinary.parseAsString(BINLOG_UPDATE_57);
+        Assert.assertFalse(json.contains("17fc9889474028063990914001f6854f6b8b5784"));
+        Assert.assertTrue(json.contains("1f3a2ea5bc1f60258df20521bee9ac636df69a3a"));
+    }
+
+    @Test
+    public void test80PartialUpdateBinlog() throws IOException {
+        final String json = JsonBinary.parseAsString(BINLOG_UPDATE_80);
+        Assert.assertFalse(json.contains("17fc9889474028063990914001f6854f6b8b5784"));
+        System.out.println(json);
+        Assert.assertTrue(json.contains("1f3a2ea5bc1f60258df20521bee9ac636df69a3a"));
+    }
+}
\ No newline at end of file

From 9ec6632c36b567f665b94b407f5eed18f72e2aa5 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Fri, 28 Jan 2022 03:32:44 -0800
Subject: [PATCH 102/217] bump changelog

---
 CHANGELOG.md | 4 ++++
 pom.xml      | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index feadff22..849bfd17 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Changelog
 
+## [0.25.5](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.25.4...0.25.5) - 2022-01-28
+
+- mysql 8 also puts JSON keys in any damn place it likes
+
 ## [0.25.4](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.25.3...0.25.4) - 2021-10-13
 
 - add debugging info to eof exception
diff --git a/pom.xml b/pom.xml
index 5920031c..72c97898 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.25.4
+    0.25.5
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From 908b5a18998907949c84a66174c8584c7e889e3f Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Thu, 24 Mar 2022 08:09:37 -0700
Subject: [PATCH 103/217] remove custom deploy stuff

---
 .mvn/maven.config |   1 -
 .mvn/settings.xml |  14 ----
 README.md         |   7 +-
 pom.xml           | 172 ++++++++++++++--------------------------------
 4 files changed, 54 insertions(+), 140 deletions(-)
 delete mode 100644 .mvn/maven.config
 delete mode 100644 .mvn/settings.xml

diff --git a/.mvn/maven.config b/.mvn/maven.config
deleted file mode 100644
index ccb1b15d..00000000
--- a/.mvn/maven.config
+++ /dev/null
@@ -1 +0,0 @@
--s .mvn/settings.xml
diff --git a/.mvn/settings.xml b/.mvn/settings.xml
deleted file mode 100644
index e25acb74..00000000
--- a/.mvn/settings.xml
+++ /dev/null
@@ -1,14 +0,0 @@
-
-
-
-    
-        
-            maven-central
-            ${env.OSS_SONATYPE_ORG_USERNAME}
-            ${env.OSS_SONATYPE_ORG_PASSWORD}
-        
-    
-
-
diff --git a/README.md b/README.md
index 407e27a3..f9dea13f 100644
--- a/README.md
+++ b/README.md
@@ -221,13 +221,10 @@ mvn # shows how to build, test, etc. project
 
 ## Deployment
 
-First:
-http://maven.apache.org/guides/mini/guide-encryption.html
+setup your settings.xml to have a "central" entry.
 
-Secondly:
 ```
-    export OSS_SONATYPE_ORG_USERNAME=username
-    export OSS_SONATYPE_ORG_PASSWORD={encryptedpasswordJF!#$flkj}
+mvn deploy
 ```
 
 ## Contributing
diff --git a/pom.xml b/pom.xml
index 72c97898..8885000f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -33,14 +33,6 @@
             Ben Osheroff
         
     
-    
-        
-            maven-central
-            Sonatype Nexus Staging
-            https://oss.sonatype.org/service/local/staging/deploy/maven2
-        
-    
-
     
         UTF-8
         UTF-8
@@ -86,20 +78,21 @@
             test
         
     
-
-    
-        
-            
-                
-                org.apache.maven.plugins
-                maven-jxr-plugin
-                2.3
-            
-        
-    
-
     
         
+              
+                org.sonatype.plugins
+                nexus-staging-maven-plugin
+                1.6.6
+                true
+                
+                  central
+                  https://oss.sonatype.org/
+                  true
+                  10
+                  true
+                
+              
             
                 org.apache.maven.plugins
                 maven-compiler-plugin
@@ -121,14 +114,6 @@
                     
                 
             
-            
-                org.apache.maven.plugins
-                maven-deploy-plugin
-                2.8.2
-                
-                    true
-                
-            
             
                 org.apache.maven.plugins
                 maven-surefire-plugin
@@ -158,35 +143,48 @@
                 
             
             
-                com.github.shyiko.usage-maven-plugin
-                usage-maven-plugin
-                1.0.0
-                
-                    
-                        # build everything (append "-DskipTests=true" if you wish to skip tests)
-                        ./mvnw clean package
-
-                        # run unit + integration tests
-                        ./mvnw -P coverage clean verify
-                        # use -Dvagrant.integration.box= to switch between MySQL sandboxes
-
-                        # publish a new version
-                        ./mvnw versions:set -DnewVersion=<version>
-                        ./mvnw -Ddeploy=maven-central
-                        git tag <version> && git push origin <version>
-                    
-                
+                org.apache.maven.plugins
+                maven-source-plugin
+                3.2.1
+                
+                    
+                        attach-sources
+                        verify
+                        
+                            jar-no-fork
+                        
+                    
+                
+            
+            
+                org.apache.maven.plugins
+                maven-javadoc-plugin
+                3.2.0
+                
+                    
+                        attach-javadocs
+                        
+                            jar
+                        
+                    
+                
+            
+            
+                org.apache.maven.plugins
+                maven-gpg-plugin
+                1.6
+                
+                    
+                        sign-artifacts
+                        verify
+                        
+                            sign
+                        
+                    
+                
             
         
-        
-            
-                com.github.shyiko.usage-maven-plugin
-                usage-maven-plugin
-                1.0.0
-            
-        
     
-
     
         
             mysql-8-compat
@@ -199,72 +197,6 @@
                 
             
         
-        
-            deploy-to-maven-central
-            
-                
-                    deploy
-                    maven-central
-                
-            
-            
-                clean deploy
-                
-                    
-                        org.apache.maven.plugins
-                        maven-source-plugin
-                        3.2.1
-                        
-                            
-                                attach-sources
-                                verify
-                                
-                                    jar-no-fork
-                                
-                            
-                        
-                    
-                    
-                        org.apache.maven.plugins
-                        maven-javadoc-plugin
-                        3.2.0
-                        
-                            
-                                attach-javadocs
-                                
-                                    jar
-                                
-                            
-                        
-                    
-                    
-                        org.apache.maven.plugins
-                        maven-gpg-plugin
-                        1.6
-                        
-                            
-                                sign-artifacts
-                                verify
-                                
-                                    sign
-                                
-                            
-                        
-                    
-                    
-                        org.sonatype.plugins
-                        nexus-staging-maven-plugin
-                        1.6.8
-                        true
-                        
-                            https://oss.sonatype.org/
-                            maven-central
-                            true
-                        
-                    
-                
-            
-        
     
 
 

From 324858397ec1cb8328d6035f67d411bb16d17740 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Thu, 14 Apr 2022 22:20:12 -0700
Subject: [PATCH 104/217] stop crashing in BufferedSocketInputStream#read

---
 .../shyiko/mysql/binlog/io/BufferedSocketInputStream.java     | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/io/BufferedSocketInputStream.java b/src/main/java/com/github/shyiko/mysql/binlog/io/BufferedSocketInputStream.java
index 69f00e1a..ad37255e 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/io/BufferedSocketInputStream.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/io/BufferedSocketInputStream.java
@@ -60,6 +60,10 @@ public int read(byte[] b, int off, int len) throws IOException {
             }
             offset = 0;
             limit = in.read(buffer, 0, buffer.length);
+            
+            if (limit == -1) {
+                return -1;
+            }
         }
         int bytesRemainingInBuffer = Math.min(len, limit - offset);
         System.arraycopy(buffer, offset, b, off, bytesRemainingInBuffer);

From 9b9b9d3447ed81800c71d9252413ccc2da8b2721 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Thu, 14 Apr 2022 22:20:50 -0700
Subject: [PATCH 105/217] v0.25.6

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 8885000f..5ff1215a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.25.5
+    0.25.6
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From c19b1add648232dbd77151e40555927997a42ff5 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Thu, 14 Apr 2022 22:21:24 -0700
Subject: [PATCH 106/217] update changelog

---
 CHANGELOG.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 849bfd17..7dbaa2b5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Changelog
 
+## [0.25.6](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.25.5...0.25.6) - 2022-04-14
+
+- stop crashing in an inopportune place
+
 ## [0.25.5](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.25.4...0.25.5) - 2022-01-28
 
 - mysql 8 also puts JSON keys in any damn place it likes

From 51fc0c9e94cd4466836623514c7da7938e9a8f5a Mon Sep 17 00:00:00 2001
From: harveyyue 
Date: Sun, 19 Jun 2022 13:18:33 +0800
Subject: [PATCH 107/217] #74 Deserialize table map metadata failed with
 java.io.EOFException: Failed to read next byte from position 15

---
 .../TableMapEventMetadataDeserializer.java    | 11 ++---
 ...TableMapEventMetadataDeserializerTest.java | 43 +++++++++++++++++++
 2 files changed, 49 insertions(+), 5 deletions(-)
 create mode 100644 src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializerTest.java

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
index 2a0705d0..61c34bce 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
@@ -51,6 +51,12 @@ public TableMapEventMetadata deserialize(ByteArrayInputStream inputStream, int n
             if (fieldType == null) {
                 throw new IOException("Unsupported table metadata field type " + code);
             }
+            if (MetadataFieldType.UNKNOWN_METADATA_FIELD_TYPE.equals(fieldType)) {
+                if (logger.isLoggable(Level.FINE)) {
+                    logger.fine("Received metadata field of unknown type");
+                }
+                continue;
+            }
 
             int fieldLength = inputStream.readPackedInteger();
 
@@ -93,11 +99,6 @@ public TableMapEventMetadata deserialize(ByteArrayInputStream inputStream, int n
                 case VISIBILITY:
                     result.setVisibility(readBooleanList(inputStream, nColumns));
                     break;
-                case UNKNOWN_METADATA_FIELD_TYPE:
-                    if (logger.isLoggable(Level.FINE)) {
-                        logger.fine("Received metadata field of unknown type");
-                    }
-                    break;
                 default:
                     inputStream.enterBlock(remainingBytes);
                     throw new IOException("Unsupported table metadata field type " + code);
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializerTest.java b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializerTest.java
new file mode 100644
index 00000000..1676293a
--- /dev/null
+++ b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializerTest.java
@@ -0,0 +1,43 @@
+package com.github.shyiko.mysql.binlog.event.deserialization;
+
+import com.github.shyiko.mysql.binlog.event.TableMapEventMetadata;
+import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import static org.testng.Assert.assertEquals;
+
+/**
+ * @author Harvey Yue
+ */
+public class TableMapEventMetadataDeserializerTest {
+
+    /**
+     * https://github.com/mysql/mysql-server/blob/8.0/libbinlogevents/include/rows_event.h#L185
+     * There are some optional metadata defined. They are listed in the table
+     * Table_table_map_event_optional_metadata. Optional metadata fields
+     * follow null_bits. Whether binlogging an optional metadata is decided by the
+     * server. The order is not defined, so they can be binlogged in any order.
+     *
+     * @throws IOException
+     */
+    @Test
+    public void deserialize() throws IOException {
+        byte[] metadataIncludingUnknownFieldType = {1, 2, 0, -128, 2, 9, 83, 6, 63, 7, 63, 8, 63, 9, 63};
+        TableMapEventMetadataDeserializer deserializer = new TableMapEventMetadataDeserializer();
+        TableMapEventMetadata tableMapEventMetadata =
+            deserializer.deserialize(new ByteArrayInputStream(metadataIncludingUnknownFieldType), 23, 8);
+
+        Map expectedCharsetCollations = new LinkedHashMap<>();
+        expectedCharsetCollations.put(6, 63);
+        expectedCharsetCollations.put(7, 63);
+        expectedCharsetCollations.put(8, 63);
+        expectedCharsetCollations.put(9, 63);
+
+        assertEquals(tableMapEventMetadata.getDefaultCharset().getDefaultCharsetCollation(), 83);
+        assertEquals(tableMapEventMetadata.getDefaultCharset().getCharsetCollations(), expectedCharsetCollations);
+    }
+}

From a7f2de33da826d64f257260c1205487666e83c9b Mon Sep 17 00:00:00 2001
From: Somesh Malviya 
Date: Thu, 30 Jun 2022 12:20:48 +0200
Subject: [PATCH 108/217] Add support for compressed binlogs
 https://issues.redhat.com/browse/DBZ-2663

---
 pom.xml                                       |   6 ++
 .../shyiko/mysql/binlog/event/EventType.java  |  12 ++-
 .../event/TransactionPayloadEventData.java    |  67 ++++++++++++
 .../deserialization/EventDeserializer.java    |  26 +++++
 ...ansactionPayloadEventDataDeserializer.java | 102 ++++++++++++++++++
 .../BinaryLogFileReaderIntegrationTest.java   |   7 ++
 ...ctionPayloadEventDataDeserializerTest.java | 100 +++++++++++++++++
 src/test/resources/mysql-bin.compressed       | Bin 0 -> 771 bytes
 8 files changed, 319 insertions(+), 1 deletion(-)
 create mode 100644 src/main/java/com/github/shyiko/mysql/binlog/event/TransactionPayloadEventData.java
 create mode 100644 src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TransactionPayloadEventDataDeserializer.java
 create mode 100644 src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/TransactionPayloadEventDataDeserializerTest.java
 create mode 100644 src/test/resources/mysql-bin.compressed

diff --git a/pom.xml b/pom.xml
index 5ff1215a..319457aa 100644
--- a/pom.xml
+++ b/pom.xml
@@ -77,6 +77,12 @@
             2.9.10.3
             test
         
+        
+            com.github.luben
+            zstd-jni
+            1.5.0-2
+            compile
+        
     
     
         
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
index 7f94ccfd..261fbd29 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
@@ -192,7 +192,17 @@ public enum EventType {
     /**
      * Prepared XA transaction terminal event similar to XID except that it is specific to XA transaction.
      */
-    XA_PREPARE;
+    XA_PREPARE,
+    /**
+     Extension of UPDATE_ROWS_EVENT, allowing partial values according
+     to binlog_row_value_options.
+     */
+    PARTIAL_UPDATE_ROWS_EVENT,
+    /**
+     * Generated when 'binlog_transaction_compression' is set to 'ON'.
+     * It encapsulates all the events of a transaction in a Zstd compressed payload.
+     */
+    TRANSACTION_PAYLOAD;
 
     public static boolean isRowMutation(EventType eventType) {
         return EventType.isWrite(eventType) ||
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/TransactionPayloadEventData.java b/src/main/java/com/github/shyiko/mysql/binlog/event/TransactionPayloadEventData.java
new file mode 100644
index 00000000..f5014cc8
--- /dev/null
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/TransactionPayloadEventData.java
@@ -0,0 +1,67 @@
+package com.github.shyiko.mysql.binlog.event;
+
+import java.util.ArrayList;
+
+
+public class TransactionPayloadEventData implements EventData {
+    private int payloadSize;
+    private int uncompressedSize;
+    private int compressionType;
+    private byte[] payload;
+    private ArrayList uncompressedEvents;
+
+    public ArrayList getUncompressedEvents() {
+        return uncompressedEvents;
+    }
+
+    public void setUncompressedEvents(ArrayList uncompressedEvents) {
+        this.uncompressedEvents = uncompressedEvents;
+    }
+
+    public int getPayloadSize() {
+        return payloadSize;
+    }
+
+    public void setPayloadSize(int payloadSize) {
+        this.payloadSize = payloadSize;
+    }
+
+    public int getUncompressedSize() {
+        return uncompressedSize;
+    }
+
+    public void setUncompressedSize(int uncompressedSize) {
+        this.uncompressedSize = uncompressedSize;
+    }
+
+    public int getCompressionType() {
+        return compressionType;
+    }
+
+    public void setCompressionType(int compressionType) {
+        this.compressionType = compressionType;
+    }
+
+    public byte[] getPayload() {
+        return payload;
+    }
+
+    public void setPayload(byte[] payload) {
+        this.payload = payload;
+    }
+
+    @Override
+    public String toString() {
+        final StringBuilder sb = new StringBuilder();
+        sb.append("TransactionPayloadEventData");
+        sb.append("{compression_type=").append(compressionType).append(", payload_size=").append(payloadSize).append(", uncompressed_size='").append(uncompressedSize).append('\'');
+        sb.append(", payload: ");
+        sb.append("\n");
+        for (Event e : uncompressedEvents) {
+            sb.append(e.toString());
+            sb.append("\n");
+        }
+        sb.append("}");
+        return sb.toString();
+    }
+}
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java
index cf936852..9ef90df9 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java
@@ -22,6 +22,7 @@
 import com.github.shyiko.mysql.binlog.event.FormatDescriptionEventData;
 import com.github.shyiko.mysql.binlog.event.LRUCache;
 import com.github.shyiko.mysql.binlog.event.TableMapEventData;
+import com.github.shyiko.mysql.binlog.event.TransactionPayloadEventData;
 import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream;
 
 import java.io.IOException;
@@ -120,6 +121,8 @@ private void registerDefaultEventDataDeserializers() {
                new PreviousGtidSetDeserializer());
         eventDataDeserializers.put(EventType.XA_PREPARE,
                 new XAPrepareEventDataDeserializer());
+        eventDataDeserializers.put(EventType.TRANSACTION_PAYLOAD,
+                new TransactionPayloadEventDataDeserializer());
     }
 
     public void setEventDataDeserializer(EventType eventType, EventDataDeserializer eventDataDeserializer) {
@@ -227,6 +230,9 @@ public Event nextEvent(ByteArrayInputStream inputStream) throws IOException {
             case TABLE_MAP:
                 eventData = deserializeTableMapEventData(inputStream, eventHeader);
                 break;
+            case TRANSACTION_PAYLOAD:
+                eventData = deserializeTransactionPayloadEventData(inputStream, eventHeader);
+                break;
             default:
                 EventDataDeserializer eventDataDeserializer = getEventDataDeserializer(eventHeader.getEventType());
                 eventData = deserializeEventData(inputStream, eventHeader, eventDataDeserializer);
@@ -272,6 +278,26 @@ private EventData deserializeFormatDescriptionEventData(ByteArrayInputStream inp
         return eventData;
     }
 
+    public EventData deserializeTransactionPayloadEventData(ByteArrayInputStream inputStream, EventHeader eventHeader)
+        throws IOException {
+        EventDataDeserializer eventDataDeserializer = eventDataDeserializers.get(EventType.TRANSACTION_PAYLOAD);
+        EventData eventData = deserializeEventData(inputStream, eventHeader, eventDataDeserializer);
+        TransactionPayloadEventData transactionPayloadEventData = (TransactionPayloadEventData) eventData;
+
+        /**
+         * Handling for TABLE_MAP events withing the transaction payload event. This is to ensure that for the table map
+         * events within the transaction payload, the target table id and the event gets added to the
+         * tableMapEventByTableId map. This is map is later used while deserializing rows.
+         */
+        for (Event event : transactionPayloadEventData.getUncompressedEvents()) {
+            if (event.getHeader().getEventType() == EventType.TABLE_MAP && event.getData() != null) {
+                TableMapEventData tableMapEvent = (TableMapEventData) event.getData();
+                tableMapEventByTableId.put(tableMapEvent.getTableId(), tableMapEvent);
+            }
+        }
+        return eventData;
+    }
+
     public EventData deserializeTableMapEventData(ByteArrayInputStream inputStream, EventHeader eventHeader)
             throws IOException {
         EventDataDeserializer eventDataDeserializer =
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TransactionPayloadEventDataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TransactionPayloadEventDataDeserializer.java
new file mode 100644
index 00000000..a8e84876
--- /dev/null
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TransactionPayloadEventDataDeserializer.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2013 Stanley Shyiko
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.github.shyiko.mysql.binlog.event.deserialization;
+
+import com.github.luben.zstd.Zstd;
+import com.github.shyiko.mysql.binlog.event.Event;
+import com.github.shyiko.mysql.binlog.event.TransactionPayloadEventData;
+import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+
+/**
+ * @author Somesh Malviya
+ * @author Debjeet Sarkar
+ */
+public class TransactionPayloadEventDataDeserializer implements EventDataDeserializer {
+    public static final int OTW_PAYLOAD_HEADER_END_MARK = 0;
+    public static final int OTW_PAYLOAD_SIZE_FIELD = 1;
+    public static final int OTW_PAYLOAD_COMPRESSION_TYPE_FIELD = 2;
+    public static final int OTW_PAYLOAD_UNCOMPRESSED_SIZE_FIELD = 3;
+
+    @Override
+    public TransactionPayloadEventData deserialize(ByteArrayInputStream inputStream) throws IOException {
+        TransactionPayloadEventData eventData = new TransactionPayloadEventData();
+        // Read the header fields from the event data
+        while (inputStream.available() > 0) {
+            int fieldType = 0;
+            int fieldLen = 0;
+            // Read the type of the field
+            if (inputStream.available() >= 1) {
+                fieldType = inputStream.readPackedInteger();
+            }
+            // We have reached the end of the Event Data Header
+            if (fieldType == OTW_PAYLOAD_HEADER_END_MARK) {
+                break;
+            }
+            // Read the size of the field
+            if (inputStream.available() >= 1) {
+                fieldLen = inputStream.readPackedInteger();
+            }
+            switch (fieldType) {
+                case OTW_PAYLOAD_SIZE_FIELD:
+                    // Fetch the payload size
+                    eventData.setPayloadSize(inputStream.readPackedInteger());
+                    break;
+                case OTW_PAYLOAD_COMPRESSION_TYPE_FIELD:
+                    // Fetch the compression type
+                    eventData.setCompressionType(inputStream.readPackedInteger());
+                    break;
+                case OTW_PAYLOAD_UNCOMPRESSED_SIZE_FIELD:
+                    // Fetch the uncompressed size
+                    eventData.setUncompressedSize(inputStream.readPackedInteger());
+                    break;
+                default:
+                    // Ignore unrecognized field
+                    inputStream.read(fieldLen);
+                    break;
+            }
+        }
+        if (eventData.getUncompressedSize() == 0) {
+            // Default the uncompressed to the payload size
+            eventData.setUncompressedSize(eventData.getPayloadSize());
+        }
+        // set the payload to the rest of the input buffer
+        eventData.setPayload(inputStream.read(eventData.getPayloadSize()));
+
+        // Decompress the payload
+        byte[] src = eventData.getPayload();
+        byte[] dst = ByteBuffer.allocate(eventData.getUncompressedSize()).array();
+        Zstd.decompressByteArray(dst, 0, dst.length, src, 0, src.length);
+
+        // Read and store events from decompressed byte array into input stream
+        ArrayList decompressedEvents = new ArrayList<>();
+        EventDeserializer transactionPayloadEventDeserializer = new EventDeserializer();
+        ByteArrayInputStream destinationInputStream = new ByteArrayInputStream(dst);
+
+        Event internalEvent = transactionPayloadEventDeserializer.nextEvent(destinationInputStream);
+        while(internalEvent != null) {
+            decompressedEvents.add(internalEvent);
+            internalEvent = transactionPayloadEventDeserializer.nextEvent(destinationInputStream);
+        }
+
+        eventData.setUncompressedEvents(decompressedEvents);
+
+        return eventData;
+    }
+}
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogFileReaderIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogFileReaderIntegrationTest.java
index ef9eca57..71c48eef 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogFileReaderIntegrationTest.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogFileReaderIntegrationTest.java
@@ -46,6 +46,13 @@ public void testNextEvent() throws Exception {
         readAll(reader, 1462);
     }
 
+    @Test
+    public void testNextEventCompressed() throws Exception {
+        BinaryLogFileReader reader = new BinaryLogFileReader(
+            new FileInputStream("src/test/resources/mysql-bin.compressed"));
+        readAll(reader, 5);
+    }
+
     @Test
     public void testChecksumNONE() throws Exception {
         EventDeserializer eventDeserializer = new EventDeserializer();
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/TransactionPayloadEventDataDeserializerTest.java b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/TransactionPayloadEventDataDeserializerTest.java
new file mode 100644
index 00000000..a3b8e76e
--- /dev/null
+++ b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/TransactionPayloadEventDataDeserializerTest.java
@@ -0,0 +1,100 @@
+/*
+ * Copyright 2013 Stanley Shyiko
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.github.shyiko.mysql.binlog.event.deserialization;
+
+import com.github.shyiko.mysql.binlog.event.EventType;
+import com.github.shyiko.mysql.binlog.event.TransactionPayloadEventData;
+import com.github.shyiko.mysql.binlog.event.XAPrepareEventData;
+import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+
+import static org.testng.Assert.assertEquals;
+
+/**
+ * @author Somesh Malviya
+ */
+public class TransactionPayloadEventDataDeserializerTest {
+
+      /* DATA is a binary representation of following:
+       TransactionPayloadEventData{compression_type=0, payload_size=451, uncompressed_size='960', payload:
+           Event{header=EventHeaderV4{timestamp=1646406641000, eventType=QUERY, serverId=223344, headerLength=19, dataLength=57, nextPosition=0, flags=8}, data=QueryEventData{threadId=12, executionTime=0, errorCode=0, database='', sql='BEGIN'}}
+           Event{header=EventHeaderV4{timestamp=1646406641000, eventType=TABLE_MAP, serverId=223344, headerLength=19, dataLength=63, nextPosition=0, flags=0}, data=TableMapEventData{tableId=84, database='demo', table='movies', columnTypes=3, 15, 3, 15, 15, 15, 15, 15, 15, 15, 15, columnMetadata=0, 1024, 0, 1024, 1024, 4096, 2048, 1024, 1024, 1024, 1024, columnNullability={}, eventMetadata=TableMapEventMetadata{signedness={}, defaultCharset=255, charsetCollations=null, columnCharsets=null, columnNames=null, setStrValues=null, enumStrValues=null, geometryTypes=null, simplePrimaryKeys=null, primaryKeysWithPrefix=null, enumAndSetDefaultCharset=null, enumAndSetColumnCharsets=null,visibility=null}}}
+           Event{header=EventHeaderV4{timestamp=1646406641000, eventType=EXT_UPDATE_ROWS, serverId=223344, headerLength=19, dataLength=756, nextPosition=0, flags=0}, data=UpdateRowsEventData{tableId=84, includedColumnsBeforeUpdate={0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, includedColumns={0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, rows=[
+               {before=[1, Once Upon a Time in the West, 1968, Italy, Western, Claudia Cardinale|Charles Bronson|Henry Fonda|Gabriele Ferzetti|Frank Wolff|Al Mulock|Jason Robards|Woody Strode|Jack Elam|Lionel Stander|Paolo Stoppa|Keenan Wynn|Aldo Sambrell, Sergio Leone, Ennio Morricone, Sergio Leone|Sergio Donati|Dario Argento|Bernardo Bertolucci, Tonino Delli Colli, Paramount Pictures], after=[1, Once Upon a Time in the West, 1968, Italy, Western|Action, Claudia Cardinale|Charles Bronson|Henry Fonda|Gabriele Ferzetti|Frank Wolff|Al Mulock|Jason Robards|Woody Strode|Jack Elam|Lionel Stander|Paolo Stoppa|Keenan Wynn|Aldo Sambrell, Sergio Leone, Ennio Morricone, Sergio Leone|Sergio Donati|Dario Argento|Bernardo Bertolucci, Tonino Delli Colli, Paramount Pictures]}
+           ]}}
+           Event{header=EventHeaderV4{timestamp=1646406641000, eventType=XID, serverId=223344, headerLength=19, dataLength=8, nextPosition=0, flags=0}, data=XidEventData{xid=31}}
+           }
+      */
+      private static final byte[] DATA = {
+        2, 1, 0, 3, 3, -4, -64, 3, 1, 3, -4, -61, 1, 0, 40, -75, 47, -3, 0, 88, -68, 13, 0, -90, -34,
+        97, 57, 96, 103, -108, 14, 32, 1, 32, 8, -126, 32, 120, 18, 103, 8, -126, -114, 45, -84, -15,
+        -9, -66, 68, 74, -118, -40, 82, 68, -110, 16, 13, -122, 26, 35, 98, 20, 123, 16, 7, -5, -10, 69,
+        -128, 37, 107, 91, -42, 50, -10, -116, -6, -79, 51, 11, 93, -14, 73, 10, 87, 0, 81, 0, 81, 0,
+        -1, -95, 63, -53, -78, 76, -31, -116, -56, -15, -88, -70, 26, 36, -55, -28, -13, 44, 66, -60,
+        56, 4, -3, 113, -122, -58, 35, -112, 8, 18, 41, 28, -37, -42, -96, -83, -124, -73, -75, 84, -29,
+        -48, 41, 62, -15, -88, -70, 6, 72, -110, -55, 71, -63, -125, 3, -90, -14, 103, -111, 67, 1, -98,
+        -3, -15, 71, -125, -126, 88, -108, -16, -1, -104, 7, 79, -24, 6, -66, -16, -57, 53, -113, -86,
+        -117, 33, 73, 38, -97, -100, -68, 96, 125, -103, -40, 32, 92, 7, 111, 51, -71, 110, -37, -109,
+        -44, 33, 42, -59, -99, 73, -49, -29, 69, 16, -71, 49, -18, 87, 73, 108, -35, -45, -54, 18, -41,
+        41, 55, -22, -87, 37, -75, 81, 29, 117, -106, 67, -32, -73, 16, 91, -50, 29, 30, -89, -16, -31,
+        0, 126, 7, 4, -120, 45, 39, -73, -126, -55, 45, -41, 106, 20, -87, -55, 125, 49, -56, -99, 120,
+        -63, 11, 4, -116, 57, 100, -71, 87, -109, -35, 44, -34, 110, -66, -32, -36, 62, -55, -46, 77,
+        54, -27, 40, -111, -39, -61, 73, 86, -34, 77, 16, -11, -70, 26, 110, -78, 93, -85, 68, 124, 75,
+        -79, -62, 77, -70, -27, 110, -102, 104, -87, -61, -28, -59, -92, 16, 113, -87, 126, 112, -109,
+        30, -86, -101, 19, 49, -22, -87, -44, 19, -55, -115, 41, 68, -68, -104, -38, 117, 34, -46, 81,
+        98, 69, -123, -21, -1, -1, -65, -31, 4, 30, 85, 23, -125, 36, -103, 124, -70, -63, -119, 18, 5,
+        96, -5, 58, 112, 106, 18, 9, -71, -45, -106, 62, -107, 120, -92, 57, -41, -106, 108, -50, -19,
+        37, -101, 27, 55, -59, 35, 109, -102, 58, -82, -31, -37, 74, 54, -11, -108, -33, 86, 98, 67, 94,
+        -117, 71, -55, 110, 79, 47, -79, 65, -27, -66, -60, 3, -53, 61, -75, -9, 58, 34, -69, 113, 18,
+        0, 9, -123, 64, 53, 121, 75, 21, -68, 7, 33, -73, -30, -127, -103, 9, 17, 66, -49, 84, 65, 2,
+        43, 16, -125, 0, 43, 55, 114, 109, 4, -50, -64, -62, -64, 99, 0, 28, -96, 53, -96, -13, 0, -68,
+        1, 0, 0
+      };
+
+    // Compression type for Zstd is 0
+    private static final int COMPRESSION_TYPE = 0;
+    private static final int PAYLOAD_SIZE = 451;
+    private static final int UNCOMPRESSED_SIZE = 960;
+    private static final int NUMBER_OF_UNCOMPRESSED_EVENTS = 4;
+    private static final String UNCOMPRESSED_UPDATE_EVENT =
+      new StringBuilder()
+          .append(
+              "UpdateRowsEventData{tableId=84, includedColumnsBeforeUpdate={0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, includedColumns={0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, rows=[\n")
+          .append(
+              "    {before=[1, Once Upon a Time in the West, 1968, Italy, Western, Claudia Cardinale|Charles Bronson|Henry Fonda|Gabriele Ferzetti|Frank Wolff|Al Mulock|Jason Robards|Woody Strode|Jack Elam|Lionel Stander|Paolo Stoppa|Keenan Wynn|Aldo Sambrell, Sergio Leone, Ennio Morricone, Sergio Leone|Sergio Donati|Dario Argento|Bernardo Bertolucci, Tonino Delli Colli, Paramount Pictures],")
+          .append(
+              " after=[1, Once Upon a Time in the West, 1968, Italy, Western|Action, Claudia Cardinale|Charles Bronson|Henry Fonda|Gabriele Ferzetti|Frank Wolff|Al Mulock|Jason Robards|Woody Strode|Jack Elam|Lionel Stander|Paolo Stoppa|Keenan Wynn|Aldo Sambrell, Sergio Leone, Ennio Morricone, Sergio Leone|Sergio Donati|Dario Argento|Bernardo Bertolucci, Tonino Delli Colli, Paramount Pictures]}\n")
+          .append("]}")
+          .toString();
+
+    @Test
+    public void deserialize() throws IOException {
+        TransactionPayloadEventDataDeserializer deserializer = new TransactionPayloadEventDataDeserializer();
+        TransactionPayloadEventData transactionPayloadEventData =
+            deserializer.deserialize(new ByteArrayInputStream(DATA));
+          assertEquals(COMPRESSION_TYPE, transactionPayloadEventData.getCompressionType());
+          assertEquals(PAYLOAD_SIZE, transactionPayloadEventData.getPayloadSize());
+          assertEquals(UNCOMPRESSED_SIZE, transactionPayloadEventData.getUncompressedSize());
+          assertEquals(NUMBER_OF_UNCOMPRESSED_EVENTS, transactionPayloadEventData.getUncompressedEvents().size());
+          assertEquals(EventType.QUERY, transactionPayloadEventData.getUncompressedEvents().get(0).getHeader().getEventType());
+          assertEquals(EventType.TABLE_MAP, transactionPayloadEventData.getUncompressedEvents().get(1).getHeader().getEventType());
+          assertEquals(EventType.EXT_UPDATE_ROWS, transactionPayloadEventData.getUncompressedEvents().get(2).getHeader().getEventType());
+          assertEquals(EventType.XID, transactionPayloadEventData.getUncompressedEvents().get(3).getHeader().getEventType());
+          assertEquals(UNCOMPRESSED_UPDATE_EVENT, transactionPayloadEventData.getUncompressedEvents().get(2).getData().toString());
+    }
+}
diff --git a/src/test/resources/mysql-bin.compressed b/src/test/resources/mysql-bin.compressed
new file mode 100644
index 0000000000000000000000000000000000000000..3e47c1774204bbe1979dc5b45b25ac772f25d701
GIT binary patch
literal 771
zcmeyDl$m!{crzD~W#`p2AUjs+uwWL&Q%$T$s;0k__$)1l*-8`pLLv$Sg`yp
zY&)hrfkQ}B=JvG(Yg@K&4S9S)(+;SF&12F@_k+#M%RZ$~bY`6Q_oI7rQ^b@H|7WoK
zzhK+<;kar4s%}M3wfS@QB-GBlp%BBKZ@e?__T(#yT1V%4o`38ru+#8exM$AY%cq2{
zYns1Wsk${#wse~FgY5#*=VaxUe|X4H$IjBBtG>PIr0(@Bk(DQF4NuIiILOV?W0|ru
zeDYnLdwKgF+_5`($=B?u#>AV4J;Uz#3VhuqmA5H&wM&in#zVflp61QUSb6x#(Io{SR5>LdBa^X4dRF*eS%C@Y|{&
zONevl|V*E0YC(*>Li%>cul04M+e

literal 0
HcmV?d00001


From 3fcbe9e3193e35718c1cf6bc4573780eccae535d Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Fri, 15 Jul 2022 07:14:27 -0700
Subject: [PATCH 109/217] v0.26.0

---
 CHANGELOG.md                                          |  7 +++++++
 .../github/shyiko/mysql/binlog/BinaryLogClient.java   |  1 +
 .../mysql/binlog/io/BufferedSocketInputStream.java    |  2 +-
 .../shyiko/mysql/binlog/BinaryLogClientTest.java      | 11 +++++++++++
 4 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7dbaa2b5..99854484 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,12 @@
 # Changelog
 
+## [0.26.0](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.25.6...0.26.0) - 2022-07-15
+
+- Compressed binlogs, thank you Somesh Malviya
+- fix crash on unknown field type
+
+# Changelog
+
 ## [0.25.6](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.25.5...0.25.6) - 2022-04-14
 
 - stop crashing in an inopportune place
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index d8b8299f..824f2a6c 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -780,6 +780,7 @@ public void run() {
                             // expected in case of disconnect
                         }
                         if (threadExecutor.isShutdown()) {
+                            logger.info("threadExecutor is shut down, terminating keepalive thread");
                             return;
                         }
                         boolean connectionLost = false;
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/io/BufferedSocketInputStream.java b/src/main/java/com/github/shyiko/mysql/binlog/io/BufferedSocketInputStream.java
index ad37255e..5095637c 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/io/BufferedSocketInputStream.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/io/BufferedSocketInputStream.java
@@ -60,7 +60,7 @@ public int read(byte[] b, int off, int len) throws IOException {
             }
             offset = 0;
             limit = in.read(buffer, 0, buffer.length);
-            
+
             if (limit == -1) {
                 return -1;
             }
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientTest.java
index 7f13b7c1..b1159b56 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientTest.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientTest.java
@@ -178,4 +178,15 @@ public void run() {
         }
     }
 
+    @Test
+    public void testDeadlockyCode() throws IOException, InterruptedException {
+        final BinaryLogClient binaryLogClient = new BinaryLogClient("localhost", 3306, "root", "123456");
+        binaryLogClient.setHeartbeatInterval(10000);
+        binaryLogClient.setKeepAlive(true);
+        binaryLogClient.setKeepAliveInterval(2000);
+
+        binaryLogClient.connect();
+
+        Thread.sleep(10000000);
+    }
 }

From 334c405122ad02b7dce34d48cc1f891d4e865ea8 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Fri, 15 Jul 2022 07:45:53 -0700
Subject: [PATCH 110/217] comment out test

---
 .../com/github/shyiko/mysql/binlog/BinaryLogClientTest.java | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientTest.java
index b1159b56..81660b71 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientTest.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientTest.java
@@ -178,6 +178,7 @@ public void run() {
         }
     }
 
+    /*
     @Test
     public void testDeadlockyCode() throws IOException, InterruptedException {
         final BinaryLogClient binaryLogClient = new BinaryLogClient("localhost", 3306, "root", "123456");
@@ -187,6 +188,9 @@ public void testDeadlockyCode() throws IOException, InterruptedException {
 
         binaryLogClient.connect();
 
-        Thread.sleep(10000000);
+        Thread.sleep(1000);
+
+        binaryLogClient.disconnect();
     }
+    */
 }

From bcf7a43d13b1dd9864358a480f929f6298a95380 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Fri, 15 Jul 2022 07:48:40 -0700
Subject: [PATCH 111/217] actually bump version, chumpstick

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 319457aa..0e05b93a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.25.6
+    0.26.0
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From 13378cbd52c8f7d706880c771e45a8d58764ef11 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Sat, 16 Jul 2022 08:00:57 -0700
Subject: [PATCH 112/217] stop holding keepalive lock while waiting for
 shutdown

the keepalive thread may be in the middle of reconnecting, which
means it tries to get the keepalive lock, which can cause
deadlock.
---
 .../shyiko/mysql/binlog/BinaryLogClient.java  | 33 ++++++++-----------
 1 file changed, 13 insertions(+), 20 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index 824f2a6c..61dc743a 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -794,17 +794,13 @@ public void run() {
                             }
                         }
                         if (connectionLost) {
-                            if (logger.isLoggable(Level.INFO)) {
-                                logger.info("Trying to restore lost connection to " + hostname + ":" + port);
-                            }
+                            logger.info("Keepalive: Trying to restore lost connection to " + hostname + ":" + port);
                             try {
                                 terminateConnect();
                                 connect(connectTimeout);
                             } catch (Exception ce) {
-                                if (logger.isLoggable(Level.WARNING)) {
-                                    logger.warning("Failed to restore connection to " + hostname + ":" + port +
-                                        ". Next attempt in " + keepAliveInterval + "ms");
-                                }
+                                logger.warning("keepalive: Failed to restore connection to " + hostname + ":" + port +
+                                    ". Next attempt in " + keepAliveInterval + "ms");
                             }
                         }
                     }
@@ -1179,19 +1175,16 @@ public void disconnect() throws IOException {
     }
 
     private void terminateKeepAliveThread() {
-        try {
-            keepAliveThreadExecutorLock.lock();
-            ExecutorService keepAliveThreadExecutor = this.keepAliveThreadExecutor;
-            if (keepAliveThreadExecutor == null) {
-                return;
-            }
-            keepAliveThreadExecutor.shutdownNow();
-            while (!awaitTerminationInterruptibly(keepAliveThreadExecutor,
-                Long.MAX_VALUE, TimeUnit.NANOSECONDS)) {
-                // ignore
-            }
-        } finally {
-            keepAliveThreadExecutorLock.unlock();
+        keepAliveThreadExecutorLock.lock();
+        ExecutorService keepAliveThreadExecutor = this.keepAliveThreadExecutor;
+        if (keepAliveThreadExecutor == null) {
+            return;
+        }
+        keepAliveThreadExecutor.shutdownNow();
+        keepAliveThreadExecutorLock.unlock();
+        while (!awaitTerminationInterruptibly(keepAliveThreadExecutor,
+            Long.MAX_VALUE, TimeUnit.NANOSECONDS)) {
+            // ignore
         }
     }
 

From fee145c37182f7f67355394c3b5f5128c3735da1 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Mon, 18 Jul 2022 08:01:29 -0700
Subject: [PATCH 113/217] try to get circle back up and running

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 9d282aa7..37fa8406 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -35,7 +35,7 @@ jobs:
           command: echo "testing under mysql $MYSQL_VERSION"
       - run:
           name: testit
-          command: mvn verify
+          command: mvn verify -Dgpg.skip
       - store_artifacts:
           path: test.log
 

From c544bc05b89940f0031d67ebfeb2eb2df8916c8e Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Mon, 18 Jul 2022 08:35:09 -0700
Subject: [PATCH 114/217] I'm dumb.

---
 .../shyiko/mysql/binlog/BinaryLogClient.java      | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index 61dc743a..4a78d990 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -1175,13 +1175,16 @@ public void disconnect() throws IOException {
     }
 
     private void terminateKeepAliveThread() {
-        keepAliveThreadExecutorLock.lock();
-        ExecutorService keepAliveThreadExecutor = this.keepAliveThreadExecutor;
-        if (keepAliveThreadExecutor == null) {
-            return;
+        try {
+            keepAliveThreadExecutorLock.lock();
+            ExecutorService keepAliveThreadExecutor = this.keepAliveThreadExecutor;
+            if ( keepAliveThreadExecutor == null ) {
+                return;
+            }
+            keepAliveThreadExecutor.shutdownNow();
+        } finally {
+            keepAliveThreadExecutorLock.unlock();
         }
-        keepAliveThreadExecutor.shutdownNow();
-        keepAliveThreadExecutorLock.unlock();
         while (!awaitTerminationInterruptibly(keepAliveThreadExecutor,
             Long.MAX_VALUE, TimeUnit.NANOSECONDS)) {
             // ignore

From 185fccc5940bac1ad6911deff8ee3928d4cfafef Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Mon, 18 Jul 2022 18:41:36 -0700
Subject: [PATCH 115/217] 0.26.1

---
 CHANGELOG.md | 3 +++
 pom.xml      | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 99854484..aebdf3d8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,7 @@
 # Changelog
+## [0.26.1](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.26.0...0.26.1) - 2022-07-18
+
+- fix deadlock with disconnect and keepalive thread.
 
 ## [0.26.0](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.25.6...0.26.0) - 2022-07-15
 
diff --git a/pom.xml b/pom.xml
index 0e05b93a..50c2922a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.26.0
+    0.26.1
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From 87103b06481c945d9f6df403840ae391b75e9df7 Mon Sep 17 00:00:00 2001
From: Meel Velliste 
Date: Wed, 8 Jul 2015 01:11:13 -0700
Subject: [PATCH 116/217] Updated gitignore

---
 .gitignore | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 88ff3803..d2d768d5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,4 @@
-.idea
+*.idea
 *.iml
 .DS_Store
 target

From c03860b88fbbcc67f51eddc0215f8945f4302f78 Mon Sep 17 00:00:00 2001
From: Meel Velliste 
Date: Thu, 30 Jul 2015 14:44:57 -0700
Subject: [PATCH 117/217] Fixed '0000-00-00 00:00:00' dates -> null in binlog

---
 .../event/deserialization/AbstractRowsEventDataDeserializer.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
index 89971365..df40459c 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
@@ -24,6 +24,7 @@
 import java.math.BigDecimal;
 import java.util.BitSet;
 import java.util.Calendar;
+import java.util.Date;
 import java.util.Map;
 import java.util.TimeZone;
 

From f2452b4b1c0be0db78e4726983de75bb33f4da27 Mon Sep 17 00:00:00 2001
From: Meel Velliste 
Date: Thu, 29 Oct 2015 21:18:32 -0700
Subject: [PATCH 118/217] Exceptions thrown in binlog processing will now cause
 whole update to fail instead of just warning in the log

---
 .../java/com/github/shyiko/mysql/binlog/BinaryLogClient.java | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index 4a78d990..9dbb8b4f 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -1121,9 +1121,8 @@ private void notifyEventListeners(Event event) {
             try {
                 eventListener.onEvent(event);
             } catch (Exception e) {
-                if (logger.isLoggable(Level.WARNING)) {
-                    logger.log(Level.WARNING, eventListener + " choked on " + event, e);
-                }
+                throw new RuntimeException("Binlog event listener " + eventListener +
+                    " choked on " + event, e);
             }
         }
     }

From 8e0752f1736a9bc46e4d226e990a0cf757e8d532 Mon Sep 17 00:00:00 2001
From: Georgie 
Date: Thu, 3 Dec 2015 19:07:21 -0800
Subject: [PATCH 119/217] Configure java.util.logging

---
 .../java/com/github/shyiko/mysql/binlog/BinaryLogClient.java    | 2 +-
 .../shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java     | 2 +-
 .../java/com/github/shyiko/mysql/binlog/TCPReverseProxy.java    | 2 +-
 .../java/com/github/shyiko/mysql/binlog/TraceEventListener.java | 2 +-
 .../com/github/shyiko/mysql/binlog/TraceLifecycleListener.java  | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index 9dbb8b4f..12a3dc2b 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -120,7 +120,7 @@ public X509Certificate[] getAcceptedIssuers() {
     // https://dev.mysql.com/doc/internals/en/sending-more-than-16mbyte.html
     private static final int MAX_PACKET_LENGTH = 16777215;
 
-    private final Logger logger = Logger.getLogger(getClass().getName());
+    private final Logger logger = Logger.getLogger("donkey");
 
     private final String hostname;
     private final int port;
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java
index 7e04312c..b5ce6772 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java
@@ -101,7 +101,7 @@ public class BinaryLogClientIntegrationTest {
 
     protected static final long DEFAULT_TIMEOUT = TimeUnit.SECONDS.toMillis(3);
 
-    private final Logger logger = Logger.getLogger(getClass().getSimpleName());
+    private final Logger logger = Logger.getLogger("donkey");
 
     {
         logger.setLevel(Level.FINEST);
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/TCPReverseProxy.java b/src/test/java/com/github/shyiko/mysql/binlog/TCPReverseProxy.java
index 883b3f25..0fe52e2c 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/TCPReverseProxy.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/TCPReverseProxy.java
@@ -36,7 +36,7 @@
  */
 public class TCPReverseProxy {
 
-    private final Logger logger = Logger.getLogger(getClass().getSimpleName());
+    private final Logger logger = Logger.getLogger("donkey");
 
     private final int port;
     private final String targetHost;
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/TraceEventListener.java b/src/test/java/com/github/shyiko/mysql/binlog/TraceEventListener.java
index 0eacd643..7ec1ed18 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/TraceEventListener.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/TraceEventListener.java
@@ -25,7 +25,7 @@
  */
 public class TraceEventListener implements BinaryLogClient.EventListener {
 
-    private final Logger logger = Logger.getLogger(getClass().getSimpleName());
+    private final Logger logger = Logger.getLogger("donkey");
 
     @Override
     public void onEvent(Event event) {
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/TraceLifecycleListener.java b/src/test/java/com/github/shyiko/mysql/binlog/TraceLifecycleListener.java
index 6e3f5702..f4fde7e2 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/TraceLifecycleListener.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/TraceLifecycleListener.java
@@ -23,7 +23,7 @@
  */
 public class TraceLifecycleListener implements BinaryLogClient.LifecycleListener {
 
-    private final Logger logger = Logger.getLogger(getClass().getSimpleName());
+    private final Logger logger = Logger.getLogger("donkey");
 
     @Override
     public void onConnect(BinaryLogClient client) {

From fc7bc6de2e1404569450e89f95bc8a6db4c5f65a Mon Sep 17 00:00:00 2001
From: Georgie 
Date: Wed, 9 Mar 2016 19:55:13 -0800
Subject: [PATCH 120/217] Deployment

---
 pom.xml | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/pom.xml b/pom.xml
index 50c2922a..22f67b92 100644
--- a/pom.xml
+++ b/pom.xml
@@ -190,6 +190,13 @@
                 
             
         
+        
+            
+                org.springframework.build
+                aws-maven
+                5.0.0.RELEASE
+            
+        
     
     
         
@@ -205,4 +212,20 @@
         
     
 
+    
+    
+        
+            fivetran-maven-release
+            s3://fivetran-maven/public/release
+        
+    
+
+    
+        
+            aws-public-release
+            AWS Public Release Repository
+            https://s3.amazonaws.com/fivetran-maven/public/release
+        
+    
+
 

From 7c59d43f75f188e187abf0b3657cc743438a940d Mon Sep 17 00:00:00 2001
From: George Fraser 
Date: Mon, 2 Jan 2017 12:26:41 -0800
Subject: [PATCH 121/217] Ignore target/

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index d2d768d5..9c209dfb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,4 @@ target
 .settings
 .vagrant
 .*.sw*
+target/

From fbce9a3939d17605134c817e3d3fb882a868fd42 Mon Sep 17 00:00:00 2001
From: George Fraser 
Date: Thu, 29 Dec 2016 12:56:15 -0800
Subject: [PATCH 122/217] Don't verify for REQUIRED or PREFERRED

---
 .../java/com/github/shyiko/mysql/binlog/BinaryLogClient.java     | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index 12a3dc2b..ab57aefe 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -758,7 +758,6 @@ private void ensureEventDataDeserializer(EventType eventType,
         }
     }
 
-
     private void spawnKeepAliveThread() {
         final ExecutorService threadExecutor =
             Executors.newSingleThreadExecutor(new ThreadFactory() {

From a485c6d1bacea4cba7bd6946d0b8879f908950c3 Mon Sep 17 00:00:00 2001
From: George Fraser 
Date: Mon, 2 Jan 2017 12:28:04 -0800
Subject: [PATCH 123/217] Latest version

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 22f67b92..c5fa4a14 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.26.1
+    0.7.3-0-FIVETRAN
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From c13fc72c5ea6f418034dbc9e31395328631759d6 Mon Sep 17 00:00:00 2001
From: George Fraser 
Date: Wed, 18 Jan 2017 14:05:49 -0800
Subject: [PATCH 124/217] Update version

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index c5fa4a14..65b0a642 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.7.3-0-FIVETRAN
+    0.7.3-1-FIVETRAN
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From d0fc54c58454be89c08c22825737e39e06ed0073 Mon Sep 17 00:00:00 2001
From: Meel Velliste 
Date: Mon, 30 Jan 2017 17:36:25 -0800
Subject: [PATCH 125/217] Fixed top-level JSON null value

---
 .../binlog/event/deserialization/json/JsonBinary.java      | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
index b167ba75..857c3c5c 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
@@ -176,7 +176,12 @@ public static String parseAsString(byte[] bytes) throws IOException {
      * @throws IOException if there is a problem reading or processing the binary representation
      */
     public static void parse(byte[] bytes, JsonFormatter formatter) throws IOException {
-        new JsonBinary(bytes).parse(formatter);
+        if (bytes.length == 0) {
+            // When the top-level value is a JSON "null", the value in java shows up as a non-null, empty byte array
+            formatter.valueNull();
+        } else {
+            new JsonBinary(bytes).parse(formatter);
+        }
     }
 
     private final ByteArrayInputStream reader;

From 7a0607707e0f31cf1e78d08deb3ad00e3e123ab7 Mon Sep 17 00:00:00 2001
From: Meel Velliste 
Date: Mon, 30 Jan 2017 17:37:22 -0800
Subject: [PATCH 126/217] Updated version

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 65b0a642..527c3b21 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.7.3-1-FIVETRAN
+    0.7.3-2-FIVETRAN
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From 27f53e1b33402399f0a3baf198c63b8f0d34661a Mon Sep 17 00:00:00 2001
From: Meel Velliste 
Date: Fri, 3 Feb 2017 13:56:23 -0800
Subject: [PATCH 127/217] Added json null test

---
 .../json/JsonBinaryValueIntegrationTest.java       | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java
index 8fb3917c..6406a63e 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java
@@ -34,6 +34,7 @@
 import org.testng.annotations.Test;
 
 import java.io.Serializable;
+import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.SQLSyntaxErrorException;
 import java.sql.Statement;
@@ -500,6 +501,19 @@ public void testScalarBinaryAsBase64() throws Exception {
         assertEquals(writeAndCaptureJSON("CAST(x'cafebabe' AS JSON)"), "\"yv66vg==\"");
     }
 
+    @Test
+    public void testJsonNull() throws Exception {
+        master.execute(new BinaryLogClientIntegrationTest.Callback() {
+            @Override
+            public void execute(Statement statement) throws SQLException {
+                ResultSet results = statement.executeQuery("SELECT version()");
+                results.next();
+                System.out.println("MySQL version = " + results.getString(1));
+            }
+        });
+        assertJSONMatchOriginal("null");
+    }
+
     private void assertJSONMatchOriginal(String value) throws Exception {
         assetJSONEquals(value, writeAndCaptureJSON("'" + value + "'"));
     }

From eb1da4754836269fbabb0910daa7f2a303aabe8d Mon Sep 17 00:00:00 2001
From: Meel Velliste 
Date: Tue, 7 Feb 2017 11:08:45 -0800
Subject: [PATCH 128/217] version++

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 527c3b21..fafab007 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.7.3-2-FIVETRAN
+    0.7.3-3-FIVETRAN
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From 0b3f203292f3e52a84db158006abb8db6ed6f5ab Mon Sep 17 00:00:00 2001
From: George Fraser 
Date: Mon, 1 Oct 2018 23:24:49 -0700
Subject: [PATCH 129/217] TLSHostnameVerifier uses no longer available API

---
 pom.xml                                       |  2 +-
 .../shyiko/mysql/binlog/BinaryLogClient.java  |  9 ++--
 .../binlog/network/TLSHostnameVerifier.java   | 47 -------------------
 3 files changed, 5 insertions(+), 53 deletions(-)
 delete mode 100644 src/main/java/com/github/shyiko/mysql/binlog/network/TLSHostnameVerifier.java

diff --git a/pom.xml b/pom.xml
index fafab007..8bbd4e54 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.7.3-3-FIVETRAN
+    0.7.3-4-FIVETRAN
 
     mysql-binlog-connector-java
     MySQL Binary Log connector
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index ab57aefe..7c5252ad 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -40,7 +40,6 @@
 import com.github.shyiko.mysql.binlog.network.SSLSocketFactory;
 import com.github.shyiko.mysql.binlog.network.ServerException;
 import com.github.shyiko.mysql.binlog.network.SocketFactory;
-import com.github.shyiko.mysql.binlog.network.TLSHostnameVerifier;
 import com.github.shyiko.mysql.binlog.network.protocol.ErrorPacket;
 import com.github.shyiko.mysql.binlog.network.protocol.GreetingPacket;
 import com.github.shyiko.mysql.binlog.network.protocol.Packet;
@@ -55,10 +54,6 @@
 import com.github.shyiko.mysql.binlog.network.protocol.command.PingCommand;
 import com.github.shyiko.mysql.binlog.network.protocol.command.QueryCommand;
 import com.github.shyiko.mysql.binlog.network.protocol.command.SSLRequestCommand;
-
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.X509TrustManager;
 import java.io.EOFException;
 import java.io.IOException;
 import java.net.InetSocketAddress;
@@ -84,6 +79,10 @@
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+
 
 /**
  * MySQL replication stream client.
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/TLSHostnameVerifier.java b/src/main/java/com/github/shyiko/mysql/binlog/network/TLSHostnameVerifier.java
deleted file mode 100644
index 6dd3aa43..00000000
--- a/src/main/java/com/github/shyiko/mysql/binlog/network/TLSHostnameVerifier.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright 2016 Stanley Shyiko
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.github.shyiko.mysql.binlog.network;
-
-import javax.net.ssl.HostnameVerifier;
-import javax.net.ssl.SSLException;
-import javax.net.ssl.SSLPeerUnverifiedException;
-import javax.net.ssl.SSLSession;
-import java.security.cert.Certificate;
-import java.security.cert.X509Certificate;
-
-/**
- * @author Stanley Shyiko
- */
-public class TLSHostnameVerifier implements HostnameVerifier {
-
-    public boolean verify(String hostname, SSLSession session) {
-        HostnameChecker checker = HostnameChecker.DEFAULT;
-        try {
-            Certificate[] peerCertificates = session.getPeerCertificates();
-            if (peerCertificates.length > 0 && peerCertificates[0] instanceof X509Certificate) {
-                X509Certificate peerCertificate = (X509Certificate) peerCertificates[0];
-                try {
-                    checker.check(hostname, peerCertificate);
-                    return true;
-                } catch (SSLException ignored) {
-                }
-            }
-        } catch (SSLPeerUnverifiedException ignored) {
-        }
-        return false;
-    }
-
-}

From 52bb8a3a9142f6a507e57b38f2864a650c325ce0 Mon Sep 17 00:00:00 2001
From: glarwood 
Date: Thu, 4 Oct 2018 17:46:00 -0700
Subject: [PATCH 130/217] refactor(BinaryLogClient): make binlogChecksum
 related methods/fields protected

---
 .../com/github/shyiko/mysql/binlog/BinaryLogClient.java     | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index 7c5252ad..021038eb 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -149,7 +149,7 @@ public X509Certificate[] getAcceptedIssuers() {
     private SocketFactory socketFactory;
     private SSLSocketFactory sslSocketFactory;
 
-    private volatile PacketChannel channel;
+    protected volatile PacketChannel channel;
     private volatile boolean connected;
     private volatile long masterServerId = -1;
 
@@ -909,7 +909,7 @@ private void fetchBinlogFilenameAndPosition() throws IOException {
         binlogPosition = Long.parseLong(resultSetRow.getValue(1));
     }
 
-    private ChecksumType fetchBinlogChecksum() throws IOException {
+    protected ChecksumType fetchBinlogChecksum() throws IOException {
         channel.write(new QueryCommand("show global variables like 'binlog_checksum'"));
         ResultSetRowPacket[] resultSet = readResultSet();
         if (resultSet.length == 0) {
@@ -1062,7 +1062,7 @@ private void commitGtid() {
         }
     }
 
-    private ResultSetRowPacket[] readResultSet() throws IOException {
+    protected ResultSetRowPacket[] readResultSet() throws IOException {
         List resultSet = new LinkedList<>();
         byte[] statementResult = channel.read();
         checkError(statementResult);

From 68619b42dfa797ffb6eae3fcfa32b11c481f3834 Mon Sep 17 00:00:00 2001
From: glarwood 
Date: Mon, 29 Apr 2019 14:17:00 -0700
Subject: [PATCH 131/217] refactor(BinaryLogClient): manually undo
 fd9f0333b6c28829c1c3d09ac6df6a0a22c548c6 to simplify upstream merge

---
 .../java/com/github/shyiko/mysql/binlog/BinaryLogClient.java   | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index 021038eb..d79e55e0 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -700,8 +700,7 @@ private boolean tryUpgradeToSSL(GreetingPacket greetingPacket) throws IOExceptio
                         sslMode == SSLMode.REQUIRED || sslMode == SSLMode.PREFERRED ?
                             DEFAULT_REQUIRED_SSL_MODE_SOCKET_FACTORY :
                             DEFAULT_VERIFY_CA_SSL_MODE_SOCKET_FACTORY;
-                channel.upgradeToSSL(sslSocketFactory,
-                    sslMode == SSLMode.VERIFY_IDENTITY ? new TLSHostnameVerifier() : null);
+                channel.upgradeToSSL(sslSocketFactory, null);
                 logger.info("SSL enabled");
                 return true;
             }

From edee5cc5f0367518b2efa0abc8b7e4558d3bd5c9 Mon Sep 17 00:00:00 2001
From: glarwood 
Date: Mon, 13 May 2019 16:49:48 -0700
Subject: [PATCH 132/217] fix(ByteArrayOutputStream): use composed
 OutputStream#write

---
 .../shyiko/mysql/binlog/network/protocol/PacketChannel.java      | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/PacketChannel.java b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/PacketChannel.java
index 95a4b2be..c64e7034 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/PacketChannel.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/PacketChannel.java
@@ -85,6 +85,7 @@ public void write(Command command) throws IOException {
         buffer.writeInteger(packetNumber++, 1);
 
         buffer.write(body, 0, body.length);
+        buffer.flush();
         outputStream.write(buffer.toByteArray());
         // though it has no effect in case of default (underlying) output stream (SocketOutputStream),
         // it may be necessary in case of non-default one

From 4b24ae5bd7fa9c546c9e4b01f241f81c8aa18f1e Mon Sep 17 00:00:00 2001
From: Ellen Peng 
Date: Fri, 11 Oct 2019 17:31:36 -0700
Subject: [PATCH 133/217] specified year zero condition

---
 .../deserialization/AbstractRowsEventDataDeserializer.java     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
index df40459c..81629c59 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
@@ -378,7 +378,8 @@ protected Serializable deserializeDatetimeV2(int meta, ByteArrayInputStream inpu
     }
 
     protected Serializable deserializeYear(ByteArrayInputStream inputStream) throws IOException {
-        return 1900 + inputStream.readInteger(1);
+        int year = inputStream.readInteger(1);
+        return year == 0 ? 0 : 1900 + year;
     }
 
     protected Serializable deserializeString(int length, ByteArrayInputStream inputStream) throws IOException {

From dfcb631e683294486e64828b70dc9f58b01a0a4c Mon Sep 17 00:00:00 2001
From: glarwood 
Date: Tue, 31 Mar 2020 15:40:42 -0700
Subject: [PATCH 134/217] fix(AbstractRowsEventDataDeserializer): add
 microsecond precision for datetime/timestamp

---
 .../AbstractRowsEventDataDeserializer.java         | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
index 81629c59..55422b65 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
@@ -24,7 +24,6 @@
 import java.math.BigDecimal;
 import java.util.BitSet;
 import java.util.Calendar;
-import java.util.Date;
 import java.util.Map;
 import java.util.TimeZone;
 
@@ -332,7 +331,7 @@ protected Serializable deserializeTimestampV2(int meta, ByteArrayInputStream inp
         if (deserializeDateAndTimeAsLong) {
             return castTimestamp(timestamp, fsp);
         }
-        return new java.sql.Timestamp(timestamp);
+        return convertLongTimestamptWithFSP(timestamp, fsp);
     }
 
     protected Serializable deserializeDatetime(ByteArrayInputStream inputStream) throws IOException {
@@ -341,7 +340,7 @@ protected Serializable deserializeDatetime(ByteArrayInputStream inputStream) thr
         if (deserializeDateAndTimeAsLong) {
             return castTimestamp(timestamp, 0);
         }
-        return timestamp != null ? new java.util.Date(timestamp) : null;
+        return timestamp != null ? new java.sql.Timestamp(timestamp) : null;
     }
 
     protected Serializable deserializeDatetimeV2(int meta, ByteArrayInputStream inputStream) throws IOException {
@@ -374,7 +373,14 @@ protected Serializable deserializeDatetimeV2(int meta, ByteArrayInputStream inpu
         if (deserializeDateAndTimeAsLong) {
             return castTimestamp(timestamp, fsp);
         }
-        return timestamp != null ? new java.util.Date(timestamp) : null;
+
+        return timestamp != null ? convertLongTimestamptWithFSP(timestamp, fsp) : null;
+    }
+
+    private java.sql.Timestamp convertLongTimestamptWithFSP(Long timestamp, int fsp) {
+        java.sql.Timestamp ts = new java.sql.Timestamp(timestamp);
+        ts.setNanos(fsp * 1000);
+        return ts;
     }
 
     protected Serializable deserializeYear(ByteArrayInputStream inputStream) throws IOException {

From d5616cfa5b570462eb960be4056191ff6a499dce Mon Sep 17 00:00:00 2001
From: glarwood 
Date: Wed, 1 Apr 2020 16:32:05 -0700
Subject: [PATCH 135/217] fix(AbstractRowsEventDataDeserializer): add
 microsecond precision for time

---
 .../deserialization/AbstractRowsEventDataDeserializer.java   | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
index 55422b65..3504a894 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
@@ -22,6 +22,7 @@
 import java.io.IOException;
 import java.io.Serializable;
 import java.math.BigDecimal;
+import java.sql.Time;
 import java.util.BitSet;
 import java.util.Calendar;
 import java.util.Map;
@@ -285,7 +286,7 @@ protected Serializable deserializeTime(ByteArrayInputStream inputStream) throws
         if (deserializeDateAndTimeAsLong) {
             return castTimestamp(timestamp, 0);
         }
-        return timestamp != null ? new java.sql.Time(timestamp) : null;
+        return timestamp != null ? new java.sql.Timestamp(timestamp) : null;
     }
 
     protected Serializable deserializeTimeV2(int meta, ByteArrayInputStream inputStream) throws IOException {
@@ -313,7 +314,7 @@ protected Serializable deserializeTimeV2(int meta, ByteArrayInputStream inputStr
         if (deserializeDateAndTimeAsLong) {
             return castTimestamp(timestamp, fsp);
         }
-        return timestamp != null ? new java.sql.Time(timestamp) : null;
+        return timestamp != null ? convertLongTimestamptWithFSP(timestamp, fsp) : null;
     }
 
     protected Serializable deserializeTimestamp(ByteArrayInputStream inputStream) throws IOException {

From 994af2de96f76572f108e9b1da30c0db9deed427 Mon Sep 17 00:00:00 2001
From: Matt Alexander 
Date: Mon, 12 Oct 2020 12:08:12 -0700
Subject: [PATCH 136/217] Add AURORA_PADDING event types and refactor event
 types to no longer use an enum ordinal to resolve event type IDs

---
 .../shyiko/mysql/binlog/event/EventType.java  | 97 +++++++++++--------
 .../EventHeaderV4Deserializer.java            | 13 ++-
 2 files changed, 62 insertions(+), 48 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
index 261fbd29..0461a171 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
@@ -25,90 +25,90 @@ public enum EventType {
     /**
      * Events of this event type should never occur. Not written to a binary log.
      */
-    UNKNOWN,
+    UNKNOWN(0),
     /**
      * A descriptor event that is written to the beginning of the each binary log file. (In MySQL 4.0 and 4.1,
      * this event is written only to the first binary log file that the server creates after startup.) This event is
      * used in MySQL 3.23 through 4.1 and superseded in MySQL 5.0 by {@link #FORMAT_DESCRIPTION}.
      */
-    START_V3,
+    START_V3(1),
     /**
      * Written when an updating statement is done.
      */
-    QUERY,
+    QUERY(2),
     /**
      * Written when mysqld stops.
      */
-    STOP,
+    STOP(3),
     /**
      * Written when mysqld switches to a new binary log file. This occurs when someone issues a FLUSH LOGS statement or
      * the current binary log file becomes larger than max_binlog_size.
      */
-    ROTATE,
+    ROTATE(4),
     /**
      * Written every time a statement uses an AUTO_INCREMENT column or the LAST_INSERT_ID() function; precedes other
      * events for the statement. This is written only before a {@link #QUERY} and is not used in case of RBR.
      */
-    INTVAR,
+    INTVAR(5),
     /**
      * Used for LOAD DATA INFILE statements in MySQL 3.23.
      */
-    LOAD,
+    LOAD(6),
     /**
      * Not used.
      */
-    SLAVE,
+    SLAVE(7),
     /**
      * Used for LOAD DATA INFILE statements in MySQL 4.0 and 4.1.
      */
-    CREATE_FILE,
+    CREATE_FILE(8),
     /**
      * Used for LOAD DATA INFILE statements as of MySQL 4.0.
      */
-    APPEND_BLOCK,
+    APPEND_BLOCK(9),
     /**
      * Used for LOAD DATA INFILE statements in 4.0 and 4.1.
      */
-    EXEC_LOAD,
+    EXEC_LOAD(10),
     /**
      * Used for LOAD DATA INFILE statements as of MySQL 4.0.
      */
-    DELETE_FILE,
+    DELETE_FILE(11),
     /**
      * Used for LOAD DATA INFILE statements in MySQL 4.0 and 4.1.
      */
-    NEW_LOAD,
+    NEW_LOAD(12),
     /**
      * Written every time a statement uses the RAND() function; precedes other events for the statement. Indicates the
      * seed values to use for generating a random number with RAND() in the next statement. This is written only
      * before a {@link #QUERY} and is not used in case of RBR.
      */
-    RAND,
+    RAND(13),
     /**
      * Written every time a statement uses a user variable; precedes other events for the statement. Indicates the
      * value to use for the user variable in the next statement. This is written only before a {@link #QUERY} and
      * is not used in case of RBR.
      */
-    USER_VAR,
+    USER_VAR(14),
     /**
      * A descriptor event that is written to the beginning of the each binary log file.
      * This event is used as of MySQL 5.0; it supersedes {@link #START_V3}.
      */
-    FORMAT_DESCRIPTION,
+    FORMAT_DESCRIPTION(15),
     /**
      * Generated for a commit of a transaction that modifies one or more tables of an XA-capable storage engine.
      * Normal transactions are implemented by sending a {@link #QUERY} containing a BEGIN statement and a {@link #QUERY}
      * containing a COMMIT statement (or a ROLLBACK statement if the transaction is rolled back).
      */
-    XID,
+    XID(16),
     /**
      * Used for LOAD DATA INFILE statements as of MySQL 5.0.
      */
-    BEGIN_LOAD_QUERY,
+    BEGIN_LOAD_QUERY(17),
     /**
      * Used for LOAD DATA INFILE statements as of MySQL 5.0.
      */
-    EXECUTE_LOAD_QUERY,
+    EXECUTE_LOAD_QUERY(18),
     /**
      * This event precedes each row operation event. It maps a table definition to a number, where the table definition
      * consists of database and table names and column definitions. The purpose of this event is to enable replication
@@ -117,92 +117,107 @@ public enum EventType {
      * of TABLE_MAP events: one per table used by events in the sequence.
      * Used in case of RBR.
      */
-    TABLE_MAP,
+    TABLE_MAP(19),
     /**
      * Describes inserted rows (within a single table).
      * Used in case of RBR (5.1.0 - 5.1.15).
      */
-    PRE_GA_WRITE_ROWS,
+    PRE_GA_WRITE_ROWS(20),
     /**
      * Describes updated rows (within a single table).
      * Used in case of RBR (5.1.0 - 5.1.15).
      */
-    PRE_GA_UPDATE_ROWS,
+    PRE_GA_UPDATE_ROWS(21),
     /**
      * Describes deleted rows (within a single table).
      * Used in case of RBR (5.1.0 - 5.1.15).
      */
-    PRE_GA_DELETE_ROWS,
+    PRE_GA_DELETE_ROWS(22),
     /**
      * Describes inserted rows (within a single table).
      * Used in case of RBR (5.1.16 - mysql-trunk).
      */
-    WRITE_ROWS,
+    WRITE_ROWS(23),
     /**
      * Describes updated rows (within a single table).
      * Used in case of RBR (5.1.16 - mysql-trunk).
      */
-    UPDATE_ROWS,
+    UPDATE_ROWS(24),
     /**
      * Describes deleted rows (within a single table).
      * Used in case of RBR (5.1.16 - mysql-trunk).
      */
-    DELETE_ROWS,
+    DELETE_ROWS(25),
     /**
      * Used to log an out of the ordinary event that occurred on the master. It notifies the slave that something
      * happened on the master that might cause data to be in an inconsistent state.
      */
-    INCIDENT,
+    INCIDENT(26),
     /**
      * Sent by a master to a slave to let the slave know that the master is still alive. Not written to a binary log.
      */
-    HEARTBEAT,
+    HEARTBEAT(27),
     /**
      * In some situations, it is necessary to send over ignorable data to the slave: data that a slave can handle in
      * case there is code for handling it, but which can be ignored if it is not recognized.
      */
-    IGNORABLE,
+    IGNORABLE(28),
     /**
      * Introduced to record the original query for rows events in RBR.
      */
-    ROWS_QUERY,
+    ROWS_QUERY(29),
     /**
      * Describes inserted rows (within a single table).
      * Used in case of RBR (5.1.18+).
      */
-    EXT_WRITE_ROWS,
+    EXT_WRITE_ROWS(30),
     /**
      * Describes updated rows (within a single table).
      * Used in case of RBR (5.1.18+).
      */
-    EXT_UPDATE_ROWS,
+    EXT_UPDATE_ROWS(31),
     /**
      * Describes deleted rows (within a single table).
      * Used in case of RBR (5.1.18+).
      */
-    EXT_DELETE_ROWS,
+    EXT_DELETE_ROWS(32),
     /**
      * Global Transaction Identifier.
      */
-    GTID,
-    ANONYMOUS_GTID,
-    PREVIOUS_GTIDS,
-    TRANSACTION_CONTEXT,
-    VIEW_CHANGE,
+    GTID(33),
+    ANONYMOUS_GTID(34),
+    PREVIOUS_GTIDS(35),
+    TRANSACTION_CONTEXT(36),
+    VIEW_CHANGE(37),
     /**
      * Prepared XA transaction terminal event similar to XID except that it is specific to XA transaction.
      */
-    XA_PREPARE,
+    XA_PREPARE(38),
     /**
      Extension of UPDATE_ROWS_EVENT, allowing partial values according
      to binlog_row_value_options.
      */
-    PARTIAL_UPDATE_ROWS_EVENT,
+    PARTIAL_UPDATE_ROWS_EVENT(39),
     /**
      * Generated when 'binlog_transaction_compression' is set to 'ON'.
      * It encapsulates all the events of a transaction in a Zstd compressed payload.
      */
-    TRANSACTION_PAYLOAD;
+    TRANSACTION_PAYLOAD(40),
+    AURORA_PADDING(100);
+
+    int eventId;
+
+    EventType(int eventId) {
+        this.eventId = eventId;
+    }
+
+    public static EventType forId(int eventId) {
+        for (EventType type : EventType.values()) {
+            if (type.eventId == eventId) return type;
+        }
+
+        return null;
+    }
 
     public static boolean isRowMutation(EventType eventType) {
         return EventType.isWrite(eventType) ||
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
index c0569f09..05d7bff5 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
@@ -26,8 +26,6 @@
  */
 public class EventHeaderV4Deserializer implements EventHeaderDeserializer {
 
-    private static final EventType[] EVENT_TYPES = EventType.values();
-
     @Override
     public EventHeaderV4 deserialize(ByteArrayInputStream inputStream) throws IOException {
         EventHeaderV4 header = new EventHeaderV4();
@@ -40,11 +38,12 @@ public EventHeaderV4 deserialize(ByteArrayInputStream inputStream) throws IOExce
         return header;
     }
 
-    private static EventType getEventType(int ordinal) {
-        if (ordinal >= EVENT_TYPES.length) {
-            return EventType.UNKNOWN;
+    private static EventType getEventType(int ordinal) throws IOException {
+        EventType eventType = EventType.forId(ordinal);
+
+        if (eventType == null) {
+            throw new IOException("Unknown event type " + ordinal);
         }
-        return EVENT_TYPES[ordinal];
+        return eventType;
     }
-
 }

From da0dbae7fdbef9bfba214a3ffa4eb0cd1a340863 Mon Sep 17 00:00:00 2001
From: Matt Alexander 
Date: Mon, 12 Oct 2020 15:04:23 -0700
Subject: [PATCH 137/217] Make event id private and final

---
 .../java/com/github/shyiko/mysql/binlog/event/EventType.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
index 0461a171..db22b038 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
@@ -205,7 +205,7 @@ public enum EventType {
     TRANSACTION_PAYLOAD(40),
     AURORA_PADDING(100);
 
-    int eventId;
+    private final int eventId;
 
     EventType(int eventId) {
         this.eventId = eventId;

From d2a6f7080c3c2da8f7c31942edd2e99c577ea1cc Mon Sep 17 00:00:00 2001
From: Shiva Vanamala 
Date: Thu, 12 Nov 2020 20:04:53 -0800
Subject: [PATCH 138/217] Create unknown event type for invalid and proprietary
 event types. Removed AURORA_PADDING event type.

---
 .../event/deserialization/EventHeaderV4Deserializer.java    | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
index 05d7bff5..e88d3221 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
@@ -38,11 +38,15 @@ public EventHeaderV4 deserialize(ByteArrayInputStream inputStream) throws IOExce
         return header;
     }
 
+    /**
+     * Parses the event type based on the ordinal.
+     * If an invalid or proprietary ordinal is passed, an unknown event is returned.
+     */
     private static EventType getEventType(int ordinal) throws IOException {
         EventType eventType = EventType.forId(ordinal);
 
         if (eventType == null) {
-            throw new IOException("Unknown event type " + ordinal);
+            return EventType.forId(0);
         }
         return eventType;
     }

From e88feff35ec9ae13daa367ca7a9cd3cb76f477cd Mon Sep 17 00:00:00 2001
From: Shiva Vanamala 
Date: Thu, 12 Nov 2020 20:11:56 -0800
Subject: [PATCH 139/217] Removed AURORA_PADDING event

---
 .../java/com/github/shyiko/mysql/binlog/event/EventType.java   | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
index db22b038..d41049e9 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
@@ -202,8 +202,7 @@ public enum EventType {
      * Generated when 'binlog_transaction_compression' is set to 'ON'.
      * It encapsulates all the events of a transaction in a Zstd compressed payload.
      */
-    TRANSACTION_PAYLOAD(40),
-    AURORA_PADDING(100);
+    TRANSACTION_PAYLOAD(40);
 
     private final int eventId;
 

From 26a2c5a01432f61e5a2e48305141598cde974f97 Mon Sep 17 00:00:00 2001
From: Shiva Vanamala 
Date: Fri, 13 Nov 2020 09:30:12 -0800
Subject: [PATCH 140/217] Changed the way unknown event type is being created
 EventType.forId(0) -> EventType.UNKNOWN

---
 .../binlog/event/deserialization/EventHeaderV4Deserializer.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
index e88d3221..74f5bc3c 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
@@ -46,7 +46,7 @@ private static EventType getEventType(int ordinal) throws IOException {
         EventType eventType = EventType.forId(ordinal);
 
         if (eventType == null) {
-            return EventType.forId(0);
+            return EventType.UNKNOWN;
         }
         return eventType;
     }

From 194d5bedffce2a44982f121fbdb88e404069f7c5 Mon Sep 17 00:00:00 2001
From: Shiva Vanamala 
Date: Wed, 18 Nov 2020 18:47:20 -0800
Subject: [PATCH 141/217] EventType.forId() returns EventType.UNKNOWN if
 invalid eventId is passed.

---
 .../com/github/shyiko/mysql/binlog/event/EventType.java    | 7 ++++++-
 .../event/deserialization/EventHeaderV4Deserializer.java   | 7 +------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
index d41049e9..edad576f 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
@@ -210,12 +210,17 @@ public enum EventType {
         this.eventId = eventId;
     }
 
+    /**
+     * Creates EventType based on the id.
+     * 
+     * 
If id passed is out of the range of the standard event types, EventType.UNKNOWN is returned.
+     */
     public static EventType forId(int eventId) {
         for (EventType type : EventType.values()) {
             if (type.eventId == eventId) return type;
         }
 
-        return null;
+        return EventType.UNKNOWN;
     }
 
     public static boolean isRowMutation(EventType eventType) {
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
index 74f5bc3c..7c8ea5b6 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
@@ -43,11 +43,6 @@ public EventHeaderV4 deserialize(ByteArrayInputStream inputStream) throws IOExce
      * If an invalid or proprietary ordinal is passed, an unknown event is returned.
      */
     private static EventType getEventType(int ordinal) throws IOException {
-        EventType eventType = EventType.forId(ordinal);
-
-        if (eventType == null) {
-            return EventType.UNKNOWN;
-        }
-        return eventType;
+        return EventType.forId(ordinal);
     }
 }

From ba5bae5630e947c6ee684fd933018b3395a289cb Mon Sep 17 00:00:00 2001
From: Shiva Vanamala 
Date: Wed, 18 Nov 2020 18:56:38 -0800
Subject: [PATCH 142/217] refactor: Removed redundant single line function

---
 .../github/shyiko/mysql/binlog/event/EventType.java    |  4 ++--
 .../deserialization/EventHeaderV4Deserializer.java     | 10 +---------
 2 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
index edad576f..148999b5 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
@@ -211,9 +211,9 @@ public enum EventType {
     }
 
     /**
-     * Creates EventType based on the id.
+     * Parses the event type based on the ordinal.
      * 
-     * 
If id passed is out of the range of the standard event types, EventType.UNKNOWN is returned.
+     * 
If an invalid or proprietary ordinal is passed, EventType.UNKNOWN is returned.
      */
     public static EventType forId(int eventId) {
         for (EventType type : EventType.values()) {
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
index 7c8ea5b6..27fa617b 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
@@ -30,19 +30,11 @@ public class EventHeaderV4Deserializer implements EventHeaderDeserializer
Date: Thu, 28 Oct 2021 13:38:11 -0700
Subject: [PATCH 143/217] feature(table_map_event_data_serializer): add support
 of ColumnVisibility metadata and handle unknown metadata field type

---
 .../event/deserialization/TableMapEventMetadataDeserializer.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
index 61c34bce..9f2f26c0 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
@@ -96,6 +96,7 @@ public TableMapEventMetadata deserialize(ByteArrayInputStream inputStream, int n
                     break;
                 case ENUM_AND_SET_COLUMN_CHARSET:
                     result.setEnumAndSetColumnCharsets(readIntegers(inputStream));
+                    break;
                 case VISIBILITY:
                     result.setVisibility(readBooleanList(inputStream, nColumns));
                     break;

From 467dbe7052bafb7662e176a6dfaedea7ed15bb73 Mon Sep 17 00:00:00 2001
From: zekailiu 
Date: Thu, 28 Oct 2021 21:13:12 -0700
Subject: [PATCH 144/217] fix(table_map_event_data_serializer): skip
 deserializing the table map event when the stream is empty

---
 .../deserialization/TableMapEventMetadataDeserializer.java     | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
index 9f2f26c0..cfd5af12 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
@@ -58,6 +58,9 @@ public TableMapEventMetadata deserialize(ByteArrayInputStream inputStream, int n
                 continue;
             }
 
+            //for some reasons, the UNKNOWN_METADATA_FIELD_TYPE will mess up the stream
+            if(inputStream.available() == 0) return result;
+
             int fieldLength = inputStream.readPackedInteger();
 
             remainingBytes = inputStream.available();

From 9bf8e2ca8922c48b190d1b3986909d4dad96b453 Mon Sep 17 00:00:00 2001
From: zekailiu 
Date: Thu, 28 Oct 2021 21:47:03 -0700
Subject: [PATCH 145/217] fix(table_map_event_data_serializer): log a message
 when skipping deserializing the table map event

---
 .../deserialization/TableMapEventMetadataDeserializer.java   | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
index cfd5af12..c6efdda4 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
@@ -59,7 +59,10 @@ public TableMapEventMetadata deserialize(ByteArrayInputStream inputStream, int n
             }
 
             //for some reasons, the UNKNOWN_METADATA_FIELD_TYPE will mess up the stream
-            if(inputStream.available() == 0) return result;
+            if(inputStream.available() == 0) {
+                logger.warning("Stream is empty so cannot read field length for field type: " + fieldType);
+                return result;
+            }
 
             int fieldLength = inputStream.readPackedInteger();
 

From e1254f595bab63d0737bbf802f8048fd56258aba Mon Sep 17 00:00:00 2001
From: Imen Graja 
Date: Wed, 9 Feb 2022 14:55:27 -0800
Subject: [PATCH 146/217] fix for json parsing error

---
 .../binlog/event/deserialization/json/JsonBinary.java      | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
index 857c3c5c..b167ba75 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
@@ -176,12 +176,7 @@ public static String parseAsString(byte[] bytes) throws IOException {
      * @throws IOException if there is a problem reading or processing the binary representation
      */
     public static void parse(byte[] bytes, JsonFormatter formatter) throws IOException {
-        if (bytes.length == 0) {
-            // When the top-level value is a JSON "null", the value in java shows up as a non-null, empty byte array
-            formatter.valueNull();
-        } else {
-            new JsonBinary(bytes).parse(formatter);
-        }
+        new JsonBinary(bytes).parse(formatter);
     }
 
     private final ByteArrayInputStream reader;

From 6120bace6172ac5b0c22314e1290d6d58771080a Mon Sep 17 00:00:00 2001
From: Matt Alexander 
Date: Thu, 17 Mar 2022 11:37:45 -0700
Subject: [PATCH 147/217] Fork BinaryLogClient to FF abortRequest change

---
 .../shyiko/mysql/binlog/BinaryLogClient.java  |  3 +-
 .../mysql/binlog/NewBinaryLogClient.java      | 30 +++++++++++++++++++
 2 files changed, 32 insertions(+), 1 deletion(-)
 create mode 100644 src/main/java/com/github/shyiko/mysql/binlog/NewBinaryLogClient.java

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index d79e55e0..23406605 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -145,6 +145,7 @@ public X509Certificate[] getAcceptedIssuers() {
 
     private final List eventListeners = new CopyOnWriteArrayList();
     private final List lifecycleListeners = new CopyOnWriteArrayList();
+    protected boolean abortRequest = false;
 
     private SocketFactory socketFactory;
     private SSLSocketFactory sslSocketFactory;
@@ -924,7 +925,7 @@ private void confirmSupportOfChecksum(ChecksumType checksumType) throws IOExcept
         eventDeserializer.setChecksumType(checksumType);
     }
 
-    private void listenForEventPackets() throws IOException {
+    protected void listenForEventPackets() throws IOException {
         ByteArrayInputStream inputStream = channel.getInputStream();
         boolean completeShutdown = false;
         try {
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/NewBinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/NewBinaryLogClient.java
new file mode 100644
index 00000000..dccccf90
--- /dev/null
+++ b/src/main/java/com/github/shyiko/mysql/binlog/NewBinaryLogClient.java
@@ -0,0 +1,30 @@
+package com.github.shyiko.mysql.binlog;
+
+import com.github.shyiko.mysql.binlog.network.protocol.PacketChannel;
+
+import java.io.IOException;
+
+public class NewBinaryLogClient extends BinaryLogClient {
+
+    public NewBinaryLogClient(String username, String password) {
+        super(username, password);
+    }
+
+    public NewBinaryLogClient(String schema, String username, String password) {
+        super(schema, username, password);
+    }
+
+    public NewBinaryLogClient(String hostname, int port, String username, String password) {
+        super(hostname, port, username, password);
+    }
+
+    public NewBinaryLogClient(String hostname, int port, String schema, String username, String password) {
+        super(hostname, port, schema, username, password);
+    }
+
+    @Override
+    protected void listenForEventPackets(PacketChannel channel) throws IOException {
+        abortRequest = false;
+        super.listenForEventPackets(channel);
+    }
+}

From 52542ab532b1cb6b7c006b8a2a48cae6319e652b Mon Sep 17 00:00:00 2001
From: Josh Wood 
Date: Mon, 16 May 2022 17:24:57 -0500
Subject: [PATCH 148/217] cleanup(mysql): [T-252649] consolidate new abort
 logic

---
 .../shyiko/mysql/binlog/BinaryLogClient.java  |  1 +
 .../mysql/binlog/NewBinaryLogClient.java      | 30 -------------------
 2 files changed, 1 insertion(+), 30 deletions(-)
 delete mode 100644 src/main/java/com/github/shyiko/mysql/binlog/NewBinaryLogClient.java

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index 23406605..be09334d 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -926,6 +926,7 @@ private void confirmSupportOfChecksum(ChecksumType checksumType) throws IOExcept
     }
 
     protected void listenForEventPackets() throws IOException {
+        abortRequest = false;
         ByteArrayInputStream inputStream = channel.getInputStream();
         boolean completeShutdown = false;
         try {
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/NewBinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/NewBinaryLogClient.java
deleted file mode 100644
index dccccf90..00000000
--- a/src/main/java/com/github/shyiko/mysql/binlog/NewBinaryLogClient.java
+++ /dev/null
@@ -1,30 +0,0 @@
-package com.github.shyiko.mysql.binlog;
-
-import com.github.shyiko.mysql.binlog.network.protocol.PacketChannel;
-
-import java.io.IOException;
-
-public class NewBinaryLogClient extends BinaryLogClient {
-
-    public NewBinaryLogClient(String username, String password) {
-        super(username, password);
-    }
-
-    public NewBinaryLogClient(String schema, String username, String password) {
-        super(schema, username, password);
-    }
-
-    public NewBinaryLogClient(String hostname, int port, String username, String password) {
-        super(hostname, port, username, password);
-    }
-
-    public NewBinaryLogClient(String hostname, int port, String schema, String username, String password) {
-        super(hostname, port, schema, username, password);
-    }
-
-    @Override
-    protected void listenForEventPackets(PacketChannel channel) throws IOException {
-        abortRequest = false;
-        super.listenForEventPackets(channel);
-    }
-}

From c3a744e8f03c9a3e7f2716cb6565918f40af74c8 Mon Sep 17 00:00:00 2001
From: Ashley Cox 
Date: Sat, 6 Aug 2022 14:15:55 -0700
Subject: [PATCH 149/217] re-add abort logic on its own (without old deadlock
 fix)

---
 .../com/github/shyiko/mysql/binlog/BinaryLogClient.java    | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index be09334d..1914c3d4 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -930,7 +930,7 @@ protected void listenForEventPackets() throws IOException {
         ByteArrayInputStream inputStream = channel.getInputStream();
         boolean completeShutdown = false;
         try {
-            while (inputStream.peek() != -1) {
+            while (!abortRequest && inputStream.peek() != -1) {
                 int packetLength = inputStream.readInteger(3);
                 inputStream.skip(1); // 1 byte for sequence
                 int marker = inputStream.read();
@@ -977,6 +977,7 @@ protected void listenForEventPackets() throws IOException {
                 }
             }
         } finally {
+            abortRequest = false;
             if (isConnected()) {
                 if (completeShutdown) {
                     disconnect(); // initiate complete shutdown sequence (which includes keep alive thread)
@@ -1172,6 +1173,10 @@ public void disconnect() throws IOException {
         terminateConnect();
     }
 
+    public void abort() {
+        abortRequest = true;
+    }
+
     private void terminateKeepAliveThread() {
         try {
             keepAliveThreadExecutorLock.lock();

From 1c214000f2f99b3b68384b0b675e2a6ec1549b13 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Fri, 12 Aug 2022 13:29:13 -0700
Subject: [PATCH 150/217] fixup javadoc stuff

---
 pom.xml                                              |  2 +-
 .../AnnotateRowsEventDataDeserializer.java           |  2 +-
 .../MariadbGtidEventDataDeserializer.java            | 12 ++++++------
 .../MariadbGtidListEventDataDeserializer.java        |  8 ++++----
 4 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/pom.xml b/pom.xml
index 50c2922a..5c5806e3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -165,7 +165,7 @@
             
                 org.apache.maven.plugins
                 maven-javadoc-plugin
-                3.2.0
+                3.4.0
                 
                     
                         attach-javadocs
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AnnotateRowsEventDataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AnnotateRowsEventDataDeserializer.java
index 688a9b29..981d9c94 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AnnotateRowsEventDataDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AnnotateRowsEventDataDeserializer.java
@@ -8,7 +8,7 @@
 /**
  * Mariadb ANNOTATE_ROWS_EVENT Fields
  * 
- *  string The SQL statement (not null-terminated)
+ *  string<EOF> The SQL statement (not null-terminated)
  * 

  *
  * @author Winger
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidEventDataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidEventDataDeserializer.java
index c57e2558..4eb45f1f 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidEventDataDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidEventDataDeserializer.java
@@ -8,14 +8,14 @@
 /**
  * Mariadb GTID_EVENT Fields
  * 
- *     uint<8> GTID sequence
- *     uint<4> Replication Domain ID
- *     uint<1> Flags
+ *     uint8 GTID sequence
+ *     uint4 Replication Domain ID
+ *     uint1 Flags
  *
- * 	if flag & FL_GROUP_COMMIT_ID
- * 	    uint<8> commit_id
+ * 	if flag & FL_GROUP_COMMIT_ID
+ * 	    uint8 commit_id
  * 	else
- * 	    uint<6> 0
+ * 	    uint6 0
  * 

  *
  * @author Winger
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidListEventDataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidListEventDataDeserializer.java
index 8a429532..25836195 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidListEventDataDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidListEventDataDeserializer.java
@@ -10,11 +10,11 @@
 /**
  * Mariadb GTID_LIST_EVENT Fields
  * 
- *  uint<4> Number of GTIDs
+ *  uint4 Number of GTIDs
  *  GTID[0]
- *      uint<4> Replication Domain ID
- *      uint<4> Server_ID
- *      uint<8> GTID sequence ...
+ *      uint4 Replication Domain ID
+ *      uint4 Server_ID
+ *      uint8 GTID sequence ...
  * GTID[n]
  * 

  *

From 67a840edd8692f8ec4ba8eb83d910f65bde85df8 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Fri, 12 Aug 2022 16:33:07 -0700
Subject: [PATCH 151/217] can we build under maria?

---
 .circleci/config.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 37fa8406..ef8eeeb2 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -56,4 +56,8 @@ workflows:
           name: "test-8.0"
           mysql: "8.0"
           requires: [ "build" ]
+      - test:
+          name: "test-mariadb"
+          mysql: "mariadb"
+          requires: [ "build" ]
 

From 5d4c0fd746ef0ec25029b80177da167ee2848796 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Fri, 12 Aug 2022 16:43:03 -0700
Subject: [PATCH 152/217] see about running tests

---
 .../MariadbBinaryLogClientIntegrationTest.java       | 12 +++++++++---
 .../shyiko/mysql/binlog/MysqlOnetimeServer.java      |  9 +++++++--
 .../com/github/shyiko/mysql/binlog/MysqlVersion.java |  6 ++++--
 3 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/src/test/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClientIntegrationTest.java
index fac8b579..ca5039e0 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClientIntegrationTest.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClientIntegrationTest.java
@@ -3,6 +3,8 @@
 import com.github.shyiko.mysql.binlog.event.AnnotateRowsEventData;
 import com.github.shyiko.mysql.binlog.event.MariadbGtidEventData;
 import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer;
+import org.testng.SkipException;
+import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
 import java.sql.ResultSet;
@@ -16,13 +18,17 @@
 /**
  * @author Winger
  */
-public class MariadbBinaryLogClientIntegrationTest {
+public class MariadbBinaryLogClientIntegrationTest extends BinaryLogClientIntegrationTest {
 
+    MysqlOnetimeServer primaryServer;
     protected BinaryLogClientIntegrationTest.MySQLConnection master;
 
     @Test
     public void testMariadbUseGTIDAndAnnotateRowsEvent() throws Exception {
-        master = new BinaryLogClientIntegrationTest.MySQLConnection("127.0.0.1", 3306, "root", "");
+        if ( !mysqlVersion.isMaria )
+            throw new SkipException("not maria");
+
+
         master.execute(new BinaryLogClientIntegrationTest.Callback() {
             @Override
             public void execute(Statement statement) throws SQLException {
@@ -45,7 +51,7 @@ public void execute(ResultSet rs) throws SQLException {
         });
 
         CountDownEventListener eventListener;
-        MariadbBinaryLogClient client = new MariadbBinaryLogClient("127.0.0.1", 3306, "root", "123456");
+        MariadbBinaryLogClient client = new MariadbBinaryLogClient(master.hostname(), master.port(), master.username(), master.password());
         client.setGtidSet(currentGtidPos[0]);
         client.setUseSendAnnotateRowsEvent(true);
 
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java
index a3c5f51f..127714b9 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java
@@ -249,8 +249,13 @@ public void shutDown() {
 	}
 
     public static MysqlVersion getVersion() {
-        String[] parts = getVersionString().split("\\.");
-        return new MysqlVersion(Integer.valueOf(parts[0]), Integer.valueOf(parts[1]));
+        String version = getVersionString();
+        if ( version == "mariadb") {
+            return new MysqlVersion(0, 0, true);
+        } else {
+            String[] parts = version.split("\\.");
+            return new MysqlVersion(Integer.valueOf(parts[0]), Integer.valueOf(parts[1]), false);
+        }
     }
 
 	private static String getVersionString() {
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/MysqlVersion.java b/src/test/java/com/github/shyiko/mysql/binlog/MysqlVersion.java
index 7213a724..949873b6 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/MysqlVersion.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/MysqlVersion.java
@@ -5,12 +5,14 @@
 import java.sql.SQLException;
 
 public class MysqlVersion {
+    public boolean isMaria;
 	private final int major;
 	private final int minor;
 
-	public MysqlVersion(int major, int minor) {
+	public MysqlVersion(int major, int minor, boolean isMaria) {
 		this.major = major;
 		this.minor = minor;
+        this.isMaria = isMaria;
 	}
 
 	public boolean atLeast(int major, int minor) {
@@ -27,7 +29,7 @@ public boolean lessThan(int major, int minor) {
 
 	public static MysqlVersion capture(Connection c) throws SQLException {
 		DatabaseMetaData meta = c.getMetaData();
-		return new MysqlVersion(meta.getDatabaseMajorVersion(), meta.getDatabaseMinorVersion());
+		return new MysqlVersion(meta.getDatabaseMajorVersion(), meta.getDatabaseMinorVersion(), false);
 	}
 
 	public int getMajor() {

From fbc9c73c8b8bb0d720abbafb7e80aa04d57c0d32 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Fri, 12 Aug 2022 16:45:11 -0700
Subject: [PATCH 153/217] streamline a touch

---
 .circleci/config.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index ef8eeeb2..f09dc95f 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -32,9 +32,7 @@ jobs:
       - restore_cache:
           key: dependency-cache-{{ checksum "pom.xml" }}
       - run:
-          command: echo "testing under mysql $MYSQL_VERSION"
-      - run:
-          name: testit
+          name: "testing under version << parameters.mysql >>"
           command: mvn verify -Dgpg.skip
       - store_artifacts:
           path: test.log

From f2ea85cf9721cc4aed54a826e31085b9bdc4b7da Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Fri, 12 Aug 2022 16:46:47 -0700
Subject: [PATCH 154/217] oh java

---
 .../java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java
index 127714b9..ea043e9d 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java
@@ -250,7 +250,7 @@ public void shutDown() {
 
     public static MysqlVersion getVersion() {
         String version = getVersionString();
-        if ( version == "mariadb") {
+        if ( version.equals("mariadb") ) {
             return new MysqlVersion(0, 0, true);
         } else {
             String[] parts = version.split("\\.");

From 85840c30fe184503d4c43931fa4656a0673e2ce7 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Fri, 12 Aug 2022 17:29:11 -0700
Subject: [PATCH 155/217] add --debug

---
 .../java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java
index ea043e9d..12d3cd90 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java
@@ -70,6 +70,7 @@ public void boot() throws Exception {
 
 		ProcessBuilder pb = new ProcessBuilder(
 			dir + "/src/test/onetimeserver",
+            "--debug",
 			"--mysql-version=" + getVersionString(),
 			"--log-slave-updates",
 			"--log-bin=master",

From 62475193129e642a9e27392904cb44a2558770fb Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Fri, 12 Aug 2022 18:39:45 -0700
Subject: [PATCH 156/217] bump


From 988750eeb8f303cc265bee8ddb4e9b9426385f2d Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Fri, 12 Aug 2022 22:22:18 -0700
Subject: [PATCH 157/217] maria fixes in JSON land

---
 .../deserialization/json/JsonBinary.java      | 10 ++++
 .../mysql/binlog/CountDownEventListener.java  | 16 +++++++
 .../json/JsonBinaryValueIntegrationTest.java  | 47 ++++++++++++++++---
 3 files changed, 66 insertions(+), 7 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
index b167ba75..8a2b8d84 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
@@ -162,11 +162,21 @@ public class JsonBinary {
      * @throws IOException if there is a problem reading or processing the binary representation
      */
     public static String parseAsString(byte[] bytes) throws IOException {
+        /* check for mariaDB-format JSON strings inside columns marked JSON */
+        if ( isJSONString(bytes) ) {
+            return new String(bytes);
+        }
         JsonStringFormatter handler = new JsonStringFormatter();
         parse(bytes, handler);
         return handler.getString();
     }
 
+    private static boolean isJSONString(byte[] bytes) {
+        if (bytes[0] > 0x0f)
+            return true;
+        else
+            return false;
+    }
     /**
      * Parse the MySQL binary representation of a {@code JSON} value and call the supplied {@link JsonFormatter}
      * for the various components of the value.
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/CountDownEventListener.java b/src/test/java/com/github/shyiko/mysql/binlog/CountDownEventListener.java
index 73a6f5b9..6256efe6 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/CountDownEventListener.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/CountDownEventListener.java
@@ -86,6 +86,22 @@ private void waitForCounterToGetZero(String counterName, AtomicInteger counter,
         }
     }
 
+    public void waitForAtLeast(EventType eventType, int numberOfEvents, long timeoutInMilliseconds)
+        throws TimeoutException, InterruptedException {
+        AtomicInteger counter = getCounter(countersByType, eventType);
+
+        synchronized (counter) {
+            if (counter.get() < numberOfEvents) {
+                counter.wait(timeoutInMilliseconds);
+                if (counter.get() < numberOfEvents) {
+                    throw new TimeoutException("Received " + (numberOfEvents + counter.get()) + " " +
+                        eventType.name() + " event(s) instead of expected " + numberOfEvents);
+                }
+            }
+            counter.set(0);
+        }
+    }
+
     public void reset() {
         synchronized (countersByType) {
             countersByType.clear();
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java
index 8fb3917c..8866c155 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java
@@ -22,17 +22,20 @@
 import com.github.shyiko.mysql.binlog.MysqlOnetimeServer;
 import com.github.shyiko.mysql.binlog.TraceEventListener;
 import com.github.shyiko.mysql.binlog.TraceLifecycleListener;
+import com.github.shyiko.mysql.binlog.event.Event;
 import com.github.shyiko.mysql.binlog.event.EventData;
 import com.github.shyiko.mysql.binlog.event.EventType;
 import com.github.shyiko.mysql.binlog.event.QueryEventData;
 import com.github.shyiko.mysql.binlog.event.UpdateRowsEventData;
 import com.github.shyiko.mysql.binlog.event.WriteRowsEventData;
 import org.skyscreamer.jsonassert.JSONAssert;
+import org.testng.SkipException;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
+import java.io.IOException;
 import java.io.Serializable;
 import java.sql.SQLException;
 import java.sql.SQLSyntaxErrorException;
@@ -66,6 +69,8 @@ public class JsonBinaryValueIntegrationTest {
     private BinaryLogClient client;
     private CountDownEventListener eventListener;
 
+    private boolean isMaria = "mariadb".equals(System.getenv("MYSQL_VERSION"));
+
     @BeforeClass
     public void setUp() throws Exception {
         TimeZone.setDefault(TimeZone.getTimeZone("GMT"));
@@ -102,10 +107,15 @@ public void execute(Statement statement) throws SQLException {
             System.err.println("skipping JSON tests (pre 5.7)");
             throw new org.testng.SkipException("JSON data type is not supported by current version of MySQL");
         }
-        eventListener.waitFor(EventType.QUERY, 3, DEFAULT_TIMEOUT);
+        eventListener.waitForAtLeast(EventType.QUERY, 3, DEFAULT_TIMEOUT);
         eventListener.reset();
     }
 
+    private String parseAndRemoveSpaces(byte[] jsonBinary) throws IOException {
+        String parsed = JsonBinary.parseAsString(jsonBinary);
+        return parsed.replaceAll(" ", "");
+    }
+
     @Test
     public void testMysql8JsonSetPartialUpdateWithHoles() throws Exception {
         CapturingEventListener capturingEventListener = new CapturingEventListener();
@@ -122,7 +132,7 @@ public void testMysql8JsonSetPartialUpdateWithHoles() throws Exception {
 
         List updateEvents = capturingEventListener.getEvents(UpdateRowsEventData.class);
         Serializable[] updateData = updateEvents.iterator().next().getRows().get(0).getValue();
-        assertEquals(JsonBinary.parseAsString((byte[]) updateData[0]), json.replace("970785C8-C299", "970785C8"));
+        assertEquals(parseAndRemoveSpaces((byte[]) updateData[0]), json.replace("970785C8-C299", "970785C8"));
     }
 
     @Test
@@ -141,7 +151,7 @@ public void testMysql8JsonRemovePartialUpdateWithHoles() throws Exception {
 
         List updateEvents = capturingEventListener.getEvents(UpdateRowsEventData.class);
         Serializable[] updateData = updateEvents.iterator().next().getRows().get(0).getValue();
-        assertEquals(JsonBinary.parseAsString((byte[]) updateData[0]), json.replace("\"ab\":\"970785C8-C299\"", ""));
+        assertEquals(parseAndRemoveSpaces((byte[]) updateData[0]), json.replace("\"ab\":\"970785C8-C299\"", ""));
 
         client.unregisterEventListener(capturingEventListener);
     }
@@ -162,7 +172,7 @@ public void testMysql8JsonRemovePartialUpdateWithHolesAndSparseKeys() throws Exc
 
         List updateEvents = capturingEventListener.getEvents(UpdateRowsEventData.class);
         Serializable[] updateData = updateEvents.iterator().next().getRows().get(0).getValue();
-        assertEquals(JsonBinary.parseAsString((byte[]) updateData[0]), json.replace(
+        assertEquals(parseAndRemoveSpaces((byte[]) updateData[0]), json.replace(
                 "\"17fc9889474028063990914001f6854f6b8b5784\":\"test_field_for_remove_fields_behaviour_2\",", ""));
 
         client.unregisterEventListener(capturingEventListener);
@@ -184,7 +194,7 @@ public void testMysql8JsonReplacePartialUpdateWithHoles() throws Exception {
 
         List updateEvents = capturingEventListener.getEvents(UpdateRowsEventData.class);
         Serializable[] updateData = updateEvents.iterator().next().getRows().get(0).getValue();
-        assertEquals(JsonBinary.parseAsString((byte[]) updateData[0]), json.replace("970785C8-C299", "9707"));
+        assertEquals(parseAndRemoveSpaces((byte[]) updateData[0]), json.replace("970785C8-C299", "9707"));
 
         client.unregisterEventListener(capturingEventListener);
     }
@@ -207,7 +217,9 @@ public void testMysql8JsonRemoveArrayValue() throws Exception {
 
         List updateEvents = capturingEventListener.getEvents(UpdateRowsEventData.class);
         Serializable[] updateData = updateEvents.iterator().next().getRows().get(0).getValue();
-        assertEquals(JsonBinary.parseAsString((byte[]) updateData[0]), "[\"foo\",\"baz\"]");
+        String parsed = parseAndRemoveSpaces((byte[]) updateData[0]);
+
+        assertEquals(parsed, "[\"foo\",\"baz\"]");
 
         client.unregisterEventListener(capturingEventListener);
     }
@@ -449,12 +461,18 @@ public void testEmptyArray() throws Exception {
 
     @Test
     public void testScalarDateTime() throws Exception {
+        if ( isMaria )
+            throw new SkipException("");
+
         assertEquals(writeAndCaptureJSON("CAST(CAST('2015-01-15 23:24:25' AS DATETIME) AS JSON)"),
             "\"2015-01-15 23:24:25\"");
     }
 
     @Test
     public void testScalarTime() throws Exception {
+        if ( isMaria )
+            throw new SkipException("");
+
         assertEquals(writeAndCaptureJSON("CAST(CAST('23:24:25' AS TIME) AS JSON)"),
             "\"23:24:25\"");
         assertEquals(writeAndCaptureJSON("CAST(CAST('23:24:25.12' AS TIME(3)) AS JSON)"),
@@ -465,12 +483,17 @@ public void testScalarTime() throws Exception {
 
     @Test
     public void testScalarDate() throws Exception {
+        if ( isMaria )
+            throw new SkipException("");
         assertEquals(writeAndCaptureJSON("CAST(CAST('2015-01-15' AS DATE) AS JSON)"),
             "\"2015-01-15\"");
     }
 
     @Test
     public void testScalarTimestamp() throws Exception {
+        if ( isMaria )
+            throw new SkipException("");
+        
         // timestamp literals are interpreted by MySQL as DATETIME values
         assertEquals(writeAndCaptureJSON("CAST(TIMESTAMP'2015-01-15 23:24:25' AS JSON)"),
             "\"2015-01-15 23:24:25\"");
@@ -485,6 +508,9 @@ public void testScalarTimestamp() throws Exception {
 
     @Test
     public void testScalarGeometry() throws Exception {
+        if ( isMaria )
+            throw new SkipException("");
+
         assertEquals(writeAndCaptureJSON("CAST(ST_GeomFromText('POINT(1 1)') AS JSON)"),
             "{\"type\":\"Point\",\"coordinates\":[1.0,1.0]}");
     }
@@ -496,6 +522,9 @@ public void testScalarStringWithCharsetConversion() throws Exception {
 
     @Test
     public void testScalarBinaryAsBase64() throws Exception {
+        if ( isMaria )
+            throw new SkipException("");
+
         assertEquals(writeAndCaptureJSON("CAST(x'cafe' AS JSON)"), "\"yv4=\"");
         assertEquals(writeAndCaptureJSON("CAST(x'cafebabe' AS JSON)"), "\"yv66vg==\"");
     }
@@ -529,10 +558,14 @@ private String writeAndCaptureJSON(final String value) throws Exception {
             System.out.println("I am about to fail an expectation...");
             assertTrue(false, "did not receive rows in json test for " + value);
         }
-        byte[] b = (byte[]) capturingEventListener.getEvents(WriteRowsEventData.class).get(0).getRows().get(0)[0];
+        WriteRowsEventData e = capturingEventListener.getEvents(WriteRowsEventData.class).get(0);
+        Serializable[] firstRow = e.getRows().get(0);
+
+        byte[] b = (byte[]) firstRow[0];
         return b == null ? null : JsonBinary.parseAsString(b);
     }
 
+
     @AfterMethod
     public void afterEachTest() throws Exception {
         final CountDownLatch latch = new CountDownLatch(1);

From 492c6a46c512094064cb488d7bac1070e9d50ecf Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Sat, 13 Aug 2022 12:27:35 -0700
Subject: [PATCH 158/217] can maven be quiet plz?

---
 .circleci/config.yml                                            | 2 +-
 .../deserialization/json/JsonBinaryValueIntegrationTest.java    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index f09dc95f..35b8d1d5 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -26,7 +26,7 @@ jobs:
         type: string
     environment:
       MYSQL_VERSION: "<< parameters.mysql >>"
-      JAVA_TOOL_OPTIONS: "-Xmx250m"
+      JAVA_TOOL_OPTIONS: "-Xmx250m -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
     steps:
       - checkout
       - restore_cache:
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java
index 8866c155..5c268cbb 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java
@@ -493,7 +493,7 @@ public void testScalarDate() throws Exception {
     public void testScalarTimestamp() throws Exception {
         if ( isMaria )
             throw new SkipException("");
-        
+
         // timestamp literals are interpreted by MySQL as DATETIME values
         assertEquals(writeAndCaptureJSON("CAST(TIMESTAMP'2015-01-15 23:24:25' AS JSON)"),
             "\"2015-01-15 23:24:25\"");

From 7c52c0abfff4d6ca3e978faa34d2193419df6945 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Sat, 13 Aug 2022 14:38:20 -0700
Subject: [PATCH 159/217] abstract out some tests, shuffle around
 maria-specific stuff

---
 .../shyiko/mysql/binlog/BinaryLogClient.java  |  13 +-
 .../mysql/binlog/MariadbBinaryLogClient.java  |  28 +++
 .../mysql/binlog/AbstractIntegrationTest.java |  69 +++++++
 .../BinaryLogClientIntegrationTest.java       | 174 +-----------------
 ...MariadbBinaryLogClientIntegrationTest.java |  16 +-
 .../shyiko/mysql/binlog/MySQLConnection.java  | 117 ++++++++++++
 .../json/JsonBinaryValueIntegrationTest.java  |   6 +-
 7 files changed, 228 insertions(+), 195 deletions(-)
 create mode 100644 src/test/java/com/github/shyiko/mysql/binlog/AbstractIntegrationTest.java
 create mode 100644 src/test/java/com/github/shyiko/mysql/binlog/MySQLConnection.java

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index 1c7f74ac..d3650bac 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -139,7 +139,7 @@ public X509Certificate[] getAcceptedIssuers() {
     protected final Object gtidSetAccessLock = new Object();
     private boolean gtidSetFallbackToPurged;
     private boolean useBinlogFilenamePositionInGtidMode;
-    private String gtid;
+    protected String gtid;
     private boolean tx;
 
     private EventDeserializer eventDeserializer = new EventDeserializer();
@@ -1010,7 +1010,7 @@ private byte[] readPacketSplitInChunks(ByteArrayInputStream inputStream, int pac
         return result;
     }
 
-    protected void updateClientBinlogFilenameAndPosition(Event event) {
+    private void updateClientBinlogFilenameAndPosition(Event event) {
         EventHeader eventHeader = event.getHeader();
         EventType eventType = eventHeader.getEventType();
         if (eventType == EventType.ROTATE) {
@@ -1041,15 +1041,6 @@ protected void updateGtidSet(Event event) {
                 GtidEventData gtidEventData = (GtidEventData) EventDataWrapper.internal(event.getData());
                 gtid = gtidEventData.getGtid();
                 break;
-            case MARIADB_GTID:
-                MariadbGtidEventData mariadbGtidEventData =  (MariadbGtidEventData) EventDataWrapper.internal(event.getData());
-                mariadbGtidEventData.setServerId(eventHeader.getServerId());
-                gtid = mariadbGtidEventData.toString();
-                break;
-            case MARIADB_GTID_LIST:
-                MariadbGtidListEventData mariadbGtidListEventData =  (MariadbGtidListEventData) EventDataWrapper.internal(event.getData());
-                gtid = mariadbGtidListEventData.getMariaGTIDSet().toString();
-                break;
             case XID:
                 commitGtid();
                 tx = false;
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClient.java
index 0c6740d6..aa6cc5af 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClient.java
@@ -1,7 +1,12 @@
 package com.github.shyiko.mysql.binlog;
 
+import com.github.shyiko.mysql.binlog.event.Event;
+import com.github.shyiko.mysql.binlog.event.EventHeader;
 import com.github.shyiko.mysql.binlog.event.EventType;
+import com.github.shyiko.mysql.binlog.event.MariadbGtidEventData;
+import com.github.shyiko.mysql.binlog.event.MariadbGtidListEventData;
 import com.github.shyiko.mysql.binlog.event.deserialization.AnnotateRowsEventDataDeserializer;
+import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer;
 import com.github.shyiko.mysql.binlog.event.deserialization.MariadbGtidEventDataDeserializer;
 import com.github.shyiko.mysql.binlog.event.deserialization.MariadbGtidListEventDataDeserializer;
 import com.github.shyiko.mysql.binlog.network.protocol.command.Command;
@@ -75,6 +80,29 @@ protected void ensureGtidEventDataDeserializer() {
         ensureEventDataDeserializer(EventType.MARIADB_GTID_LIST, MariadbGtidListEventDataDeserializer.class);
     }
 
+    @Override
+    protected void updateGtidSet(Event event) {
+        synchronized (gtidSetAccessLock) {
+            if (gtidSet == null) {
+                return;
+            }
+        }
+        EventHeader eventHeader = event.getHeader();
+        switch(eventHeader.getEventType()) {
+            case MARIADB_GTID:
+                MariadbGtidEventData mariadbGtidEventData = (MariadbGtidEventData) EventDeserializer.EventDataWrapper.internal(event.getData());
+                mariadbGtidEventData.setServerId(eventHeader.getServerId());
+                gtid = mariadbGtidEventData.toString();
+                break;
+            case MARIADB_GTID_LIST:
+                MariadbGtidListEventData mariadbGtidListEventData = (MariadbGtidListEventData) EventDeserializer.EventDataWrapper.internal(event.getData());
+                gtid = mariadbGtidListEventData.getMariaGTIDSet().toString();
+                break;
+            default:
+                super.updateGtidSet(event);
+        }
+    }
+
     public boolean isUseSendAnnotateRowsEvent() {
         return useSendAnnotateRowsEvent;
     }
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/AbstractIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/AbstractIntegrationTest.java
new file mode 100644
index 00000000..802052f6
--- /dev/null
+++ b/src/test/java/com/github/shyiko/mysql/binlog/AbstractIntegrationTest.java
@@ -0,0 +1,69 @@
+package com.github.shyiko.mysql.binlog;
+
+import com.github.shyiko.mysql.binlog.event.EventType;
+import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer;
+import org.testng.annotations.BeforeClass;
+
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.TimeZone;
+
+public abstract class AbstractIntegrationTest {
+    protected MySQLConnection master;
+    protected MySQLConnection slave;
+    protected BinaryLogClient client;
+    protected CountDownEventListener eventListener;
+    protected MysqlVersion mysqlVersion;
+
+    protected MysqlOnetimeServerOptions getOptions() {
+        MysqlOnetimeServerOptions options = new MysqlOnetimeServerOptions();
+        options.fullRowMetaData = true;
+        return options;
+    }
+
+    @BeforeClass
+    public void setUp() throws Exception {
+        TimeZone.setDefault(TimeZone.getTimeZone("GMT"));
+        mysqlVersion = MysqlOnetimeServer.getVersion();
+        MysqlOnetimeServer masterServer = new MysqlOnetimeServer(getOptions());
+        MysqlOnetimeServer slaveServer = new MysqlOnetimeServer(getOptions());
+
+        masterServer.boot();
+        slaveServer.boot();
+        slaveServer.setupSlave(masterServer.getPort());
+
+        master = new MySQLConnection("127.0.0.1", masterServer.getPort(), "root", "");
+        slave = new MySQLConnection("127.0.0.1", slaveServer.getPort(), "root", "");
+
+        client = new BinaryLogClient(slave.hostname, slave.port, slave.username, slave.password);
+        EventDeserializer eventDeserializer = new EventDeserializer();
+        eventDeserializer.setCompatibilityMode(EventDeserializer.CompatibilityMode.CHAR_AND_BINARY_AS_BYTE_ARRAY,
+            EventDeserializer.CompatibilityMode.DATE_AND_TIME_AS_LONG);
+        client.setEventDeserializer(eventDeserializer);
+        client.setServerId(client.getServerId() - 1); // avoid clashes between BinaryLogClient instances
+        client.setKeepAlive(false);
+        client.registerEventListener(new TraceEventListener());
+        client.registerEventListener(eventListener = new CountDownEventListener());
+        client.registerLifecycleListener(new TraceLifecycleListener());
+        client.connect(BinaryLogClientIntegrationTest.DEFAULT_TIMEOUT);
+        master.execute(new BinaryLogClientIntegrationTest.Callback() {
+            @Override
+            public void execute(Statement statement) throws SQLException {
+                statement.execute("drop database if exists mbcj_test");
+                statement.execute("create database mbcj_test");
+                statement.execute("use mbcj_test");
+            }
+        });
+        eventListener.waitFor(EventType.QUERY, 2, BinaryLogClientIntegrationTest.DEFAULT_TIMEOUT);
+
+        if ( mysqlVersion.atLeast(8, 0) ) {
+            setupMysql8Login(master);
+            eventListener.waitFor(EventType.QUERY, 2, BinaryLogClientIntegrationTest.DEFAULT_TIMEOUT);
+        }
+    }
+
+    protected void setupMysql8Login(MySQLConnection server) throws Exception {
+        server.execute("create user 'mysql8' IDENTIFIED WITH caching_sha2_password BY 'testpass'");
+        server.execute("grant replication slave, replication client on *.* to 'mysql8'");
+    }
+}
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java
index 7e04312c..cba20577 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java
@@ -35,16 +35,13 @@
 import com.github.shyiko.mysql.binlog.network.SSLMode;
 import com.github.shyiko.mysql.binlog.network.ServerException;
 import com.github.shyiko.mysql.binlog.network.SocketFactory;
-import com.mysql.cj.MysqlConnection;
 import org.mockito.InOrder;
 import org.testng.SkipException;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeClass;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
-import java.io.Closeable;
 import java.io.EOFException;
 import java.io.FilterInputStream;
 import java.io.FilterOutputStream;
@@ -56,8 +53,6 @@
 import java.math.MathContext;
 import java.net.Socket;
 import java.net.SocketException;
-import java.sql.Connection;
-import java.sql.DriverManager;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.SQLSyntaxErrorException;
@@ -97,7 +92,7 @@
 /**
  * @author Stanley Shyiko
  */
-public class BinaryLogClientIntegrationTest {
+public class BinaryLogClientIntegrationTest extends AbstractIntegrationTest {
 
     protected static final long DEFAULT_TIMEOUT = TimeUnit.SECONDS.toMillis(3);
 
@@ -109,58 +104,6 @@ public class BinaryLogClientIntegrationTest {
 
     private final TimeZone timeZoneBeforeTheTest = TimeZone.getDefault();
 
-    protected MySQLConnection master, slave;
-    protected BinaryLogClient client;
-    protected CountDownEventListener eventListener;
-    protected MysqlVersion mysqlVersion;
-
-    protected MysqlOnetimeServerOptions getOptions() {
-        MysqlOnetimeServerOptions options = new MysqlOnetimeServerOptions();
-        options.fullRowMetaData = true;
-        return options;
-    }
-
-    @BeforeClass
-    public void setUp() throws Exception {
-        TimeZone.setDefault(TimeZone.getTimeZone("GMT"));
-        mysqlVersion = MysqlOnetimeServer.getVersion();
-        MysqlOnetimeServer masterServer = new MysqlOnetimeServer(getOptions());
-        MysqlOnetimeServer slaveServer = new MysqlOnetimeServer(getOptions());
-
-        masterServer.boot();
-        slaveServer.boot();
-        slaveServer.setupSlave(masterServer.getPort());
-
-        master = new MySQLConnection("127.0.0.1", masterServer.getPort(), "root", "");
-        slave = new MySQLConnection("127.0.0.1", slaveServer.getPort(), "root", "");
-
-        client = new BinaryLogClient(slave.hostname, slave.port, slave.username, slave.password);
-        EventDeserializer eventDeserializer = new EventDeserializer();
-        eventDeserializer.setCompatibilityMode(CompatibilityMode.CHAR_AND_BINARY_AS_BYTE_ARRAY,
-            CompatibilityMode.DATE_AND_TIME_AS_LONG);
-        client.setEventDeserializer(eventDeserializer);
-        client.setServerId(client.getServerId() - 1); // avoid clashes between BinaryLogClient instances
-        client.setKeepAlive(false);
-        client.registerEventListener(new TraceEventListener());
-        client.registerEventListener(eventListener = new CountDownEventListener());
-        client.registerLifecycleListener(new TraceLifecycleListener());
-        client.connect(DEFAULT_TIMEOUT);
-        master.execute(new Callback() {
-            @Override
-            public void execute(Statement statement) throws SQLException {
-                statement.execute("drop database if exists mbcj_test");
-                statement.execute("create database mbcj_test");
-                statement.execute("use mbcj_test");
-            }
-        });
-        eventListener.waitFor(EventType.QUERY, 2, DEFAULT_TIMEOUT);
-
-        if ( mysqlVersion.atLeast(8, 0) ) {
-            setupMysql8Login(master);
-            eventListener.waitFor(EventType.QUERY, 2, DEFAULT_TIMEOUT);
-        }
-    }
-
     @BeforeMethod
     public void beforeEachTest() throws Exception {
         master.execute(new Callback() {
@@ -170,7 +113,7 @@ public void execute(Statement statement) throws SQLException {
                 statement.execute("create table bikini_bottom (name varchar(255) primary key)");
             }
         });
-        eventListener.waitFor(EventType.QUERY, 2, DEFAULT_TIMEOUT);
+        eventListener.waitForAtLeast(EventType.QUERY, 2, DEFAULT_TIMEOUT);
         eventListener.reset();
     }
 
@@ -1061,11 +1004,6 @@ public void execute(Statement statement) throws SQLException {
         }
     }
 
-    private void setupMysql8Login(MySQLConnection server) throws Exception {
-        server.execute("create user 'mysql8' IDENTIFIED WITH caching_sha2_password BY 'testpass'");
-        server.execute("grant replication slave, replication client on *.* to 'mysql8'");
-    }
-
     @Test
     public void testMysql8Auth() throws Exception {
         if ( !mysqlVersion.atLeast(8, 0) )
@@ -1207,114 +1145,6 @@ public void execute(Statement statement) throws SQLException {
         }
     }
 
-    /**
-     * Representation of a MySQL connection.
-     */
-    public static final class MySQLConnection implements Closeable {
-
-        private final String hostname;
-        private final int port;
-        private final String username;
-        private final String password;
-        private Connection connection;
-
-        public MySQLConnection(String hostname, int port, String username, String password)
-            throws ClassNotFoundException, SQLException {
-            this.hostname = hostname;
-            this.port = port;
-            this.username = username;
-            this.password = password;
-            Class.forName("com.mysql.jdbc.Driver");
-            connect();
-        }
-
-        private void connect() throws SQLException {
-            this.connection = DriverManager.getConnection("jdbc:mysql://" + hostname + ":" + port +
-                "?serverTimezone=UTC", username, password);
-            execute(new Callback() {
-
-                @Override
-                public void execute(Statement statement) throws SQLException {
-                    statement.execute("SET time_zone = '+00:00'");
-                }
-            });
-        }
-
-        public String hostname() {
-            return hostname;
-        }
-
-        public int port() {
-            return port;
-        }
-
-        public String username() {
-            return username;
-        }
-
-        public String password() {
-            return password;
-        }
-
-        public void execute(Callback callback, boolean autocommit) throws SQLException {
-            connection.setAutoCommit(autocommit);
-            Statement statement = connection.createStatement();
-            try {
-                callback.execute(statement);
-                if (!autocommit) {
-                    connection.commit();
-                }
-            } finally {
-                statement.close();
-            }
-        }
-
-        public void execute(Callback callback) throws SQLException {
-            execute(callback, false);
-        }
-
-        public void execute(final String...statements) throws SQLException {
-            execute(new Callback() {
-                @Override
-                public void execute(Statement statement) throws SQLException {
-                    for (String command : statements) {
-                        statement.execute(command);
-                    }
-                }
-            });
-        }
-
-        public void query(String sql, Callback callback) throws SQLException {
-            connection.setAutoCommit(false);
-            Statement statement = connection.createStatement();
-            try {
-                ResultSet rs = statement.executeQuery(sql);
-                try {
-                    callback.execute(rs);
-                    connection.commit();
-                } finally {
-                    rs.close();
-                }
-            } finally {
-                statement.close();
-            }
-        }
-
-        @Override
-        public void close() throws IOException {
-            try {
-                connection.close();
-            } catch (SQLException e) {
-                throw new IOException(e);
-            }
-        }
-
-        public void reconnect() throws IOException, SQLException {
-            close();
-            connect();
-        }
-    }
-
     /**
      * Callback used in the {@link MySQLConnection#execute(Callback)} method.
      *
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClientIntegrationTest.java
index ca5039e0..6cf81817 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClientIntegrationTest.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClientIntegrationTest.java
@@ -4,7 +4,7 @@
 import com.github.shyiko.mysql.binlog.event.MariadbGtidEventData;
 import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer;
 import org.testng.SkipException;
-import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
 import java.sql.ResultSet;
@@ -18,17 +18,15 @@
 /**
  * @author Winger
  */
-public class MariadbBinaryLogClientIntegrationTest extends BinaryLogClientIntegrationTest {
-
-    MysqlOnetimeServer primaryServer;
-    protected BinaryLogClientIntegrationTest.MySQLConnection master;
-
-    @Test
-    public void testMariadbUseGTIDAndAnnotateRowsEvent() throws Exception {
+public class MariadbBinaryLogClientIntegrationTest extends AbstractIntegrationTest {
+    @BeforeMethod
+    public void checkMariaDB() throws Exception {
         if ( !mysqlVersion.isMaria )
             throw new SkipException("not maria");
+    }
 
-
+    @Test
+    public void testMariadbUseGTIDAndAnnotateRowsEvent() throws Exception {
         master.execute(new BinaryLogClientIntegrationTest.Callback() {
             @Override
             public void execute(Statement statement) throws SQLException {
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/MySQLConnection.java b/src/test/java/com/github/shyiko/mysql/binlog/MySQLConnection.java
new file mode 100644
index 00000000..3ba13a8e
--- /dev/null
+++ b/src/test/java/com/github/shyiko/mysql/binlog/MySQLConnection.java
@@ -0,0 +1,117 @@
+package com.github.shyiko.mysql.binlog;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+/**
+ * Representation of a MySQL connection.
+ */
+public final class MySQLConnection implements Closeable {
+
+    public final String hostname;
+    public final int port;
+    public final String username;
+    public final String password;
+    public Connection connection;
+
+    public MySQLConnection(String hostname, int port, String username, String password)
+        throws ClassNotFoundException, SQLException {
+        this.hostname = hostname;
+        this.port = port;
+        this.username = username;
+        this.password = password;
+        Class.forName("com.mysql.jdbc.Driver");
+        connect();
+    }
+
+    private void connect() throws SQLException {
+        this.connection = DriverManager.getConnection("jdbc:mysql://" + hostname + ":" + port +
+            "?serverTimezone=UTC", username, password);
+        execute(new BinaryLogClientIntegrationTest.Callback() {
+
+            @Override
+            public void execute(Statement statement) throws SQLException {
+                statement.execute("SET time_zone = '+00:00'");
+            }
+        });
+    }
+
+    public String hostname() {
+        return hostname;
+    }
+
+    public int port() {
+        return port;
+    }
+
+    public String username() {
+        return username;
+    }
+
+    public String password() {
+        return password;
+    }
+
+    public void execute(BinaryLogClientIntegrationTest.Callback callback, boolean autocommit) throws SQLException {
+        connection.setAutoCommit(autocommit);
+        Statement statement = connection.createStatement();
+        try {
+            callback.execute(statement);
+            if ( !autocommit ) {
+                connection.commit();
+            }
+        } finally {
+            statement.close();
+        }
+    }
+
+    public void execute(BinaryLogClientIntegrationTest.Callback callback) throws SQLException {
+        execute(callback, false);
+    }
+
+    public void execute(final String... statements) throws SQLException {
+        execute(new BinaryLogClientIntegrationTest.Callback() {
+            @Override
+            public void execute(Statement statement) throws SQLException {
+                for ( String command : statements ) {
+                    statement.execute(command);
+                }
+            }
+        });
+    }
+
+    public void query(String sql, BinaryLogClientIntegrationTest.Callback callback) throws SQLException {
+        connection.setAutoCommit(false);
+        Statement statement = connection.createStatement();
+        try {
+            ResultSet rs = statement.executeQuery(sql);
+            try {
+                callback.execute(rs);
+                connection.commit();
+            } finally {
+                rs.close();
+            }
+        } finally {
+            statement.close();
+        }
+    }
+
+    @Override
+    public void close() throws IOException {
+        try {
+            connection.close();
+        } catch ( SQLException e ) {
+            throw new IOException(e);
+        }
+    }
+
+    public void reconnect() throws IOException, SQLException {
+        close();
+        connect();
+    }
+}
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java
index 5c268cbb..0ef0cded 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java
@@ -19,10 +19,10 @@
 import com.github.shyiko.mysql.binlog.BinaryLogClientIntegrationTest;
 import com.github.shyiko.mysql.binlog.CapturingEventListener;
 import com.github.shyiko.mysql.binlog.CountDownEventListener;
+import com.github.shyiko.mysql.binlog.MySQLConnection;
 import com.github.shyiko.mysql.binlog.MysqlOnetimeServer;
 import com.github.shyiko.mysql.binlog.TraceEventListener;
 import com.github.shyiko.mysql.binlog.TraceLifecycleListener;
-import com.github.shyiko.mysql.binlog.event.Event;
 import com.github.shyiko.mysql.binlog.event.EventData;
 import com.github.shyiko.mysql.binlog.event.EventType;
 import com.github.shyiko.mysql.binlog.event.QueryEventData;
@@ -65,7 +65,7 @@ public class JsonBinaryValueIntegrationTest {
 
     private final TimeZone timeZoneBeforeTheTest = TimeZone.getDefault();
 
-    private BinaryLogClientIntegrationTest.MySQLConnection master;
+    private MySQLConnection master;
     private BinaryLogClient client;
     private CountDownEventListener eventListener;
 
@@ -78,7 +78,7 @@ public void setUp() throws Exception {
         MysqlOnetimeServer masterServer = new MysqlOnetimeServer();
         masterServer.boot();
 
-        master = new BinaryLogClientIntegrationTest.MySQLConnection("127.0.0.1", masterServer.getPort(), "root", "");
+        master = new MySQLConnection("127.0.0.1", masterServer.getPort(), "root", "");
 
         client = new BinaryLogClient(master.hostname(), master.port(), master.username(), master.password());
         client.setServerId(client.getServerId() - 1); // avoid clashes between BinaryLogClient instances

From 2ae2eb801d196f86471a7998a7c7a8566b743d57 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Sat, 13 Aug 2022 21:44:13 -0700
Subject: [PATCH 160/217] refactor: bring mariaDB back into the mainline path

no need to know what sever you're connecting to.  we can figure it out.

Also, these tests never worked.  fix them.
---
 .../shyiko/mysql/binlog/BinaryLogClient.java  | 120 ++++++++++++++++--
 .../mysql/binlog/MariadbBinaryLogClient.java  | 113 -----------------
 .../shyiko/mysql/binlog/MariadbGtidSet.java   |   7 +
 ...MariadbBinaryLogClientIntegrationTest.java |  17 ++-
 4 files changed, 129 insertions(+), 128 deletions(-)
 delete mode 100644 src/main/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClient.java

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index d3650bac..3591674b 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -25,12 +25,15 @@
 import com.github.shyiko.mysql.binlog.event.MariadbGtidListEventData;
 import com.github.shyiko.mysql.binlog.event.QueryEventData;
 import com.github.shyiko.mysql.binlog.event.RotateEventData;
+import com.github.shyiko.mysql.binlog.event.deserialization.AnnotateRowsEventDataDeserializer;
 import com.github.shyiko.mysql.binlog.event.deserialization.ChecksumType;
 import com.github.shyiko.mysql.binlog.event.deserialization.EventDataDeserializationException;
 import com.github.shyiko.mysql.binlog.event.deserialization.EventDataDeserializer;
 import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer;
 import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer.EventDataWrapper;
 import com.github.shyiko.mysql.binlog.event.deserialization.GtidEventDataDeserializer;
+import com.github.shyiko.mysql.binlog.event.deserialization.MariadbGtidEventDataDeserializer;
+import com.github.shyiko.mysql.binlog.event.deserialization.MariadbGtidListEventDataDeserializer;
 import com.github.shyiko.mysql.binlog.event.deserialization.QueryEventDataDeserializer;
 import com.github.shyiko.mysql.binlog.event.deserialization.RotateEventDataDeserializer;
 import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream;
@@ -71,6 +74,7 @@
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Locale;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
@@ -138,6 +142,7 @@ public X509Certificate[] getAcceptedIssuers() {
     protected GtidSet gtidSet;
     protected final Object gtidSetAccessLock = new Object();
     private boolean gtidSetFallbackToPurged;
+    private boolean gtidEnabled = false;
     private boolean useBinlogFilenamePositionInGtidMode;
     protected String gtid;
     private boolean tx;
@@ -168,6 +173,10 @@ public X509Certificate[] getAcceptedIssuers() {
 
     private final Lock connectLock = new ReentrantLock();
     private final Lock keepAliveThreadExecutorLock = new ReentrantLock();
+    private boolean useSendAnnotateRowsEvent;
+
+
+    private Boolean isMariaDB;
 
     /**
      * Alias for BinaryLogClient("localhost", 3306, <no schema> = null, username, password).
@@ -321,7 +330,7 @@ public String getGtidSet() {
     }
 
     /**
-     * @param gtidSet GTID set (can be an empty string).
+     * @param gtidStr GTID set string (can be an empty string).
      * 
NOTE #1: Any value but null will switch BinaryLogClient into a GTID mode (this will also set binlogFilename
      * to "" (provided it's null) forcing MySQL to send events starting from the oldest known binlog (keep in mind
      * that connection will fail if gtid_purged is anything but empty (unless
@@ -330,19 +339,27 @@ public String getGtidSet() {
      * @see #getGtidSet()
      * @see #setGtidSetFallbackToPurged(boolean)
      */
-    public void setGtidSet(String gtidSet) {
-        if (gtidSet != null && this.binlogFilename == null) {
+    public void setGtidSet(String gtidStr) {
+        if ( gtidStr == null )
+            return;
+
+        this.gtidEnabled = true;
+
+        if (this.binlogFilename == null) {
             this.binlogFilename = "";
         }
+
         synchronized (gtidSetAccessLock) {
-            this.gtidSet = gtidSet != null ? buildGtidSet(gtidSet) : null;
+            if ( !gtidStr.equals("") ) {
+                if ( MariadbGtidSet.isMariaGtidSet(gtidStr) ) {
+                    this.gtidSet = new MariadbGtidSet(gtidStr);
+                } else {
+                    this.gtidSet = new GtidSet(gtidStr);
+                }
+            }
         }
     }
 
-    protected GtidSet buildGtidSet(String gtidSet) {
-        return new GtidSet(gtidSet);
-    }
-
     /**
      * @see #setGtidSetFallbackToPurged(boolean)
 	 * @return whether gtid_purged is used as a fallback
@@ -505,6 +522,21 @@ public void setThreadFactory(ThreadFactory threadFactory) {
         this.threadFactory = threadFactory;
     }
 
+
+    /**
+     * @return true/false depending on whether we've connected to MariaDB.  NULL if not connected.
+     */
+    public Boolean getMariaDB() {
+        return isMariaDB;
+    }
+
+    public boolean isUseSendAnnotateRowsEvent() {
+        return useSendAnnotateRowsEvent;
+    }
+
+    public void setUseSendAnnotateRowsEvent(boolean useSendAnnotateRowsEvent) {
+        this.useSendAnnotateRowsEvent = useSendAnnotateRowsEvent;
+    }
     /**
      * Connect to the replication stream. Note that this method blocks until disconnected.
      * @throws AuthenticationException if authentication fails
@@ -536,6 +568,7 @@ public void connect() throws IOException, IllegalStateException {
                 }
                 GreetingPacket greetingPacket = receiveGreeting();
 
+                detectMariaDB(greetingPacket);
                 tryUpgradeToSSL(greetingPacket);
 
                 new Authenticator(greetingPacket, channel, schema, username, password).authenticate();
@@ -591,7 +624,7 @@ public void connect() throws IOException, IllegalStateException {
             }
             ensureEventDataDeserializer(EventType.ROTATE, RotateEventDataDeserializer.class);
             synchronized (gtidSetAccessLock) {
-                if (gtidSet != null) {
+                if (this.gtidEnabled) {
                     ensureGtidEventDataDeserializer();
                 }
             }
@@ -606,6 +639,13 @@ public void connect() throws IOException, IllegalStateException {
         }
     }
 
+    private void detectMariaDB(GreetingPacket packet) {
+        String serverVersion = packet.getServerVersion();
+        if ( serverVersion == null )
+            return;
+
+        this.isMariaDB = serverVersion.toLowerCase().contains("mariadb");
+    }
     /**
      * Apply additional options for connection before requesting binlog stream.
      */
@@ -725,9 +765,16 @@ private void setMasterServerId() throws IOException {
 
     protected void requestBinaryLogStream() throws IOException {
         long serverId = blocking ? this.serverId : 0; // http://bugs.mysql.com/bug.php?id=71178
+        if ( this.isMariaDB )
+            requestBinaryLogStreamMaria(serverId);
+        else
+            requestBinaryLogStreamMysql(serverId);
+    }
+
+    private void requestBinaryLogStreamMysql(long serverId) throws IOException {
         Command dumpBinaryLogCommand;
         synchronized (gtidSetAccessLock) {
-            if (gtidSet != null) {
+            if (this.gtidEnabled) {
                 dumpBinaryLogCommand = new DumpBinaryLogGtidCommand(serverId,
                     useBinlogFilenamePositionInGtidMode ? binlogFilename : "",
                     useBinlogFilenamePositionInGtidMode ? binlogPosition : 4,
@@ -737,6 +784,27 @@ protected void requestBinaryLogStream() throws IOException {
             }
         }
         channel.write(dumpBinaryLogCommand);
+
+    }
+
+    protected void requestBinaryLogStreamMaria(long serverId) throws IOException {
+        Command dumpBinaryLogCommand;
+        synchronized (gtidSetAccessLock) {
+            if (this.gtidEnabled) {
+                channel.write(new QueryCommand("SET @mariadb_slave_capability=4"));
+                checkError(channel.read());
+                channel.write(new QueryCommand("SET @slave_connect_state = '" + gtidSet.toString() + "'"));
+                checkError(channel.read());
+                channel.write(new QueryCommand("SET @slave_gtid_strict_mode = 0"));
+                checkError(channel.read());
+                channel.write(new QueryCommand("SET @slave_gtid_ignore_duplicates = 0"));
+                checkError(channel.read());
+                dumpBinaryLogCommand = new DumpBinaryLogCommand(serverId, "", 0L, isUseSendAnnotateRowsEvent());
+            } else {
+                dumpBinaryLogCommand = new DumpBinaryLogCommand(serverId, binlogFilename, binlogPosition);
+            }
+        }
+        channel.write(dumpBinaryLogCommand);
     }
 
     protected void ensureEventDataDeserializer(EventType eventType,
@@ -759,6 +827,9 @@ protected void ensureEventDataDeserializer(EventType eventType,
     protected void ensureGtidEventDataDeserializer() {
         ensureEventDataDeserializer(EventType.GTID, GtidEventDataDeserializer.class);
         ensureEventDataDeserializer(EventType.QUERY, QueryEventDataDeserializer.class);
+        ensureEventDataDeserializer(EventType.ANNOTATE_ROWS, AnnotateRowsEventDataDeserializer.class);
+        ensureEventDataDeserializer(EventType.MARIADB_GTID, MariadbGtidEventDataDeserializer.class);
+        ensureEventDataDeserializer(EventType.MARIADB_GTID_LIST, MariadbGtidListEventDataDeserializer.class);
     }
 
     private void spawnKeepAliveThread() {
@@ -902,11 +973,27 @@ private String fetchGtidPurged() throws IOException {
     }
 
     protected void setupGtidSet() throws IOException{
+        if (!this.gtidEnabled)
+            return;
+
         synchronized (gtidSetAccessLock) {
-            if (gtidSet != null && "".equals(gtidSet.toString()) && gtidSetFallbackToPurged) {
-                gtidSet = new GtidSet(fetchGtidPurged());
+            if ( this.isMariaDB ) {
+                if ( gtidSet == null ) {
+                    gtidSet = new MariadbGtidSet("");
+                } else if ( !(gtidSet instanceof MariadbGtidSet) ) {
+                    throw new RuntimeException("Connected to MariaDB but given a mysql GTID set!");
+                }
+            } else {
+                if ( gtidSet == null && gtidSetFallbackToPurged ) {
+                    gtidSet = new GtidSet(fetchGtidPurged());
+                } else if ( gtidSet == null ){
+                    gtidSet = new GtidSet("");
+                } else if ( gtidSet instanceof MariadbGtidSet ) {
+                    throw new RuntimeException("Connected to Mysql but given a MariaDB GTID set!");
+                }
             }
         }
+
     }
 
     private void fetchBinlogFilenameAndPosition() throws IOException {
@@ -1061,6 +1148,15 @@ protected void updateGtidSet(Event event) {
                 }
                 commitGtid(sql);
                 break;
+            case MARIADB_GTID:
+                MariadbGtidEventData mariadbGtidEventData = (MariadbGtidEventData) EventDeserializer.EventDataWrapper.internal(event.getData());
+                mariadbGtidEventData.setServerId(eventHeader.getServerId());
+                gtid = mariadbGtidEventData.toString();
+                break;
+            case MARIADB_GTID_LIST:
+                MariadbGtidListEventData mariadbGtidListEventData = (MariadbGtidListEventData) EventDeserializer.EventDataWrapper.internal(event.getData());
+                gtid = mariadbGtidListEventData.getMariaGTIDSet().toString();
+                break;
             default:
         }
     }
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClient.java
deleted file mode 100644
index aa6cc5af..00000000
--- a/src/main/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClient.java
+++ /dev/null
@@ -1,113 +0,0 @@
-package com.github.shyiko.mysql.binlog;
-
-import com.github.shyiko.mysql.binlog.event.Event;
-import com.github.shyiko.mysql.binlog.event.EventHeader;
-import com.github.shyiko.mysql.binlog.event.EventType;
-import com.github.shyiko.mysql.binlog.event.MariadbGtidEventData;
-import com.github.shyiko.mysql.binlog.event.MariadbGtidListEventData;
-import com.github.shyiko.mysql.binlog.event.deserialization.AnnotateRowsEventDataDeserializer;
-import com.github.shyiko.mysql.binlog.event.deserialization.EventDeserializer;
-import com.github.shyiko.mysql.binlog.event.deserialization.MariadbGtidEventDataDeserializer;
-import com.github.shyiko.mysql.binlog.event.deserialization.MariadbGtidListEventDataDeserializer;
-import com.github.shyiko.mysql.binlog.network.protocol.command.Command;
-import com.github.shyiko.mysql.binlog.network.protocol.command.DumpBinaryLogCommand;
-import com.github.shyiko.mysql.binlog.network.protocol.command.QueryCommand;
-
-import java.io.IOException;
-
-/**
- * Mariadb replication stream client.
- *
- * @author Winger
- */
-public class MariadbBinaryLogClient extends BinaryLogClient {
-
-    private boolean useSendAnnotateRowsEvent;
-
-    public MariadbBinaryLogClient(String username, String password) {
-        super(username, password);
-    }
-
-    public MariadbBinaryLogClient(String schema, String username, String password) {
-        super(schema, username, password);
-    }
-
-    public MariadbBinaryLogClient(String hostname, int port, String username, String password) {
-        super(hostname, port, username, password);
-    }
-
-    public MariadbBinaryLogClient(String hostname, int port, String schema, String username, String password) {
-        super(hostname, port, schema, username, password);
-    }
-
-    @Override
-    protected GtidSet buildGtidSet(String gtidSet) {
-        return new MariadbGtidSet(gtidSet);
-    }
-
-    @Override
-    protected void setupGtidSet() throws IOException {
-        //Mariadb ignore
-    }
-
-    @Override
-    protected void requestBinaryLogStream() throws IOException {
-        long serverId = isBlocking() ? this.getServerId() : 0; // http://bugs.mysql.com/bug.php?id=71178
-        Command dumpBinaryLogCommand;
-        synchronized (gtidSetAccessLock) {
-            if (gtidSet != null) {
-                channel.write(new QueryCommand("SET @mariadb_slave_capability=4"));
-                checkError(channel.read());
-                channel.write(new QueryCommand("SET @slave_connect_state = '" + gtidSet.toString() + "'"));
-                checkError(channel.read());
-                channel.write(new QueryCommand("SET @slave_gtid_strict_mode = 0"));
-                checkError(channel.read());
-                channel.write(new QueryCommand("SET @slave_gtid_ignore_duplicates = 0"));
-                checkError(channel.read());
-                dumpBinaryLogCommand = new DumpBinaryLogCommand(serverId, "", 0L, isUseSendAnnotateRowsEvent());
-
-            } else {
-                dumpBinaryLogCommand = new DumpBinaryLogCommand(serverId, getBinlogFilename(), getBinlogPosition());
-            }
-        }
-        channel.write(dumpBinaryLogCommand);
-    }
-
-    @Override
-    protected void ensureGtidEventDataDeserializer() {
-        ensureEventDataDeserializer(EventType.ANNOTATE_ROWS, AnnotateRowsEventDataDeserializer.class);
-        ensureEventDataDeserializer(EventType.MARIADB_GTID, MariadbGtidEventDataDeserializer.class);
-        ensureEventDataDeserializer(EventType.MARIADB_GTID_LIST, MariadbGtidListEventDataDeserializer.class);
-    }
-
-    @Override
-    protected void updateGtidSet(Event event) {
-        synchronized (gtidSetAccessLock) {
-            if (gtidSet == null) {
-                return;
-            }
-        }
-        EventHeader eventHeader = event.getHeader();
-        switch(eventHeader.getEventType()) {
-            case MARIADB_GTID:
-                MariadbGtidEventData mariadbGtidEventData = (MariadbGtidEventData) EventDeserializer.EventDataWrapper.internal(event.getData());
-                mariadbGtidEventData.setServerId(eventHeader.getServerId());
-                gtid = mariadbGtidEventData.toString();
-                break;
-            case MARIADB_GTID_LIST:
-                MariadbGtidListEventData mariadbGtidListEventData = (MariadbGtidListEventData) EventDeserializer.EventDataWrapper.internal(event.getData());
-                gtid = mariadbGtidListEventData.getMariaGTIDSet().toString();
-                break;
-            default:
-                super.updateGtidSet(event);
-        }
-    }
-
-    public boolean isUseSendAnnotateRowsEvent() {
-        return useSendAnnotateRowsEvent;
-    }
-
-    public void setUseSendAnnotateRowsEvent(boolean useSendAnnotateRowsEvent) {
-        this.useSendAnnotateRowsEvent = useSendAnnotateRowsEvent;
-    }
-}
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/MariadbGtidSet.java b/src/main/java/com/github/shyiko/mysql/binlog/MariadbGtidSet.java
index c86ee54a..49fede51 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/MariadbGtidSet.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/MariadbGtidSet.java
@@ -1,6 +1,7 @@
 package com.github.shyiko.mysql.binlog;
 
 import java.util.*;
+import java.util.regex.Pattern;
 
 /**
  * Mariadb Global Transaction ID
@@ -27,6 +28,12 @@ public MariadbGtidSet(String gtidSet) {
         }
     }
 
+    static Pattern MARIA_GTID_PATTERN = Pattern.compile("^\\d+-\\d+-\\d+");
+
+    public static boolean isMariaGtidSet(String gtidSet) {
+        return MARIA_GTID_PATTERN.matcher(gtidSet).find();
+    }
+
     @Override
     public String toString() {
         StringBuilder sb = new StringBuilder();
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClientIntegrationTest.java
index 6cf81817..e35a1d6f 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClientIntegrationTest.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClientIntegrationTest.java
@@ -19,6 +19,18 @@
  * @author Winger
  */
 public class MariadbBinaryLogClientIntegrationTest extends AbstractIntegrationTest {
+    @Override
+    protected MysqlOnetimeServerOptions getOptions() {
+        MysqlOnetimeServerOptions options = super.getOptions();
+        if ( options.extraParams == null )
+            options.extraParams = "";
+        else
+            options.extraParams += " ";
+
+        options.extraParams += "--binlog-annotate-row-events";
+        return options;
+    }
+
     @BeforeMethod
     public void checkMariaDB() throws Exception {
         if ( !mysqlVersion.isMaria )
@@ -49,7 +61,7 @@ public void execute(ResultSet rs) throws SQLException {
         });
 
         CountDownEventListener eventListener;
-        MariadbBinaryLogClient client = new MariadbBinaryLogClient(master.hostname(), master.port(), master.username(), master.password());
+        BinaryLogClient client = new BinaryLogClient(master.hostname(), master.port(), master.username(), master.password());
         client.setGtidSet(currentGtidPos[0]);
         client.setUseSendAnnotateRowsEvent(true);
 
@@ -71,13 +83,12 @@ public void execute(Statement statement) throws SQLException {
 
         try {
             eventListener.reset();
-            client.connect();
+            client.connect(5000);
 
             eventListener.waitFor(MariadbGtidEventData.class, 1, TimeUnit.SECONDS.toMillis(4));
             String gtidSet = client.getGtidSet();
             assertNotNull(gtidSet);
 
-            eventListener.reset();
             eventListener.waitFor(AnnotateRowsEventData.class, 1, TimeUnit.SECONDS.toMillis(4));
             gtidSet = client.getGtidSet();
             assertNotEquals(currentGtidPos[0], gtidSet);

From 9e1926772f35e7fbe0b8efeddbc7dae27c18cf28 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Sun, 14 Aug 2022 04:17:50 -0700
Subject: [PATCH 161/217] update README

---
 README.md | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 53aa87d2..761d17d3 100644
--- a/README.md
+++ b/README.md
@@ -84,13 +84,15 @@ kick off from a specific filename or position, use `client.setBinlogFilename(fil
 > `client.connect()` is blocking (meaning that client will listen for events in the current thread).
 `client.connect(timeout)`, on the other hand, spawns a separate thread.
 
-> Note difference between MariaDB and MySQL
-```
-BinaryLogClient client = new MariadbBinaryLogClient("hostname", 3306, "username", "password");
-// ... as same as BinaryLogClient
-```
-> `client.setGtidSet(gtid)` meaning that client kick off from a specific gtid, MariaDB also support.
-> `client.setUseSendAnnotateRowsEvent(true)` meaning that client will send annotate rows events(describe the query which caused the row event), and 'false' by default
+
+#### MariaDB
+
+The stock BinaryLogClient works out of the box with MariaDB but there's two differences;
+
+One, MariaDB's GTIDs are different.  They're still strings but parse differently.
+Two, Maria can send the ANNOTATE_ROWS events which allows you to recover the SQL used to generate rows in row-based replication.
+
+See https://mariadb.com/kb/en/annotate_rows_log_event/ and `client.setUseSendAnnotateRowsEvent(true)` 
 
 #### Controlling event deserialization
 

From 420d192ad96cd9528bd282c1a337e6140d7452ef Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Sun, 14 Aug 2022 13:55:33 -0700
Subject: [PATCH 162/217] checkpoint

---
 pom.xml                                                    | 2 +-
 .../com/github/shyiko/mysql/binlog/BinaryLogClient.java    | 2 +-
 src/main/java/com/github/shyiko/mysql/binlog/GtidSet.java  | 7 +++++++
 .../binlog/MariadbBinaryLogClientIntegrationTest.java      | 3 +++
 4 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 5c5806e3..68635492 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.26.1
+    0.27.0-PRE1
 
     mysql-binlog-connector-java
     MySQL Binary Log connector
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index 3591674b..bd21d883 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -784,7 +784,6 @@ private void requestBinaryLogStreamMysql(long serverId) throws IOException {
             }
         }
         channel.write(dumpBinaryLogCommand);
-
     }
 
     protected void requestBinaryLogStreamMaria(long serverId) throws IOException {
@@ -793,6 +792,7 @@ protected void requestBinaryLogStreamMaria(long serverId) throws IOException {
             if (this.gtidEnabled) {
                 channel.write(new QueryCommand("SET @mariadb_slave_capability=4"));
                 checkError(channel.read());
+                logger.info(gtidSet.toString());
                 channel.write(new QueryCommand("SET @slave_connect_state = '" + gtidSet.toString() + "'"));
                 checkError(channel.read());
                 channel.write(new QueryCommand("SET @slave_gtid_strict_mode = 0"));
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/GtidSet.java b/src/main/java/com/github/shyiko/mysql/binlog/GtidSet.java
index 55d619e1..d1d70f8d 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/GtidSet.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/GtidSet.java
@@ -40,6 +40,13 @@ public class GtidSet {
 
     private final Map map = new LinkedHashMap();
 
+    public static GtidSet parse(String gtidStr) {
+        if ( MariadbGtidSet.isMariaGtidSet(gtidStr) ) {
+            return new MariadbGtidSet(gtidStr);
+        } else {
+            return new GtidSet(gtidStr);
+        }
+    }
     /**
      * @param gtidSet gtid set comprised of closed intervals (like MySQL's executed_gtid_set).
      */
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClientIntegrationTest.java
index e35a1d6f..c021ec47 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClientIntegrationTest.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/MariadbBinaryLogClientIntegrationTest.java
@@ -22,6 +22,9 @@ public class MariadbBinaryLogClientIntegrationTest extends AbstractIntegrationTe
     @Override
     protected MysqlOnetimeServerOptions getOptions() {
         MysqlOnetimeServerOptions options = super.getOptions();
+        if ( !mysqlVersion.isMaria )
+            return options;
+
         if ( options.extraParams == null )
             options.extraParams = "";
         else

From 669087892cd217fbbfc447b02af4b4ebfff38aa8 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Sun, 21 Aug 2022 21:25:44 -0700
Subject: [PATCH 163/217] pre-3

This resolves the current/historical split and parses the flags
---
 pom.xml                                       |  2 +-
 .../github/shyiko/mysql/binlog/GtidSet.java   |  4 +
 .../shyiko/mysql/binlog/MariadbGtidSet.java   | 87 +++++++++++++++++--
 .../binlog/event/MariadbGtidEventData.java    | 16 ++++
 .../MariadbGtidEventDataDeserializer.java     |  1 +
 5 files changed, 101 insertions(+), 9 deletions(-)

diff --git a/pom.xml b/pom.xml
index 68635492..3e8a149e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.27.0-PRE1
+    0.27.0-PRE3
 
     mysql-binlog-connector-java
     MySQL Binary Log connector
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/GtidSet.java b/src/main/java/com/github/shyiko/mysql/binlog/GtidSet.java
index d1d70f8d..fb68b54c 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/GtidSet.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/GtidSet.java
@@ -167,6 +167,10 @@ public String toString() {
         return join(gtids, ",");
     }
 
+    public String toSeenString() {
+        return this.toString();
+    }
+
     private static String join(Collection o, String delimiter) {
         if (o.isEmpty()) {
             return "";
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/MariadbGtidSet.java b/src/main/java/com/github/shyiko/mysql/binlog/MariadbGtidSet.java
index 49fede51..4a0bf453 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/MariadbGtidSet.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/MariadbGtidSet.java
@@ -10,20 +10,36 @@
  * @see GTID for the original doc
  */
 public class MariadbGtidSet extends GtidSet {
+    /*
+        we keep two maps; one of them contains the current GTID position for
+        each domain.  The other contains all the "seen" GTID positions for each
+        domain and can be used to compare against another gtid postion.
+     */
+    protected Map positionMap = new HashMap<>();
 
-    private Map map = new HashMap<>();
+    protected Map> seenMap = new LinkedHashMap<>();
 
     public MariadbGtidSet() {
         super(null); //
     }
 
+    /**
+     * Initialize a new MariaDB gtid set from a string, like:
+     * 0-1-24,0-555555-9709
+     * DOMAIN_ID-SERVER_ID-SEQUENCE[,DOMAIN_ID-SERVER_ID-SEQUENCE]
+     *
+     * note that for duplicate domain ids it's "last one wins" for the current position
+     * @param gtidSet a string representing the gtid set.
+     */
     public MariadbGtidSet(String gtidSet) {
         super(null);
         if (gtidSet != null && gtidSet.length() > 0) {
             String[] gtids = gtidSet.replaceAll("\n", "").split(",");
             for (String gtid : gtids) {
                 MariaGtid mariaGtid = MariaGtid.parse(gtid);
-                map.put(mariaGtid.getDomainId(), mariaGtid);
+
+                positionMap.put(mariaGtid.getDomainId(), mariaGtid);
+                addToSeenSet(mariaGtid);
             }
         }
     }
@@ -34,10 +50,20 @@ public static boolean isMariaGtidSet(String gtidSet) {
         return MARIA_GTID_PATTERN.matcher(gtidSet).find();
     }
 
+    private void addToSeenSet(MariaGtid gtid) {
+        if ( !this.seenMap.containsKey(gtid.domainId) ) {
+            this.seenMap.put(gtid.domainId, new LinkedHashMap<>());
+        }
+
+        LinkedHashMap domainMap = this.seenMap.get(gtid.domainId);
+        domainMap.put(gtid.serverId, gtid);
+    }
+
+
     @Override
     public String toString() {
         StringBuilder sb = new StringBuilder();
-        for (MariaGtid gtid : map.values()) {
+        for (MariaGtid gtid : positionMap.values()) {
             if (sb.length() > 0) {
                 sb.append(",");
             }
@@ -46,6 +72,21 @@ public String toString() {
         return sb.toString();
     }
 
+    @Override
+    public String toSeenString() {
+        StringBuilder sb = new StringBuilder();
+        for (Long domainID : seenMap.keySet()) {
+            for( MariaGtid gtid: seenMap.get(domainID).values() ) {
+                if (sb.length() > 0) {
+                    sb.append(",");
+                }
+
+                sb.append(gtid.toString());
+            }
+        }
+        return sb.toString();
+    }
+
     @Override
     public Collection getUUIDSets() {
         throw new UnsupportedOperationException("Mariadb gtid not support this method");
@@ -64,22 +105,52 @@ public UUIDSet putUUIDSet(UUIDSet uuidSet) {
     @Override
     public boolean add(String gtid) {
         MariaGtid mariaGtid = MariaGtid.parse(gtid);
-        map.put(mariaGtid.getDomainId(), mariaGtid);
+        add(mariaGtid);
         return true;
     }
 
     public void add(MariaGtid gtid) {
-        map.put(gtid.getDomainId(), gtid);
+        positionMap.put(gtid.getDomainId(), gtid);
+        addToSeenSet(gtid);
     }
 
+    /*
+        we're trying to ask "is this position behind the other position?"
+        - if we have a domain that the other doesn't, we're probably "ahead".
+        - the inverse is true too
+     */
     @Override
     public boolean isContainedWithin(GtidSet other) {
-        throw new UnsupportedOperationException("Mariadb gtid not support this method");
+        if (!(other instanceof MariadbGtidSet))
+            return false;
+
+        MariadbGtidSet o = (MariadbGtidSet) other;
+
+        for ( Long domainID : this.seenMap.keySet() ) {
+            if ( !o.seenMap.containsKey(domainID) ) {
+                return false;
+            }
+
+            LinkedHashMap thisDomainMap = this.seenMap.get(domainID);
+            LinkedHashMap otherDomainMap = o.seenMap.get(domainID);
+
+            for ( Long serverID : thisDomainMap.keySet() ) {
+                if ( !otherDomainMap.containsKey(serverID)) {
+                    return false;
+                }
+
+                MariaGtid thisGtid = thisDomainMap.get(serverID);
+                MariaGtid otherGtid = otherDomainMap.get(serverID);
+                if ( thisGtid.sequence >= otherGtid.sequence )
+                    return false;
+            }
+        }
+        return true;
     }
 
     @Override
     public int hashCode() {
-        return map.keySet().hashCode();
+        return this.seenMap.keySet().hashCode();
     }
 
     @Override
@@ -89,7 +160,7 @@ public boolean equals(Object obj) {
         }
         if (obj instanceof MariadbGtidSet) {
             MariadbGtidSet that = (MariadbGtidSet) obj;
-            return this.map.equals(that.map);
+            return this.positionMap.equals(that.positionMap);
         }
         return false;
     }
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/MariadbGtidEventData.java b/src/main/java/com/github/shyiko/mysql/binlog/event/MariadbGtidEventData.java
index 54838d03..43f0c55c 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/MariadbGtidEventData.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/MariadbGtidEventData.java
@@ -7,11 +7,19 @@
  * @see GTID_EVENT for the original doc
  */
 public class MariadbGtidEventData implements EventData {
+    public static int FL_STANDALONE = 1;
+    public static int FL_GROUP_COMMIT_ID = 2;
+    public static int FL_TRANSACTIONAL = 4;
+    public static int FL_ALLOW_PARALLEL = 8;
+    public static int FL_WAITED = 16;
+    public static int FL_DDL = 32;
 
     private long sequence;
     private long domainId;
     private long serverId;
 
+    private int flags;
+
     public long getSequence() {
         return sequence;
     }
@@ -36,6 +44,14 @@ public void setServerId(long serverId) {
         this.serverId = serverId;
     }
 
+    public int getFlags() {
+        return flags;
+    }
+
+    public void setFlags(int flags) {
+        this.flags = flags;
+    }
+
     @Override
     public String toString() {
         return domainId + "-" + serverId + "-" + sequence;
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidEventDataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidEventDataDeserializer.java
index 4eb45f1f..4f619b54 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidEventDataDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/MariadbGtidEventDataDeserializer.java
@@ -27,6 +27,7 @@ public MariadbGtidEventData deserialize(ByteArrayInputStream inputStream) throws
         MariadbGtidEventData event = new MariadbGtidEventData();
         event.setSequence(inputStream.readLong(8));
         event.setDomainId(inputStream.readInteger(4));
+        event.setFlags(inputStream.readInteger(1));
         // Flags ignore
         return event;
     }

From 4c83012ffe981d4705c1d8a98e00e72925aef316 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Sat, 27 Aug 2022 06:17:37 -0700
Subject: [PATCH 164/217] 0.27.0

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 3e8a149e..161e6243 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.27.0-PRE3
+    0.27.0
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From c6ab6e02ee726497d5601a22359ab96ad962422a Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Sat, 27 Aug 2022 06:20:43 -0700
Subject: [PATCH 165/217] update CHANGELOG

---
 CHANGELOG.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index aebdf3d8..d1245208 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,12 @@
 # Changelog
+## [0.27.0](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.26.1...0.27.0) - 2022-09-27
+
+- Add "official" MariaDB support.  @wingerx started this worked and @ivapiv bugged me until it was
+  done, thanks all.  This includes:
+- MariaDB GTID support
+- support for the ANNOTATE_ROWS_EVENTS
+- MariaDB detection in getMariaDB()
+
 ## [0.26.1](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.26.0...0.26.1) - 2022-07-18
 
 - fix deadlock with disconnect and keepalive thread.

From 2e4686f2080a01bfdc954e202100d6f3eeb9813e Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Sat, 27 Aug 2022 06:41:21 -0700
Subject: [PATCH 166/217] fix test speed

---
 .../github/shyiko/mysql/binlog/CountDownEventListener.java   | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/test/java/com/github/shyiko/mysql/binlog/CountDownEventListener.java b/src/test/java/com/github/shyiko/mysql/binlog/CountDownEventListener.java
index 6256efe6..6ceaa7d0 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/CountDownEventListener.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/CountDownEventListener.java
@@ -91,9 +91,10 @@ public void waitForAtLeast(EventType eventType, int numberOfEvents, long timeout
         AtomicInteger counter = getCounter(countersByType, eventType);
 
         synchronized (counter) {
-            if (counter.get() < numberOfEvents) {
+            counter.set(counter.get() - numberOfEvents);
+            if (counter.get() < 0) {
                 counter.wait(timeoutInMilliseconds);
-                if (counter.get() < numberOfEvents) {
+                if (counter.get() < 0) {
                     throw new TimeoutException("Received " + (numberOfEvents + counter.get()) + " " +
                         eventType.name() + " event(s) instead of expected " + numberOfEvents);
                 }

From eaa9e568931db9452e342715523550c40b108152 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Sun, 28 Aug 2022 04:57:18 -0700
Subject: [PATCH 167/217] correct mariadb_slave_capability value

I think it's fair to say we understand ANNOTATE_ROWS_EVENT, but none of
the other stuff represented by this variable is true.
---
 .../com/github/shyiko/mysql/binlog/BinaryLogClient.java  | 9 +++++++--
 .../mysql/binlog/BinaryLogClientIntegrationTest.java     | 2 +-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index bd21d883..a0f2f430 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -788,10 +788,15 @@ private void requestBinaryLogStreamMysql(long serverId) throws IOException {
 
     protected void requestBinaryLogStreamMaria(long serverId) throws IOException {
         Command dumpBinaryLogCommand;
+
+        /*
+            https://jira.mariadb.org/browse/MDEV-225
+         */
+        channel.write(new QueryCommand("SET @mariadb_slave_capability=1"));
+        checkError(channel.read());
+
         synchronized (gtidSetAccessLock) {
             if (this.gtidEnabled) {
-                channel.write(new QueryCommand("SET @mariadb_slave_capability=4"));
-                checkError(channel.read());
                 logger.info(gtidSet.toString());
                 channel.write(new QueryCommand("SET @slave_connect_state = '" + gtidSet.toString() + "'"));
                 checkError(channel.read());
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java
index cba20577..f664ff63 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java
@@ -806,7 +806,7 @@ public void execute(Statement statement) throws SQLException {
                         statement.execute("flush logs");
                     }
                 });
-                eventListener.waitFor(EventType.QUERY, 1, DEFAULT_TIMEOUT);
+                eventListener.waitForAtLeast(EventType.QUERY, 1, DEFAULT_TIMEOUT);
                 eventListener.waitFor(EventType.ROTATE, 3, DEFAULT_TIMEOUT); /* 2 with timestamp 0 */
                 eventListener.waitFor(ByteArrayEventData.class, 5, DEFAULT_TIMEOUT);
             } finally {

From faed36015d9ec16abd846bbeb60f487af5e5bfa9 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Sun, 28 Aug 2022 14:26:41 -0700
Subject: [PATCH 168/217] update changelog

---
 CHANGELOG.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d1245208..abdc1308 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,8 @@
 # Changelog
+## [0.27.1](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.27.0...0.27.1) - 2022-09-28
+
+- fix a bug around the capability that we send maria
+
 ## [0.27.0](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.26.1...0.27.0) - 2022-09-27
 
 - Add "official" MariaDB support.  @wingerx started this worked and @ivapiv bugged me until it was

From 2774bedde7960ba20a66e0bf0699ac4f66f63825 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Wed, 31 Aug 2022 08:15:57 -0700
Subject: [PATCH 169/217] 0.27.1

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 161e6243..728a255e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.27.0
+    0.27.1
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From 561b18af3a8602ce7ea5e341a768c1e38b5e7f34 Mon Sep 17 00:00:00 2001
From: Ashley Cox 
Date: Tue, 6 Sep 2022 15:35:33 -0700
Subject: [PATCH 170/217] add client identity method

---
 .../java/com/github/shyiko/mysql/binlog/BinaryLogClient.java  | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index 1914c3d4..f413d9fa 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -218,6 +218,10 @@ public BinaryLogClient(String hostname, int port, String schema, String username
         this.password = password;
     }
 
+    public String fivetranClientIdentity() {
+        return "mysql-binlog-connector-java-osheroff";
+    }
+
     public boolean isBlocking() {
         return blocking;
     }

From d76c7eef687ece96608413e2db66027758e8aa8f Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Fri, 16 Sep 2022 05:53:41 -0700
Subject: [PATCH 171/217] implement a better regex for detecting mariadb gtid
 strs

---
 .../com/github/shyiko/mysql/binlog/MariadbGtidSet.java |  6 +++++-
 .../com/github/shyiko/mysql/binlog/GtidSetTest.java    |  1 +
 .../github/shyiko/mysql/binlog/MariadbGtidSetTest.java | 10 ++++++++++
 3 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/MariadbGtidSet.java b/src/main/java/com/github/shyiko/mysql/binlog/MariadbGtidSet.java
index 4a0bf453..224030b8 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/MariadbGtidSet.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/MariadbGtidSet.java
@@ -44,7 +44,11 @@ public MariadbGtidSet(String gtidSet) {
         }
     }
 
-    static Pattern MARIA_GTID_PATTERN = Pattern.compile("^\\d+-\\d+-\\d+");
+    static String threeDashes = "\\d{1,10}-\\d{1,10}-\\d{1,20}";
+
+    static Pattern MARIA_GTID_PATTERN = Pattern.compile(
+        "^" + threeDashes + "(\\s*,\\s*" + threeDashes + ")*$"
+    );
 
     public static boolean isMariaGtidSet(String gtidSet) {
         return MARIA_GTID_PATTERN.matcher(gtidSet).find();
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/GtidSetTest.java b/src/test/java/com/github/shyiko/mysql/binlog/GtidSetTest.java
index 79318aa6..cfb9f49e 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/GtidSetTest.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/GtidSetTest.java
@@ -164,4 +164,5 @@ public void testPutUUIDSet() {
         assertEquals(gtidSet, gtidSet2);
     }
 
+
 }
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/MariadbGtidSetTest.java b/src/test/java/com/github/shyiko/mysql/binlog/MariadbGtidSetTest.java
index 71d5753a..0c282405 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/MariadbGtidSetTest.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/MariadbGtidSetTest.java
@@ -2,6 +2,8 @@
 
 import org.testng.annotations.Test;
 
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertNotEquals;
 
@@ -33,4 +35,12 @@ public void testEquals() {
         assertEquals(new MariadbGtidSet(""), new MariadbGtidSet(""));
         assertEquals(new MariadbGtidSet("0-0-7404"), new MariadbGtidSet("0-0-7404"));
     }
+
+    @Test
+    public void testMatcher() {
+        assertTrue(MariadbGtidSet.isMariaGtidSet("0-0-3323"));
+        assertTrue(MariadbGtidSet.isMariaGtidSet("0-0-3323,4-33-12342134,444-33-13412341233"));
+        assertTrue(MariadbGtidSet.isMariaGtidSet("0-0-3323, 4-33-12342134, 444-33-13412341233"));
+        assertFalse(MariadbGtidSet.isMariaGtidSet("07212070-4330-3bc8-8a3a-01e34be47bc3:1-141692942,a0c4a949-fae8-30f3-a4d2-fee56a1a9307:1-1427643460,a16ef643-1d4a-3fd9-a86e-1adeb836eb2d:1-1411988930,b0d822f4-5a84-30d3-a929-61f64740d7ac:1-59364"));
+    }
 }

From 1c6b8f90f60e611ee5c87468fec73da1c5d018ca Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Fri, 16 Sep 2022 06:08:41 -0700
Subject: [PATCH 172/217] 0.27.2

---
 CHANGELOG.md | 8 ++++++--
 pom.xml      | 2 +-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index abdc1308..f91fbfab 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,9 +1,13 @@
 # Changelog
-## [0.27.1](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.27.0...0.27.1) - 2022-09-28
+## [0.27.2](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.27.2...0.27.1) - 2022-09-16
+
+- Fix the maria gtid detection regex to avoid erroneously detecting mysql gtids as maria
+
+## [0.27.1](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.27.0...0.27.1) - 2022-08-28
 
 - fix a bug around the capability that we send maria
 
-## [0.27.0](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.26.1...0.27.0) - 2022-09-27
+## [0.27.0](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.26.1...0.27.0) - 2022-08-27
 
 - Add "official" MariaDB support.  @wingerx started this worked and @ivapiv bugged me until it was
   done, thanks all.  This includes:
diff --git a/pom.xml b/pom.xml
index 728a255e..9a1fa643 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.27.1
+    0.27.2
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From 2ea9f6c26e3dd60d23f37282aa4204653d3cb04a Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Sun, 25 Sep 2022 22:29:45 -0700
Subject: [PATCH 173/217] pass useSendAnnotate etc through non-gtid commands

---
 .../java/com/github/shyiko/mysql/binlog/BinaryLogClient.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index a0f2f430..e349d20a 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -806,7 +806,7 @@ protected void requestBinaryLogStreamMaria(long serverId) throws IOException {
                 checkError(channel.read());
                 dumpBinaryLogCommand = new DumpBinaryLogCommand(serverId, "", 0L, isUseSendAnnotateRowsEvent());
             } else {
-                dumpBinaryLogCommand = new DumpBinaryLogCommand(serverId, binlogFilename, binlogPosition);
+                dumpBinaryLogCommand = new DumpBinaryLogCommand(serverId, binlogFilename, binlogPosition, isUseSendAnnotateRowsEvent());
             }
         }
         channel.write(dumpBinaryLogCommand);

From 8bf89739952ec1951758e1b6cc08f8175ecc5782 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Sun, 25 Sep 2022 22:31:01 -0700
Subject: [PATCH 174/217] 0.27.3

---
 CHANGELOG.md | 5 +++++
 pom.xml      | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f91fbfab..10472eae 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,8 @@
+# Changelog
+## [0.27.3](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.27.3...0.27.2) - 2022-09-25
+
+- pass use-annotate-rows through non-gtid mariadb connections
+
 # Changelog
 ## [0.27.2](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.27.2...0.27.1) - 2022-09-16
 
diff --git a/pom.xml b/pom.xml
index 9a1fa643..5fc05ea1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.27.2
+    0.27.3
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From 4eb294a6d64b69271fa615375367e6e607650b95 Mon Sep 17 00:00:00 2001
From: Jukka Pihl 
Date: Sun, 2 Oct 2022 17:19:14 +0300
Subject: [PATCH 175/217] Add MariaDB BINLOG_CHECKPOINT(161) event.

---
 .../event/BinlogCheckpointEventData.java      | 43 +++++++++++++++++++
 .../shyiko/mysql/binlog/event/EventType.java  |  1 +
 ...BinlogCheckpointEventDataDeserializer.java | 36 ++++++++++++++++
 .../deserialization/EventDeserializer.java    |  2 +
 4 files changed, 82 insertions(+)
 create mode 100644 src/main/java/com/github/shyiko/mysql/binlog/event/BinlogCheckpointEventData.java
 create mode 100644 src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/BinlogCheckpointEventDataDeserializer.java

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/BinlogCheckpointEventData.java b/src/main/java/com/github/shyiko/mysql/binlog/event/BinlogCheckpointEventData.java
new file mode 100644
index 00000000..bd1fceed
--- /dev/null
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/BinlogCheckpointEventData.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2015 Stanley Shyiko
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.github.shyiko.mysql.binlog.event;
+
+/**
+ * @author Stanley Shyiko
+ */
+
+public class BinlogCheckpointEventData implements EventData {
+
+    private String logFileName;
+
+    public void setLogFileName(String logFileName) {
+        this.logFileName = logFileName;
+    }
+
+    public String getLogFileName() {
+        return this.logFileName;
+    }
+
+    @Override
+    public String toString() {
+        final StringBuilder sb = new StringBuilder();
+        sb.append("BinlogCheckpointEventData");
+        sb.append("{logFileName=").append(logFileName);
+        sb.append('}');
+        return sb.toString();
+    }
+
+}
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
index 80ca58e0..77360bac 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
@@ -212,6 +212,7 @@ public enum EventType {
      * @see Replication Protocol for the original doc.
      */
     ANNOTATE_ROWS(160), //
+    BINLOG_CHECKPOINT(161),
     MARIADB_GTID(162),
     MARIADB_GTID_LIST(163);
 
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/BinlogCheckpointEventDataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/BinlogCheckpointEventDataDeserializer.java
new file mode 100644
index 00000000..ae456b4d
--- /dev/null
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/BinlogCheckpointEventDataDeserializer.java
@@ -0,0 +1,36 @@
+
+/*
+ * Copyright 2013 Stanley Shyiko
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.github.shyiko.mysql.binlog.event.deserialization;
+
+import com.github.shyiko.mysql.binlog.event.BinlogCheckpointEventData;
+import com.github.shyiko.mysql.binlog.io.ByteArrayInputStream;
+
+import java.io.IOException;
+
+/**
+ * @author Stanley Shyiko
+ */
+public class BinlogCheckpointEventDataDeserializer implements EventDataDeserializer {
+
+    @Override
+    public BinlogCheckpointEventData deserialize(ByteArrayInputStream inputStream) throws IOException {
+        BinlogCheckpointEventData eventData = new BinlogCheckpointEventData();
+        int length = inputStream.readInteger(4);
+        eventData.setLogFileName(inputStream.readString(length));
+        return eventData;
+    }
+}
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java
index 47c94c47..81dea256 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventDeserializer.java
@@ -125,6 +125,8 @@ private void registerDefaultEventDataDeserializers() {
             new AnnotateRowsEventDataDeserializer());
         eventDataDeserializers.put(EventType.MARIADB_GTID,
             new MariadbGtidEventDataDeserializer());
+        eventDataDeserializers.put(EventType.BINLOG_CHECKPOINT,
+            new BinlogCheckpointEventDataDeserializer());
         eventDataDeserializers.put(EventType.MARIADB_GTID_LIST,
             new MariadbGtidListEventDataDeserializer());
         eventDataDeserializers.put(EventType.TRANSACTION_PAYLOAD,

From 6dec4fea3d7969e62b6aa4baea9e38692d904a13 Mon Sep 17 00:00:00 2001
From: UcanInfosec <107478475+UcanInfosec@users.noreply.github.com>
Date: Sun, 30 Oct 2022 09:44:59 -0400
Subject: [PATCH 176/217] Update Databind to Remediate CVE

---
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index 5fc05ea1..f6b3822b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -68,13 +68,13 @@
         
             com.fasterxml.jackson.core
             jackson-core
-            2.9.10
+            2.13.4
             test
         
         
             com.fasterxml.jackson.core
             jackson-databind
-            2.9.10.3
+            2.13.4.2
             test
         
         

From 7057e6a6568bedbc253433c50379f904ee31ece2 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Tue, 1 Nov 2022 15:23:23 -0700
Subject: [PATCH 177/217] set maria slave capability to 4

---
 .../java/com/github/shyiko/mysql/binlog/BinaryLogClient.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index e349d20a..d36b05e8 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -792,7 +792,7 @@ protected void requestBinaryLogStreamMaria(long serverId) throws IOException {
         /*
             https://jira.mariadb.org/browse/MDEV-225
          */
-        channel.write(new QueryCommand("SET @mariadb_slave_capability=1"));
+        channel.write(new QueryCommand("SET @mariadb_slave_capability=4"));
         checkError(channel.read());
 
         synchronized (gtidSetAccessLock) {

From 48dd5f8e00989e1da62995e2b7bcfa06e8d48ef5 Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Tue, 1 Nov 2022 15:24:32 -0700
Subject: [PATCH 178/217] 0.27.4

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 5fc05ea1..0ca3e87c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.27.3
+    0.27.4
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From 93b257047b155265a9d572862903beaf33363c6d Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Tue, 1 Nov 2022 15:25:41 -0700
Subject: [PATCH 179/217] update changelog

---
 CHANGELOG.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 10472eae..03fb7bf4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,8 @@
 # Changelog
+## [0.27.4](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.27.4...0.27.3) - 2022-11-01
+
+- move mariadb_slave_capability back to 4
+
 ## [0.27.3](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.27.3...0.27.2) - 2022-09-25
 
 - pass use-annotate-rows through non-gtid mariadb connections

From f1f0e9c3a8dac94d1fcba0200762fc387c1b83dc Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Tue, 1 Nov 2022 18:25:32 -0700
Subject: [PATCH 180/217] 0.27.5

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 29506997..f090d9d9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.27.4
+    0.27.5
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From ccca3973c3d1e07fb2a3bac1c24bb4b5d32d1d8c Mon Sep 17 00:00:00 2001
From: Ben Osheroff 
Date: Tue, 1 Nov 2022 18:26:15 -0700
Subject: [PATCH 181/217] update CHANGELOG

---
 CHANGELOG.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 03fb7bf4..c845d140 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,8 @@
 # Changelog
+## [0.27.5](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.27.5...0.27.4) - 2022-11-01
+
+- add mariadb BINLOG_CHECKPOINT event 
+
 ## [0.27.4](https://github.com/osheroff/mysql-binlog-connector-java/compare/0.27.4...0.27.3) - 2022-11-01
 
 - move mariadb_slave_capability back to 4

From d711ecbe77f978ab723bd5df188974f37fda9b26 Mon Sep 17 00:00:00 2001
From: Meel Velliste 
Date: Wed, 8 Jul 2015 01:11:13 -0700
Subject: [PATCH 182/217] Updated gitignore

---
 .gitignore | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 88ff3803..d2d768d5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,4 @@
-.idea
+*.idea
 *.iml
 .DS_Store
 target

From c09f4444b8ca219f9082ea644a999203bc6fc485 Mon Sep 17 00:00:00 2001
From: Meel Velliste 
Date: Thu, 30 Jul 2015 14:44:57 -0700
Subject: [PATCH 183/217] Fixed '0000-00-00 00:00:00' dates -> null in binlog

---
 .../event/deserialization/AbstractRowsEventDataDeserializer.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
index 89971365..df40459c 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
@@ -24,6 +24,7 @@
 import java.math.BigDecimal;
 import java.util.BitSet;
 import java.util.Calendar;
+import java.util.Date;
 import java.util.Map;
 import java.util.TimeZone;
 

From e10fa850266fc9ba3421c3c572f457ceccb7e03e Mon Sep 17 00:00:00 2001
From: Meel Velliste 
Date: Thu, 29 Oct 2015 21:18:32 -0700
Subject: [PATCH 184/217] Exceptions thrown in binlog processing will now cause
 whole update to fail instead of just warning in the log

---
 .../java/com/github/shyiko/mysql/binlog/BinaryLogClient.java | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index d36b05e8..52bf37e7 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -1245,9 +1245,8 @@ private void notifyEventListeners(Event event) {
             try {
                 eventListener.onEvent(event);
             } catch (Exception e) {
-                if (logger.isLoggable(Level.WARNING)) {
-                    logger.log(Level.WARNING, eventListener + " choked on " + event, e);
-                }
+                throw new RuntimeException("Binlog event listener " + eventListener +
+                    " choked on " + event, e);
             }
         }
     }

From e1a873df8767e3cb23b2574c700c5da09fa6bd8b Mon Sep 17 00:00:00 2001
From: Georgie 
Date: Thu, 3 Dec 2015 19:07:21 -0800
Subject: [PATCH 185/217] Configure java.util.logging

---
 .../java/com/github/shyiko/mysql/binlog/BinaryLogClient.java    | 2 +-
 .../shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java     | 2 +-
 .../java/com/github/shyiko/mysql/binlog/TCPReverseProxy.java    | 2 +-
 .../java/com/github/shyiko/mysql/binlog/TraceEventListener.java | 2 +-
 .../com/github/shyiko/mysql/binlog/TraceLifecycleListener.java  | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index 52bf37e7..0a3bfea0 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -124,7 +124,7 @@ public X509Certificate[] getAcceptedIssuers() {
     // https://dev.mysql.com/doc/internals/en/sending-more-than-16mbyte.html
     private static final int MAX_PACKET_LENGTH = 16777215;
 
-    private final Logger logger = Logger.getLogger(getClass().getName());
+    private final Logger logger = Logger.getLogger("donkey");
 
     private final String hostname;
     private final int port;
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java
index f664ff63..f720e4ee 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java
@@ -96,7 +96,7 @@ public class BinaryLogClientIntegrationTest extends AbstractIntegrationTest {
 
     protected static final long DEFAULT_TIMEOUT = TimeUnit.SECONDS.toMillis(3);
 
-    private final Logger logger = Logger.getLogger(getClass().getSimpleName());
+    private final Logger logger = Logger.getLogger("donkey");
 
     {
         logger.setLevel(Level.FINEST);
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/TCPReverseProxy.java b/src/test/java/com/github/shyiko/mysql/binlog/TCPReverseProxy.java
index 883b3f25..0fe52e2c 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/TCPReverseProxy.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/TCPReverseProxy.java
@@ -36,7 +36,7 @@
  */
 public class TCPReverseProxy {
 
-    private final Logger logger = Logger.getLogger(getClass().getSimpleName());
+    private final Logger logger = Logger.getLogger("donkey");
 
     private final int port;
     private final String targetHost;
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/TraceEventListener.java b/src/test/java/com/github/shyiko/mysql/binlog/TraceEventListener.java
index 0eacd643..7ec1ed18 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/TraceEventListener.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/TraceEventListener.java
@@ -25,7 +25,7 @@
  */
 public class TraceEventListener implements BinaryLogClient.EventListener {
 
-    private final Logger logger = Logger.getLogger(getClass().getSimpleName());
+    private final Logger logger = Logger.getLogger("donkey");
 
     @Override
     public void onEvent(Event event) {
diff --git a/src/test/java/com/github/shyiko/mysql/binlog/TraceLifecycleListener.java b/src/test/java/com/github/shyiko/mysql/binlog/TraceLifecycleListener.java
index 6e3f5702..f4fde7e2 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/TraceLifecycleListener.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/TraceLifecycleListener.java
@@ -23,7 +23,7 @@
  */
 public class TraceLifecycleListener implements BinaryLogClient.LifecycleListener {
 
-    private final Logger logger = Logger.getLogger(getClass().getSimpleName());
+    private final Logger logger = Logger.getLogger("donkey");
 
     @Override
     public void onConnect(BinaryLogClient client) {

From 8b8c2f8f29c9c3254ab2e3b2822531afb3ef0fc3 Mon Sep 17 00:00:00 2001
From: Georgie 
Date: Wed, 9 Mar 2016 19:55:13 -0800
Subject: [PATCH 186/217] Deployment

---
 pom.xml | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/pom.xml b/pom.xml
index f090d9d9..f2524651 100644
--- a/pom.xml
+++ b/pom.xml
@@ -190,6 +190,13 @@
                 
             
         
+        
+            
+                org.springframework.build
+                aws-maven
+                5.0.0.RELEASE
+            
+        
     
     
         
@@ -205,4 +212,20 @@
         
     
 
+    
+    
+        
+            fivetran-maven-release
+            s3://fivetran-maven/public/release
+        
+    
+
+    
+        
+            aws-public-release
+            AWS Public Release Repository
+            https://s3.amazonaws.com/fivetran-maven/public/release
+        
+    
+
 

From 39997e63f63806c3360133e936e2fe7d155e9914 Mon Sep 17 00:00:00 2001
From: George Fraser 
Date: Mon, 2 Jan 2017 12:26:41 -0800
Subject: [PATCH 187/217] Ignore target/

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index d2d768d5..9c209dfb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,4 @@ target
 .settings
 .vagrant
 .*.sw*
+target/

From 1369c2f8271ae61f6670afab8bc863d1bfbf744d Mon Sep 17 00:00:00 2001
From: George Fraser 
Date: Mon, 2 Jan 2017 12:28:04 -0800
Subject: [PATCH 188/217] Latest version

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index f2524651..8c54941e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.27.5
+    0.7.3-0-FIVETRAN
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From 8487db5c200ab8ec79733b5ffba5603741faa621 Mon Sep 17 00:00:00 2001
From: George Fraser 
Date: Wed, 18 Jan 2017 14:05:49 -0800
Subject: [PATCH 189/217] Update version

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 8c54941e..f23a69a6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.7.3-0-FIVETRAN
+    0.7.3-1-FIVETRAN
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From f42e8accfdd8bb3a96b15e99c41eadaa7acb2893 Mon Sep 17 00:00:00 2001
From: Meel Velliste 
Date: Mon, 30 Jan 2017 17:36:25 -0800
Subject: [PATCH 190/217] Fixed top-level JSON null value

---
 .../binlog/event/deserialization/json/JsonBinary.java      | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
index 8a2b8d84..6a4dd48e 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
@@ -186,7 +186,12 @@ private static boolean isJSONString(byte[] bytes) {
      * @throws IOException if there is a problem reading or processing the binary representation
      */
     public static void parse(byte[] bytes, JsonFormatter formatter) throws IOException {
-        new JsonBinary(bytes).parse(formatter);
+        if (bytes.length == 0) {
+            // When the top-level value is a JSON "null", the value in java shows up as a non-null, empty byte array
+            formatter.valueNull();
+        } else {
+            new JsonBinary(bytes).parse(formatter);
+        }
     }
 
     private final ByteArrayInputStream reader;

From a8f584bc1f79ed164ec6362c53918d1be66fce4f Mon Sep 17 00:00:00 2001
From: Meel Velliste 
Date: Mon, 30 Jan 2017 17:37:22 -0800
Subject: [PATCH 191/217] Updated version

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index f23a69a6..6de6e45a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.7.3-1-FIVETRAN
+    0.7.3-2-FIVETRAN
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From 05ac2e534f66cb314b9b2f7ac2c439493c2d4ab0 Mon Sep 17 00:00:00 2001
From: Meel Velliste 
Date: Fri, 3 Feb 2017 13:56:23 -0800
Subject: [PATCH 192/217] Added json null test

---
 .../json/JsonBinaryValueIntegrationTest.java       | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java
index 0ef0cded..d5589ad0 100644
--- a/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java
+++ b/src/test/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java
@@ -37,6 +37,7 @@
 
 import java.io.IOException;
 import java.io.Serializable;
+import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.SQLSyntaxErrorException;
 import java.sql.Statement;
@@ -529,6 +530,19 @@ public void testScalarBinaryAsBase64() throws Exception {
         assertEquals(writeAndCaptureJSON("CAST(x'cafebabe' AS JSON)"), "\"yv66vg==\"");
     }
 
+    @Test
+    public void testJsonNull() throws Exception {
+        master.execute(new BinaryLogClientIntegrationTest.Callback() {
+            @Override
+            public void execute(Statement statement) throws SQLException {
+                ResultSet results = statement.executeQuery("SELECT version()");
+                results.next();
+                System.out.println("MySQL version = " + results.getString(1));
+            }
+        });
+        assertJSONMatchOriginal("null");
+    }
+
     private void assertJSONMatchOriginal(String value) throws Exception {
         assetJSONEquals(value, writeAndCaptureJSON("'" + value + "'"));
     }

From 48c75b3190fc3668eae6f4cb17294884e7e70a4d Mon Sep 17 00:00:00 2001
From: Meel Velliste 
Date: Tue, 7 Feb 2017 11:08:45 -0800
Subject: [PATCH 193/217] version++

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 6de6e45a..47df6718 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.7.3-2-FIVETRAN
+    0.7.3-3-FIVETRAN
 
     mysql-binlog-connector-java
     MySQL Binary Log connector

From 7bf4e4f8959a68bc5c9b03fa1a11f3b005e257fb Mon Sep 17 00:00:00 2001
From: George Fraser 
Date: Mon, 1 Oct 2018 23:24:49 -0700
Subject: [PATCH 194/217] TLSHostnameVerifier uses no longer available API

---
 pom.xml                                       |  2 +-
 .../shyiko/mysql/binlog/BinaryLogClient.java  |  9 ++--
 .../binlog/network/TLSHostnameVerifier.java   | 47 -------------------
 3 files changed, 5 insertions(+), 53 deletions(-)
 delete mode 100644 src/main/java/com/github/shyiko/mysql/binlog/network/TLSHostnameVerifier.java

diff --git a/pom.xml b/pom.xml
index 47df6718..d0665633 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
 
     com.zendesk
     mysql-binlog-connector-java
-    0.7.3-3-FIVETRAN
+    0.7.3-4-FIVETRAN
 
     mysql-binlog-connector-java
     MySQL Binary Log connector
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index 0a3bfea0..992d06d2 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -46,7 +46,6 @@
 import com.github.shyiko.mysql.binlog.network.SSLSocketFactory;
 import com.github.shyiko.mysql.binlog.network.ServerException;
 import com.github.shyiko.mysql.binlog.network.SocketFactory;
-import com.github.shyiko.mysql.binlog.network.TLSHostnameVerifier;
 import com.github.shyiko.mysql.binlog.network.protocol.ErrorPacket;
 import com.github.shyiko.mysql.binlog.network.protocol.GreetingPacket;
 import com.github.shyiko.mysql.binlog.network.protocol.Packet;
@@ -58,10 +57,6 @@
 import com.github.shyiko.mysql.binlog.network.protocol.command.PingCommand;
 import com.github.shyiko.mysql.binlog.network.protocol.command.QueryCommand;
 import com.github.shyiko.mysql.binlog.network.protocol.command.SSLRequestCommand;
-
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.X509TrustManager;
 import java.io.EOFException;
 import java.io.IOException;
 import java.net.InetSocketAddress;
@@ -88,6 +83,10 @@
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+
 
 /**
  * MySQL replication stream client.
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/TLSHostnameVerifier.java b/src/main/java/com/github/shyiko/mysql/binlog/network/TLSHostnameVerifier.java
deleted file mode 100644
index 6dd3aa43..00000000
--- a/src/main/java/com/github/shyiko/mysql/binlog/network/TLSHostnameVerifier.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright 2016 Stanley Shyiko
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.github.shyiko.mysql.binlog.network;
-
-import javax.net.ssl.HostnameVerifier;
-import javax.net.ssl.SSLException;
-import javax.net.ssl.SSLPeerUnverifiedException;
-import javax.net.ssl.SSLSession;
-import java.security.cert.Certificate;
-import java.security.cert.X509Certificate;
-
-/**
- * @author Stanley Shyiko
- */
-public class TLSHostnameVerifier implements HostnameVerifier {
-
-    public boolean verify(String hostname, SSLSession session) {
-        HostnameChecker checker = HostnameChecker.DEFAULT;
-        try {
-            Certificate[] peerCertificates = session.getPeerCertificates();
-            if (peerCertificates.length > 0 && peerCertificates[0] instanceof X509Certificate) {
-                X509Certificate peerCertificate = (X509Certificate) peerCertificates[0];
-                try {
-                    checker.check(hostname, peerCertificate);
-                    return true;
-                } catch (SSLException ignored) {
-                }
-            }
-        } catch (SSLPeerUnverifiedException ignored) {
-        }
-        return false;
-    }
-
-}

From c73f9ce7f2bb882cd7e78346f7686afae05128c2 Mon Sep 17 00:00:00 2001
From: glarwood 
Date: Thu, 4 Oct 2018 17:46:00 -0700
Subject: [PATCH 195/217] refactor(BinaryLogClient): make binlogChecksum
 related methods/fields protected

---
 .../java/com/github/shyiko/mysql/binlog/BinaryLogClient.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index 992d06d2..776dd91a 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -1012,7 +1012,7 @@ private void fetchBinlogFilenameAndPosition() throws IOException {
         binlogPosition = Long.parseLong(resultSetRow.getValue(1));
     }
 
-    private ChecksumType fetchBinlogChecksum() throws IOException {
+    protected ChecksumType fetchBinlogChecksum() throws IOException {
         channel.write(new QueryCommand("show global variables like 'binlog_checksum'"));
         ResultSetRowPacket[] resultSet = readResultSet();
         if (resultSet.length == 0) {
@@ -1187,7 +1187,7 @@ private void commitGtid() {
         }
     }
 
-    private ResultSetRowPacket[] readResultSet() throws IOException {
+    protected ResultSetRowPacket[] readResultSet() throws IOException {
         List resultSet = new LinkedList<>();
         byte[] statementResult = channel.read();
         checkError(statementResult);

From d06003ddbd23f09846e3bfd98115c9fbd9e1b0bd Mon Sep 17 00:00:00 2001
From: glarwood 
Date: Mon, 29 Apr 2019 14:17:00 -0700
Subject: [PATCH 196/217] refactor(BinaryLogClient): manually undo
 fd9f0333b6c28829c1c3d09ac6df6a0a22c548c6 to simplify upstream merge

---
 .../java/com/github/shyiko/mysql/binlog/BinaryLogClient.java   | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index 776dd91a..70e8ef8d 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -739,8 +739,7 @@ private boolean tryUpgradeToSSL(GreetingPacket greetingPacket) throws IOExceptio
                         sslMode == SSLMode.REQUIRED || sslMode == SSLMode.PREFERRED ?
                             DEFAULT_REQUIRED_SSL_MODE_SOCKET_FACTORY :
                             DEFAULT_VERIFY_CA_SSL_MODE_SOCKET_FACTORY;
-                channel.upgradeToSSL(sslSocketFactory,
-                    sslMode == SSLMode.VERIFY_IDENTITY ? new TLSHostnameVerifier() : null);
+                channel.upgradeToSSL(sslSocketFactory, null);
                 logger.info("SSL enabled");
                 return true;
             }

From b0370a9dba192430ddeccfab7aed126be8a546de Mon Sep 17 00:00:00 2001
From: glarwood 
Date: Mon, 13 May 2019 16:49:48 -0700
Subject: [PATCH 197/217] fix(ByteArrayOutputStream): use composed
 OutputStream#write

---
 .../shyiko/mysql/binlog/network/protocol/PacketChannel.java      | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/PacketChannel.java b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/PacketChannel.java
index 95a4b2be..c64e7034 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/PacketChannel.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/network/protocol/PacketChannel.java
@@ -85,6 +85,7 @@ public void write(Command command) throws IOException {
         buffer.writeInteger(packetNumber++, 1);
 
         buffer.write(body, 0, body.length);
+        buffer.flush();
         outputStream.write(buffer.toByteArray());
         // though it has no effect in case of default (underlying) output stream (SocketOutputStream),
         // it may be necessary in case of non-default one

From 336b7715c98de9669b707a1203c28f0b498b35c5 Mon Sep 17 00:00:00 2001
From: Ellen Peng 
Date: Fri, 11 Oct 2019 17:31:36 -0700
Subject: [PATCH 198/217] specified year zero condition

---
 .../deserialization/AbstractRowsEventDataDeserializer.java     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
index df40459c..81629c59 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
@@ -378,7 +378,8 @@ protected Serializable deserializeDatetimeV2(int meta, ByteArrayInputStream inpu
     }
 
     protected Serializable deserializeYear(ByteArrayInputStream inputStream) throws IOException {
-        return 1900 + inputStream.readInteger(1);
+        int year = inputStream.readInteger(1);
+        return year == 0 ? 0 : 1900 + year;
     }
 
     protected Serializable deserializeString(int length, ByteArrayInputStream inputStream) throws IOException {

From cdabe05b92077f1cb96a3ac52622e4dd980a9b3d Mon Sep 17 00:00:00 2001
From: glarwood 
Date: Tue, 31 Mar 2020 15:40:42 -0700
Subject: [PATCH 199/217] fix(AbstractRowsEventDataDeserializer): add
 microsecond precision for datetime/timestamp

---
 .../AbstractRowsEventDataDeserializer.java         | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
index 81629c59..55422b65 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
@@ -24,7 +24,6 @@
 import java.math.BigDecimal;
 import java.util.BitSet;
 import java.util.Calendar;
-import java.util.Date;
 import java.util.Map;
 import java.util.TimeZone;
 
@@ -332,7 +331,7 @@ protected Serializable deserializeTimestampV2(int meta, ByteArrayInputStream inp
         if (deserializeDateAndTimeAsLong) {
             return castTimestamp(timestamp, fsp);
         }
-        return new java.sql.Timestamp(timestamp);
+        return convertLongTimestamptWithFSP(timestamp, fsp);
     }
 
     protected Serializable deserializeDatetime(ByteArrayInputStream inputStream) throws IOException {
@@ -341,7 +340,7 @@ protected Serializable deserializeDatetime(ByteArrayInputStream inputStream) thr
         if (deserializeDateAndTimeAsLong) {
             return castTimestamp(timestamp, 0);
         }
-        return timestamp != null ? new java.util.Date(timestamp) : null;
+        return timestamp != null ? new java.sql.Timestamp(timestamp) : null;
     }
 
     protected Serializable deserializeDatetimeV2(int meta, ByteArrayInputStream inputStream) throws IOException {
@@ -374,7 +373,14 @@ protected Serializable deserializeDatetimeV2(int meta, ByteArrayInputStream inpu
         if (deserializeDateAndTimeAsLong) {
             return castTimestamp(timestamp, fsp);
         }
-        return timestamp != null ? new java.util.Date(timestamp) : null;
+
+        return timestamp != null ? convertLongTimestamptWithFSP(timestamp, fsp) : null;
+    }
+
+    private java.sql.Timestamp convertLongTimestamptWithFSP(Long timestamp, int fsp) {
+        java.sql.Timestamp ts = new java.sql.Timestamp(timestamp);
+        ts.setNanos(fsp * 1000);
+        return ts;
     }
 
     protected Serializable deserializeYear(ByteArrayInputStream inputStream) throws IOException {

From 7b8bba4c71d7b6f01ac6dd08aaf6df1195c0eff5 Mon Sep 17 00:00:00 2001
From: glarwood 
Date: Wed, 1 Apr 2020 16:32:05 -0700
Subject: [PATCH 200/217] fix(AbstractRowsEventDataDeserializer): add
 microsecond precision for time

---
 .../deserialization/AbstractRowsEventDataDeserializer.java   | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
index 55422b65..3504a894 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java
@@ -22,6 +22,7 @@
 import java.io.IOException;
 import java.io.Serializable;
 import java.math.BigDecimal;
+import java.sql.Time;
 import java.util.BitSet;
 import java.util.Calendar;
 import java.util.Map;
@@ -285,7 +286,7 @@ protected Serializable deserializeTime(ByteArrayInputStream inputStream) throws
         if (deserializeDateAndTimeAsLong) {
             return castTimestamp(timestamp, 0);
         }
-        return timestamp != null ? new java.sql.Time(timestamp) : null;
+        return timestamp != null ? new java.sql.Timestamp(timestamp) : null;
     }
 
     protected Serializable deserializeTimeV2(int meta, ByteArrayInputStream inputStream) throws IOException {
@@ -313,7 +314,7 @@ protected Serializable deserializeTimeV2(int meta, ByteArrayInputStream inputStr
         if (deserializeDateAndTimeAsLong) {
             return castTimestamp(timestamp, fsp);
         }
-        return timestamp != null ? new java.sql.Time(timestamp) : null;
+        return timestamp != null ? convertLongTimestamptWithFSP(timestamp, fsp) : null;
     }
 
     protected Serializable deserializeTimestamp(ByteArrayInputStream inputStream) throws IOException {

From 6b3bb726826b6b9d74fcc417bc8a87901a864962 Mon Sep 17 00:00:00 2001
From: Matt Alexander 
Date: Mon, 12 Oct 2020 12:08:12 -0700
Subject: [PATCH 201/217] Add AURORA_PADDING event types and refactor event
 types to no longer use an enum ordinal to resolve event type IDs

---
 .../shyiko/mysql/binlog/event/EventType.java    | 17 ++++++++++++-----
 .../EventHeaderV4Deserializer.java              |  1 -
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
index 77360bac..393421cb 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
@@ -193,18 +193,17 @@ public enum EventType {
      * Prepared XA transaction terminal event similar to XID except that it is specific to XA transaction.
      */
     XA_PREPARE(38),
-
     /**
      Extension of UPDATE_ROWS_EVENT, allowing partial values according
      to binlog_row_value_options.
      */
     PARTIAL_UPDATE_ROWS_EVENT(39),
-
     /**
      * Generated when 'binlog_transaction_compression' is set to 'ON'.
      * It encapsulates all the events of a transaction in a Zstd compressed payload.
      */
     TRANSACTION_PAYLOAD(40),
+    AURORA_PADDING(100);
 
     /**
      * MariaDB Support Events
@@ -216,10 +215,18 @@ public enum EventType {
     MARIADB_GTID(162),
     MARIADB_GTID_LIST(163);
 
-    private final int eventNumber;
+    int eventId;
+
+    EventType(int eventId) {
+        this.eventId = eventId;
+    }
+
+    public static EventType forId(int eventId) {
+        for (EventType type : EventType.values()) {
+            if (type.eventId == eventId) return type;
+        }
 
-    EventType(int eventNumber) {
-        this.eventNumber = eventNumber;
+        return null;
     }
 
     public static boolean isRowMutation(EventType eventType) {
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
index e9533629..edfc143e 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
@@ -42,5 +42,4 @@ private static EventType getEventType(int ordinal) {
         EventType eventType = EventType.byEventNumber(ordinal);
         return eventType == null ? EventType.UNKNOWN : eventType;
     }
-
 }

From dfa9de18c627d490b71eecd8042396d351b7616e Mon Sep 17 00:00:00 2001
From: Matt Alexander 
Date: Mon, 12 Oct 2020 15:04:23 -0700
Subject: [PATCH 202/217] Make event id private and final

---
 .../java/com/github/shyiko/mysql/binlog/event/EventType.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
index 393421cb..78142acd 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
@@ -215,7 +215,7 @@ public enum EventType {
     MARIADB_GTID(162),
     MARIADB_GTID_LIST(163);
 
-    int eventId;
+    private final int eventId;
 
     EventType(int eventId) {
         this.eventId = eventId;

From 30987eb7697da833082352af4f3a7bfdd892d9b0 Mon Sep 17 00:00:00 2001
From: Shiva Vanamala 
Date: Thu, 12 Nov 2020 20:04:53 -0800
Subject: [PATCH 203/217] Create unknown event type for invalid and proprietary
 event types. Removed AURORA_PADDING event type.

---
 .../deserialization/EventHeaderV4Deserializer.java | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
index edfc143e..e88d3221 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
@@ -38,8 +38,16 @@ public EventHeaderV4 deserialize(ByteArrayInputStream inputStream) throws IOExce
         return header;
     }
 
-    private static EventType getEventType(int ordinal) {
-        EventType eventType = EventType.byEventNumber(ordinal);
-        return eventType == null ? EventType.UNKNOWN : eventType;
+    /**
+     * Parses the event type based on the ordinal.
+     * If an invalid or proprietary ordinal is passed, an unknown event is returned.
+     */
+    private static EventType getEventType(int ordinal) throws IOException {
+        EventType eventType = EventType.forId(ordinal);
+
+        if (eventType == null) {
+            return EventType.forId(0);
+        }
+        return eventType;
     }
 }

From c7c70fd39a36dd040ea4bc8b004bc88f231013af Mon Sep 17 00:00:00 2001
From: Shiva Vanamala 
Date: Thu, 12 Nov 2020 20:11:56 -0800
Subject: [PATCH 204/217] Removed AURORA_PADDING event

---
 .../java/com/github/shyiko/mysql/binlog/event/EventType.java   | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
index 78142acd..48ca9823 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
@@ -202,8 +202,7 @@ public enum EventType {
      * Generated when 'binlog_transaction_compression' is set to 'ON'.
      * It encapsulates all the events of a transaction in a Zstd compressed payload.
      */
-    TRANSACTION_PAYLOAD(40),
-    AURORA_PADDING(100);
+    TRANSACTION_PAYLOAD(40);
 
     /**
      * MariaDB Support Events

From c11cb2e41ca938277fb6e496fe4bad55d423fec5 Mon Sep 17 00:00:00 2001
From: Shiva Vanamala 
Date: Fri, 13 Nov 2020 09:30:12 -0800
Subject: [PATCH 205/217] Changed the way unknown event type is being created
 EventType.forId(0) -> EventType.UNKNOWN

---
 .../binlog/event/deserialization/EventHeaderV4Deserializer.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
index e88d3221..74f5bc3c 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
@@ -46,7 +46,7 @@ private static EventType getEventType(int ordinal) throws IOException {
         EventType eventType = EventType.forId(ordinal);
 
         if (eventType == null) {
-            return EventType.forId(0);
+            return EventType.UNKNOWN;
         }
         return eventType;
     }

From 41fc68b14db23891d4cb6f743afa69dcb1563499 Mon Sep 17 00:00:00 2001
From: Shiva Vanamala 
Date: Wed, 18 Nov 2020 18:47:20 -0800
Subject: [PATCH 206/217] EventType.forId() returns EventType.UNKNOWN if
 invalid eventId is passed.

---
 .../com/github/shyiko/mysql/binlog/event/EventType.java    | 7 ++++++-
 .../event/deserialization/EventHeaderV4Deserializer.java   | 7 +------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
index 48ca9823..f1db36e1 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
@@ -220,12 +220,17 @@ public enum EventType {
         this.eventId = eventId;
     }
 
+    /**
+     * Creates EventType based on the id.
+     * 
+     * 
If id passed is out of the range of the standard event types, EventType.UNKNOWN is returned.
+     */
     public static EventType forId(int eventId) {
         for (EventType type : EventType.values()) {
             if (type.eventId == eventId) return type;
         }
 
-        return null;
+        return EventType.UNKNOWN;
     }
 
     public static boolean isRowMutation(EventType eventType) {
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
index 74f5bc3c..7c8ea5b6 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
@@ -43,11 +43,6 @@ public EventHeaderV4 deserialize(ByteArrayInputStream inputStream) throws IOExce
      * If an invalid or proprietary ordinal is passed, an unknown event is returned.
      */
     private static EventType getEventType(int ordinal) throws IOException {
-        EventType eventType = EventType.forId(ordinal);
-
-        if (eventType == null) {
-            return EventType.UNKNOWN;
-        }
-        return eventType;
+        return EventType.forId(ordinal);
     }
 }

From 4e3ac3c31183e765d5b89c1194437b135448b3a8 Mon Sep 17 00:00:00 2001
From: Shiva Vanamala 
Date: Wed, 18 Nov 2020 18:56:38 -0800
Subject: [PATCH 207/217] refactor: Removed redundant single line function

---
 .../github/shyiko/mysql/binlog/event/EventType.java    |  4 ++--
 .../deserialization/EventHeaderV4Deserializer.java     | 10 +---------
 2 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
index f1db36e1..5f3f5878 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
@@ -221,9 +221,9 @@ public enum EventType {
     }
 
     /**
-     * Creates EventType based on the id.
+     * Parses the event type based on the ordinal.
      * 
-     * 
If id passed is out of the range of the standard event types, EventType.UNKNOWN is returned.
+     * 
If an invalid or proprietary ordinal is passed, EventType.UNKNOWN is returned.
      */
     public static EventType forId(int eventId) {
         for (EventType type : EventType.values()) {
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
index 7c8ea5b6..27fa617b 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/EventHeaderV4Deserializer.java
@@ -30,19 +30,11 @@ public class EventHeaderV4Deserializer implements EventHeaderDeserializer
Date: Thu, 28 Oct 2021 13:38:11 -0700
Subject: [PATCH 208/217] feature(table_map_event_data_serializer): add support
 of ColumnVisibility metadata and handle unknown metadata field type

---
 .../event/deserialization/TableMapEventMetadataDeserializer.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
index 61c34bce..9f2f26c0 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
@@ -96,6 +96,7 @@ public TableMapEventMetadata deserialize(ByteArrayInputStream inputStream, int n
                     break;
                 case ENUM_AND_SET_COLUMN_CHARSET:
                     result.setEnumAndSetColumnCharsets(readIntegers(inputStream));
+                    break;
                 case VISIBILITY:
                     result.setVisibility(readBooleanList(inputStream, nColumns));
                     break;

From c835a5b9962853817f2ce6575335ebcd08d527fc Mon Sep 17 00:00:00 2001
From: zekailiu 
Date: Thu, 28 Oct 2021 21:13:12 -0700
Subject: [PATCH 209/217] fix(table_map_event_data_serializer): skip
 deserializing the table map event when the stream is empty

---
 .../deserialization/TableMapEventMetadataDeserializer.java     | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
index 9f2f26c0..cfd5af12 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
@@ -58,6 +58,9 @@ public TableMapEventMetadata deserialize(ByteArrayInputStream inputStream, int n
                 continue;
             }
 
+            //for some reasons, the UNKNOWN_METADATA_FIELD_TYPE will mess up the stream
+            if(inputStream.available() == 0) return result;
+
             int fieldLength = inputStream.readPackedInteger();
 
             remainingBytes = inputStream.available();

From f0f73e343ad9f509f0cfc93217f43326538bca65 Mon Sep 17 00:00:00 2001
From: zekailiu 
Date: Thu, 28 Oct 2021 21:47:03 -0700
Subject: [PATCH 210/217] fix(table_map_event_data_serializer): log a message
 when skipping deserializing the table map event

---
 .../deserialization/TableMapEventMetadataDeserializer.java   | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
index cfd5af12..c6efdda4 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java
@@ -59,7 +59,10 @@ public TableMapEventMetadata deserialize(ByteArrayInputStream inputStream, int n
             }
 
             //for some reasons, the UNKNOWN_METADATA_FIELD_TYPE will mess up the stream
-            if(inputStream.available() == 0) return result;
+            if(inputStream.available() == 0) {
+                logger.warning("Stream is empty so cannot read field length for field type: " + fieldType);
+                return result;
+            }
 
             int fieldLength = inputStream.readPackedInteger();
 

From cfe843e280b493ccbd22bc2c58e1684c1b39194e Mon Sep 17 00:00:00 2001
From: Imen Graja 
Date: Wed, 9 Feb 2022 14:55:27 -0800
Subject: [PATCH 211/217] fix for json parsing error

---
 .../binlog/event/deserialization/json/JsonBinary.java      | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
index 6a4dd48e..8a2b8d84 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
@@ -186,12 +186,7 @@ private static boolean isJSONString(byte[] bytes) {
      * @throws IOException if there is a problem reading or processing the binary representation
      */
     public static void parse(byte[] bytes, JsonFormatter formatter) throws IOException {
-        if (bytes.length == 0) {
-            // When the top-level value is a JSON "null", the value in java shows up as a non-null, empty byte array
-            formatter.valueNull();
-        } else {
-            new JsonBinary(bytes).parse(formatter);
-        }
+        new JsonBinary(bytes).parse(formatter);
     }
 
     private final ByteArrayInputStream reader;

From 710f48b0b902ab46b7c53248c7f69e9ab5a5a186 Mon Sep 17 00:00:00 2001
From: Matt Alexander 
Date: Thu, 17 Mar 2022 11:37:45 -0700
Subject: [PATCH 212/217] Fork BinaryLogClient to FF abortRequest change

---
 .../shyiko/mysql/binlog/BinaryLogClient.java  |  3 +-
 .../mysql/binlog/NewBinaryLogClient.java      | 30 +++++++++++++++++++
 2 files changed, 32 insertions(+), 1 deletion(-)
 create mode 100644 src/main/java/com/github/shyiko/mysql/binlog/NewBinaryLogClient.java

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index 70e8ef8d..53695932 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -150,6 +150,7 @@ public X509Certificate[] getAcceptedIssuers() {
 
     private final List eventListeners = new CopyOnWriteArrayList();
     private final List lifecycleListeners = new CopyOnWriteArrayList();
+    protected boolean abortRequest = false;
 
     private SocketFactory socketFactory;
     private SSLSocketFactory sslSocketFactory;
@@ -1027,7 +1028,7 @@ private void confirmSupportOfChecksum(ChecksumType checksumType) throws IOExcept
         eventDeserializer.setChecksumType(checksumType);
     }
 
-    private void listenForEventPackets() throws IOException {
+    protected void listenForEventPackets() throws IOException {
         ByteArrayInputStream inputStream = channel.getInputStream();
         boolean completeShutdown = false;
         try {
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/NewBinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/NewBinaryLogClient.java
new file mode 100644
index 00000000..dccccf90
--- /dev/null
+++ b/src/main/java/com/github/shyiko/mysql/binlog/NewBinaryLogClient.java
@@ -0,0 +1,30 @@
+package com.github.shyiko.mysql.binlog;
+
+import com.github.shyiko.mysql.binlog.network.protocol.PacketChannel;
+
+import java.io.IOException;
+
+public class NewBinaryLogClient extends BinaryLogClient {
+
+    public NewBinaryLogClient(String username, String password) {
+        super(username, password);
+    }
+
+    public NewBinaryLogClient(String schema, String username, String password) {
+        super(schema, username, password);
+    }
+
+    public NewBinaryLogClient(String hostname, int port, String username, String password) {
+        super(hostname, port, username, password);
+    }
+
+    public NewBinaryLogClient(String hostname, int port, String schema, String username, String password) {
+        super(hostname, port, schema, username, password);
+    }
+
+    @Override
+    protected void listenForEventPackets(PacketChannel channel) throws IOException {
+        abortRequest = false;
+        super.listenForEventPackets(channel);
+    }
+}

From ecd7088fe206798d7fd3b0e7ff08d9e672b01630 Mon Sep 17 00:00:00 2001
From: Josh Wood 
Date: Mon, 16 May 2022 17:24:57 -0500
Subject: [PATCH 213/217] cleanup(mysql): [T-252649] consolidate new abort
 logic

---
 .../shyiko/mysql/binlog/BinaryLogClient.java  |  1 +
 .../mysql/binlog/NewBinaryLogClient.java      | 30 -------------------
 2 files changed, 1 insertion(+), 30 deletions(-)
 delete mode 100644 src/main/java/com/github/shyiko/mysql/binlog/NewBinaryLogClient.java

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index 53695932..e6e5a54b 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -1029,6 +1029,7 @@ private void confirmSupportOfChecksum(ChecksumType checksumType) throws IOExcept
     }
 
     protected void listenForEventPackets() throws IOException {
+        abortRequest = false;
         ByteArrayInputStream inputStream = channel.getInputStream();
         boolean completeShutdown = false;
         try {
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/NewBinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/NewBinaryLogClient.java
deleted file mode 100644
index dccccf90..00000000
--- a/src/main/java/com/github/shyiko/mysql/binlog/NewBinaryLogClient.java
+++ /dev/null
@@ -1,30 +0,0 @@
-package com.github.shyiko.mysql.binlog;
-
-import com.github.shyiko.mysql.binlog.network.protocol.PacketChannel;
-
-import java.io.IOException;
-
-public class NewBinaryLogClient extends BinaryLogClient {
-
-    public NewBinaryLogClient(String username, String password) {
-        super(username, password);
-    }
-
-    public NewBinaryLogClient(String schema, String username, String password) {
-        super(schema, username, password);
-    }
-
-    public NewBinaryLogClient(String hostname, int port, String username, String password) {
-        super(hostname, port, username, password);
-    }
-
-    public NewBinaryLogClient(String hostname, int port, String schema, String username, String password) {
-        super(hostname, port, schema, username, password);
-    }
-
-    @Override
-    protected void listenForEventPackets(PacketChannel channel) throws IOException {
-        abortRequest = false;
-        super.listenForEventPackets(channel);
-    }
-}

From c9186f052b88265358c87e5948575a27b1decdf1 Mon Sep 17 00:00:00 2001
From: Ashley Cox 
Date: Sat, 6 Aug 2022 14:15:55 -0700
Subject: [PATCH 214/217] re-add abort logic on its own (without old deadlock
 fix)

---
 .../com/github/shyiko/mysql/binlog/BinaryLogClient.java    | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index e6e5a54b..87ecd095 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -1033,7 +1033,7 @@ protected void listenForEventPackets() throws IOException {
         ByteArrayInputStream inputStream = channel.getInputStream();
         boolean completeShutdown = false;
         try {
-            while (inputStream.peek() != -1) {
+            while (!abortRequest && inputStream.peek() != -1) {
                 int packetLength = inputStream.readInteger(3);
                 inputStream.skip(1); // 1 byte for sequence
                 int marker = inputStream.read();
@@ -1080,6 +1080,7 @@ protected void listenForEventPackets() throws IOException {
                 }
             }
         } finally {
+            abortRequest = false;
             if (isConnected()) {
                 if (completeShutdown) {
                     disconnect(); // initiate complete shutdown sequence (which includes keep alive thread)
@@ -1297,6 +1298,10 @@ public void disconnect() throws IOException {
         terminateConnect();
     }
 
+    public void abort() {
+        abortRequest = true;
+    }
+
     private void terminateKeepAliveThread() {
         try {
             keepAliveThreadExecutorLock.lock();

From 655337d313d1e24b924bf84fd3fd5dade568b381 Mon Sep 17 00:00:00 2001
From: Ashley Cox 
Date: Tue, 6 Sep 2022 15:35:33 -0700
Subject: [PATCH 215/217] add client identity method

---
 .../java/com/github/shyiko/mysql/binlog/BinaryLogClient.java  | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
index 87ecd095..43bd362a 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/BinaryLogClient.java
@@ -227,6 +227,10 @@ public BinaryLogClient(String hostname, int port, String schema, String username
         this.password = password;
     }
 
+    public String fivetranClientIdentity() {
+        return "mysql-binlog-connector-java-osheroff";
+    }
+
     public boolean isBlocking() {
         return blocking;
     }

From 812e3a983736095f30a7231e721c673490be68b1 Mon Sep 17 00:00:00 2001
From: Matthew Alexander
 <61435459+fivetran-mattalexander@users.noreply.github.com>
Date: Wed, 16 Nov 2022 13:47:16 -0800
Subject: [PATCH 216/217] Fix merge errors

---
 .../com/github/shyiko/mysql/binlog/event/EventType.java     | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
index 5f3f5878..e6e9564a 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/event/EventType.java
@@ -202,7 +202,7 @@ public enum EventType {
      * Generated when 'binlog_transaction_compression' is set to 'ON'.
      * It encapsulates all the events of a transaction in a Zstd compressed payload.
      */
-    TRANSACTION_PAYLOAD(40);
+    TRANSACTION_PAYLOAD(40),
 
     /**
      * MariaDB Support Events
@@ -222,7 +222,7 @@ public enum EventType {
 
     /**
      * Parses the event type based on the ordinal.
-     * 
+     *
      * 
If an invalid or proprietary ordinal is passed, EventType.UNKNOWN is returned.
      */
     public static EventType forId(int eventId) {
@@ -259,7 +259,7 @@ public static boolean isDelete(EventType eventType) {
 
     public static EventType byEventNumber(int num) {
         for (EventType type : EventType.values()) {
-            if (type.eventNumber == num) {
+            if (type.eventId == num) {
                 return type;
             }
         }

From 8fba1921b301c428ca6d8eb686f52fedff49d75e Mon Sep 17 00:00:00 2001
From: Matthew Alexander
 <61435459+fivetran-mattalexander@users.noreply.github.com>
Date: Thu, 17 Nov 2022 11:38:05 -0800
Subject: [PATCH 217/217] Fix isContainedWithin to correctly return True when
 positions are equivalent

---
 .../java/com/github/shyiko/mysql/binlog/MariadbGtidSet.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/com/github/shyiko/mysql/binlog/MariadbGtidSet.java b/src/main/java/com/github/shyiko/mysql/binlog/MariadbGtidSet.java
index 224030b8..91c854e1 100644
--- a/src/main/java/com/github/shyiko/mysql/binlog/MariadbGtidSet.java
+++ b/src/main/java/com/github/shyiko/mysql/binlog/MariadbGtidSet.java
@@ -145,7 +145,7 @@ public boolean isContainedWithin(GtidSet other) {
 
                 MariaGtid thisGtid = thisDomainMap.get(serverID);
                 MariaGtid otherGtid = otherDomainMap.get(serverID);
-                if ( thisGtid.sequence >= otherGtid.sequence )
+                if ( thisGtid.sequence > otherGtid.sequence )
                     return false;
             }
         }