Merge pull request #64 from jpechane/json-8.0

Process JSON field names using offsets
osheroff · Jan 28, 2022 · 1a560fb · 1a560fb
2 parents a93b785 + 0292696
commit 1a560fb
Show file tree

Hide file tree

Showing 3 changed files with 163 additions and 7 deletions.
diff --git a/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java b/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/json/JsonBinary.java
@@ -326,10 +326,11 @@ protected void parseObject(boolean small, JsonFormatter formatter)
         int valueSize = small ? 2 : 4;
 
         // Read each key-entry, consisting of the offset and length of each key ...
-        int[] keyLengths = new int[numElements];
+        KeyEntry[] keys = new KeyEntry[numElements];
         for (int i = 0; i != numElements; ++i) {
-            readUnsignedIndex(numBytes, small, "key offset in"); // unused
-            keyLengths[i] = readUInt16();
+            keys[i] = new KeyEntry(
+                    readUnsignedIndex(numBytes, small, "key offset in"),
+                    readUInt16());
         }
 
         // Read each key value value-entry
@@ -374,9 +375,14 @@ protected void parseObject(boolean small, JsonFormatter formatter)
         }
 
         // Read each key ...
-        String[] keys = new String[numElements];
         for (int i = 0; i != numElements; ++i) {
-            keys[i] = reader.readString(keyLengths[i]);
+            final int skipBytes = keys[i].index + objectOffset - reader.getPosition();
+            // Skip to a start of a field name if the current position does not point to it
+            // This can happen for MySQL 8
+            if (skipBytes != 0) {
+                reader.fastSkip(skipBytes);
+            }
+            keys[i].name = reader.readString(keys[i].length);
         }
 
         // Now parse the values ...
@@ -385,7 +391,7 @@ protected void parseObject(boolean small, JsonFormatter formatter)
             if (i != 0) {
                 formatter.nextEntry();
             }
-            formatter.name(keys[i]);
+            formatter.name(keys[i].name);
             ValueEntry entry = entries[i];
             if (entry.resolved) {
                 Object value = entry.value;
@@ -996,6 +1002,26 @@ protected static String asHex(int value) {
         return Integer.toHexString(value);
     }
 
+    /**
+     * Class used internally to hold key entry information.
+     */
+    protected static final class KeyEntry {
+
+        protected final int index;
+        protected final int length;
+        protected String name;
+
+        public KeyEntry(int index, int length) {
+            this.index = index;
+            this.length = length;
+        }
+
+        public KeyEntry setKey(String key) {
+            this.name = key;
+            return this;
+        }
+    }
+
     /**
      * Class used internally to hold value entry information.
      */

diff --git a/...github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java b/...github/shyiko/mysql/binlog/event/deserialization/json/JsonBinaryValueIntegrationTest.java
@@ -22,7 +22,6 @@
 import com.github.shyiko.mysql.binlog.MysqlOnetimeServer;
 import com.github.shyiko.mysql.binlog.TraceEventListener;
 import com.github.shyiko.mysql.binlog.TraceLifecycleListener;
-import com.github.shyiko.mysql.binlog.event.Event;
 import com.github.shyiko.mysql.binlog.event.EventData;
 import com.github.shyiko.mysql.binlog.event.EventType;
 import com.github.shyiko.mysql.binlog.event.QueryEventData;
@@ -147,6 +146,28 @@ public void testMysql8JsonRemovePartialUpdateWithHoles() throws Exception {
         client.unregisterEventListener(capturingEventListener);
     }
 
+    @Test
+    public void testMysql8JsonRemovePartialUpdateWithHolesAndSparseKeys() throws Exception {
+        CapturingEventListener capturingEventListener = new CapturingEventListener();
+        client.registerEventListener(capturingEventListener);
+        String json = "{\"17fc9889474028063990914001f6854f6b8b5784\":\"test_field_for_remove_fields_behaviour_2\",\"1f3a2ea5bc1f60258df20521bee9ac636df69a3a\":{\"currency\":\"USD\"},\"4f4d99a438f334d7dbf83a1816015b361b848b3b\":{\"currency\":\"USD\"},\"9021162291be72f5a8025480f44bf44d5d81d07c\":\"test_field_for_remove_fields_behaviour_3_will_be_removed\",\"9b0ed11532efea688fdf12b28f142b9eb08a80c5\":{\"currency\":\"USD\"},\"e65ad0762c259b05b4866f7249eabecabadbe577\":\"test_field_for_remove_fields_behaviour_1_updated\",\"ff2c07edcaa3e987c23fb5cc4fe860bb52becf00\":{\"currency\":\"USD\"}}";
+        master.execute("DROP TABLE IF EXISTS json_test", "create table json_test (j JSON)",
+                "INSERT INTO json_test VALUES ('" + json + "')",
+                "UPDATE json_test SET j = JSON_REMOVE(j, '$.\"17fc9889474028063990914001f6854f6b8b5784\"')");
+        capturingEventListener.waitFor(WriteRowsEventData.class, 1, DEFAULT_TIMEOUT);
+        capturingEventListener.waitFor(UpdateRowsEventData.class, 1, DEFAULT_TIMEOUT);
+        List<WriteRowsEventData> events = capturingEventListener.getEvents(WriteRowsEventData.class);
+        Serializable[] insertData = events.iterator().next().getRows().get(0);
+        assertEquals(JsonBinary.parseAsString((byte[]) insertData[0]), json);
+
+        List<UpdateRowsEventData> updateEvents = capturingEventListener.getEvents(UpdateRowsEventData.class);
+        Serializable[] updateData = updateEvents.iterator().next().getRows().get(0).getValue();
+        assertEquals(JsonBinary.parseAsString((byte[]) updateData[0]), json.replace(
+                "\"17fc9889474028063990914001f6854f6b8b5784\":\"test_field_for_remove_fields_behaviour_2\",", ""));
+
+        client.unregisterEventListener(capturingEventListener);
+    }
+
     @Test
     public void testMysql8JsonReplacePartialUpdateWithHoles() throws Exception {
         CapturingEventListener capturingEventListener = new CapturingEventListener();

diff --git a/...com/github/shyiko/mysql/binlog/event/deserialization/json/JsonPartialUpdateParseTest.java b/...com/github/shyiko/mysql/binlog/event/deserialization/json/JsonPartialUpdateParseTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.github.shyiko.mysql.binlog.event.deserialization.json;
+
+import java.io.IOException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * The column value
+ * <pre>
+ * {
+ *    "17fc9889474028063990914001f6854f6b8b5784":"test_field_for_remove_fields_behaviour_2",
+ *    "1f3a2ea5bc1f60258df20521bee9ac636df69a3a":{
+ *       "currency":"USD"
+ *    },
+ *    "4f4d99a438f334d7dbf83a1816015b361b848b3b":{
+ *       "currency":"USD"
+ *    },
+ *    "9021162291be72f5a8025480f44bf44d5d81d07c":"test_field_for_remove_fields_behaviour_3_will_be_removed",
+ *    "9b0ed11532efea688fdf12b28f142b9eb08a80c5":{
+ *       "currency":"USD"
+ *    },
+ *    "e65ad0762c259b05b4866f7249eabecabadbe577":"test_field_for_remove_fields_behaviour_1_updated",
+ *    "ff2c07edcaa3e987c23fb5cc4fe860bb52becf00":{
+ *       "currency":"USD"
+ *    }
+ * }
+ * </pre>
+ * is partially updated using
+ * <pre>
+ * JSON_REMOVE(custom_fields, '$.\"17fc9889474028063990914001f6854f6b8b5784\"')
+ * </pre>
+ * MySQL 5.7 and MySQL 8.0 emits different values in binlog (MySQL 8 value is sparse and requires strict using of offsets) 
+ */
+public class JsonPartialUpdateParseTest {
+
+    private static final byte[] BINLOG_UPDATE_57 = { 0, 6, 0, -28, 1, 46, 0, 40, 0, 86, 0, 40, 0, 126, 0, 40, 0, -90, 0,
+            40, 0, -50, 0, 40, 0, -10, 0, 40, 0, 0, 30, 1, 0, 53, 1, 12, 76, 1, 0, -123, 1, 12, -100, 1, 0, -51, 1, 49,
+            102, 51, 97, 50, 101, 97, 53, 98, 99, 49, 102, 54, 48, 50, 53, 56, 100, 102, 50, 48, 53, 50, 49, 98, 101,
+            101, 57, 97, 99, 54, 51, 54, 100, 102, 54, 57, 97, 51, 97, 52, 102, 52, 100, 57, 57, 97, 52, 51, 56, 102,
+            51, 51, 52, 100, 55, 100, 98, 102, 56, 51, 97, 49, 56, 49, 54, 48, 49, 53, 98, 51, 54, 49, 98, 56, 52, 56,
+            98, 51, 98, 57, 48, 50, 49, 49, 54, 50, 50, 57, 49, 98, 101, 55, 50, 102, 53, 97, 56, 48, 50, 53, 52, 56,
+            48, 102, 52, 52, 98, 102, 52, 52, 100, 53, 100, 56, 49, 100, 48, 55, 99, 57, 98, 48, 101, 100, 49, 49, 53,
+            51, 50, 101, 102, 101, 97, 54, 56, 56, 102, 100, 102, 49, 50, 98, 50, 56, 102, 49, 52, 50, 98, 57, 101, 98,
+            48, 56, 97, 56, 48, 99, 53, 101, 54, 53, 97, 100, 48, 55, 54, 50, 99, 50, 53, 57, 98, 48, 53, 98, 52, 56,
+            54, 54, 102, 55, 50, 52, 57, 101, 97, 98, 101, 99, 97, 98, 97, 100, 98, 101, 53, 55, 55, 102, 102, 50, 99,
+            48, 55, 101, 100, 99, 97, 97, 51, 101, 57, 56, 55, 99, 50, 51, 102, 98, 53, 99, 99, 52, 102, 101, 56, 54,
+            48, 98, 98, 53, 50, 98, 101, 99, 102, 48, 48, 1, 0, 23, 0, 11, 0, 8, 0, 12, 19, 0, 99, 117, 114, 114, 101,
+            110, 99, 121, 3, 85, 83, 68, 1, 0, 23, 0, 11, 0, 8, 0, 12, 19, 0, 99, 117, 114, 114, 101, 110, 99, 121, 3,
+            85, 83, 68, 56, 116, 101, 115, 116, 95, 102, 105, 101, 108, 100, 95, 102, 111, 114, 95, 114, 101, 109, 111,
+            118, 101, 95, 102, 105, 101, 108, 100, 115, 95, 98, 101, 104, 97, 118, 105, 111, 117, 114, 95, 51, 95, 119,
+            105, 108, 108, 95, 98, 101, 95, 114, 101, 109, 111, 118, 101, 100, 1, 0, 23, 0, 11, 0, 8, 0, 12, 19, 0, 99,
+            117, 114, 114, 101, 110, 99, 121, 3, 85, 83, 68, 48, 116, 101, 115, 116, 95, 102, 105, 101, 108, 100, 95,
+            102, 111, 114, 95, 114, 101, 109, 111, 118, 101, 95, 102, 105, 101, 108, 100, 115, 95, 98, 101, 104, 97,
+            118, 105, 111, 117, 114, 95, 49, 95, 117, 112, 100, 97, 116, 101, 100, 1, 0, 23, 0, 11, 0, 8, 0, 12, 19, 0,
+            99, 117, 114, 114, 101, 110, 99, 121, 3, 85, 83, 68 };
+
+    private static final byte[] BINLOG_UPDATE_80 = { 0, 6, 0, 60, 2, 93, 0, 40, 0, -123, 0, 40, 0, -83, 0, 40, 0, -43,
+            0, 40, 0, -3, 0, 40, 0, 37, 1, 40, 0, 0, 118, 1, 0, -115, 1, 12, -92, 1, 0, -35, 1, 12, -12, 1, 0, 37, 2, 1,
+            12, -12, 1, 0, 37, 2, 49, 55, 102, 99, 57, 56, 56, 57, 52, 55, 52, 48, 50, 56, 48, 54, 51, 57, 57, 48, 57,
+            49, 52, 48, 48, 49, 102, 54, 56, 53, 52, 102, 54, 98, 56, 98, 53, 55, 56, 52, 49, 102, 51, 97, 50, 101, 97,
+            53, 98, 99, 49, 102, 54, 48, 50, 53, 56, 100, 102, 50, 48, 53, 50, 49, 98, 101, 101, 57, 97, 99, 54, 51, 54,
+            100, 102, 54, 57, 97, 51, 97, 52, 102, 52, 100, 57, 57, 97, 52, 51, 56, 102, 51, 51, 52, 100, 55, 100, 98,
+            102, 56, 51, 97, 49, 56, 49, 54, 48, 49, 53, 98, 51, 54, 49, 98, 56, 52, 56, 98, 51, 98, 57, 48, 50, 49, 49,
+            54, 50, 50, 57, 49, 98, 101, 55, 50, 102, 53, 97, 56, 48, 50, 53, 52, 56, 48, 102, 52, 52, 98, 102, 52, 52,
+            100, 53, 100, 56, 49, 100, 48, 55, 99, 57, 98, 48, 101, 100, 49, 49, 53, 51, 50, 101, 102, 101, 97, 54, 56,
+            56, 102, 100, 102, 49, 50, 98, 50, 56, 102, 49, 52, 50, 98, 57, 101, 98, 48, 56, 97, 56, 48, 99, 53, 101,
+            54, 53, 97, 100, 48, 55, 54, 50, 99, 50, 53, 57, 98, 48, 53, 98, 52, 56, 54, 54, 102, 55, 50, 52, 57, 101,
+            97, 98, 101, 99, 97, 98, 97, 100, 98, 101, 53, 55, 55, 102, 102, 50, 99, 48, 55, 101, 100, 99, 97, 97, 51,
+            101, 57, 56, 55, 99, 50, 51, 102, 98, 53, 99, 99, 52, 102, 101, 56, 54, 48, 98, 98, 53, 50, 98, 101, 99,
+            102, 48, 48, 40, 116, 101, 115, 116, 95, 102, 105, 101, 108, 100, 95, 102, 111, 114, 95, 114, 101, 109, 111,
+            118, 101, 95, 102, 105, 101, 108, 100, 115, 95, 98, 101, 104, 97, 118, 105, 111, 117, 114, 95, 50, 1, 0, 23,
+            0, 11, 0, 8, 0, 12, 19, 0, 99, 117, 114, 114, 101, 110, 99, 121, 3, 85, 83, 68, 1, 0, 23, 0, 11, 0, 8, 0,
+            12, 19, 0, 99, 117, 114, 114, 101, 110, 99, 121, 3, 85, 83, 68, 56, 116, 101, 115, 116, 95, 102, 105, 101,
+            108, 100, 95, 102, 111, 114, 95, 114, 101, 109, 111, 118, 101, 95, 102, 105, 101, 108, 100, 115, 95, 98,
+            101, 104, 97, 118, 105, 111, 117, 114, 95, 51, 95, 119, 105, 108, 108, 95, 98, 101, 95, 114, 101, 109, 111,
+            118, 101, 100, 1, 0, 23, 0, 11, 0, 8, 0, 12, 19, 0, 99, 117, 114, 114, 101, 110, 99, 121, 3, 85, 83, 68, 48,
+            116, 101, 115, 116, 95, 102, 105, 101, 108, 100, 95, 102, 111, 114, 95, 114, 101, 109, 111, 118, 101, 95,
+            102, 105, 101, 108, 100, 115, 95, 98, 101, 104, 97, 118, 105, 111, 117, 114, 95, 49, 95, 117, 112, 100, 97,
+            116, 101, 100, 1, 0, 23, 0, 11, 0, 8, 0, 12, 19, 0, 99, 117, 114, 114, 101, 110, 99, 121, 3, 85, 83, 68 };
+
+    @Test
+    public void test57PartialUpdateBinlog() throws IOException {
+        final String json = JsonBinary.parseAsString(BINLOG_UPDATE_57);
+        Assert.assertFalse(json.contains("17fc9889474028063990914001f6854f6b8b5784"));
+        Assert.assertTrue(json.contains("1f3a2ea5bc1f60258df20521bee9ac636df69a3a"));
+    }
+
+    @Test
+    public void test80PartialUpdateBinlog() throws IOException {
+        final String json = JsonBinary.parseAsString(BINLOG_UPDATE_80);
+        Assert.assertFalse(json.contains("17fc9889474028063990914001f6854f6b8b5784"));
+        System.out.println(json);
+        Assert.assertTrue(json.contains("1f3a2ea5bc1f60258df20521bee9ac636df69a3a"));
+    }
+}