WL#3549: Binlog Compression

This worklog implements binary log compression, making use of the
compression algorithm ZSTD. Compression is done per
transaction. Compressed transactions remain compressed when they are
replicated between primaries and secondaries. This means that both
storage and network footprint is reduced for handling binary log
files.

Approved by: Pedro Gomes <pedro.gomes@oracle.com>
Approved by: Pedro Figueiredo <pedro.figueiredo@oracle.com>

libbinlogevents/include/binlog_event.h

@@ -345,6 +345,8 @@ enum Log_event_type {
   */
   PARTIAL_UPDATE_ROWS_EVENT = 39,
 
+  TRANSACTION_PAYLOAD_EVENT = 40,
+
   /**
     Add new events here - right above this comment!
     Existing events (except ENUM_END_EVENT) should never change their numbers
@@ -833,7 +835,8 @@ class Binary_log_event {
     ROWS_HEADER_LEN_V2 = 10,
     TRANSACTION_CONTEXT_HEADER_LEN = 18,
     VIEW_CHANGE_HEADER_LEN = 52,
-    XA_PREPARE_HEADER_LEN = 0
+    XA_PREPARE_HEADER_LEN = 0,
+    TRANSACTION_PAYLOAD_HEADER_LEN = 0,
   };  // end enum_post_header_length
  protected:
   /**

libbinlogevents/include/codecs/base.h

@@ -0,0 +1,76 @@
+/* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License, version 2.0,
+   as published by the Free Software Foundation.
+
+   This program is also distributed with certain software (including
+   but not limited to OpenSSL) that is licensed under separate terms,
+   as designated in a particular file or component or in included license
+   documentation.  The authors of MySQL hereby grant you an additional
+   permission to link the program and your derivative works with the
+   separately licensed software that they have included with MySQL.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License, version 2.0, for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
+#ifndef CODECS_BASE_INCLUDED
+#define CODECS_BASE_INCLUDED
+
+#include <utility>
+#include "libbinlogevents/include/binary_log.h"
+
+namespace binary_log {
+namespace codecs {
+
+/**
+  This is the abstract and base class for binary log codecs.
+
+  It defines the codec API. Implementations of this class must
+  be stateless.
+ */
+class Codec {
+ public:
+  /**
+    Member function that shall decode the contents of the given buffer into a
+    binary log event.
+
+    @param from the buffer containing the encoded event.
+    @param size the length of the data in the buffer.
+    @param to the binary log event to populate.
+
+    @return a pair containing the amount of bytes decoded from the buffer and a
+            boolean stating whether there was an error or not. False if no
+            error, true otherwise.
+  */
+  virtual std::pair<std::size_t, bool> decode(const unsigned char *from,
+                                              std::size_t size,
+                                              Binary_log_event &to) const = 0;
+
+  /**
+    Member function that shall encode the contents of the given binary log event
+    into an in memory buffer.
+
+    @param from the binary log event to encode.
+    @param to the buffer where the encoded event should be saved into.
+    @param size the size of the buffer.
+
+    @return a pair containing the amount of bytes encoded and whether there was
+            an error or not. False if no error, true otherwise.
+  */
+  virtual std::pair<std::size_t, bool> encode(const Binary_log_event &from,
+                                              unsigned char *to,
+                                              std::size_t size) const = 0;
+  virtual ~Codec() {}
+};
+
+}  // namespace codecs
+}  // namespace binary_log
+
+#endif

libbinlogevents/include/codecs/binary.h

@@ -0,0 +1,99 @@
+/* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License, version 2.0,
+   as published by the Free Software Foundation.
+
+   This program is also distributed with certain software (including
+   but not limited to OpenSSL) that is licensed under separate terms,
+   as designated in a particular file or component or in included license
+   documentation.  The authors of MySQL hereby grant you an additional
+   permission to link the program and your derivative works with the
+   separately licensed software that they have included with MySQL.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License, version 2.0, for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
+#ifndef CODECS_BINARY_INCLUDED
+#define CODECS_BINARY_INCLUDED
+
+#include "base.h"
+#include "libbinlogevents/include/binary_log.h"
+
+namespace binary_log {
+namespace codecs {
+namespace binary {
+
+/**
+  This is the abstract and base class for binary log BINARY codecs.
+ */
+class Base_codec : public binary_log::codecs::Codec {
+ public:
+  static const unsigned short UINT_64T_MIN_SIZE = 1;
+  static const unsigned short UINT_64T_MAX_SIZE = 1;
+
+ protected:
+  Event_reader *m_reader;
+  inline Event_reader &reader() { return *m_reader; }
+
+ public:
+  virtual std::pair<std::size_t, bool> decode(const unsigned char *from,
+                                              std::size_t size,
+                                              Binary_log_event &to) const = 0;
+  virtual std::pair<std::size_t, bool> encode(const Binary_log_event &from,
+                                              unsigned char *to,
+                                              std::size_t size) const = 0;
+
+  virtual ~Base_codec() {}
+};
+
+/**
+  Binary codec for the transaction payload log event.
+ */
+class Transaction_payload : public Base_codec {
+ public:
+  Transaction_payload() {}
+  /**
+     This member function shall decode the contents of the buffer provided and
+     fill in the event referenced. Note that the event provided needs to be of
+     type TRANSACTION_PAYLOAD_EVENT.
+
+     @param from the buffer to decode
+     @param size the size of the buffer to decode.
+     @param to the event to store the decoded information into.
+
+     @return a pair containing the amount of bytes decoded and whether there was
+             an error or not. False if no error, true otherwise.
+   */
+  virtual std::pair<std::size_t, bool> decode(const unsigned char *from,
+                                              std::size_t size,
+                                              Binary_log_event &to) const;
+
+  /**
+     This member function shall encode the contents of the event referenced and
+     store the result in the buffer provided. Note that the event referenced
+     needs to be of type TRANSACTION_PAYLOAD_EVENT.
+
+     @param from the event to encode.
+     @param to the buffer where to store the encoded event.
+     @param size the size of the buffer.
+
+     @return a pair containing the amount of bytes encoded and whether there was
+             an error or not.
+  */
+  virtual std::pair<std::size_t, bool> encode(const Binary_log_event &from,
+                                              unsigned char *to,
+                                              std::size_t size) const;
+};
+
+}  // namespace binary
+}  // namespace codecs
+}  // namespace binary_log
+
+#endif

libbinlogevents/include/codecs/factory.h

@@ -0,0 +1,41 @@
+/* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License, version 2.0,
+   as published by the Free Software Foundation.
+
+   This program is also distributed with certain software (including
+   but not limited to OpenSSL) that is licensed under separate terms,
+   as designated in a particular file or component or in included license
+   documentation.  The authors of MySQL hereby grant you an additional
+   permission to link the program and your derivative works with the
+   separately licensed software that they have included with MySQL.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License, version 2.0, for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
+#ifndef CODECS_FACTORY_INCLUDED
+#define CODECS_FACTORY_INCLUDED
+
+#include <memory>
+#include "base.h"
+#include "libbinlogevents/include/binary_log.h"
+
+namespace binary_log {
+namespace codecs {
+
+class Factory {
+ public:
+  static std::unique_ptr<Codec> build_codec(Log_event_type type);
+};
+
+}  // namespace codecs
+}  // namespace binary_log
+
+#endif