Skip to content

Commit 1e5ae34

Browse files
author
Luis Soares
committed
WL#3549: Binlog Compression
This worklog implements binary log compression, making use of the compression algorithm ZSTD. Compression is done per transaction. Compressed transactions remain compressed when they are replicated between primaries and secondaries. This means that both storage and network footprint is reduced for handling binary log files. Approved by: Pedro Gomes <pedro.gomes@oracle.com> Approved by: Pedro Figueiredo <pedro.figueiredo@oracle.com>
1 parent 4b4166b commit 1e5ae34

File tree

217 files changed

+22177
-14203
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

217 files changed

+22177
-14203
lines changed

client/mysqlbinlog.cc

+708-144
Large diffs are not rendered by default.

libbinlogevents/include/binlog_event.h

+4-1
Original file line numberDiff line numberDiff line change
@@ -345,6 +345,8 @@ enum Log_event_type {
345345
*/
346346
PARTIAL_UPDATE_ROWS_EVENT = 39,
347347

348+
TRANSACTION_PAYLOAD_EVENT = 40,
349+
348350
/**
349351
Add new events here - right above this comment!
350352
Existing events (except ENUM_END_EVENT) should never change their numbers
@@ -833,7 +835,8 @@ class Binary_log_event {
833835
ROWS_HEADER_LEN_V2 = 10,
834836
TRANSACTION_CONTEXT_HEADER_LEN = 18,
835837
VIEW_CHANGE_HEADER_LEN = 52,
836-
XA_PREPARE_HEADER_LEN = 0
838+
XA_PREPARE_HEADER_LEN = 0,
839+
TRANSACTION_PAYLOAD_HEADER_LEN = 0,
837840
}; // end enum_post_header_length
838841
protected:
839842
/**

libbinlogevents/include/codecs/base.h

+76
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
/* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
2+
3+
This program is free software; you can redistribute it and/or modify
4+
it under the terms of the GNU General Public License, version 2.0,
5+
as published by the Free Software Foundation.
6+
7+
This program is also distributed with certain software (including
8+
but not limited to OpenSSL) that is licensed under separate terms,
9+
as designated in a particular file or component or in included license
10+
documentation. The authors of MySQL hereby grant you an additional
11+
permission to link the program and your derivative works with the
12+
separately licensed software that they have included with MySQL.
13+
14+
This program is distributed in the hope that it will be useful,
15+
but WITHOUT ANY WARRANTY; without even the implied warranty of
16+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17+
GNU General Public License, version 2.0, for more details.
18+
19+
You should have received a copy of the GNU General Public License
20+
along with this program; if not, write to the Free Software
21+
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
22+
23+
#ifndef CODECS_BASE_INCLUDED
24+
#define CODECS_BASE_INCLUDED
25+
26+
#include <utility>
27+
#include "libbinlogevents/include/binary_log.h"
28+
29+
namespace binary_log {
30+
namespace codecs {
31+
32+
/**
33+
This is the abstract and base class for binary log codecs.
34+
35+
It defines the codec API. Implementations of this class must
36+
be stateless.
37+
*/
38+
class Codec {
39+
public:
40+
/**
41+
Member function that shall decode the contents of the given buffer into a
42+
binary log event.
43+
44+
@param from the buffer containing the encoded event.
45+
@param size the length of the data in the buffer.
46+
@param to the binary log event to populate.
47+
48+
@return a pair containing the amount of bytes decoded from the buffer and a
49+
boolean stating whether there was an error or not. False if no
50+
error, true otherwise.
51+
*/
52+
virtual std::pair<std::size_t, bool> decode(const unsigned char *from,
53+
std::size_t size,
54+
Binary_log_event &to) const = 0;
55+
56+
/**
57+
Member function that shall encode the contents of the given binary log event
58+
into an in memory buffer.
59+
60+
@param from the binary log event to encode.
61+
@param to the buffer where the encoded event should be saved into.
62+
@param size the size of the buffer.
63+
64+
@return a pair containing the amount of bytes encoded and whether there was
65+
an error or not. False if no error, true otherwise.
66+
*/
67+
virtual std::pair<std::size_t, bool> encode(const Binary_log_event &from,
68+
unsigned char *to,
69+
std::size_t size) const = 0;
70+
virtual ~Codec() {}
71+
};
72+
73+
} // namespace codecs
74+
} // namespace binary_log
75+
76+
#endif
+99
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
/* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
2+
3+
This program is free software; you can redistribute it and/or modify
4+
it under the terms of the GNU General Public License, version 2.0,
5+
as published by the Free Software Foundation.
6+
7+
This program is also distributed with certain software (including
8+
but not limited to OpenSSL) that is licensed under separate terms,
9+
as designated in a particular file or component or in included license
10+
documentation. The authors of MySQL hereby grant you an additional
11+
permission to link the program and your derivative works with the
12+
separately licensed software that they have included with MySQL.
13+
14+
This program is distributed in the hope that it will be useful,
15+
but WITHOUT ANY WARRANTY; without even the implied warranty of
16+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17+
GNU General Public License, version 2.0, for more details.
18+
19+
You should have received a copy of the GNU General Public License
20+
along with this program; if not, write to the Free Software
21+
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
22+
23+
#ifndef CODECS_BINARY_INCLUDED
24+
#define CODECS_BINARY_INCLUDED
25+
26+
#include "base.h"
27+
#include "libbinlogevents/include/binary_log.h"
28+
29+
namespace binary_log {
30+
namespace codecs {
31+
namespace binary {
32+
33+
/**
34+
This is the abstract and base class for binary log BINARY codecs.
35+
*/
36+
class Base_codec : public binary_log::codecs::Codec {
37+
public:
38+
static const unsigned short UINT_64T_MIN_SIZE = 1;
39+
static const unsigned short UINT_64T_MAX_SIZE = 1;
40+
41+
protected:
42+
Event_reader *m_reader;
43+
inline Event_reader &reader() { return *m_reader; }
44+
45+
public:
46+
virtual std::pair<std::size_t, bool> decode(const unsigned char *from,
47+
std::size_t size,
48+
Binary_log_event &to) const = 0;
49+
virtual std::pair<std::size_t, bool> encode(const Binary_log_event &from,
50+
unsigned char *to,
51+
std::size_t size) const = 0;
52+
53+
virtual ~Base_codec() {}
54+
};
55+
56+
/**
57+
Binary codec for the transaction payload log event.
58+
*/
59+
class Transaction_payload : public Base_codec {
60+
public:
61+
Transaction_payload() {}
62+
/**
63+
This member function shall decode the contents of the buffer provided and
64+
fill in the event referenced. Note that the event provided needs to be of
65+
type TRANSACTION_PAYLOAD_EVENT.
66+
67+
@param from the buffer to decode
68+
@param size the size of the buffer to decode.
69+
@param to the event to store the decoded information into.
70+
71+
@return a pair containing the amount of bytes decoded and whether there was
72+
an error or not. False if no error, true otherwise.
73+
*/
74+
virtual std::pair<std::size_t, bool> decode(const unsigned char *from,
75+
std::size_t size,
76+
Binary_log_event &to) const;
77+
78+
/**
79+
This member function shall encode the contents of the event referenced and
80+
store the result in the buffer provided. Note that the event referenced
81+
needs to be of type TRANSACTION_PAYLOAD_EVENT.
82+
83+
@param from the event to encode.
84+
@param to the buffer where to store the encoded event.
85+
@param size the size of the buffer.
86+
87+
@return a pair containing the amount of bytes encoded and whether there was
88+
an error or not.
89+
*/
90+
virtual std::pair<std::size_t, bool> encode(const Binary_log_event &from,
91+
unsigned char *to,
92+
std::size_t size) const;
93+
};
94+
95+
} // namespace binary
96+
} // namespace codecs
97+
} // namespace binary_log
98+
99+
#endif
+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/* Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
2+
3+
This program is free software; you can redistribute it and/or modify
4+
it under the terms of the GNU General Public License, version 2.0,
5+
as published by the Free Software Foundation.
6+
7+
This program is also distributed with certain software (including
8+
but not limited to OpenSSL) that is licensed under separate terms,
9+
as designated in a particular file or component or in included license
10+
documentation. The authors of MySQL hereby grant you an additional
11+
permission to link the program and your derivative works with the
12+
separately licensed software that they have included with MySQL.
13+
14+
This program is distributed in the hope that it will be useful,
15+
but WITHOUT ANY WARRANTY; without even the implied warranty of
16+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17+
GNU General Public License, version 2.0, for more details.
18+
19+
You should have received a copy of the GNU General Public License
20+
along with this program; if not, write to the Free Software
21+
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
22+
23+
#ifndef CODECS_FACTORY_INCLUDED
24+
#define CODECS_FACTORY_INCLUDED
25+
26+
#include <memory>
27+
#include "base.h"
28+
#include "libbinlogevents/include/binary_log.h"
29+
30+
namespace binary_log {
31+
namespace codecs {
32+
33+
class Factory {
34+
public:
35+
static std::unique_ptr<Codec> build_codec(Log_event_type type);
36+
};
37+
38+
} // namespace codecs
39+
} // namespace binary_log
40+
41+
#endif

0 commit comments

Comments
 (0)