Skip to content

Commit

Permalink
MDEV-12615: InnoDB page compression method snappy mostly does not com…
Browse files Browse the repository at this point in the history 
…press pages

Snappy compression method require that output buffer
used for compression is bigger than input buffer.
Similarly lzo require additional work memory buffer.
Increase the allocated buffer accordingly.

buf_tmp_buffer_t: removed unnecessary lzo_mem, crypt_buf_free and
comp_buf_free.

buf_pool_reserve_tmp_slot: use alligned_alloc and if snappy
available allocate size based on snappy_max_compressed_length and
if lzo is available increase buffer by LZO1X_1_15_MEM_COMPRESS.

fil_compress_page: Remove unneeded lzo mem (we use same buffer)
and if output buffer is not yet allocated allocate based similarly
as above.

Decompression does not require additional work area.

    Modify test to use same test as other compression method tests.
  • Loading branch information
Jan Lindström committed May 20, 2017
1 parent f880200 commit 90c52e5
Show file tree
Hide file tree
Showing 12 changed files with 276 additions and 811 deletions.
491 changes: 76 additions & 415 deletions mysql-test/suite/innodb/r/innodb-page_compression_snappy.result
View file

Large diffs are not rendered by default.

252 changes: 6 additions & 246 deletions mysql-test/suite/innodb/t/innodb-page_compression_snappy.test
View file
Original file line number Diff line number Diff line change
@@ -1,253 +1,13 @@
-- source include/have_innodb.inc
-- source include/have_innodb_snappy.inc
--source include/not_embedded.inc

call mtr.add_suppression("Compression failed for space*");

--disable_query_log
let $innodb_compression_algorithm_orig=`select @@innodb_compression_algorithm`;
let $innodb_file_format_orig = `select @@innodb_file_format`;
let $innodb_file_per_table_orig = `select @@innodb_file_per_table`;
--enable_query_log

set global innodb_file_format = `barracuda`;
set global innodb_file_per_table = on;
call mtr.add_suppression("InnoDB: Compression failed for space [0-9]+ name test/innodb_page_compressed[0-9] len [0-9]+ err 2 write_size [0-9]+.");

# snappy
set global innodb_compression_algorithm = 6;

create table innodb_compressed(c1 int, b char(20)) engine=innodb row_format=compressed key_block_size=8;
show warnings;
create table innodb_normal (c1 int, b char(20)) engine=innodb;
show warnings;
create table innodb_page_compressed1 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=1;
show warnings;
show create table innodb_page_compressed1;
create table innodb_page_compressed2 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=2;
show warnings;
show create table innodb_page_compressed2;
create table innodb_page_compressed3 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=3;
show warnings;
show create table innodb_page_compressed3;
create table innodb_page_compressed4 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=4;
show warnings;
show create table innodb_page_compressed4;
create table innodb_page_compressed5 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=5;
show warnings;
show create table innodb_page_compressed5;
create table innodb_page_compressed6 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=6;
show warnings;
show create table innodb_page_compressed6;
create table innodb_page_compressed7 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=7;
show warnings;
show create table innodb_page_compressed7;
create table innodb_page_compressed8 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=8;
show warnings;
show create table innodb_page_compressed8;
create table innodb_page_compressed9 (c1 int, b char(20)) engine=innodb page_compressed=1 page_compression_level=9;
show warnings;
show create table innodb_page_compressed9;
delimiter //;
create procedure innodb_insert_proc (repeat_count int)
begin
declare current_num int;
set current_num = 0;
while current_num < repeat_count do
insert into innodb_normal values(current_num,'testing..');
set current_num = current_num + 1;
end while;
end//
delimiter ;//
commit;

set autocommit=0;
call innodb_insert_proc(5000);
commit;
set autocommit=1;
select count(*) from innodb_normal;
insert into innodb_compressed select * from innodb_normal;
insert into innodb_page_compressed1 select * from innodb_normal;
insert into innodb_page_compressed2 select * from innodb_normal;
insert into innodb_page_compressed3 select * from innodb_normal;
insert into innodb_page_compressed4 select * from innodb_normal;
insert into innodb_page_compressed5 select * from innodb_normal;
insert into innodb_page_compressed6 select * from innodb_normal;
insert into innodb_page_compressed7 select * from innodb_normal;
insert into innodb_page_compressed8 select * from innodb_normal;
insert into innodb_page_compressed9 select * from innodb_normal;
commit;
select count(*) from innodb_compressed;
select count(*) from innodb_page_compressed1;
select count(*) from innodb_page_compressed1 where c1 < 500000;
select count(*) from innodb_page_compressed2 where c1 < 500000;
select count(*) from innodb_page_compressed3 where c1 < 500000;
select count(*) from innodb_page_compressed4 where c1 < 500000;
select count(*) from innodb_page_compressed5 where c1 < 500000;
select count(*) from innodb_page_compressed6 where c1 < 500000;
select count(*) from innodb_page_compressed7 where c1 < 500000;
select count(*) from innodb_page_compressed8 where c1 < 500000;
select count(*) from innodb_page_compressed9 where c1 < 500000;

alter table innodb_normal page_compressed=1 page_compression_level=8;
show warnings;
show create table innodb_normal;
alter table innodb_compressed row_format=default page_compressed=1 page_compression_level=8 key_block_size=0;
show warnings;
show create table innodb_compressed;

update innodb_page_compressed1 set c1 = c1 + 1;
update innodb_page_compressed2 set c1 = c1 + 1;
update innodb_page_compressed3 set c1 = c1 + 1;
update innodb_page_compressed4 set c1 = c1 + 1;
update innodb_page_compressed5 set c1 = c1 + 1;
update innodb_page_compressed6 set c1 = c1 + 1;
update innodb_page_compressed7 set c1 = c1 + 1;
update innodb_page_compressed8 set c1 = c1 + 1;
update innodb_page_compressed9 set c1 = c1 + 1;
select count(*) from innodb_compressed;
select count(*) from innodb_page_compressed1;
select count(*) from innodb_page_compressed1 where c1 < 500000;
select count(*) from innodb_page_compressed2 where c1 < 500000;
select count(*) from innodb_page_compressed3 where c1 < 500000;
select count(*) from innodb_page_compressed4 where c1 < 500000;
select count(*) from innodb_page_compressed5 where c1 < 500000;
select count(*) from innodb_page_compressed6 where c1 < 500000;
select count(*) from innodb_page_compressed7 where c1 < 500000;
select count(*) from innodb_page_compressed8 where c1 < 500000;
select count(*) from innodb_page_compressed9 where c1 < 500000;

--source include/restart_mysqld.inc

update innodb_page_compressed1 set c1 = c1 + 1;
update innodb_page_compressed2 set c1 = c1 + 1;
update innodb_page_compressed3 set c1 = c1 + 1;
update innodb_page_compressed4 set c1 = c1 + 1;
update innodb_page_compressed5 set c1 = c1 + 1;
update innodb_page_compressed6 set c1 = c1 + 1;
update innodb_page_compressed7 set c1 = c1 + 1;
update innodb_page_compressed8 set c1 = c1 + 1;
update innodb_page_compressed9 set c1 = c1 + 1;
select count(*) from innodb_compressed;
select count(*) from innodb_page_compressed1;
select count(*) from innodb_page_compressed1 where c1 < 500000;
select count(*) from innodb_page_compressed2 where c1 < 500000;
select count(*) from innodb_page_compressed3 where c1 < 500000;
select count(*) from innodb_page_compressed4 where c1 < 500000;
select count(*) from innodb_page_compressed5 where c1 < 500000;
select count(*) from innodb_page_compressed6 where c1 < 500000;
select count(*) from innodb_page_compressed7 where c1 < 500000;
select count(*) from innodb_page_compressed8 where c1 < 500000;
select count(*) from innodb_page_compressed9 where c1 < 500000;

# zlib
set global innodb_compression_algorithm = 1;
update innodb_page_compressed1 set c1 = c1 + 1;
update innodb_page_compressed2 set c1 = c1 + 1;
update innodb_page_compressed3 set c1 = c1 + 1;
update innodb_page_compressed4 set c1 = c1 + 1;
update innodb_page_compressed5 set c1 = c1 + 1;
update innodb_page_compressed6 set c1 = c1 + 1;
update innodb_page_compressed7 set c1 = c1 + 1;
update innodb_page_compressed8 set c1 = c1 + 1;
update innodb_page_compressed9 set c1 = c1 + 1;
commit;
select count(*) from innodb_compressed;
select count(*) from innodb_page_compressed1;
select count(*) from innodb_page_compressed1 where c1 < 500000;
select count(*) from innodb_page_compressed2 where c1 < 500000;
select count(*) from innodb_page_compressed3 where c1 < 500000;
select count(*) from innodb_page_compressed4 where c1 < 500000;
select count(*) from innodb_page_compressed5 where c1 < 500000;
select count(*) from innodb_page_compressed6 where c1 < 500000;
select count(*) from innodb_page_compressed7 where c1 < 500000;
select count(*) from innodb_page_compressed8 where c1 < 500000;
select count(*) from innodb_page_compressed9 where c1 < 500000;

--source include/restart_mysqld.inc

update innodb_page_compressed1 set c1 = c1 + 1;
update innodb_page_compressed2 set c1 = c1 + 1;
update innodb_page_compressed3 set c1 = c1 + 1;
update innodb_page_compressed4 set c1 = c1 + 1;
update innodb_page_compressed5 set c1 = c1 + 1;
update innodb_page_compressed6 set c1 = c1 + 1;
update innodb_page_compressed7 set c1 = c1 + 1;
update innodb_page_compressed8 set c1 = c1 + 1;
update innodb_page_compressed9 set c1 = c1 + 1;
select count(*) from innodb_compressed;
select count(*) from innodb_page_compressed1;
select count(*) from innodb_page_compressed1 where c1 < 500000;
select count(*) from innodb_page_compressed2 where c1 < 500000;
select count(*) from innodb_page_compressed3 where c1 < 500000;
select count(*) from innodb_page_compressed4 where c1 < 500000;
select count(*) from innodb_page_compressed5 where c1 < 500000;
select count(*) from innodb_page_compressed6 where c1 < 500000;
select count(*) from innodb_page_compressed7 where c1 < 500000;
select count(*) from innodb_page_compressed8 where c1 < 500000;
select count(*) from innodb_page_compressed9 where c1 < 500000;

# none
set global innodb_compression_algorithm = 0;
update innodb_page_compressed1 set c1 = c1 + 1;
update innodb_page_compressed2 set c1 = c1 + 1;
update innodb_page_compressed3 set c1 = c1 + 1;
update innodb_page_compressed4 set c1 = c1 + 1;
update innodb_page_compressed5 set c1 = c1 + 1;
update innodb_page_compressed6 set c1 = c1 + 1;
update innodb_page_compressed7 set c1 = c1 + 1;
update innodb_page_compressed8 set c1 = c1 + 1;
update innodb_page_compressed9 set c1 = c1 + 1;
commit;
select count(*) from innodb_compressed;
select count(*) from innodb_page_compressed1;
select count(*) from innodb_page_compressed1 where c1 < 500000;
select count(*) from innodb_page_compressed2 where c1 < 500000;
select count(*) from innodb_page_compressed3 where c1 < 500000;
select count(*) from innodb_page_compressed4 where c1 < 500000;
select count(*) from innodb_page_compressed5 where c1 < 500000;
select count(*) from innodb_page_compressed6 where c1 < 500000;
select count(*) from innodb_page_compressed7 where c1 < 500000;
select count(*) from innodb_page_compressed8 where c1 < 500000;
select count(*) from innodb_page_compressed9 where c1 < 500000;

--source include/restart_mysqld.inc

update innodb_page_compressed1 set c1 = c1 + 1;
update innodb_page_compressed2 set c1 = c1 + 1;
update innodb_page_compressed3 set c1 = c1 + 1;
update innodb_page_compressed4 set c1 = c1 + 1;
update innodb_page_compressed5 set c1 = c1 + 1;
update innodb_page_compressed6 set c1 = c1 + 1;
update innodb_page_compressed7 set c1 = c1 + 1;
update innodb_page_compressed8 set c1 = c1 + 1;
update innodb_page_compressed9 set c1 = c1 + 1;
select count(*) from innodb_compressed;
select count(*) from innodb_page_compressed1;
select count(*) from innodb_page_compressed1 where c1 < 500000;
select count(*) from innodb_page_compressed2 where c1 < 500000;
select count(*) from innodb_page_compressed3 where c1 < 500000;
select count(*) from innodb_page_compressed4 where c1 < 500000;
select count(*) from innodb_page_compressed5 where c1 < 500000;
select count(*) from innodb_page_compressed6 where c1 < 500000;
select count(*) from innodb_page_compressed7 where c1 < 500000;
select count(*) from innodb_page_compressed8 where c1 < 500000;
select count(*) from innodb_page_compressed9 where c1 < 500000;
set global innodb_compression_algorithm = snappy;

drop procedure innodb_insert_proc;
drop table innodb_normal;
drop table innodb_compressed;
drop table innodb_page_compressed1;
drop table innodb_page_compressed2;
drop table innodb_page_compressed3;
drop table innodb_page_compressed4;
drop table innodb_page_compressed5;
drop table innodb_page_compressed6;
drop table innodb_page_compressed7;
drop table innodb_page_compressed8;
drop table innodb_page_compressed9;
# All page compression test use the same
--source include/innodb-page-compression.inc

# reset system
--disable_query_log
EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algorithm_orig;
EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig;
EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig;
--enable_query_log
-- echo #done
76 changes: 51 additions & 25 deletions storage/innobase/buf/buf0buf.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,10 +64,38 @@ Created 11/5/1995 Heikki Tuuri
#include "ut0byte.h"
#include <new>

#ifdef UNIV_LINUX
#include <stdlib.h>
#endif

#ifdef HAVE_LZO
#include "lzo/lzo1x.h"
#endif

#ifdef HAVE_SNAPPY
#include "snappy-c.h"
#endif

inline void* aligned_malloc(size_t size, size_t align) {
void *result;
#ifdef _MSC_VER
result = _aligned_malloc(size, align);
#else
if(posix_memalign(&result, align, size)) {
result = 0;
}
#endif
return result;
}

inline void aligned_free(void *ptr) {
#ifdef _MSC_VER
_aligned_free(ptr);
#else
free(ptr);
#endif
}

/*
IMPLEMENTATION OF THE BUFFER POOL
=================================
Expand Down Expand Up @@ -1555,20 +1583,14 @@ buf_pool_free_instance(
if (buf_pool->tmp_arr) {
for(ulint i = 0; i < buf_pool->tmp_arr->n_slots; i++) {
buf_tmp_buffer_t* slot = &(buf_pool->tmp_arr->slots[i]);
#ifdef HAVE_LZO
if (slot && slot->lzo_mem) {
ut_free(slot->lzo_mem);
slot->lzo_mem = NULL;
}
#endif
if (slot && slot->crypt_buf_free) {
ut_free(slot->crypt_buf_free);
slot->crypt_buf_free = NULL;
if (slot && slot->crypt_buf) {
aligned_free(slot->crypt_buf);
slot->crypt_buf = NULL;
}

if (slot && slot->comp_buf_free) {
ut_free(slot->comp_buf_free);
slot->comp_buf_free = NULL;
if (slot && slot->comp_buf) {
aligned_free(slot->comp_buf);
slot->comp_buf = NULL;
}
}
}
Expand Down Expand Up @@ -6024,22 +6046,27 @@ buf_pool_reserve_tmp_slot(
buf_pool_mutex_exit(buf_pool);

/* Allocate temporary memory for encryption/decryption */
if (free_slot->crypt_buf_free == NULL) {
free_slot->crypt_buf_free = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*2));
free_slot->crypt_buf = static_cast<byte *>(ut_align(free_slot->crypt_buf_free, UNIV_PAGE_SIZE));
memset(free_slot->crypt_buf_free, 0, UNIV_PAGE_SIZE *2);
if (free_slot->crypt_buf == NULL) {
free_slot->crypt_buf = static_cast<byte*>(aligned_malloc(UNIV_PAGE_SIZE, UNIV_PAGE_SIZE));
memset(free_slot->crypt_buf, 0, UNIV_PAGE_SIZE);
}

/* For page compressed tables allocate temporary memory for
compression/decompression */
if (compressed && free_slot->comp_buf_free == NULL) {
free_slot->comp_buf_free = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*2));
free_slot->comp_buf = static_cast<byte *>(ut_align(free_slot->comp_buf_free, UNIV_PAGE_SIZE));
memset(free_slot->comp_buf_free, 0, UNIV_PAGE_SIZE *2);
#ifdef HAVE_LZO
free_slot->lzo_mem = static_cast<byte *>(ut_malloc(LZO1X_1_15_MEM_COMPRESS));
memset(free_slot->lzo_mem, 0, LZO1X_1_15_MEM_COMPRESS);
if (compressed && free_slot->comp_buf == NULL) {
ulint size = UNIV_PAGE_SIZE;

/* Both snappy and lzo compression methods require that
output buffer used for compression is bigger than input
buffer. Increase the allocated buffer size accordingly. */
#if HAVE_SNAPPY
size = snappy_max_compressed_length(size);
#endif
#if HAVE_LZO
size += LZO1X_1_15_MEM_COMPRESS;
#endif
free_slot->comp_buf = static_cast<byte*>(aligned_malloc(size, UNIV_PAGE_SIZE));
memset(free_slot->comp_buf, 0, size);
}

return (free_slot);
Expand Down Expand Up @@ -6127,8 +6154,7 @@ buf_page_encrypt_before_write(
fsp_flags_get_page_compression_level(space->flags),
fil_space_get_block_size(space, bpage->offset),
encrypted,
&out_len,
IF_LZO(slot->lzo_mem, NULL));
&out_len);

bpage->real_size = out_len;

Expand Down
3 changes: 1 addition & 2 deletions storage/innobase/fil/fil0fil.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6744,8 +6744,7 @@ fil_iterate(
0,/* FIXME: compression level */
512,/* FIXME: use proper block size */
encrypted,
&len,
NULL);
&len);

updated = true;
}
Expand Down
Loading

0 comments on commit 90c52e5

Please sign in to comment.