Skip to content

Commit

Permalink
Added new SQL function GET_INDEX_SIZE_BY_PREFIX
Browse files Browse the repository at this point in the history
Summary:
**Motivation**
Core Data Demand Efficiency team wants to track physical size of the assoc types and fbtypes, as well as existing logical size. Tracking physical size of the specific assoc/fb types is not available as of now.

**Change**
Added the new function GET_INDEX_SIZE_BY_PREFIX which returns the index size by prefix. Currently we support only numeric keys.

Reviewed By: yizhang82

Differential Revision: D27419569

fbshipit-source-id: 84604ac03bd
  • Loading branch information
Pushapgl authored and facebook-github-bot committed Apr 22, 2021
1 parent bc53cf1 commit 78923eb
Show file tree
Hide file tree
Showing 8 changed files with 631 additions and 31 deletions.
129 changes: 129 additions & 0 deletions mysql-test/suite/rocksdb/r/get_index_size_by_prefix.result
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
CREATE TABLE t1 (i INT, j INT, l CHAR(30), k BIGINT, PRIMARY KEY(i,j) COMMENT "cf1", KEY sk(k), KEY sk1(k,l)) ENGINE = ROCKSDB;
SET @@GLOBAL.ROCKSDB_UPDATE_CF_OPTIONS=
'cf1={write_buffer_size=8m;target_file_size_base=1m};';
SET rocksdb_bulk_load_allow_sk=1;
set rocksdb_bulk_load=1;
set rocksdb_bulk_load_size=100000;
LOAD DATA INFILE <input_file> INTO TABLE t1;
set rocksdb_bulk_load=0;
include/assert.inc ['Expected prefix size > 70000']
include/assert.inc ['Expected prefix size > 70000']
include/assert.inc ['Expected prefix size > 70000']
include/assert.inc ['Expected prefix size > 70000']
include/assert.inc ['Expected prefix size > 70000']
include/assert.inc ['Expected prefix size = 0']
include/assert.inc ['Expected prefix size > 40000']
include/assert.inc ['Expected prefix size > 40000']
include/assert.inc ['Expected prefix size > 40000']
include/assert.inc ['Expected prefix size > 40000']
include/assert.inc ['Expected prefix size > 40000']
include/assert.inc ['Expected prefix size > 40000']
include/assert.inc ['Expected prefix size = 0']
include/assert.inc ['Expected prefix size > 60000']
include/assert.inc ['Expected prefix size > 60000']
include/assert.inc ['Expected prefix size > 60000']
include/assert.inc ['Expected prefix size > 60000']
include/assert.inc ['Expected prefix size > 60000']
include/assert.inc ['Expected prefix size = 0']
SELECT GET_INDEX_SIZE_BY_PREFIX(null, null, null, null);
ERROR HY000: Incorrect arguments to get_index_size_by_prefix
SELECT GET_INDEX_SIZE_BY_PREFIX("test", null, null, null);
ERROR HY000: Incorrect arguments to get_index_size_by_prefix
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", null, null);
ERROR HY000: Incorrect arguments to get_index_size_by_prefix
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "k", null);
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: 4th argument expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("random", "t1", "PRIMARY", 1);
ERROR 42000: Unknown database 'random'
SELECT GET_INDEX_SIZE_BY_PREFIX("random", "random", "PRIMARY", 1);
ERROR 42000: Unknown database 'random'
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "random", "PRIMARY", 1);
ERROR 42S02: Table 'test.random' doesn't exist
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "random", 1);
ERROR 42S12: Table 't1' has no index like the one used in CREATE INDEX; recreate the table
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", "random string");
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: 4th argument expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", CAST("2007-07-18" AS DATE));
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: 4th argument expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", CAST("2007-07-18" AS DATETIME));
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: 4th argument expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", CAST(3.33333333 AS FLOAT));
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: 4th argument expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", CAST(3.33333333 AS DOUBLE));
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: 4th argument expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", CAST(3.33333333 AS REAL));
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: 4th argument expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", CAST(3.33333333 AS DECIMAL));
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: 4th argument expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", TIME('2:59:58.999'));
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: 4th argument expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", TIMESTAMP(20000303000000));
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: 4th argument expected to be numeric
CREATE TABLE t2 (
f1 CHAR(20) PRIMARY KEY,
f2 CHAR(20),
f3 BLOB,
f4 DATE,
f5 FLOAT,
f6 LONGTEXT,
f7 REAL,
f8 TIME,
f9 BINARY(20),
f10 BIT(64),
f11 DOUBLE,
f12 ENUM ('a', 'b', 'c') DEFAULT 'a',
f13 NUMERIC(17, 9),
f14
SET ('a', 'b', 'c') default 'b',
f15 VARBINARY(20),
f16 VARCHAR(20),
KEY sk2(f2),
KEY sk3(f3(10)),
KEY sk4(f4),
KEY sk5(f5),
KEY sk6(f6(10)),
KEY sk7(f7),
KEY sk8(f8),
KEY sk9(f9),
KEY sk10(f10),
KEY sk11(f11),
KEY sk12(f12),
KEY sk13(f13),
KEY sk14(f14),
KEY sk15(f15),
KEY sk16(f16)) ENGINE = ROCKSDB;
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "PRIMARY", 1);
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: Index keypart expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk2", 1);
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: Index keypart expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk3", 1);
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: Index keypart expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk4", 1);
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: Index keypart expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk5", 1);
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: Index keypart expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk6", 1);
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: Index keypart expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk7", 1);
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: Index keypart expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk8", 1);
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: Index keypart expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk9", 1);
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: Index keypart expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk10", 1);
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: Index keypart expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk11", 1);
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: Index keypart expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk12", 1);
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: Index keypart expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk13", 1);
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: Index keypart expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk14", 1);
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: Index keypart expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk15", 1);
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: Index keypart expected to be numeric
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk16", 1);
ERROR HY000: Incorrect arguments to GET_INDEX_SIZE_BY_PREFIX: Index keypart expected to be numeric
DROP TABLE t1;
DROP TABLE t2;
set @@global.rocksdb_update_cf_options= '';
255 changes: 255 additions & 0 deletions mysql-test/suite/rocksdb/t/get_index_size_by_prefix.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,255 @@
# Get Index Prefix size
CREATE TABLE t1 (i INT, j INT, l CHAR(30), k BIGINT, PRIMARY KEY(i,j) COMMENT "cf1", KEY sk(k), KEY sk1(k,l)) ENGINE = ROCKSDB;

# Dump all the insertion data in file
--let $file = `SELECT CONCAT(@@datadir, "test_loadfile.txt")`
--let MTR_DATA_ORDER_DESC = $data_order_desc;

# Create a text file with data to import into the table.
# The primary key is in sorted order and the secondary keys are randomly generated
--let ROCKSDB_INFILE = $file
perl;
my $fn = $ENV{'ROCKSDB_INFILE'};
open(my $fh, '>', $fn) || die "perl open($fn): $!";

# Change back to 5000000 once perf issue is resolved
my $max = 50;
my $inner_max = 10000;
my $inner_max_2 = 11000;
my $inner_max_3 = 12000;
my $tmp = "ABCD ABCD ABCD ABCD ABCD";
my $space = $ENV{'SPACE'};
for (my $ii = 0; $ii < $max; $ii++)
{
for (my $jj = 0; $jj < $inner_max; $jj++)
{
print $fh "$ii\t$jj\t$tmp\t$ii\n";
}
# Insert NULL char values
for (my $jj = $inner_max; $jj < $inner_max_2; $jj++)
{
print $fh "$ii\t$jj\t$space\t$ii\n";
}
# Insert NULL int values
for (my $jj = $inner_max_2; $jj < $inner_max_3; $jj++)
{
print $fh "$ii\t$jj\t$space\t$space\n";
}
}

close($fh);
EOF
--file_exists $file

# Update CF to smaller value to create multiple SST in ingestion
eval SET @@GLOBAL.ROCKSDB_UPDATE_CF_OPTIONS=
'cf1={write_buffer_size=8m;target_file_size_base=1m};';

SET rocksdb_bulk_load_allow_sk=1;
set rocksdb_bulk_load=1;
set rocksdb_bulk_load_size=100000;
--disable_query_log
--echo LOAD DATA INFILE <input_file> INTO TABLE t1;

eval LOAD DATA INFILE '$file'
INTO TABLE t1
FIELDS TERMINATED BY "\t"
LINES TERMINATED BY "\n"
(i, j, @vl, @vk)
SET
l = NULLIF(@vl,''),
k = NULLIF(@vk,'')
;

--enable_query_log
set rocksdb_bulk_load=0;
--remove_file $file

# Get index prefix size by primary key
--let $assert_text= 'Expected prefix size > 70000'
--let $assert_cond= [SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", 1)] > 70000;
--source include/assert.inc

--let $assert_text= 'Expected prefix size > 70000'
--let $assert_cond= [SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", 2)] > 70000;
--source include/assert.inc

--let $assert_text= 'Expected prefix size > 70000'
--let $assert_cond= [SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", 3)] > 70000;
--source include/assert.inc

--let $assert_text= 'Expected prefix size > 70000'
--let $assert_cond= [SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", 4)] > 70000;
--source include/assert.inc


--let $assert_text= 'Expected prefix size > 70000'
--let $assert_cond= [SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", 5)] > 70000;
--source include/assert.inc

--let $assert_text= 'Expected prefix size = 0'
--let $assert_cond= [SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", 2147483647)] = 0;
--source include/assert.inc

# Get index prefix size by sk
--let $assert_text= 'Expected prefix size > 40000'
--let $assert_cond= [SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "sk", 1)] > 40000;
--source include/assert.inc

--let $assert_text= 'Expected prefix size > 40000'
--let $assert_cond= [SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "sk", 2)] > 40000;
--source include/assert.inc

--let $assert_text= 'Expected prefix size > 40000'
--let $assert_cond= [SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "sk", 3)] > 40000;
--source include/assert.inc

--let $assert_text= 'Expected prefix size > 40000'
--let $assert_cond= [SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "sk", 4)] > 40000;
--source include/assert.inc

--let $assert_text= 'Expected prefix size > 40000'
--let $assert_cond= [SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "sk", 5)] > 40000;
--source include/assert.inc

--let $assert_text= 'Expected prefix size > 40000'
--let $assert_cond= [SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "sk", 5)] > 40000;
--source include/assert.inc

--let $assert_text= 'Expected prefix size = 0'
--let $assert_cond= [SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "sk", 2147483647)] = 0;
--source include/assert.inc

# Get index prefix size by sk1
--let $assert_text= 'Expected prefix size > 60000'
--let $assert_cond= [SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "sk1", 1)] > 60000;
--source include/assert.inc

--let $assert_text= 'Expected prefix size > 60000'
--let $assert_cond= [SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "sk1", 2)] > 60000;
--source include/assert.inc

--let $assert_text= 'Expected prefix size > 60000'
--let $assert_cond= [SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "sk1", 3)] > 60000;
--source include/assert.inc

--let $assert_text= 'Expected prefix size > 60000'
--let $assert_cond= [SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "sk1", 4)] > 60000;
--source include/assert.inc

--let $assert_text= 'Expected prefix size > 60000'
--let $assert_cond= [SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "sk1", 5)] > 60000;
--source include/assert.inc

--let $assert_text= 'Expected prefix size = 0'
--let $assert_cond= [SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "sk1", 2147483647)] = 0;
--source include/assert.inc

# Test null params
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX(null, null, null, null);
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", null, null, null);
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", null, null);
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "k", null);

# Test on invalid params
--error ER_BAD_DB_ERROR
SELECT GET_INDEX_SIZE_BY_PREFIX("random", "t1", "PRIMARY", 1);
--error ER_BAD_DB_ERROR
SELECT GET_INDEX_SIZE_BY_PREFIX("random", "random", "PRIMARY", 1);
--error ER_NO_SUCH_TABLE
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "random", "PRIMARY", 1);
--error ER_NO_SUCH_INDEX
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "random", 1);
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", "random string");
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", CAST("2007-07-18" AS DATE));
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", CAST("2007-07-18" AS DATETIME));
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", CAST(3.33333333 AS FLOAT));
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", CAST(3.33333333 AS DOUBLE));
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", CAST(3.33333333 AS REAL));
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", CAST(3.33333333 AS DECIMAL));
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", TIME('2:59:58.999'));
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t1", "PRIMARY", TIMESTAMP(20000303000000));

# Index keys are not int
CREATE TABLE t2 (
f1 CHAR(20) PRIMARY KEY,
f2 CHAR(20),
f3 BLOB,
f4 DATE,
f5 FLOAT,
f6 LONGTEXT,
f7 REAL,
f8 TIME,
f9 BINARY(20),
f10 BIT(64),
f11 DOUBLE,
f12 ENUM ('a', 'b', 'c') DEFAULT 'a',
f13 NUMERIC(17, 9),
f14
SET ('a', 'b', 'c') default 'b',
f15 VARBINARY(20),
f16 VARCHAR(20),
KEY sk2(f2),
KEY sk3(f3(10)),
KEY sk4(f4),
KEY sk5(f5),
KEY sk6(f6(10)),
KEY sk7(f7),
KEY sk8(f8),
KEY sk9(f9),
KEY sk10(f10),
KEY sk11(f11),
KEY sk12(f12),
KEY sk13(f13),
KEY sk14(f14),
KEY sk15(f15),
KEY sk16(f16)) ENGINE = ROCKSDB;
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "PRIMARY", 1);
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk2", 1);
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk3", 1);
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk4", 1);
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk5", 1);
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk6", 1);
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk7", 1);
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk8", 1);
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk9", 1);
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk10", 1);
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk11", 1);
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk12", 1);
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk13", 1);
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk14", 1);
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk15", 1);
--error ER_WRONG_ARGUMENTS
SELECT GET_INDEX_SIZE_BY_PREFIX("test", "t2", "sk16", 1);

DROP TABLE t1;
DROP TABLE t2;
set @@global.rocksdb_update_cf_options= '';
Loading

0 comments on commit 78923eb

Please sign in to comment.