From 33b56eca26006be60108fed6588f60f21b4ef802 Mon Sep 17 00:00:00 2001 From: slowtech Date: Sun, 27 Mar 2022 12:29:34 +0000 Subject: [PATCH 1/4] add LOAD DATA LOCAL INFILE to speed up data insert --- src/drivers/mysql/drv_mysql.c | 9 +++++ src/lua/oltp_common.lua | 70 ++++++++++++++++++++++++++++++----- 2 files changed, 69 insertions(+), 10 deletions(-) diff --git a/src/drivers/mysql/drv_mysql.c b/src/drivers/mysql/drv_mysql.c index 58600b8e0..395a41c14 100644 --- a/src/drivers/mysql/drv_mysql.c +++ b/src/drivers/mysql/drv_mysql.c @@ -86,6 +86,7 @@ static sb_arg_t mysql_drv_args[] = "1213,1020,1205", LIST), SB_OPT("mysql-dry-run", "Dry run, pretend that all MySQL client API " "calls are successful without executing them", "off", BOOL), + SB_OPT("mysql-local-infile", "Enable/disable LOAD DATA LOCAL INFILE.", "on", BOOL), SB_OPT_END }; @@ -110,6 +111,7 @@ typedef struct unsigned char debug; sb_list_t *ignored_errors; unsigned int dry_run; + unsigned char use_local_infile; } mysql_drv_args_t; typedef struct @@ -334,6 +336,7 @@ int mysql_drv_init(void) #endif args.use_compression = sb_get_value_flag("mysql-compression"); + args.use_local_infile = sb_get_value_flag("mysql-local-infile"); args.debug = sb_get_value_flag("mysql-debug"); if (args.debug) sb_globals.verbosity = LOG_DEBUG; @@ -411,6 +414,12 @@ static int mysql_drv_real_connect(db_mysql_conn_t *db_mysql_con) DEBUG("mysql_options(%p, %s, %s)",con, "MYSQL_OPT_COMPRESS", "NULL"); mysql_options(con, MYSQL_OPT_COMPRESS, NULL); } + + if (args.use_local_infile) + { + DEBUG("mysql_options(%p, %s, %s)",con, "MYSQL_OPT_LOCAL_INFILE", "NULL"); + mysql_options(con, MYSQL_OPT_LOCAL_INFILE, NULL); + } DEBUG("mysql_real_connect(%p, \"%s\", \"%s\", \"%s\", \"%s\", %u, \"%s\", %s)", con, diff --git a/src/lua/oltp_common.lua b/src/lua/oltp_common.lua index fdc5c83ed..df2e7fdff 100644 --- a/src/lua/oltp_common.lua +++ b/src/lua/oltp_common.lua @@ -78,7 +78,11 @@ sysbench.cmdline.options = { "PostgreSQL driver. The only currently supported " .. "variant is 'redshift'. When enabled, " .. "create_secondary is automatically disabled, and " .. - "delete_inserts is set to 0"} + "delete_inserts is set to 0"}, + fast = + {"Use LOAD DATA LOCAL to import data, only works with MySQL",false}, + csv_dir = + {"The directory to save CSV file","/tmp"} } -- Prepare the dataset. This command supports parallel execution, i.e. will @@ -176,6 +180,9 @@ function create_table(drv, con, table_num) engine_def = "/*! ENGINE = " .. sysbench.opt.mysql_storage_engine .. " */" elseif drv:name() == "pgsql" then + if sysbench.opt.fast then + error("--fast is not supported for pgsql") + end if not sysbench.opt.auto_inc then id_def = "INTEGER NOT NULL" elseif pgsql_variant == 'redshift' then @@ -207,37 +214,80 @@ CREATE TABLE sbtest%d( sysbench.opt.table_size, table_num)) end - if sysbench.opt.auto_inc then - query = "INSERT INTO sbtest" .. table_num .. "(k, c, pad) VALUES" - else - query = "INSERT INTO sbtest" .. table_num .. "(id, k, c, pad) VALUES" + if not (sysbench.opt.fast) then + if sysbench.opt.auto_inc then + query = "INSERT INTO sbtest" .. table_num .. "(k, c, pad) VALUES" + else + query = "INSERT INTO sbtest" .. table_num .. "(id, k, c, pad) VALUES" + end + con:bulk_insert_init(query) end - con:bulk_insert_init(query) local c_val local pad_val + local f + if (sysbench.opt.fast) then + f = assert(io.open(string.format("/%s/sbtest%d",sysbench.opt.csv_dir,table_num),'w')) + end + for i = 1, sysbench.opt.table_size do c_val = get_c_value() pad_val = get_pad_value() if (sysbench.opt.auto_inc) then - query = string.format("(%d, '%s', '%s')", + if (sysbench.opt.fast) then + query = string.format("%d,%s,%s\n", + sysbench.rand.default(1, sysbench.opt.table_size), + c_val, pad_val) + else + query = string.format("(%d, '%s', '%s')", sysbench.rand.default(1, sysbench.opt.table_size), c_val, pad_val) + + end else - query = string.format("(%d, %d, '%s', '%s')", + if (sysbench.opt.fast) then + query = string.format("%d,%d,%s,%s\n", + i, + sysbench.rand.default(1, sysbench.opt.table_size), + c_val, pad_val) + else + query = string.format("(%d, %d, '%s', '%s')", i, sysbench.rand.default(1, sysbench.opt.table_size), c_val, pad_val) + end end - con:bulk_insert_next(query) + if (sysbench.opt.fast) then + f:write(query) + else + con:bulk_insert_next(query) + end + end - con:bulk_insert_done() + if (sysbench.opt.fast) then + + f:close() + local column_name + if (sysbench.opt.auto_inc) then + column_name="k, c, pad" + else + column_name="id, k, c, pad" + end + + query = string.format("LOAD DATA LOCAL INFILE '/%s/sbtest%d' INTO TABLE sbtest%d (%s)", + sysbench.opt.csv_dir,table_num,table_num,column_name) + con:query("SET unique_checks = 0") + con:query("SET foreign_key_checks = 0") + con:query(query) + else + con:bulk_insert_done() + end if sysbench.opt.create_secondary then print(string.format("Creating a secondary index on 'sbtest%d'...", From f28a4822ccae3b5a87557c55dbd64521cef5c292 Mon Sep 17 00:00:00 2001 From: "ivictor@foxmail.com" Date: Tue, 29 Mar 2022 18:44:25 +0800 Subject: [PATCH 2/4] specify csv separator --- src/lua/oltp_common.lua | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lua/oltp_common.lua b/src/lua/oltp_common.lua index df2e7fdff..c188b1cb3 100644 --- a/src/lua/oltp_common.lua +++ b/src/lua/oltp_common.lua @@ -280,8 +280,10 @@ CREATE TABLE sbtest%d( column_name="id, k, c, pad" end - query = string.format("LOAD DATA LOCAL INFILE '/%s/sbtest%d' INTO TABLE sbtest%d (%s)", - sysbench.opt.csv_dir,table_num,table_num,column_name) + query = string.format("LOAD DATA LOCAL INFILE '/%s/sbtest%d' " .. + "INTO TABLE sbtest%d FIELDS TERMINATED BY ',' LINES TERMINATED BY '\\n' " .. + "(%s)", sysbench.opt.csv_dir,table_num,table_num,column_name) + print(query) con:query("SET unique_checks = 0") con:query("SET foreign_key_checks = 0") con:query(query) From 53e31fedd275ea24e19ea3a55a3359adc8a9f350 Mon Sep 17 00:00:00 2001 From: "ivictor@foxmail.com" Date: Fri, 1 Apr 2022 11:16:45 +0800 Subject: [PATCH 3/4] Modify the type of use_local_infile --- src/drivers/mysql/drv_mysql.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/drivers/mysql/drv_mysql.c b/src/drivers/mysql/drv_mysql.c index 395a41c14..d36adf3e3 100644 --- a/src/drivers/mysql/drv_mysql.c +++ b/src/drivers/mysql/drv_mysql.c @@ -111,7 +111,7 @@ typedef struct unsigned char debug; sb_list_t *ignored_errors; unsigned int dry_run; - unsigned char use_local_infile; + unsigned int use_local_infile; } mysql_drv_args_t; typedef struct @@ -417,8 +417,8 @@ static int mysql_drv_real_connect(db_mysql_conn_t *db_mysql_con) if (args.use_local_infile) { - DEBUG("mysql_options(%p, %s, %s)",con, "MYSQL_OPT_LOCAL_INFILE", "NULL"); - mysql_options(con, MYSQL_OPT_LOCAL_INFILE, NULL); + DEBUG("mysql_options(%p, %s, %d)",con, "MYSQL_OPT_LOCAL_INFILE", args.use_local_infile); + mysql_options(con, MYSQL_OPT_LOCAL_INFILE, &args.use_local_infile); } DEBUG("mysql_real_connect(%p, \"%s\", \"%s\", \"%s\", \"%s\", %u, \"%s\", %s)", From 70eaa34d67aa6648ed0744bb4ab379a373e9201c Mon Sep 17 00:00:00 2001 From: "ivictor@foxmail.com" Date: Fri, 1 Apr 2022 21:37:20 +0800 Subject: [PATCH 4/4] remove print(query) --- src/drivers/mysql/drv_mysql.c | 2 +- src/lua/oltp_common.lua | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/drivers/mysql/drv_mysql.c b/src/drivers/mysql/drv_mysql.c index d36adf3e3..145dc657d 100644 --- a/src/drivers/mysql/drv_mysql.c +++ b/src/drivers/mysql/drv_mysql.c @@ -111,7 +111,7 @@ typedef struct unsigned char debug; sb_list_t *ignored_errors; unsigned int dry_run; - unsigned int use_local_infile; + unsigned int use_local_infile; } mysql_drv_args_t; typedef struct diff --git a/src/lua/oltp_common.lua b/src/lua/oltp_common.lua index c188b1cb3..6f85959a9 100644 --- a/src/lua/oltp_common.lua +++ b/src/lua/oltp_common.lua @@ -283,7 +283,6 @@ CREATE TABLE sbtest%d( query = string.format("LOAD DATA LOCAL INFILE '/%s/sbtest%d' " .. "INTO TABLE sbtest%d FIELDS TERMINATED BY ',' LINES TERMINATED BY '\\n' " .. "(%s)", sysbench.opt.csv_dir,table_num,table_num,column_name) - print(query) con:query("SET unique_checks = 0") con:query("SET foreign_key_checks = 0") con:query(query)