Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

add LOAD DATA LOCAL INFILE to speed up data insert #450

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

add LOAD DATA LOCAL INFILE to speed up data insert #450

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
33b56ec
add LOAD DATA LOCAL INFILE to speed up data insert
slowtech Mar 27, 2022
f28a482
specify csv separator
Mar 29, 2022
53e31fe
Modify the type of use_local_infile
Apr 1, 2022
70eaa34
remove print(query)
Apr 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions src/drivers/mysql/drv_mysql.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ static sb_arg_t mysql_drv_args[] =
"1213,1020,1205", LIST),
SB_OPT("mysql-dry-run", "Dry run, pretend that all MySQL client API "
"calls are successful without executing them", "off", BOOL),
SB_OPT("mysql-local-infile", "Enable/disable LOAD DATA LOCAL INFILE.", "on", BOOL),

SB_OPT_END
};
Expand All @@ -110,6 +111,7 @@ typedef struct
unsigned char debug;
sb_list_t *ignored_errors;
unsigned int dry_run;
unsigned int use_local_infile;
} mysql_drv_args_t;

typedef struct
Expand Down Expand Up @@ -334,6 +336,7 @@ int mysql_drv_init(void)
#endif

args.use_compression = sb_get_value_flag("mysql-compression");
args.use_local_infile = sb_get_value_flag("mysql-local-infile");
args.debug = sb_get_value_flag("mysql-debug");
if (args.debug)
sb_globals.verbosity = LOG_DEBUG;
Expand Down Expand Up @@ -411,6 +414,12 @@ static int mysql_drv_real_connect(db_mysql_conn_t *db_mysql_con)
DEBUG("mysql_options(%p, %s, %s)",con, "MYSQL_OPT_COMPRESS", "NULL");
mysql_options(con, MYSQL_OPT_COMPRESS, NULL);
}

if (args.use_local_infile)
{
DEBUG("mysql_options(%p, %s, %d)",con, "MYSQL_OPT_LOCAL_INFILE", args.use_local_infile);
mysql_options(con, MYSQL_OPT_LOCAL_INFILE, &args.use_local_infile);
}

DEBUG("mysql_real_connect(%p, \"%s\", \"%s\", \"%s\", \"%s\", %u, \"%s\", %s)",
con,
Expand Down
71 changes: 61 additions & 10 deletions src/lua/oltp_common.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,11 @@ sysbench.cmdline.options = {
"PostgreSQL driver. The only currently supported " ..
"variant is 'redshift'. When enabled, " ..
"create_secondary is automatically disabled, and " ..
"delete_inserts is set to 0"}
"delete_inserts is set to 0"},
fast =
{"Use LOAD DATA LOCAL to import data, only works with MySQL",false},
csv_dir =
{"The directory to save CSV file","/tmp"}
}

-- Prepare the dataset. This command supports parallel execution, i.e. will
Expand Down Expand Up @@ -176,6 +180,9 @@ function create_table(drv, con, table_num)
engine_def = "/*! ENGINE = " .. sysbench.opt.mysql_storage_engine .. " */"
elseif drv:name() == "pgsql"
then
if sysbench.opt.fast then
error("--fast is not supported for pgsql")
end
if not sysbench.opt.auto_inc then
id_def = "INTEGER NOT NULL"
elseif pgsql_variant == 'redshift' then
Expand Down Expand Up @@ -207,37 +214,81 @@ CREATE TABLE sbtest%d(
sysbench.opt.table_size, table_num))
end

if sysbench.opt.auto_inc then
query = "INSERT INTO sbtest" .. table_num .. "(k, c, pad) VALUES"
else
query = "INSERT INTO sbtest" .. table_num .. "(id, k, c, pad) VALUES"
if not (sysbench.opt.fast) then
if sysbench.opt.auto_inc then
query = "INSERT INTO sbtest" .. table_num .. "(k, c, pad) VALUES"
else
query = "INSERT INTO sbtest" .. table_num .. "(id, k, c, pad) VALUES"
end
con:bulk_insert_init(query)
end

con:bulk_insert_init(query)

local c_val
local pad_val

local f
if (sysbench.opt.fast) then
f = assert(io.open(string.format("/%s/sbtest%d",sysbench.opt.csv_dir,table_num),'w'))
end

for i = 1, sysbench.opt.table_size do

c_val = get_c_value()
pad_val = get_pad_value()

if (sysbench.opt.auto_inc) then
query = string.format("(%d, '%s', '%s')",
if (sysbench.opt.fast) then
query = string.format("%d,%s,%s\n",
sysbench.rand.default(1, sysbench.opt.table_size),
c_val, pad_val)
else
query = string.format("(%d, '%s', '%s')",
sysbench.rand.default(1, sysbench.opt.table_size),
c_val, pad_val)

end
else
query = string.format("(%d, %d, '%s', '%s')",
if (sysbench.opt.fast) then
query = string.format("%d,%d,%s,%s\n",
i,
sysbench.rand.default(1, sysbench.opt.table_size),
c_val, pad_val)
else
query = string.format("(%d, %d, '%s', '%s')",
i,
sysbench.rand.default(1, sysbench.opt.table_size),
c_val, pad_val)
end
end

con:bulk_insert_next(query)
if (sysbench.opt.fast) then
f:write(query)
else
con:bulk_insert_next(query)
end

end

con:bulk_insert_done()
if (sysbench.opt.fast) then

f:close()
local column_name
if (sysbench.opt.auto_inc) then
column_name="k, c, pad"
else
column_name="id, k, c, pad"
end

query = string.format("LOAD DATA LOCAL INFILE '/%s/sbtest%d' " ..
"INTO TABLE sbtest%d FIELDS TERMINATED BY ',' LINES TERMINATED BY '\\n' " ..
"(%s)", sysbench.opt.csv_dir,table_num,table_num,column_name)
con:query("SET unique_checks = 0")
con:query("SET foreign_key_checks = 0")
con:query(query)
else
con:bulk_insert_done()
end

if sysbench.opt.create_secondary then
print(string.format("Creating a secondary index on 'sbtest%d'...",
Expand Down