diff --git a/man/ndb_import.1 b/man/ndb_import.1 index 7142ac5e472..24823e8adfa 100644 --- a/man/ndb_import.1 +++ b/man/ndb_import.1 @@ -2,12 +2,12 @@ .\" Title: \fBndb_import\fR .\" Author: [FIXME: author] [see http://docbook.sf.net/el/author] .\" Generator: DocBook XSL Stylesheets v1.79.1 -.\" Date: 11/26/2020 -.\" Manual: MySQL Database System -.\" Source: MySQL 8.0 +.\" Date: 05/20/2021 +.\" Manual: RonDB Database System +.\" Source: RonDB 21.04 .\" Language: English .\" -.TH "\FBNDB_IMPORT\FR" "1" "11/26/2020" "MySQL 8\&.0" "MySQL Database System" +.TH "\FBNDB_IMPORT\FR" "1" "05/20/2021" "RonDB 21\&.04" "RonDB Database System" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- @@ -28,7 +28,7 @@ .\" * MAIN CONTENT STARTS HERE * .\" ----------------------------------------------------------------- .SH "NAME" -ndb_import \- Import CSV data into NDB +ndb_import \- Import CSV data into RonDB .SH "SYNOPSIS" .HP \w'\fBndb_import\ \fR\fB\fIoptions\fR\fR\ 'u \fBndb_import \fR\fB\fIoptions\fR\fR @@ -38,10 +38,10 @@ ndb_import \- Import CSV data into NDB imports CSV\-formatted data, such as that produced by \fBmysqldump\fR \fB\-\-tab\fR, directly into -NDB +RonDB using the NDB API\&. \fBndb_import\fR -requires a connection to an NDB management server (\fBndb_mgmd\fR) to function; it does not require a connection to a MySQL Server\&. +requires a connection to an RonDB management server (\fBndb_mgmd\fR) to function; it does not require a connection to a MySQL Server\&. Usage .sp .if n \{\ @@ -63,7 +63,7 @@ is the name of the CSV file from which to read the data; this must include the p \fBndb_import\fR include those for specifying field separators, escapes, and line terminators, and are described later in this section\&. \fBndb_import\fR -must be able to connect to an NDB Cluster management server; for this reason, there must be an unused +must be able to connect to an RonDB Cluster management server; for this reason, there must be an unused [api] slot in the cluster config\&.ini @@ -71,7 +71,7 @@ file\&. .PP To duplicate an existing table that uses a different storage engine, such as InnoDB, as an -NDB +RonDB table, use the \fBmysql\fR client to perform a @@ -83,7 +83,7 @@ ALTER TABLE \&.\&.\&. ENGINE=NDB on the new table; after this, from the system shell, invoke \fBndb_import\fR to load the data into the new -NDB +RonDB table\&. For example, an existing InnoDB table named @@ -91,7 +91,7 @@ myinnodb_table in a database named myinnodb can be exported into an -NDB +RonDB table named myndb_table in a database named @@ -251,6 +251,16 @@ T}:T{ T} T{ .PP +\fB \fR\fB--use-auto-increment=#\fR\fB \fR +T}:T{ +For table with auto increment, ignore value in CSV file and generate +normal auto increment value +T}:T{ +.PP +(Supported in all RonDB 21.04.1 based releases) +T} +T{ +.PP \fB \fR\fB--ai-increment=#\fR\fB \fR T}:T{ For table with hidden PK, specify autoincrement increment. See mysqld @@ -472,8 +482,8 @@ T{ .PP \fB \fR\fB--opbatch=#\fR\fB \fR T}:T{ -A db execution batch is a set of transactions and operations sent to NDB - kernel. This option limits NDB operations (including blob +A db execution batch is a set of transactions and operations sent to RonDB + kernel. This option limits RonDB operations (including blob operations) in a db execution batch. Therefore it also limits number of asynch transactions. Value 0 is not valid T}:T{ @@ -674,6 +684,43 @@ Dump core on any fatal error; used for debugging only\&. .sp -1 .IP \(bu 2.3 .\} +\fB\-\-use\-auto\-increment\fR=\fI#\fR +.TS +allbox tab(:); +lB l +lB l +lB l +lB l +lB l. +T{ +Command-Line Format +T}:T{ +--use-auto-increment=# +T} +T{ +Type +T}:T{ +Boolealn +T} +T{ +Default Value +T}:T{ +TRUE +T} +.TE +.sp 1 +For a table with auto increment, ignore the value in the CSV file and instead +generate a normal auto increment value\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} \fB\-\-ai\-increment\fR=\fI#\fR .TS allbox tab(:); @@ -1107,7 +1154,7 @@ T} T{ Default Value T}:T{ -\ +Backslash T} .TE .sp 1 @@ -1184,7 +1231,7 @@ T} T{ Default Value T}:T{ -\t +TAB T} .TE .sp 1 @@ -1496,7 +1543,7 @@ T} T{ Default Value T}:T{ -\n +CR T} .TE .sp 1 @@ -1552,7 +1599,7 @@ T} .sp 1 Performs internal logging at the given level\&. This option is intended primarily for internal and development use\&. .sp -In debug builds of NDB only, the logging level can be set using this option to a maximum of 4\&. +In debug builds of RonDB only, the logging level can be set using this option to a maximum of 4\&. .RE .sp .RS 4 @@ -2488,6 +2535,8 @@ was added in NDB 7\&.6\&.2\&. .br .PP Copyright \(co 1997, 2020, Oracle and/or its affiliates. +.br +Copyright \(co 2021, 2021, Logical Clocks AB and/or its affiliates. .PP This documentation is free software; you can redistribute it and/or modify it only under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 of the License. .PP @@ -2496,8 +2545,10 @@ This documentation is distributed in the hope that it will be useful, but WITHOU You should have received a copy of the GNU General Public License along with the program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA or see http://www.gnu.org/licenses/. .sp .SH "SEE ALSO" -For more information, please refer to the MySQL Reference Manual, +For more information, please refer to the MySQL Reference Manual and the RonDB Documentation which may already be installed locally and which is also available -online at http://dev.mysql.com/doc/. +online at http://dev.mysql.com/doc/ and at http://docs.rondb.com. .SH AUTHOR Oracle Corporation (http://dev.mysql.com/). +.br +Logical Clocks AB (http://logicalclocks.com/). diff --git a/mysql-test/suite/ndb/t/ndb_import0.test b/mysql-test/suite/ndb/t/ndb_import0.test index b1dc489ff86..f4ad604c352 100644 --- a/mysql-test/suite/ndb/t/ndb_import0.test +++ b/mysql-test/suite/ndb/t/ndb_import0.test @@ -648,6 +648,86 @@ exec $NDB_IMPORT --state-dir=$MYSQLTEST_VARDIR/tmp --csvopt=cqn --pagesize=256 --pagebuffer=512 test $MYSQLTEST_VARDIR/tmp/tpersons.csv >> $NDB_TOOLS_OUTPUT 2>&1; +perl; +use strict; +use Symbol; +my $lt = !$ENV{IS_WINDOWS} ? "\n" : "\r\n"; +my $vardir = $ENV{MYSQLTEST_VARDIR} + or die "need MYSQLTEST_VARDIR"; +my $file = "$vardir/tmp/tauto_inc1.csv"; +my $fh = gensym(); +open($fh, ">:raw", $file) + or die "$file: open for write failed: $!"; +for my $i (1..5) { + print $fh $i, "\t", $i, $lt; +} +close($fh) + or die "$file: close after write failed: $!"; +exit(0) +EOF + +perl; +use strict; +use Symbol; +my $lt = !$ENV{IS_WINDOWS} ? "\n" : "\r\n"; +my $vardir = $ENV{MYSQLTEST_VARDIR} + or die "need MYSQLTEST_VARDIR"; +my $file = "$vardir/tmp/tauto_inc2.csv"; +my $fh = gensym(); +open($fh, ">:raw", $file) + or die "$file: open for write failed: $!"; +for my $i (1..5) { + print $fh $i+4, "\t", $i, $lt; +} +close($fh) + or die "$file: close after write failed: $!"; +exit(0) +EOF + +perl; +use strict; +use Symbol; +my $lt = !$ENV{IS_WINDOWS} ? "\n" : "\r\n"; +my $vardir = $ENV{MYSQLTEST_VARDIR} + or die "need MYSQLTEST_VARDIR"; +my $file = "$vardir/tmp/tauto_inc3.csv"; +my $fh = gensym(); +open($fh, ">:raw", $file) + or die "$file: open for write failed: $!"; +for my $i (1..5) { + print $fh $i+4, "\t", $i, $lt; +} +close($fh) + or die "$file: close after write failed: $!"; +exit(0) +EOF + +CREATE TABLE tauto_inc1 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +val INT NOT NULL +) engine=NDB; + +CREATE TABLE tauto_inc2 ( +id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, +val INT NOT NULL +) engine=NDB; + +CREATE TABLE tauto_inc3 ( +id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, +val INT NOT NULL +) engine=NDB; + +exec $NDB_IMPORT --state-dir=$MYSQLTEST_VARDIR/tmp --ai-prefetch-sz=1 + test $MYSQLTEST_VARDIR/tmp/tauto_inc1.csv >> $NDB_TOOLS_OUTPUT 2>&1; +exec $NDB_IMPORT --state-dir=$MYSQLTEST_VARDIR/tmp --ai-prefetch-sz=1 + test $MYSQLTEST_VARDIR/tmp/tauto_inc2.csv >> $NDB_TOOLS_OUTPUT 2>&1; +exec $NDB_IMPORT --state-dir=$MYSQLTEST_VARDIR/tmp --ai-prefetch-sz=1 + test $MYSQLTEST_VARDIR/tmp/tauto_inc3.csv >> $NDB_TOOLS_OUTPUT 2>&1; + +select id from tauto_inc1 order by id; +select id from tauto_inc2 order by id; +select id from tauto_inc3 order by id; + # Look for the specific error message from ndb_import --let $assert_file=$NDB_TOOLS_OUTPUT --let $assert_only_after=import test.tpersons @@ -656,6 +736,10 @@ exec $NDB_IMPORT --state-dir=$MYSQLTEST_VARDIR/tmp --let $assert_count=1 --source include/assert_grep.inc +drop table tauto_inc1; +drop table tauto_inc2; +drop table tauto_inc3; + drop table tpersons; drop table t1, t1ver, t2, t2ver, t3, t3ver, t4, diff --git a/storage/ndb/tools/NdbImport.cpp b/storage/ndb/tools/NdbImport.cpp index 8cdaa2490b9..7727e3405b9 100644 --- a/storage/ndb/tools/NdbImport.cpp +++ b/storage/ndb/tools/NdbImport.cpp @@ -91,6 +91,7 @@ NdbImport::Opt::Opt() m_monitor = 2; m_ai_prefetch_sz = 1024; m_ai_increment = 1; + m_use_auto_increment = true; m_ai_offset = 1; m_no_asynch = false; m_no_hint = false; diff --git a/storage/ndb/tools/NdbImport.hpp b/storage/ndb/tools/NdbImport.hpp index b04690d6204..daf6dd6ed38 100644 --- a/storage/ndb/tools/NdbImport.hpp +++ b/storage/ndb/tools/NdbImport.hpp @@ -83,6 +83,7 @@ class NdbImport { uint m_ai_prefetch_sz; uint m_ai_increment; uint m_ai_offset; + bool m_use_auto_increment; bool m_no_asynch; bool m_no_hint; uint m_pagesize; diff --git a/storage/ndb/tools/NdbImportImpl.cpp b/storage/ndb/tools/NdbImportImpl.cpp index 76f5d890800..065409b04e1 100644 --- a/storage/ndb/tools/NdbImportImpl.cpp +++ b/storage/ndb/tools/NdbImportImpl.cpp @@ -3265,12 +3265,11 @@ NdbImportImpl::ExecOpWorkerAsynch::state_define() Row* row = op->m_row; require(row != 0); const Table& table = m_util.get_table(row->m_tabid); - if (table.m_has_hidden_pk) + if (table.m_has_hidden_pk || + table.m_has_auto_increment) { const Attrs& attrs = table.m_attrs; const uint attrcnt = attrs.size(); - const Attr& attr = attrs[attrcnt - 1]; - require(attr.m_type == NdbDictionary::Column::Bigunsigned); Uint64 val; if (m_ndb->getAutoIncrementValue(table.m_tab, val, opt.m_ai_prefetch_sz, @@ -3304,7 +3303,39 @@ NdbImportImpl::ExecOpWorkerAsynch::state_define() break; } } - attr.set_value(row, &val, 8); + if (table.m_has_hidden_pk) + { + const Attr& attr = attrs[attrcnt - 1]; + require(attr.m_type == NdbDictionary::Column::Bigunsigned); + attr.set_value(row, &val, 8); + } + else + { + const Attr& attr = attrs[table.m_auto_increment_col]; + if (attr.m_type == NdbDictionary::Column::Bigunsigned) + { + attr.set_value(row, &val, 8); + } + else if (attr.m_type == NdbDictionary::Column::Bigint) + { + Int64 val64 = Int64(val); + attr.set_value(row, &val64, 8); + } + else if (attr.m_type == NdbDictionary::Column::Int) + { + Int32 val32 = Uint32(val); + attr.set_value(row, &val32, 4); + } + else if (attr.m_type == NdbDictionary::Column::Unsigned) + { + Uint32 val32 = Uint32(val); + attr.set_value(row, &val32, 4); + } + else + { + require(false); + } + } } const bool no_hint = opt.m_no_hint; Tx* tx = 0; diff --git a/storage/ndb/tools/NdbImportUtil.cpp b/storage/ndb/tools/NdbImportUtil.cpp index fdadbb38048..04ecd84fd83 100644 --- a/storage/ndb/tools/NdbImportUtil.cpp +++ b/storage/ndb/tools/NdbImportUtil.cpp @@ -782,6 +782,8 @@ NdbImportUtil::Table::Table() m_keyrec = NULL; m_recsize = 0; m_has_hidden_pk = false; + m_has_auto_increment = false; + m_auto_increment_col = 0; } void @@ -928,6 +930,7 @@ NdbImportUtil::add_table(NdbDictionary::Dictionary* dic, attr.m_attrno = i; attr.m_attrid = i; attr.m_type = col->getType(); + attr.m_auto_increment = col->getAutoIncrement(); attr.m_pk = col->getPrimaryKey(); attr.m_nullable = col->getNullable(); attr.m_precision = col->getPrecision(); @@ -1041,6 +1044,23 @@ NdbImportUtil::add_table(NdbDictionary::Dictionary* dic, break; } } + else if (attr.m_auto_increment && + c_opt.m_use_auto_increment) + { + table.m_has_auto_increment = true; + table.m_auto_increment_col = i; + if (!(attr.m_type == NdbDictionary::Column::Bigunsigned || + attr.m_type == NdbDictionary::Column::Bigint || + attr.m_type == NdbDictionary::Column::Int || + attr.m_type == NdbDictionary::Column::Unsigned)) + { + set_error_usage(error, __LINE__, + "column %u: " + "invalid auto_increment data type ", i); + ok = false; + break; + } + } } } if (!ok) diff --git a/storage/ndb/tools/NdbImportUtil.hpp b/storage/ndb/tools/NdbImportUtil.hpp index 5ed077cf64a..74cb7e2f371 100644 --- a/storage/ndb/tools/NdbImportUtil.hpp +++ b/storage/ndb/tools/NdbImportUtil.hpp @@ -239,6 +239,7 @@ class NdbImportUtil { uint m_offset; uint m_null_byte; uint m_null_bit; + bool m_auto_increment; }; typedef std::vector Attrs; @@ -258,6 +259,8 @@ class NdbImportUtil { const NdbRecord* m_keyrec; uint m_recsize; // size of main record bool m_has_hidden_pk; + bool m_has_auto_increment; + uint m_auto_increment_col; Attrs m_attrs; std::vector m_blobids; // map fragid to nodeid diff --git a/storage/ndb/tools/ndb_import.cpp b/storage/ndb/tools/ndb_import.cpp index 372905de8a6..a40d1393644 100644 --- a/storage/ndb/tools/ndb_import.cpp +++ b/storage/ndb/tools/ndb_import.cpp @@ -146,18 +146,23 @@ my_long_options[] = " up to some pre-defined limit", &g_opt.m_monitor, &g_opt.m_monitor, 0, GET_UINT, REQUIRED_ARG, g_opt.m_monitor, 0, 0, 0, 0, 0 }, + { "use-auto-increment", NDB_OPT_NOSHORT, + "Ignore value in CSV file for auto increment fields, " + " instead use an automatically incremented value", + &g_opt.m_use_auto_increment, &g_opt.m_use_auto_increment, 0, + GET_UINT, REQUIRED_ARG, g_opt.m_ai_increment, 0, 0, 0, 0, 0 }, { "ai-prefetch-sz", NDB_OPT_NOSHORT, - "For table with hidden PK, specify number of autoincrement values" - " that are prefetched. See mysqld", + "For table with hidden PK/auto increment, specify number of autoincrement" + " values that are prefetched. See mysqld", &g_opt.m_ai_prefetch_sz, &g_opt.m_ai_prefetch_sz, 0, GET_UINT, REQUIRED_ARG, g_opt.m_ai_prefetch_sz, 0, 0, 0, 0, 0 }, { "ai-increment", NDB_OPT_NOSHORT, - "For table with hidden PK, specify autoincrement increment." + "For table with hidden PK/auto increment, specify autoincrement increment." " See mysqld", &g_opt.m_ai_increment, &g_opt.m_ai_increment, 0, GET_UINT, REQUIRED_ARG, g_opt.m_ai_increment, 0, 0, 0, 0, 0 }, { "ai-offset", NDB_OPT_NOSHORT, - "For table with hidden PK, specify autoincrement offset." + "For table with hidden PK/auto increment, specify autoincrement offset." " See mysqld", &g_opt.m_ai_offset, &g_opt.m_ai_offset, 0, GET_UINT, REQUIRED_ARG, g_opt.m_ai_offset, 0, 0, 0, 0, 0 },