WL#14717 - InnoDB: Bulk Ingest Feature V1

   WL#14772 InnoDB: Bulk Load - Parallel build Infrastructure
   WL#15131 Innodb: Support Bulk Load with Sorted data
   WL#15132 Innodb: Support Bulk Load with Unsorted Data
   WL#15133 Innodb: Support Bulk Load from OCI Object Store
   WL#15612 Innodb: Support Bulk Load from AWS S3

    LOAD DATA FROM INFILE | URL | S3
       'file_prefix' [COUNT N]
       INTO TABLE table_name
       [CHARACTER SET charset_name]
       [{COLUMNS | FIELDS}
           [TERMINATED BY 'string']
           [[OPTIONALLY] ENCLOSED BY 'char']
           [ESCAPED BY 'char']
       ]
       [LINES TERMINATED BY 'line_term_string']
       [IGNORE number {LINES | ROWS}]
       [PARALLEL = C]
       [MEMORY = M]
       [ALGORITHM = BULK];

Developed By:
    Niksa Skeledzija <niksa.skeledzija@oracle.com>
    Annamalai Gurusami <annamalai.gurusami@oracle.com>
    Debarun Banerjee <debarun.banerjee@oracle.com>

Change-Id: Id131b86fe39daece93a9d08150731c04ab552193

Author: Debarun Banerjee

include/mysql/components/services/bulk_data_service.h

@@ -0,0 +1,156 @@
+/* Copyright (c) 2022, 2023, Oracle and/or its affiliates.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License, version 2.0,
+   as published by the Free Software Foundation.
+
+   This program is also distributed with certain software (including
+   but not limited to OpenSSL) that is licensed under separate terms,
+   as designated in a particular file or component or in included license
+   documentation.  The authors of MySQL hereby grant you an additional
+   permission to link the program and your derivative works with the
+   separately licensed software that they have included with MySQL.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License, version 2.0, for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA */
+
+#pragma once
+
+/**
+  @file
+  This service provides interface for loading data in bulk from CSV files.
+
+*/
+
+#include <mysql/components/service.h>
+#include <string>
+
+/* Forward declaration for opaque types. */
+class THD;
+struct TABLE;
+struct CHARSET_INFO;
+
+using Bulk_loader = void;
+
+/** Bulk loader source. */
+enum class Bulk_source {
+  /** Local file system. */
+  LOCAL,
+  /** OCI object store. */
+  OCI,
+  /** Amazon S3. */
+  S3
+};
+
+/** Bulk loader string attributes. */
+enum class Bulk_string {
+  /** Schema name */
+  SCHEMA_NAME,
+  /* Table name */
+  TABLE_NAME,
+  /* File prefix URL */
+  FILE_PREFIX,
+  /** Column terminator */
+  COLUMN_TERM,
+  /** Row terminator */
+  ROW_TERM,
+};
+
+/** Bulk loader boolean attributes. */
+enum class Bulk_condition {
+  /** The algorithm used is different based on whether the data is in sorted
+  primary key order. This option tells whether to expect sorted input. */
+  ORDERED_DATA,
+  /** If enclosing is optional. */
+  OPTIONAL_ENCLOSE
+};
+
+/** Bulk loader size attributes. */
+enum class Bulk_size {
+  /** Number of input files. */
+  COUNT_FILES,
+  /** Number of rows to skip. */
+  COUNT_ROW_SKIP,
+  /** Number of columns in the table. */
+  COUNT_COLUMNS,
+  /** Number of concurrent loaders to use, */
+  CONCURRENCY,
+  /** Total memory size to use for LOAD in bytes. */
+  MEMORY
+};
+
+/** Bulk loader single byte attributes. */
+enum class Bulk_char {
+  /** Escape character. */
+  ESCAPE_CHAR,
+  /** Column enclosing character. */
+  ENCLOSE_CHAR
+};
+
+/** Bulk load driver service. */
+BEGIN_SERVICE_DEFINITION(bulk_load_driver)
+
+/**
+  Create bulk loader.
+  @param[in]  thd     mysql THD
+  @param[in]  table   mysql TABLE object
+  @param[in]  src     bulk loader source
+  @param[in]  charset source data character set
+  @return bulk loader object, opaque type.
+*/
+DECLARE_METHOD(Bulk_loader *, create_bulk_loader,
+               (THD * thd, const TABLE *table, Bulk_source src,
+                const CHARSET_INFO *charset));
+/**
+  Set string attribute for loading data.
+  @param[in,out]  loader  bulk loader
+  @param[in]      type    attribute type
+  @param[in]      value   attribute value
+*/
+DECLARE_METHOD(void, set_string,
+               (Bulk_loader * loader, Bulk_string type, std::string value));
+/**
+  Set single byte character attribute for loading data.
+  @param[in,out]  loader  bulk loader
+  @param[in]      type    attribute type
+  @param[in]      value   attribute value
+*/
+DECLARE_METHOD(void, set_char,
+               (Bulk_loader * loader, Bulk_char type, unsigned char value));
+/**
+  Set size attribute for loading data.
+  @param[in,out]  loader  bulk loader
+  @param[in]      type    attribute type
+  @param[in]      value   attribute value
+*/
+DECLARE_METHOD(void, set_size,
+               (Bulk_loader * loader, Bulk_size type, size_t value));
+/**
+  Set boolean condition attribute for loading data.
+  @param[in,out]  loader  bulk loader
+  @param[in]      type    attribute type
+  @param[in]      value   attribute value
+*/
+DECLARE_METHOD(void, set_condition,
+               (Bulk_loader * loader, Bulk_condition type, bool value));
+/**
+  Load data from CSV files.
+  @param[in,out]  loader  bulk loader
+  @return true if successful.
+*/
+DECLARE_METHOD(bool, load, (Bulk_loader * loader, size_t &affected_rows));
+
+/**
+  Drop bulk loader.
+  @param[in,out]  thd     mysql THD
+  @param[in,out]  loader  loader object to drop
+*/
+DECLARE_METHOD(void, drop_bulk_loader, (THD * thd, Bulk_loader *loader));
+
+END_SERVICE_DEFINITION(bulk_load_driver)

include/mysql/components/services/bulk_load_service.h

@@ -3609,14 +3609,18 @@ create table t3 as select * from t2 limit 0;
 insert into t3 select @a:=@a+1, t2.a from t2, t0;
 insert into t3 select @a:=@a+1, t2.a from t2, t0;
 insert into t3 select @a:=@a+1, t2.a from t2, t0;
-
 alter table t3 add primary key(id), add key(a);
+
+# The number of leaf pages in the primary key is less than 60.
+SET GLOBAL innodb_stats_persistent_sample_pages = 60;
+ANALYZE TABLE t2;
+ANALYZE TABLE t3;
 --echo The following must use loose index scan over t3, key a:
 explain select count(a) from t2 where a in ( SELECT  a FROM t3);
 select count(a) from t2 where a in ( SELECT  a FROM t3);
 
 drop table t0,t1,t2,t3;
-
+SET GLOBAL innodb_stats_persistent_sample_pages = DEFAULT;
 
 --echo #
 --echo # Bug#33062: subquery in stored routine cause crash

mysql-test/include/subquery_sj.inc

@@ -443,6 +443,7 @@ OVER	1
 OWNER	0
 PACK_KEYS	0
 PAGE	0
+PARALLEL	0
 PARSER	0
 PARSE_TREE	0
 PARTIAL	0
@@ -554,6 +555,7 @@ ROW_COUNT	0
 ROW_FORMAT	0
 ROW_NUMBER	1
 RTREE	0
+S3	0
 SAVEPOINT	0
 SCHEDULE	0
 SCHEMA	1

mysql-test/r/information_schema_keywords.result

@@ -1,4 +1,3 @@
-drop table if exists t1, t2;
 create table t1 (a date, b date, c date not null, d date);
 load data infile '../../std_data/loaddata1.dat' ignore into table t1 fields terminated by ',';
 Warnings:
@@ -20,6 +19,28 @@ a	b	c	d
 2003-03-03	2003-03-03	2003-03-03	NULL
 2003-03-03	2003-03-03	2003-03-03	NULL
 truncate table t1;
+load data from infile '../../std_data/loaddata1.dat'
+into table t1 fields terminated by ',' ignore 2 lines;
+Warnings:
+Warning	4096	Delimiter ' ' in position 0 in datetime value ' 20030303' at row 1 is superfluous and is deprecated. Please remove.
+SELECT * from t1;
+a	b	c	d
+2003-03-03	2003-03-03	2003-03-03	NULL
+truncate table t1;
+load data infile '../../std_data/loaddata1.dat' in primary key order
+into table t1 fields terminated by ',' ignore 2 lines;
+Warnings:
+Warning	4096	Delimiter ' ' in position 0 in datetime value ' 20030303' at row 1 is superfluous and is deprecated. Please remove.
+SELECT * from t1;
+a	b	c	d
+2003-03-03	2003-03-03	2003-03-03	NULL
+truncate table t1;
+load data infile '../../std_data/loaddata1.dat' count 1 into table t1;
+ERROR HY000: Incorrect usage of LOAD DATA without BULK Algorithm and multiple files
+load data infile '../../std_data/loaddata1.dat' count 10 into table t1;
+ERROR HY000: Incorrect usage of LOAD DATA without BULK Algorithm and multiple files
+load data url '../../std_data/loaddata1.dat' into table t1;
+ERROR HY000: Incorrect usage of LOAD DATA without BULK Algorithm and URL source
 load data infile '../../std_data/loaddata1.dat' ignore into table t1 fields terminated by ',' LINES STARTING BY ',' (b,c,d);
 Warnings:
 Warning	1265	Data truncated for column 'c' at row 1

mysql-test/r/loaddata.result

@@ -2823,7 +2823,7 @@ SUBPARTITION BY HASH (to_days("col2"))
   SUBPARTITION s1 TABLESPACE = "innodb_file_per_table" ENGINE = InnoDB),
  PARTITION p1 VALUES LESS THAN MAXVALUE
  (SUBPARTITION s2 TABLESPACE = "innodb_file_per_table" ENGINE = InnoDB,
-  SUBPARTITION s3 TABLESPACE = "innodb_file_per_table" ENGINE = InnoDB)) */
+  SUBPARTITION "s3" TABLESPACE = "innodb_file_per_table" ENGINE = InnoDB)) */
 SELECT partition_expression FROM information_schema.partitions
 WHERE table_schema = 'test' AND table_name = 't1';
 PARTITION_EXPRESSION
@@ -2846,7 +2846,7 @@ SUBPARTITION BY HASH (to_days(`col2`))
   SUBPARTITION s1 TABLESPACE = `innodb_file_per_table` ENGINE = InnoDB),
  PARTITION p1 VALUES LESS THAN MAXVALUE
  (SUBPARTITION s2 TABLESPACE = `innodb_file_per_table` ENGINE = InnoDB,
-  SUBPARTITION s3 TABLESPACE = `innodb_file_per_table` ENGINE = InnoDB)) */
+  SUBPARTITION `s3` TABLESPACE = `innodb_file_per_table` ENGINE = InnoDB)) */
 SELECT partition_expression FROM information_schema.partitions
 WHERE table_schema = 'test' AND table_name = 't1';
 PARTITION_EXPRESSION
@@ -2868,7 +2868,7 @@ SUBPARTITION BY HASH (to_days(col2))
   SUBPARTITION s1 TABLESPACE = innodb_file_per_table ENGINE = InnoDB),
  PARTITION p1 VALUES LESS THAN MAXVALUE
  (SUBPARTITION s2 TABLESPACE = innodb_file_per_table ENGINE = InnoDB,
-  SUBPARTITION s3 TABLESPACE = innodb_file_per_table ENGINE = InnoDB)) */
+  SUBPARTITION `s3` TABLESPACE = innodb_file_per_table ENGINE = InnoDB)) */
 SELECT partition_expression FROM information_schema.partitions
 WHERE table_schema = 'test' AND table_name = 't1';
 PARTITION_EXPRESSION

mysql-test/r/partition.result

@@ -1907,7 +1907,7 @@ SUBPARTITION BY HASH (to_days(`purchased`))
   SUBPARTITION s1 ENGINE = InnoDB),
  PARTITION p1 VALUES LESS THAN (2000)
  (SUBPARTITION s2 ENGINE = InnoDB,
-  SUBPARTITION s3 DATA DIRECTORY = 'MYSQL_TMP_DIR/sp3_dir/' ENGINE = InnoDB)) */
+  SUBPARTITION `s3` DATA DIRECTORY = 'MYSQL_TMP_DIR/sp3_dir/' ENGINE = InnoDB)) */
 # List files from t1_dir/test
 t1.ibd
 # List files from sp0_dir/test
@@ -1950,7 +1950,7 @@ SUBPARTITION BY HASH (to_days(`purchased`))
   SUBPARTITION s1 ENGINE = InnoDB),
  PARTITION p1 VALUES LESS THAN (2000)
  (SUBPARTITION s2 ENGINE = InnoDB,
-  SUBPARTITION s3 DATA DIRECTORY = 'MYSQL_TMP_DIR/sp3_dir/' ENGINE = InnoDB)) */
+  SUBPARTITION `s3` DATA DIRECTORY = 'MYSQL_TMP_DIR/sp3_dir/' ENGINE = InnoDB)) */
 ALTER TABLE t2 EXCHANGE PARTITION s0 WITH TABLE t1;
 SHOW CREATE TABLE t1;
 Table	Create Table
@@ -1971,7 +1971,7 @@ SUBPARTITION BY HASH (to_days(`purchased`))
   SUBPARTITION s1 ENGINE = InnoDB),
  PARTITION p1 VALUES LESS THAN (2000)
  (SUBPARTITION s2 ENGINE = InnoDB,
-  SUBPARTITION s3 DATA DIRECTORY = 'MYSQL_TMP_DIR/sp3_dir/' ENGINE = InnoDB)) */
+  SUBPARTITION `s3` DATA DIRECTORY = 'MYSQL_TMP_DIR/sp3_dir/' ENGINE = InnoDB)) */
 # List files from t1_dir/test
 t1.ibd
 # List files from sp0_dir/test
@@ -2018,7 +2018,7 @@ SUBPARTITION BY HASH (to_days(`purchased`))
   SUBPARTITION s1 ENGINE = InnoDB),
  PARTITION p1 VALUES LESS THAN (2000)
  (SUBPARTITION s2 ENGINE = InnoDB,
-  SUBPARTITION s3 DATA DIRECTORY = 'MYSQL_TMP_DIR/t1_dir/' ENGINE = InnoDB)) */
+  SUBPARTITION `s3` DATA DIRECTORY = 'MYSQL_TMP_DIR/t1_dir/' ENGINE = InnoDB)) */
 ALTER TABLE t2 EXCHANGE PARTITION s3 WITH TABLE t1;
 SHOW CREATE TABLE t1;
 Table	Create Table
@@ -2039,7 +2039,7 @@ SUBPARTITION BY HASH (to_days(`purchased`))
   SUBPARTITION s1 ENGINE = InnoDB),
  PARTITION p1 VALUES LESS THAN (2000)
  (SUBPARTITION s2 ENGINE = InnoDB,
-  SUBPARTITION s3 DATA DIRECTORY = 'MYSQL_TMP_DIR/sp3_dir/' ENGINE = InnoDB)) */
+  SUBPARTITION `s3` DATA DIRECTORY = 'MYSQL_TMP_DIR/sp3_dir/' ENGINE = InnoDB)) */
 # List files from t1_dir/test
 t1.ibd
 # List files from sp0_dir/test

mysql-test/r/partition_exchange.result

@@ -236,7 +236,7 @@ SUBPARTITION BY HASH (to_days(`purchased`))
   SUBPARTITION s1 DATA DIRECTORY = 'MYSQL_TMP_DIR/alternate_dir/data2/' ENGINE = InnoDB),
  PARTITION p1 VALUES LESS THAN (2000)
  (SUBPARTITION s2 DATA DIRECTORY = 'MYSQL_TMP_DIR/alternate_dir/data/' ENGINE = InnoDB,
-  SUBPARTITION s3 DATA DIRECTORY = 'MYSQL_TMP_DIR/alternate_dir/data2/' ENGINE = InnoDB),
+  SUBPARTITION `s3` DATA DIRECTORY = 'MYSQL_TMP_DIR/alternate_dir/data2/' ENGINE = InnoDB),
  PARTITION p2 VALUES LESS THAN MAXVALUE
  (SUBPARTITION s4 DATA DIRECTORY = 'MYSQL_TMP_DIR/alternate_dir/data/' ENGINE = InnoDB,
   SUBPARTITION s5 DATA DIRECTORY = 'MYSQL_TMP_DIR/alternate_dir/data2/' ENGINE = InnoDB)) */
@@ -326,7 +326,7 @@ SUBPARTITION BY HASH (to_days(`purchased`))
   SUBPARTITION s1 DATA DIRECTORY = 'MYSQL_TMP_DIR/alternate_dir/data2/' ENGINE = InnoDB),
  PARTITION p1 VALUES LESS THAN (2000)
  (SUBPARTITION s2 DATA DIRECTORY = 'MYSQL_TMP_DIR/alternate_dir/data/' ENGINE = InnoDB,
-  SUBPARTITION s3 DATA DIRECTORY = 'MYSQL_TMP_DIR/alternate_dir/data2/' ENGINE = InnoDB),
+  SUBPARTITION `s3` DATA DIRECTORY = 'MYSQL_TMP_DIR/alternate_dir/data2/' ENGINE = InnoDB),
  PARTITION p2 VALUES LESS THAN MAXVALUE
  (SUBPARTITION s4 DATA DIRECTORY = 'MYSQL_TMP_DIR/alternate_dir/data/' ENGINE = InnoDB,
   SUBPARTITION s5 DATA DIRECTORY = 'MYSQL_TMP_DIR/alternate_dir/data2/' ENGINE = InnoDB)) */

mysql-test/r/partition_innodb_tablespace.result

@@ -8344,6 +8344,13 @@ insert into t3 select @a:=@a+1, t2.a from t2, t0;
 Warnings:
 Warning	1287	Setting user variables within expressions is deprecated and will be removed in a future release. Consider alternatives: 'SET variable=expression, ...', or 'SELECT expression(s) INTO variables(s)'.
 alter table t3 add primary key(id), add key(a);
+SET GLOBAL innodb_stats_persistent_sample_pages = 60;
+ANALYZE TABLE t2;
+Table	Op	Msg_type	Msg_text
+test.t2	analyze	status	OK
+ANALYZE TABLE t3;
+Table	Op	Msg_type	Msg_text
+test.t3	analyze	status	OK
 The following must use loose index scan over t3, key a:
 explain select count(a) from t2 where a in ( SELECT  a FROM t3);
 id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
@@ -8355,6 +8362,7 @@ select count(a) from t2 where a in ( SELECT  a FROM t3);
 count(a)
 1000
 drop table t0,t1,t2,t3;
+SET GLOBAL innodb_stats_persistent_sample_pages = DEFAULT;
 #
 # Bug#33062: subquery in stored routine cause crash
 #

mysql-test/r/subquery_sj_all.result

@@ -8346,6 +8346,13 @@ insert into t3 select @a:=@a+1, t2.a from t2, t0;
 Warnings:
 Warning	1287	Setting user variables within expressions is deprecated and will be removed in a future release. Consider alternatives: 'SET variable=expression, ...', or 'SELECT expression(s) INTO variables(s)'.
 alter table t3 add primary key(id), add key(a);
+SET GLOBAL innodb_stats_persistent_sample_pages = 60;
+ANALYZE TABLE t2;
+Table	Op	Msg_type	Msg_text
+test.t2	analyze	status	OK
+ANALYZE TABLE t3;
+Table	Op	Msg_type	Msg_text
+test.t3	analyze	status	OK
 The following must use loose index scan over t3, key a:
 explain select count(a) from t2 where a in ( SELECT  a FROM t3);
 id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
@@ -8357,6 +8364,7 @@ select count(a) from t2 where a in ( SELECT  a FROM t3);
 count(a)
 1000
 drop table t0,t1,t2,t3;
+SET GLOBAL innodb_stats_persistent_sample_pages = DEFAULT;
 #
 # Bug#33062: subquery in stored routine cause crash
 #

mysql-test/r/subquery_sj_all_bka.result

@@ -8344,6 +8344,13 @@ insert into t3 select @a:=@a+1, t2.a from t2, t0;
 Warnings:
 Warning	1287	Setting user variables within expressions is deprecated and will be removed in a future release. Consider alternatives: 'SET variable=expression, ...', or 'SELECT expression(s) INTO variables(s)'.
 alter table t3 add primary key(id), add key(a);
+SET GLOBAL innodb_stats_persistent_sample_pages = 60;
+ANALYZE TABLE t2;
+Table	Op	Msg_type	Msg_text
+test.t2	analyze	status	OK
+ANALYZE TABLE t3;
+Table	Op	Msg_type	Msg_text
+test.t3	analyze	status	OK
 The following must use loose index scan over t3, key a:
 explain select count(a) from t2 where a in ( SELECT  a FROM t3);
 id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
@@ -8355,6 +8362,7 @@ select count(a) from t2 where a in ( SELECT  a FROM t3);
 count(a)
 1000
 drop table t0,t1,t2,t3;
+SET GLOBAL innodb_stats_persistent_sample_pages = DEFAULT;
 #
 # Bug#33062: subquery in stored routine cause crash
 #