Splitting rocksdb_bulk_load variable into two (#143)

Summary:
rocksdb_bulk_load changes two behaviors -- skipping unique check, and
implicitly committing every rocksdb_bulk_load_size records. These are
different features, and sometimes people want only one of them. This
diff adds two new session variable rocksdb_skip_unique_check and
rocksdb_commit_in_the_middle to take care of them. rocksdb_bulk_load
still exists for compatibility, but will be removed in the future.

This diff also adds safety check to return errors on REPLACE or INSERT
ON DUPLICATE KEY UPDATE when skipping unique check.

Test Plan: mtr

Reviewers: spetrunia, hermanlee4, jkedgar

Reviewed By: hermanlee4

Subscribers: MarkCallaghan, webscalesql-eng

Differential Revision: https://reviews.facebook.net/D53061

mysql-test/r/mysqld--help-notwin-profiling.result

@@ -975,7 +975,10 @@ The following options may be given as the first argument:
  BlockBasedTableOptions::block_size for RocksDB
  --rocksdb-block-size-deviation=# 
  BlockBasedTableOptions::block_size_deviation for RocksDB
- --rocksdb-bulk-load Use bulk-load mode for inserts
+ --rocksdb-bulk-load Use bulk-load mode for inserts. This enables both
+ rocksdb_skip_unique_check and
+ rocksdb_commit_in_the_middle. This parameter will be
+ deprecated in future release.
  --rocksdb-bulk-load-size=# 
  Max #records in a batch for bulk-load mode
  --rocksdb-bytes-per-sync=# 
@@ -997,6 +1000,9 @@ The following options may be given as the first argument:
  --rocksdb-collect-sst-properties 
  Enables collecting SST file properties on each flush
  (Defaults to on; use --skip-rocksdb-collect-sst-properties to disable.)
+ --rocksdb-commit-in-the-middle 
+ Commit rows implicitly every rocksdb_bulk_load_size, on
+ bulk load/insert
  --rocksdb-compact-cf=name 
  Compact column family
  --rocksdb-compaction-sequential-deletes=# 
@@ -1125,6 +1131,8 @@ The following options may be given as the first argument:
  Wake up drop index thread
  --rocksdb-skip-fill-cache 
  Skip filling block cache on read requests
+ --rocksdb-skip-unique-check 
+ Skip unique constraint checking
  --rocksdb-stats-dump-period-sec=# 
  DBOptions::stats_dump_period_sec for RocksDB
  --rocksdb-store-checksums 
@@ -1713,6 +1721,7 @@ rocksdb-cf-options ON
 rocksdb-cfstats ON
 rocksdb-checksums-pct 100
 rocksdb-collect-sst-properties TRUE
+rocksdb-commit-in-the-middle FALSE
 rocksdb-compact-cf 
 rocksdb-compaction-sequential-deletes 1900
 rocksdb-compaction-sequential-deletes-file-size 0
@@ -1762,6 +1771,7 @@ rocksdb-records-in-range 0
 rocksdb-seconds-between-stat-computes 3600
 rocksdb-signal-drop-index-thread FALSE
 rocksdb-skip-fill-cache FALSE
+rocksdb-skip-unique-check FALSE
 rocksdb-stats-dump-period-sec 600
 rocksdb-store-checksums FALSE
 rocksdb-strict-collation-check TRUE

mysql-test/r/mysqld--help-notwin.result

@@ -973,7 +973,10 @@ The following options may be given as the first argument:
  BlockBasedTableOptions::block_size for RocksDB
  --rocksdb-block-size-deviation=# 
  BlockBasedTableOptions::block_size_deviation for RocksDB
- --rocksdb-bulk-load Use bulk-load mode for inserts
+ --rocksdb-bulk-load Use bulk-load mode for inserts. This enables both
+ rocksdb_skip_unique_check and
+ rocksdb_commit_in_the_middle. This parameter will be
+ deprecated in future release.
  --rocksdb-bulk-load-size=# 
  Max #records in a batch for bulk-load mode
  --rocksdb-bytes-per-sync=# 
@@ -995,6 +998,9 @@ The following options may be given as the first argument:
  --rocksdb-collect-sst-properties 
  Enables collecting SST file properties on each flush
  (Defaults to on; use --skip-rocksdb-collect-sst-properties to disable.)
+ --rocksdb-commit-in-the-middle 
+ Commit rows implicitly every rocksdb_bulk_load_size, on
+ bulk load/insert
  --rocksdb-compact-cf=name 
  Compact column family
  --rocksdb-compaction-sequential-deletes=# 
@@ -1123,6 +1129,8 @@ The following options may be given as the first argument:
  Wake up drop index thread
  --rocksdb-skip-fill-cache 
  Skip filling block cache on read requests
+ --rocksdb-skip-unique-check 
+ Skip unique constraint checking
  --rocksdb-stats-dump-period-sec=# 
  DBOptions::stats_dump_period_sec for RocksDB
  --rocksdb-store-checksums 
@@ -1710,6 +1718,7 @@ rocksdb-cf-options ON
 rocksdb-cfstats ON
 rocksdb-checksums-pct 100
 rocksdb-collect-sst-properties TRUE
+rocksdb-commit-in-the-middle FALSE
 rocksdb-compact-cf 
 rocksdb-compaction-sequential-deletes 1900
 rocksdb-compaction-sequential-deletes-file-size 0
@@ -1759,6 +1768,7 @@ rocksdb-records-in-range 0
 rocksdb-seconds-between-stat-computes 3600
 rocksdb-signal-drop-index-thread FALSE
 rocksdb-skip-fill-cache FALSE
+rocksdb-skip-unique-check FALSE
 rocksdb-stats-dump-period-sec 600
 rocksdb-store-checksums FALSE
 rocksdb-strict-collation-check TRUE

mysql-test/suite/rocksdb/r/rocksdb.result

@@ -872,6 +872,7 @@ rocksdb_bytes_per_sync	0
 rocksdb_cache_index_and_filter_blocks	ON
 rocksdb_checksums_pct	100
 rocksdb_collect_sst_properties	ON
+rocksdb_commit_in_the_middle	OFF
 rocksdb_compact_cf	
 rocksdb_compaction_sequential_deletes	1900
 rocksdb_compaction_sequential_deletes_file_size	0
@@ -914,6 +915,7 @@ rocksdb_records_in_range	0
 rocksdb_seconds_between_stat_computes	3600
 rocksdb_signal_drop_index_thread	OFF
 rocksdb_skip_fill_cache	OFF
+rocksdb_skip_unique_check	OFF
 rocksdb_stats_dump_period_sec	600
 rocksdb_store_checksums	OFF
 rocksdb_strict_collation_check	OFF
@@ -2196,4 +2198,42 @@ SELECT c1 FROM test_db.t1;
 c1
 START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY;
 DROP DATABASE test_db;
+#
+# Issue #143: Split rocksdb_bulk_load option into two
+#
+CREATE TABLE t1 (id int primary key, value int) engine=RocksDB;
+SET rocksdb_skip_unique_check=1;
+INSERT INTO t1 VALUES(1, 1);
+INSERT INTO t1 VALUES(1, 2);
+INSERT INTO t1 VALUES(1, 3);
+SELECT * FROM t1;
+id	value
+REPLACE INTO t1 VALUES(4, 4);
+ERROR HY000: When disables unique check in MyRocks, 'INSERT or UPDATE' statement (INSERT ON DUPLICATE KEY UPDATE, REPLACE) is not allowed.
+INSERT INTO t1 VALUES(5, 5) ON DUPLICATE KEY UPDATE value=value+1;
+ERROR HY000: When disables unique check in MyRocks, 'INSERT or UPDATE' statement (INSERT ON DUPLICATE KEY UPDATE, REPLACE) is not allowed.
+TRUNCATE TABLE t1;
+SET @save_rocksdb_bulk_load_size= @@rocksdb_bulk_load_size;
+SET rocksdb_skip_unique_check=0;
+SET rocksdb_commit_in_the_middle=1;
+SET rocksdb_bulk_load_size=10;
+BEGIN;
+INSERT INTO t1 (id) VALUES(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),
+(11),(12),(13),(14),(15),(16),(17),(18),(19);
+ROLLBACK;
+SELECT * FROM t1;
+id	value
+1	NULL
+2	NULL
+3	NULL
+4	NULL
+5	NULL
+6	NULL
+7	NULL
+8	NULL
+9	NULL
+10	NULL
+SET rocksdb_commit_in_the_middle=0;
+SET rocksdb_bulk_load_size= @save_rocksdb_bulk_load_size;
+DROP TABLE t1;
 SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK = @ORIG_PAUSE_BACKGROUND_WORK;

mysql-test/suite/rocksdb/t/rocksdb.test

@@ -1767,4 +1767,32 @@ SELECT c1 FROM test_db.t1;
 START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY;
 DROP DATABASE test_db;
 
+--echo #
+--echo # Issue #143: Split rocksdb_bulk_load option into two
+--echo #
+CREATE TABLE t1 (id int primary key, value int) engine=RocksDB;
+SET rocksdb_skip_unique_check=1;
+INSERT INTO t1 VALUES(1, 1);
+INSERT INTO t1 VALUES(1, 2);
+INSERT INTO t1 VALUES(1, 3);
+SELECT * FROM t1;
+--error ER_UNKNOWN_ERROR
+REPLACE INTO t1 VALUES(4, 4);
+--error ER_UNKNOWN_ERROR
+INSERT INTO t1 VALUES(5, 5) ON DUPLICATE KEY UPDATE value=value+1;
+TRUNCATE TABLE t1;
+SET @save_rocksdb_bulk_load_size= @@rocksdb_bulk_load_size;
+SET rocksdb_skip_unique_check=0;
+SET rocksdb_commit_in_the_middle=1;
+SET rocksdb_bulk_load_size=10;
+BEGIN;
+INSERT INTO t1 (id) VALUES(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),
+  (11),(12),(13),(14),(15),(16),(17),(18),(19);
+ROLLBACK;
+SELECT * FROM t1;
+SET rocksdb_commit_in_the_middle=0;
+SET rocksdb_bulk_load_size= @save_rocksdb_bulk_load_size;
+DROP TABLE t1;
+
+
 SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK = @ORIG_PAUSE_BACKGROUND_WORK;

mysql-test/suite/rocksdb_sys_vars/r/rocksdb_commit_in_the_middle_basic.result

@@ -0,0 +1,100 @@
+CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO valid_values VALUES(1);
+INSERT INTO valid_values VALUES(0);
+INSERT INTO valid_values VALUES('on');
+CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO invalid_values VALUES('\'aaa\'');
+INSERT INTO invalid_values VALUES('\'bbb\'');
+SET @start_global_value = @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE;
+SELECT @start_global_value;
+@start_global_value
+0
+SET @start_session_value = @@session.ROCKSDB_COMMIT_IN_THE_MIDDLE;
+SELECT @start_session_value;
+@start_session_value
+0
+'# Setting to valid values in global scope#'
+"Trying to set variable @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE to 1"
+SET @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE   = 1;
+SELECT @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE;
+@@global.ROCKSDB_COMMIT_IN_THE_MIDDLE
+1
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE = DEFAULT;
+SELECT @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE;
+@@global.ROCKSDB_COMMIT_IN_THE_MIDDLE
+0
+"Trying to set variable @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE to 0"
+SET @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE   = 0;
+SELECT @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE;
+@@global.ROCKSDB_COMMIT_IN_THE_MIDDLE
+0
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE = DEFAULT;
+SELECT @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE;
+@@global.ROCKSDB_COMMIT_IN_THE_MIDDLE
+0
+"Trying to set variable @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE to on"
+SET @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE   = on;
+SELECT @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE;
+@@global.ROCKSDB_COMMIT_IN_THE_MIDDLE
+1
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE = DEFAULT;
+SELECT @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE;
+@@global.ROCKSDB_COMMIT_IN_THE_MIDDLE
+0
+'# Setting to valid values in session scope#'
+"Trying to set variable @@session.ROCKSDB_COMMIT_IN_THE_MIDDLE to 1"
+SET @@session.ROCKSDB_COMMIT_IN_THE_MIDDLE   = 1;
+SELECT @@session.ROCKSDB_COMMIT_IN_THE_MIDDLE;
+@@session.ROCKSDB_COMMIT_IN_THE_MIDDLE
+1
+"Setting the session scope variable back to default"
+SET @@session.ROCKSDB_COMMIT_IN_THE_MIDDLE = DEFAULT;
+SELECT @@session.ROCKSDB_COMMIT_IN_THE_MIDDLE;
+@@session.ROCKSDB_COMMIT_IN_THE_MIDDLE
+0
+"Trying to set variable @@session.ROCKSDB_COMMIT_IN_THE_MIDDLE to 0"
+SET @@session.ROCKSDB_COMMIT_IN_THE_MIDDLE   = 0;
+SELECT @@session.ROCKSDB_COMMIT_IN_THE_MIDDLE;
+@@session.ROCKSDB_COMMIT_IN_THE_MIDDLE
+0
+"Setting the session scope variable back to default"
+SET @@session.ROCKSDB_COMMIT_IN_THE_MIDDLE = DEFAULT;
+SELECT @@session.ROCKSDB_COMMIT_IN_THE_MIDDLE;
+@@session.ROCKSDB_COMMIT_IN_THE_MIDDLE
+0
+"Trying to set variable @@session.ROCKSDB_COMMIT_IN_THE_MIDDLE to on"
+SET @@session.ROCKSDB_COMMIT_IN_THE_MIDDLE   = on;
+SELECT @@session.ROCKSDB_COMMIT_IN_THE_MIDDLE;
+@@session.ROCKSDB_COMMIT_IN_THE_MIDDLE
+1
+"Setting the session scope variable back to default"
+SET @@session.ROCKSDB_COMMIT_IN_THE_MIDDLE = DEFAULT;
+SELECT @@session.ROCKSDB_COMMIT_IN_THE_MIDDLE;
+@@session.ROCKSDB_COMMIT_IN_THE_MIDDLE
+0
+'# Testing with invalid values in global scope #'
+"Trying to set variable @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE to 'aaa'"
+SET @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE   = 'aaa';
+Got one of the listed errors
+SELECT @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE;
+@@global.ROCKSDB_COMMIT_IN_THE_MIDDLE
+0
+"Trying to set variable @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE to 'bbb'"
+SET @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE   = 'bbb';
+Got one of the listed errors
+SELECT @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE;
+@@global.ROCKSDB_COMMIT_IN_THE_MIDDLE
+0
+SET @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE = @start_global_value;
+SELECT @@global.ROCKSDB_COMMIT_IN_THE_MIDDLE;
+@@global.ROCKSDB_COMMIT_IN_THE_MIDDLE
+0
+SET @@session.ROCKSDB_COMMIT_IN_THE_MIDDLE = @start_session_value;
+SELECT @@session.ROCKSDB_COMMIT_IN_THE_MIDDLE;
+@@session.ROCKSDB_COMMIT_IN_THE_MIDDLE
+0
+DROP TABLE valid_values;
+DROP TABLE invalid_values;

mysql-test/suite/rocksdb_sys_vars/r/rocksdb_skip_unique_check_basic.result

@@ -0,0 +1,100 @@
+CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO valid_values VALUES(1);
+INSERT INTO valid_values VALUES(0);
+INSERT INTO valid_values VALUES('on');
+CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO invalid_values VALUES('\'aaa\'');
+INSERT INTO invalid_values VALUES('\'bbb\'');
+SET @start_global_value = @@global.ROCKSDB_SKIP_UNIQUE_CHECK;
+SELECT @start_global_value;
+@start_global_value
+0
+SET @start_session_value = @@session.ROCKSDB_SKIP_UNIQUE_CHECK;
+SELECT @start_session_value;
+@start_session_value
+0
+'# Setting to valid values in global scope#'
+"Trying to set variable @@global.ROCKSDB_SKIP_UNIQUE_CHECK to 1"
+SET @@global.ROCKSDB_SKIP_UNIQUE_CHECK   = 1;
+SELECT @@global.ROCKSDB_SKIP_UNIQUE_CHECK;
+@@global.ROCKSDB_SKIP_UNIQUE_CHECK
+1
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_SKIP_UNIQUE_CHECK = DEFAULT;
+SELECT @@global.ROCKSDB_SKIP_UNIQUE_CHECK;
+@@global.ROCKSDB_SKIP_UNIQUE_CHECK
+0
+"Trying to set variable @@global.ROCKSDB_SKIP_UNIQUE_CHECK to 0"
+SET @@global.ROCKSDB_SKIP_UNIQUE_CHECK   = 0;
+SELECT @@global.ROCKSDB_SKIP_UNIQUE_CHECK;
+@@global.ROCKSDB_SKIP_UNIQUE_CHECK
+0
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_SKIP_UNIQUE_CHECK = DEFAULT;
+SELECT @@global.ROCKSDB_SKIP_UNIQUE_CHECK;
+@@global.ROCKSDB_SKIP_UNIQUE_CHECK
+0
+"Trying to set variable @@global.ROCKSDB_SKIP_UNIQUE_CHECK to on"
+SET @@global.ROCKSDB_SKIP_UNIQUE_CHECK   = on;
+SELECT @@global.ROCKSDB_SKIP_UNIQUE_CHECK;
+@@global.ROCKSDB_SKIP_UNIQUE_CHECK
+1
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_SKIP_UNIQUE_CHECK = DEFAULT;
+SELECT @@global.ROCKSDB_SKIP_UNIQUE_CHECK;
+@@global.ROCKSDB_SKIP_UNIQUE_CHECK
+0
+'# Setting to valid values in session scope#'
+"Trying to set variable @@session.ROCKSDB_SKIP_UNIQUE_CHECK to 1"
+SET @@session.ROCKSDB_SKIP_UNIQUE_CHECK   = 1;
+SELECT @@session.ROCKSDB_SKIP_UNIQUE_CHECK;
+@@session.ROCKSDB_SKIP_UNIQUE_CHECK
+1
+"Setting the session scope variable back to default"
+SET @@session.ROCKSDB_SKIP_UNIQUE_CHECK = DEFAULT;
+SELECT @@session.ROCKSDB_SKIP_UNIQUE_CHECK;
+@@session.ROCKSDB_SKIP_UNIQUE_CHECK
+0
+"Trying to set variable @@session.ROCKSDB_SKIP_UNIQUE_CHECK to 0"
+SET @@session.ROCKSDB_SKIP_UNIQUE_CHECK   = 0;
+SELECT @@session.ROCKSDB_SKIP_UNIQUE_CHECK;
+@@session.ROCKSDB_SKIP_UNIQUE_CHECK
+0
+"Setting the session scope variable back to default"
+SET @@session.ROCKSDB_SKIP_UNIQUE_CHECK = DEFAULT;
+SELECT @@session.ROCKSDB_SKIP_UNIQUE_CHECK;
+@@session.ROCKSDB_SKIP_UNIQUE_CHECK
+0
+"Trying to set variable @@session.ROCKSDB_SKIP_UNIQUE_CHECK to on"
+SET @@session.ROCKSDB_SKIP_UNIQUE_CHECK   = on;
+SELECT @@session.ROCKSDB_SKIP_UNIQUE_CHECK;
+@@session.ROCKSDB_SKIP_UNIQUE_CHECK
+1
+"Setting the session scope variable back to default"
+SET @@session.ROCKSDB_SKIP_UNIQUE_CHECK = DEFAULT;
+SELECT @@session.ROCKSDB_SKIP_UNIQUE_CHECK;
+@@session.ROCKSDB_SKIP_UNIQUE_CHECK
+0
+'# Testing with invalid values in global scope #'
+"Trying to set variable @@global.ROCKSDB_SKIP_UNIQUE_CHECK to 'aaa'"
+SET @@global.ROCKSDB_SKIP_UNIQUE_CHECK   = 'aaa';
+Got one of the listed errors
+SELECT @@global.ROCKSDB_SKIP_UNIQUE_CHECK;
+@@global.ROCKSDB_SKIP_UNIQUE_CHECK
+0
+"Trying to set variable @@global.ROCKSDB_SKIP_UNIQUE_CHECK to 'bbb'"
+SET @@global.ROCKSDB_SKIP_UNIQUE_CHECK   = 'bbb';
+Got one of the listed errors
+SELECT @@global.ROCKSDB_SKIP_UNIQUE_CHECK;
+@@global.ROCKSDB_SKIP_UNIQUE_CHECK
+0
+SET @@global.ROCKSDB_SKIP_UNIQUE_CHECK = @start_global_value;
+SELECT @@global.ROCKSDB_SKIP_UNIQUE_CHECK;
+@@global.ROCKSDB_SKIP_UNIQUE_CHECK
+0
+SET @@session.ROCKSDB_SKIP_UNIQUE_CHECK = @start_session_value;
+SELECT @@session.ROCKSDB_SKIP_UNIQUE_CHECK;
+@@session.ROCKSDB_SKIP_UNIQUE_CHECK
+0
+DROP TABLE valid_values;
+DROP TABLE invalid_values;