Copy of

commit 184a4a2d82f4f6f3cbcb1015bcdb32bebe73315c
Author: Abhinav Sharma <abhinavsharma@fb.com>
Date:   Thu Sep 14 11:40:08 2017 -0700

    Bump rocksdb submodule

    Summary:
    Bump rocksdb to include the fix for rocksdb.trx_info_rpl

    The bug was introduced in: facebook/rocksdb#2850
    Fixed in: facebook/rocksdb#2881

    update-submodule: rocksdb

    Reviewed By: lth

    Differential Revision: D5834658

    fbshipit-source-id: d1551bf

storage/rocksdb/CMakeLists.txt

@@ -106,6 +106,12 @@ IF (NOT "$ENV{WITH_ZSTD}" STREQUAL "")
   ADD_DEFINITIONS(-DZSTD)
 ENDIF()
 
+IF (NOT "$ENV{WITH_TBB}" STREQUAL "")
+  SET(rocksdb_static_libs ${rocksdb_static_libs}
+  $ENV{WITH_TBB}/libtbb${PIC_EXT}.a)
+  ADD_DEFINITIONS(-DTBB)
+ENDIF()
+
 CHECK_INCLUDE_FILES(zlib.h HAVE_ZLIB_H)
 
 IF (HAVE_ZLIB_H)

storage/rocksdb/ha_rocksdb.h

@@ -22,6 +22,7 @@
 /* C++ standard header files */
 #include <set>
 #include <string>
+#include <unordered_map>
 #include <unordered_set>
 #include <vector>
 
@@ -235,6 +236,12 @@ const char *const RDB_TTL_COL_QUALIFIER = "ttl_col";
 */
 #define ROCKSDB_SIZEOF_TTL_RECORD sizeof(longlong)
 
+/*
+  Maximum index prefix length in bytes.
+*/
+#define MAX_INDEX_COL_LEN_LARGE 3072
+#define MAX_INDEX_COL_LEN_SMALL 767
+
 /*
   MyRocks specific error codes. NB! Please make sure that you will update
   HA_ERR_ROCKSDB_LAST when adding new ones.  Also update the strings in
@@ -293,7 +300,14 @@ struct Rdb_table_handler {
 
   /* Stores cumulative table statistics */
   my_io_perf_atomic_t m_io_perf_read;
+  my_io_perf_atomic_t m_io_perf_write;
   Rdb_atomic_perf_counters m_table_perf_context;
+
+  /* Stores cached memtable estimate statistics */
+  std::atomic_uint m_mtcache_lock;
+  uint64_t m_mtcache_count;
+  uint64_t m_mtcache_size;
+  uint64_t m_mtcache_last_update;
 };
 
 class Rdb_key_def;
@@ -360,6 +374,8 @@ struct st_global_stats {
   ib_counter_t<ulonglong, 64, RDB_INDEXER> system_rows[ROWS_MAX];
 
   ib_counter_t<ulonglong, 64, RDB_INDEXER> queries[QUERIES_MAX];
+
+  ib_counter_t<ulonglong, 64, RDB_INDEXER> covered_secondary_key_lookups;
 };
 
 /* Struct used for exporting status to MySQL */
@@ -379,6 +395,8 @@ struct st_export_stats {
 
   ulonglong queries_point;
   ulonglong queries_range;
+
+  ulonglong covered_secondary_key_lookups;
 };
 
 /* Struct used for exporting RocksDB memory status */
@@ -387,6 +405,27 @@ struct st_memory_stats {
   ulonglong memtable_unflushed;
 };
 
+/* Struct used for exporting RocksDB IO stalls stats */
+struct st_io_stall_stats {
+  ulonglong level0_slowdown;
+  ulonglong level0_slowdown_with_compaction;
+  ulonglong level0_numfiles;
+  ulonglong level0_numfiles_with_compaction;
+  ulonglong stop_for_pending_compaction_bytes;
+  ulonglong slowdown_for_pending_compaction_bytes;
+  ulonglong memtable_compaction;
+  ulonglong memtable_slowdown;
+  ulonglong total_stop;
+  ulonglong total_slowdown;
+
+  st_io_stall_stats()
+      : level0_slowdown(0), level0_slowdown_with_compaction(0),
+        level0_numfiles(0), level0_numfiles_with_compaction(0),
+        stop_for_pending_compaction_bytes(0),
+        slowdown_for_pending_compaction_bytes(0), memtable_compaction(0),
+        memtable_slowdown(0), total_stop(0), total_slowdown(0) {}
+};
+
 } // namespace myrocks
 
 #include "./rdb_buff.h"
@@ -495,6 +534,12 @@ class ha_rocksdb : public my_core::handler {
     Pointer to the original TTL timestamp value (8 bytes) during UPDATE.
   */
   char m_ttl_bytes[ROCKSDB_SIZEOF_TTL_RECORD];
+  /*
+    The TTL timestamp value can change if the explicit TTL column is
+    updated. If we detect this when updating the PK, we indicate it here so
+    we know we must always update any SK's.
+  */
+  bool m_ttl_bytes_updated;
 
   /* rowkey of the last record we've read, in StorageFormat. */
   String m_last_rowkey;
@@ -541,7 +586,9 @@ class ha_rocksdb : public my_core::handler {
   bool m_update_scope_is_valid;
 
   /* SST information used for bulk loading the primary key */
-  std::shared_ptr<Rdb_sst_info> m_sst_info;
+  std::unique_ptr<Rdb_sst_info> m_sst_info;
+  /* External merge sorts for bulk load: key ID -> merge sort instance */
+  std::unordered_map<GL_INDEX_ID, Rdb_index_merge> m_key_merge;
   Rdb_transaction *m_bulk_load_tx;
   /* Mutex to protect finalizing bulk load */
   mysql_mutex_t m_bulk_load_mutex;
@@ -636,6 +683,13 @@ class ha_rocksdb : public my_core::handler {
   /* Setup field_decoders based on type of scan and table->read_set */
   void setup_read_decoders();
 
+  /*
+    For the active index, indicates which columns must be covered for the
+    current lookup to be covered. If the bitmap field is null, that means this
+    index does not cover the current lookup for any record.
+   */
+  MY_BITMAP m_lookup_bitmap = {nullptr, 0, 0, nullptr, nullptr};
+
   /*
     Number of bytes in on-disk (storage) record format that are used for
     storing SQL NULL flags.
@@ -854,11 +908,7 @@ class ha_rocksdb : public my_core::handler {
     DBUG_RETURN(MAX_REF_PARTS);
   }
 
-  uint max_supported_key_part_length() const override {
-    DBUG_ENTER_FUNC();
-
-    DBUG_RETURN(2048);
-  }
+  uint max_supported_key_part_length() const override;
 
   /** @brief
     unireg.cc will call this to make sure that the storage engine can handle
@@ -1040,9 +1090,13 @@ class ha_rocksdb : public my_core::handler {
                                        rocksdb::Slice *const packed_rec)
       MY_ATTRIBUTE((__nonnull__));
 
-  bool should_hide_ttl_rec(const rocksdb::Slice &ttl_rec_val,
+  bool should_hide_ttl_rec(const Rdb_key_def &kd,
+                           const rocksdb::Slice &ttl_rec_val,
                            const int64_t curr_ts)
       MY_ATTRIBUTE((__warn_unused_result__));
+  void rocksdb_skip_expired_records(const Rdb_key_def &kd,
+                                    rocksdb::Iterator *const iter,
+                                    bool seek_backward);
 
   int index_first_intern(uchar *buf)
       MY_ATTRIBUTE((__nonnull__, __warn_unused_result__));
@@ -1076,8 +1130,10 @@ class ha_rocksdb : public my_core::handler {
                          struct unique_sk_buf_info *sk_info)
       MY_ATTRIBUTE((__nonnull__, __warn_unused_result__));
   int bulk_load_key(Rdb_transaction *const tx, const Rdb_key_def &kd,
-                    const rocksdb::Slice &key, const rocksdb::Slice &value)
+                    const rocksdb::Slice &key, const rocksdb::Slice &value,
+                    bool sort)
       MY_ATTRIBUTE((__nonnull__, __warn_unused_result__));
+  void update_bytes_written(ulonglong bytes_written);
   int update_pk(const Rdb_key_def &kd, const struct update_row_info &row_info,
                 const bool &pk_changed) MY_ATTRIBUTE((__warn_unused_result__));
   int update_sk(const TABLE *const table_arg, const Rdb_key_def &kd,

storage/rocksdb/ha_rocksdb_proto.h

@@ -76,6 +76,7 @@ bool rdb_is_ttl_read_filtering_enabled();
 int rdb_dbug_set_ttl_rec_ts();
 int rdb_dbug_set_ttl_snapshot_ts();
 int rdb_dbug_set_ttl_read_filter_ts();
+bool rdb_dbug_set_ttl_ignore_pk();
 #endif
 
 enum operation_type : int;

storage/rocksdb/mysql-test/rocksdb/include/prefix_index_only_query_check.inc

@@ -0,0 +1,21 @@
+#
+# A helper include file for prefix index index-only query tests
+#
+# Parameters:
+# $prefix_index_check_title - title of the test
+# $prefix_index_check_query - test query
+# $prefix_index_check_read_avoided_delta - expected change of
+#   'rocksdb_covered_secondary_key_lookups' status variable
+#   value after running the query
+
+--let $show_count_statement = show status like 'rocksdb_covered_secondary_key_lookups'
+
+--echo # $prefix_index_check_title
+--let $base_count = query_get_value($show_count_statement, Value, 1)
+
+--eval $prefix_index_check_query
+
+--let $count = query_get_value($show_count_statement, Value, 1)
+--let $assert_text= $prefix_index_check_title: $prefix_index_check_read_avoided_delta rocksdb_covered_secondary_key_lookups
+--let $assert_cond= $count - $base_count = $prefix_index_check_read_avoided_delta
+--source include/assert.inc

storage/rocksdb/mysql-test/rocksdb/my.cnf

@@ -5,3 +5,4 @@ sql-mode=NO_ENGINE_SUBSTITUTION
 explicit-defaults-for-timestamp=1
 rocksdb_lock_wait_timeout=1
 rocksdb_strict_collation_check=0
+rocksdb_force_compute_memtable_stats_cachetime=0

storage/rocksdb/mysql-test/rocksdb/r/2pc_group_commit.result

@@ -4,20 +4,20 @@ CREATE DATABASE mysqlslap;
 USE mysqlslap;
 CREATE TABLE t1(id BIGINT AUTO_INCREMENT, value BIGINT, PRIMARY KEY(id)) ENGINE=rocksdb;
 # 2PC enabled, MyRocks durability enabled
-SET GLOBAL rocksdb_enable_2pc=0;
+SET GLOBAL rocksdb_enable_2pc=1;
 SET GLOBAL rocksdb_flush_log_at_trx_commit=1;
 ## 2PC + durability + single thread
 select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs';
 select case when variable_value-@c = 1000 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs';
 case when variable_value-@c = 1000 then 'true' else 'false' end
-false
+true
 ## 2PC + durability + group commit
 select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs';
 select case when variable_value-@c > 0 and variable_value-@c < 10000 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs';
 case when variable_value-@c > 0 and variable_value-@c < 10000 then 'true' else 'false' end
-false
+true
 # 2PC enabled, MyRocks durability disabled
-SET GLOBAL rocksdb_enable_2pc=0;
+SET GLOBAL rocksdb_enable_2pc=1;
 SET GLOBAL rocksdb_flush_log_at_trx_commit=0;
 select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs';
 select case when variable_value-@c = 0 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs';
@@ -28,16 +28,16 @@ select case when variable_value-@c = 0 then 'true' else 'false' end from informa
 case when variable_value-@c = 0 then 'true' else 'false' end
 true
 # 2PC disabled, MyRocks durability enabled
-SET GLOBAL rocksdb_enable_2pc=1;
+SET GLOBAL rocksdb_enable_2pc=0;
 SET GLOBAL rocksdb_flush_log_at_trx_commit=1;
 select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs';
 select case when variable_value-@c = 0 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs';
 case when variable_value-@c = 0 then 'true' else 'false' end
-false
+true
 select variable_value into @c from information_schema.global_status where variable_name='rocksdb_wal_group_syncs';
 select case when variable_value-@c = 0 then 'true' else 'false' end from information_schema.global_status where variable_name='rocksdb_wal_group_syncs';
 case when variable_value-@c = 0 then 'true' else 'false' end
-false
+true
 SET GLOBAL rocksdb_enable_2pc=1;
 SET GLOBAL rocksdb_flush_log_at_trx_commit=1;
 DROP TABLE t1;

storage/rocksdb/mysql-test/rocksdb/r/add_index_inplace.result

@@ -284,12 +284,16 @@ DROP TABLE t1;
 set global rocksdb_bulk_load=1;
 # Establish connection con1 (user=root)
 # Switch to connection con1
-show global variables like 'rocksdb_bulk_load';
+show global variables like 'rocksdb_bulk_load%';
 Variable_name	Value
 rocksdb_bulk_load	ON
-show session variables like 'rocksdb_bulk_load';
+rocksdb_bulk_load_allow_unsorted	OFF
+rocksdb_bulk_load_size	1000
+show session variables like 'rocksdb_bulk_load%';
 Variable_name	Value
 rocksdb_bulk_load	ON
+rocksdb_bulk_load_allow_unsorted	OFF
+rocksdb_bulk_load_size	1000
 CREATE TABLE t1 (i INT, j INT, PRIMARY KEY (i)) ENGINE = ROCKSDB;
 INSERT INTO t1 VALUES (1,1);
 # Disconnecting on con1
@@ -327,10 +331,11 @@ SET @prior_rocksdb_strict_collation_check= @@rocksdb_strict_collation_check;
 SET @prior_rocksdb_merge_buf_size = @@rocksdb_merge_buf_size;
 SET global rocksdb_strict_collation_check = off;
 SET session rocksdb_merge_combine_read_size = 566;
-SET session rocksdb_merge_buf_size = 336;
-show variables like '%rocksdb_bulk_load%';
+SET session rocksdb_merge_buf_size = 340;
+show variables like 'rocksdb_bulk_load%';
 Variable_name	Value
 rocksdb_bulk_load	OFF
+rocksdb_bulk_load_allow_unsorted	OFF
 rocksdb_bulk_load_size	1000
 CREATE TABLE t1 (a VARCHAR(80)) ENGINE=RocksDB;
 INSERT INTO t1 (a) VALUES (REPEAT("a", 80));

storage/rocksdb/mysql-test/rocksdb/r/bulk_load_errors.result

@@ -16,4 +16,30 @@ INSERT INTO t1 VALUES(20);
 INSERT INTO t1 VALUES(21);
 SET rocksdb_bulk_load=0;
 ERROR HY000: Lost connection to MySQL server during query
+TRUNCATE TABLE t1;
+SET rocksdb_bulk_load_allow_unsorted=1;
+SET rocksdb_bulk_load=1;
+INSERT INTO t1 VALUES(100);
+INSERT INTO t1 VALUES(101);
+INSERT INTO t1 VALUES(99);
+SET rocksdb_bulk_load=0;
+SELECT * FROM t1;
+pk
+99
+100
+101
+TRUNCATE TABLE t1;
+SET rocksdb_bulk_load=1;
+INSERT INTO t1 VALUES(201);
+INSERT INTO t1 VALUES(200);
+INSERT INTO t1 VALUES(202);
+INSERT INTO t1 VALUES(201);
+ERROR 23000: Failed to insert the record: the key already exists
+SET rocksdb_bulk_load=0;
+SELECT * FROM t1;
+pk
+200
+201
+202
+SET rocksdb_bulk_load_allow_unsorted=DEFAULT;
 DROP TABLE t1;