Merge tag 'tokudb-engine/tokudb-7.5.7' into 5.5

.gitignore

@@ -193,6 +193,7 @@ storage/tokudb/ft-index/tools/tokudb_dump
 storage/tokudb/ft-index/tools/tokudb_gen
 storage/tokudb/ft-index/tools/tokudb_load
 storage/tokudb/ft-index/tools/tokuftdump
+storage/tokudb/ft-index/tools/tokuft_logprint
 storage/tokudb/ft-index/ft/ftverify
 storage/tokudb/ft-index/ft/tdb-recover
 storage/tokudb/ft-index/ft/tdb_logprint

mysql-test/suite/tokudb.bugs/r/db805.result

@@ -0,0 +1,18 @@
+drop table if exists t1,t3;
+create table t3(a3 int,b3 decimal(0,0),c3 int,d3 int,primary key(a3,b3)) engine=TOKUDB;
+LOCK TABLES t3 WRITE;
+create temporary table t1(f1 int,index(f1)) engine=innodb;
+INSERT INTO t1 VALUES(1),(1),(1);
+select * from t1;
+f1
+1
+1
+1
+ALTER TABLE t1 engine=TOKUDB;
+select * from t1;
+f1
+1
+1
+1
+unlock tables;
+drop table t1,t3;

mysql-test/suite/tokudb.bugs/r/db806.result

@@ -0,0 +1,9 @@
+drop table if exists t1,t3;
+CREATE TABLE t3(a int,c int,d int)engine=TOKUDB;
+lock table t3 read;
+create temporary table t1 engine=tokudb as SELECT 1;
+select * from t1;
+1
+1
+unlock tables;
+drop table t1,t3;

mysql-test/suite/tokudb.bugs/r/db811.result

@@ -0,0 +1,14 @@
+drop table if exists t2,t3,t4;
+CREATE TABLE t3(a INT,b INT,UNIQUE KEY (a,b)) engine=TOKUDB;
+CREATE TABLE t4(c1 FLOAT ZEROFILL) engine=innodb;
+CREATE TABLE t2(a int KEY,b CHAR (1)) engine=TOKUDB PARTITION BY HASH (a) PARTITIONS 13;
+LOCK TABLES t4 WRITE,t3 WRITE,t2 WRITE;
+INSERT INTO t2(a)VALUES (REPEAT(0,1));
+ALTER TABLE t2 ADD COLUMN(c INT);
+alter table t4 add column c int;
+UPDATE t2 SET a=1;
+select * from t2;
+a	b	c
+1	NULL	NULL
+unlock tables;
+drop table t2,t3,t4;

mysql-test/suite/tokudb.bugs/r/db811s.result

@@ -0,0 +1,14 @@
+drop table if exists t2,t3,t4;
+CREATE TABLE t3(a INT,b INT,UNIQUE KEY (a,b)) engine=TOKUDB;
+CREATE TABLE t4(c1 FLOAT ZEROFILL) engine=innodb;
+CREATE TABLE t2(a int KEY,b CHAR (1)) engine=TOKUDB PARTITION BY HASH (a) PARTITIONS 1;
+LOCK TABLES t4 WRITE,t3 WRITE,t2 WRITE;
+INSERT INTO t2(a)VALUES (REPEAT(0,1));
+ALTER TABLE t2 ADD COLUMN(c INT);
+alter table t4 add column c int;
+UPDATE t2 SET a=1;
+select * from t2;
+a	b	c
+1	NULL	NULL
+unlock tables;
+drop table t2,t3,t4;

mysql-test/suite/tokudb.bugs/r/db823.result

@@ -0,0 +1,11 @@
+drop table if exists s,t;
+create table s (id int) engine=tokudb;
+lock tables s write;
+create temporary table t (id int, key(id)) engine=innodb;
+insert into t values (1);
+alter table t engine=tokudb;
+select * from t;
+id
+1
+unlock tables;
+drop table s, t;

mysql-test/suite/tokudb.bugs/t/db805.test

@@ -0,0 +1,17 @@
+# DB-805 test that conversion of t1 from innodb to tokudb can write rows
+source include/have_tokudb.inc;
+source include/have_innodb.inc;
+disable_warnings;
+drop table if exists t1,t3;
+enable_warnings;
+
+create table t3(a3 int,b3 decimal(0,0),c3 int,d3 int,primary key(a3,b3)) engine=TOKUDB;
+LOCK TABLES t3 WRITE;
+create temporary table t1(f1 int,index(f1)) engine=innodb;
+INSERT INTO t1 VALUES(1),(1),(1);
+select * from t1;
+ALTER TABLE t1 engine=TOKUDB;
+select * from t1;
+unlock tables;
+
+drop table t1,t3;

mysql-test/suite/tokudb.bugs/t/db806.test

@@ -0,0 +1,13 @@
+# DB-806 test that lock tables and create select can write rows to the new table
+source include/have_tokudb.inc;
+disable_warnings;
+drop table if exists t1,t3;
+enable_warnings;
+
+CREATE TABLE t3(a int,c int,d int)engine=TOKUDB;
+lock table t3 read;
+create temporary table t1 engine=tokudb as SELECT 1;
+select * from t1;
+unlock tables;
+
+drop table t1,t3;

mysql-test/suite/tokudb.bugs/t/db811.test

@@ -0,0 +1,22 @@
+# DB-811 test that alter table t2 updates both the schema (FRM) and the data (tokudb files)
+
+source include/have_tokudb.inc;
+source include/have_innodb.inc;
+source include/have_partition.inc;
+disable_warnings;
+drop table if exists t2,t3,t4;
+enable_warnings;
+
+CREATE TABLE t3(a INT,b INT,UNIQUE KEY (a,b)) engine=TOKUDB;
+CREATE TABLE t4(c1 FLOAT ZEROFILL) engine=innodb;
+CREATE TABLE t2(a int KEY,b CHAR (1)) engine=TOKUDB PARTITION BY HASH (a) PARTITIONS 13;
+LOCK TABLES t4 WRITE,t3 WRITE,t2 WRITE;
+INSERT INTO t2(a)VALUES (REPEAT(0,1));
+ALTER TABLE t2 ADD COLUMN(c INT);
+alter table t4 add column c int;
+UPDATE t2 SET a=1;
+select * from t2;
+unlock tables;
+
+drop table t2,t3,t4;
+

mysql-test/suite/tokudb.bugs/t/db811s.test

@@ -0,0 +1,22 @@
+# DB-811 test that alter table t2 updates both the schema (FRM) and the data (tokudb files)
+
+source include/have_tokudb.inc;
+source include/have_innodb.inc;
+source include/have_partition.inc;
+disable_warnings;
+drop table if exists t2,t3,t4;
+enable_warnings;
+
+CREATE TABLE t3(a INT,b INT,UNIQUE KEY (a,b)) engine=TOKUDB;
+CREATE TABLE t4(c1 FLOAT ZEROFILL) engine=innodb;
+CREATE TABLE t2(a int KEY,b CHAR (1)) engine=TOKUDB PARTITION BY HASH (a) PARTITIONS 1;
+LOCK TABLES t4 WRITE,t3 WRITE,t2 WRITE;
+INSERT INTO t2(a)VALUES (REPEAT(0,1));
+ALTER TABLE t2 ADD COLUMN(c INT);
+alter table t4 add column c int;
+UPDATE t2 SET a=1;
+select * from t2;
+unlock tables;
+
+drop table t2,t3,t4;
+

mysql-test/suite/tokudb.bugs/t/db823.test

@@ -0,0 +1,16 @@
+# test DB-823
+# test that the conversion of table t from innodb to tokudb succeeds.
+source include/have_tokudb.inc;
+source include/have_innodb.inc;
+disable_warnings;
+drop table if exists s,t;
+enable_warnings;
+create table s (id int) engine=tokudb;
+lock tables s write;
+create temporary table t (id int, key(id)) engine=innodb;
+insert into t values (1);
+alter table t engine=tokudb;
+select * from t;
+unlock tables;
+drop table s, t;
+

storage/tokudb/ha_tokudb.cc

@@ -6207,6 +6207,12 @@ int ha_tokudb::external_lock(THD * thd, int lock_type) {
         if (error) { goto cleanup; }
         thd_set_ha_data(thd, tokudb_hton, trx);
     }
+
+    if (tokudb_debug & TOKUDB_DEBUG_TXN) {
+        TOKUDB_HANDLER_TRACE("trx %p %p %p %p %u %u", trx->all, trx->stmt, trx->sp_level, trx->sub_sp_level, 
+                             trx->tokudb_lock_count, trx->create_lock_count);
+    }
+
     if (trx->all == NULL) {
         trx->sp_level = NULL;
     }
@@ -6215,22 +6221,16 @@ int ha_tokudb::external_lock(THD * thd, int lock_type) {
         if (lock_type == F_WRLCK) {
             use_write_locks = true;
         }
-        if (!trx->tokudb_lock_count++) {
-            if (trx->stmt) {
-                if (tokudb_debug & TOKUDB_DEBUG_TXN) {
-                    TOKUDB_HANDLER_TRACE("stmt already set %p %p %p %p", trx->all, trx->stmt, trx->sp_level, trx->sub_sp_level);
-                }
-            } else {
-                assert(trx->stmt == 0);
-                transaction = NULL;    // Safety
-                error = create_txn(thd, trx);
-                if (error) {
-                    trx->tokudb_lock_count--;  // We didn't get the lock
-                    goto cleanup;
-                }
+        if (!trx->stmt) {
+            transaction = NULL;    // Safety
+            error = create_txn(thd, trx);
+            if (error) {
+                goto cleanup;
             }
+            trx->create_lock_count = trx->tokudb_lock_count;
         }
         transaction = trx->sub_sp_level;
+        trx->tokudb_lock_count++;
     }
     else {
         tokudb_pthread_mutex_lock(&share->mutex);
@@ -6245,21 +6245,24 @@ int ha_tokudb::external_lock(THD * thd, int lock_type) {
         added_rows = 0;
         deleted_rows = 0;
         share->rows_from_locked_table = 0;
-        if (trx->tokudb_lock_count > 0 && !--trx->tokudb_lock_count) {
-            if (trx->stmt) {
-                /*
-                   F_UNLCK is done without a transaction commit / rollback.
-                   This happens if the thread didn't update any rows
-                   We must in this case commit the work to keep the row locks
-                 */
-                DBUG_PRINT("trans", ("commiting non-updating transaction"));
-                reset_stmt_progress(&trx->stmt_progress);
-                commit_txn(trx->stmt, 0);
-                trx->stmt = NULL;
-                trx->sub_sp_level = NULL;
+        if (trx->tokudb_lock_count > 0) {
+            if (--trx->tokudb_lock_count <= trx->create_lock_count) {
+                trx->create_lock_count = 0;
+                if (trx->stmt) {
+                    /*
+                      F_UNLCK is done without a transaction commit / rollback.
+                      This happens if the thread didn't update any rows
+                      We must in this case commit the work to keep the row locks
+                    */
+                    DBUG_PRINT("trans", ("commiting non-updating transaction"));
+                    reset_stmt_progress(&trx->stmt_progress);
+                    commit_txn(trx->stmt, 0);
+                    trx->stmt = NULL;
+                    trx->sub_sp_level = NULL;
+                }
             }
+            transaction = NULL;
         }
-        transaction = NULL;
     }
 cleanup:
     if (tokudb_debug & TOKUDB_DEBUG_LOCK)
@@ -6274,8 +6277,9 @@ int ha_tokudb::external_lock(THD * thd, int lock_type) {
 */
 int ha_tokudb::start_stmt(THD * thd, thr_lock_type lock_type) {
     TOKUDB_HANDLER_DBUG_ENTER("cmd %d lock %d %s", thd_sql_command(thd), lock_type, share->table_name);
-    if (0)
+    if (tokudb_debug & TOKUDB_DEBUG_LOCK) {
         TOKUDB_HANDLER_TRACE("q %s", thd->query());
+    }
 
     int error = 0;
     tokudb_trx_data *trx = (tokudb_trx_data *) thd_get_ha_data(thd, tokudb_hton);
@@ -6285,6 +6289,11 @@ int ha_tokudb::start_stmt(THD * thd, thr_lock_type lock_type) {
         thd_set_ha_data(thd, tokudb_hton, trx);
     }
 
+    if (tokudb_debug & TOKUDB_DEBUG_TXN) {
+        TOKUDB_HANDLER_TRACE("trx %p %p %p %p %u %u", trx->all, trx->stmt, trx->sp_level, trx->sub_sp_level, 
+                             trx->tokudb_lock_count, trx->create_lock_count);
+    }
+
     /*
        note that trx->stmt may have been already initialized as start_stmt()
        is called for *each table* not for each storage engine,
@@ -6295,9 +6304,7 @@ int ha_tokudb::start_stmt(THD * thd, thr_lock_type lock_type) {
         if (error) {
             goto cleanup;
         }
-        if (tokudb_debug & TOKUDB_DEBUG_TXN) {
-            TOKUDB_HANDLER_TRACE("%p %p %p %p %u", trx->all, trx->stmt, trx->sp_level, trx->sub_sp_level, trx->tokudb_lock_count);
-        }
+        trx->create_lock_count = trx->tokudb_lock_count;
     }
     else {
         if (tokudb_debug & TOKUDB_DEBUG_TXN) {

storage/tokudb/ha_tokudb_admin.cc

@@ -121,9 +121,10 @@ static int analyze_progress(void *v_extra, uint64_t rows) {
         progress_time = (float) (t_now - t_start) / (float) t_limit;
     char *write_status_msg = extra->write_status_msg;
     TABLE_SHARE *table_share = extra->table_share;
-    sprintf(write_status_msg, "%s.%s.%s %u of %u %.lf%% rows %.lf%% time", 
-            table_share->db.str, table_share->table_name.str, extra->key_name,
-            extra->key_i, table_share->keys, progress_rows * 100.0, progress_time * 100.0);
+    sprintf(write_status_msg, "%.*s.%.*s.%s %u of %u %.lf%% rows %.lf%% time", 
+            (int) table_share->db.length, table_share->db.str,
+            (int) table_share->table_name.length, table_share->table_name.str,
+            extra->key_name, extra->key_i, table_share->keys, progress_rows * 100.0, progress_time * 100.0);
     thd_proc_info(thd, write_status_msg);
     return 0;
 }
@@ -338,8 +339,10 @@ static int ha_tokudb_check_progress(void *extra, float progress) {
 
 static void ha_tokudb_check_info(THD *thd, TABLE *table, const char *msg) {
     if (thd->vio_ok()) {
-        char tablename[256];
-        snprintf(tablename, sizeof tablename, "%s.%s", table->s->db.str, table->s->table_name.str);
+        char tablename[table->s->db.length + 1 + table->s->table_name.length + 1];
+        snprintf(tablename, sizeof tablename, "%.*s.%.*s",
+                 (int) table->s->db.length, table->s->db.str,
+                 (int) table->s->table_name.length, table->s->table_name.str);
         thd->protocol->prepare_for_resend();
         thd->protocol->store(tablename, strlen(tablename), system_charset_info);
         thd->protocol->store("check", 5, system_charset_info);
@@ -388,6 +391,11 @@ int ha_tokudb::check(THD *thd, HA_CHECK_OPT *check_opt) {
             }
             struct check_context check_context = { thd };
             r = db->verify_with_progress(db, ha_tokudb_check_progress, &check_context, (tokudb_debug & TOKUDB_DEBUG_CHECK) != 0, keep_going);
+            if (r != 0) {
+                char msg[32 + strlen(kname)];
+                sprintf(msg, "Corrupt %s", kname);
+                ha_tokudb_check_info(thd, table, msg);
+            }
             snprintf(write_status_msg, sizeof write_status_msg, "%s key=%s %u result=%d", share->table_name, kname, i, r);
             thd_proc_info(thd, write_status_msg);
             if (tokudb_debug & TOKUDB_DEBUG_CHECK) {

storage/tokudb/ha_tokudb_alter_56.cc

@@ -784,13 +784,16 @@ bool ha_tokudb::commit_inplace_alter_table(TABLE *altered_table, Alter_inplace_i
         assert(trx->tokudb_lock_count > 0);
         // for partitioned tables, we use a single transaction to do all of the partition changes.  the tokudb_lock_count
         // is a reference count for each of the handlers to the same transaction.  obviously, we want to only abort once.
-        if (!--trx->tokudb_lock_count) {
-            abort_txn(ctx->alter_txn);
-            ctx->alter_txn = NULL;
-            trx->stmt = NULL;
-            trx->sub_sp_level = NULL;
+        if (trx->tokudb_lock_count > 0) {
+            if (--trx->tokudb_lock_count <= trx->create_lock_count) {
+                trx->create_lock_count = 0;
+                abort_txn(ctx->alter_txn);
+                ctx->alter_txn = NULL;
+                trx->stmt = NULL;
+                trx->sub_sp_level = NULL;
+            }
+            transaction = NULL;
         }
-        transaction = NULL;
 
         if (ctx->add_index_changed) {
             restore_add_index(table, ha_alter_info->index_add_count, ctx->incremented_num_DBs, ctx->modified_DBs);

storage/tokudb/hatoku_defines.h

@@ -355,6 +355,7 @@ typedef struct st_tokudb_trx_data {
     DB_TXN *sp_level;
     DB_TXN *sub_sp_level;
     uint tokudb_lock_count;
+    uint create_lock_count;
     tokudb_stmt_progress stmt_progress;
     bool checkpoint_lock_taken;
     LIST *handlers;