FT-701 keep all FT files consistency before next checkpoint for backup

ft/cachetable/cachetable-internal.h

@@ -411,6 +411,8 @@ class checkpointer {
     void add_background_job();
     void remove_background_job();
     void end_checkpoint(void (*testcallback_f)(void*),  void* testextra);
+    void begin_backup();
+    void end_backup();
     TOKULOGGER get_logger();
     // used during begin_checkpoint
     void increment_num_txns();
@@ -600,6 +602,7 @@ struct cachetable {
     KIBBUTZ client_kibbutz; // pool of worker threads and jobs to do asynchronously for the client.
     KIBBUTZ ct_kibbutz; // pool of worker threads and jobs to do asynchronously for the cachetable
     KIBBUTZ checkpointing_kibbutz; // small pool for checkpointing cloned pairs
+    bool in_backup; // we are in back up or NOT, default is false
 
     char *env_dir;
 };

ft/cachetable/cachetable.cc

@@ -250,6 +250,7 @@ int toku_cachetable_create_ex(CACHETABLE *ct_result, long size_limit,
     CACHETABLE XCALLOC(ct);
     ct->list.init();
     ct->cf_list.init();
+    ct->in_backup = false;
 
     int num_processors = toku_os_get_number_active_processors();
     int checkpointing_nworkers = (num_processors/4) ? num_processors/4 : 1;
@@ -2749,6 +2750,27 @@ void toku_cachetable_end_checkpoint(CHECKPOINTER cp, TOKULOGGER UU(logger),
     cp->end_checkpoint(testcallback_f, testextra);
 }
 
+// in_backup begin
+void toku_cachetable_begin_backup(CACHETABLE ct)
+{
+    ct->cf_list.read_lock();
+    ct->in_backup = true;
+    ct->cf_list.read_unlock();
+}
+
+void toku_cachetable_end_backup(CACHETABLE ct)
+{
+    ct->cf_list.read_lock();
+    ct->in_backup = false;
+    ct->cf_list.read_unlock();
+}
+
+bool toku_cachefile_in_backup(CACHEFILE cf)
+{
+    return cf->cachetable->in_backup;
+}
+// in_backup end
+
 TOKULOGGER toku_cachefile_logger (CACHEFILE cf) {
     return cf->cachetable->cp.get_logger();
 }

ft/cachetable/cachetable.h

@@ -147,6 +147,10 @@ void toku_cachetable_begin_checkpoint (CHECKPOINTER cp, struct tokulogger *logge
 void toku_cachetable_end_checkpoint(CHECKPOINTER cp, struct tokulogger *logger, 
                                    void (*testcallback_f)(void*),  void * testextra);
 
+void toku_cachetable_begin_backup(CACHETABLE ct);
+void toku_cachetable_end_backup(CACHETABLE ct);
+// cachefile is in backup or not
+bool toku_cachefile_in_backup(CACHEFILE cf);
 
 // Shuts down checkpoint thread
 // Requires no locks be held that are taken by the checkpoint function

ft/ft.cc

@@ -282,7 +282,13 @@ static void ft_note_pin_by_checkpoint (CACHEFILE UU(cachefile), void *header_v)
     FT ft = (FT) header_v;
     toku_ft_grab_reflock(ft);
     assert(!ft->pinned_by_checkpoint);
+
+    // because we add in_backup for cachetable, with this case:
+    // a cachefile in_backup is true and then do a checkpoint
+    // this assertion fails since nobody references to the ft
+#if 0
     assert(toku_ft_needed_unlocked(ft));
+#endif
     ft->pinned_by_checkpoint = true;
     toku_ft_release_reflock(ft);
 }
@@ -902,8 +908,12 @@ toku_ft_remove_reference(FT ft, bool oplsn_valid, LSN oplsn, remove_ft_ref_callb
             assert(!needed);
         }
         if (!needed) {
-            // close header
-            toku_ft_evict_from_memory(ft, oplsn_valid, oplsn);
+            // close header if we are not in backup
+            // if close header during backup, the FT files would be inconsistency
+            bool in_backup = toku_cachefile_in_backup(ft->cf);
+            if (!in_backup) {
+                toku_ft_evict_from_memory(ft, oplsn_valid, oplsn);
+            }
         }
 
         toku_ft_open_close_unlock();

ft/tests/ft-in-backup-test.cc

@@ -0,0 +1,155 @@
+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+#ident "$Id$"
+/*======
+This file is part of PerconaFT.
+
+
+Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
+
+    PerconaFT is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License, version 2,
+    as published by the Free Software Foundation.
+
+    PerconaFT is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with PerconaFT.  If not, see <http://www.gnu.org/licenses/>.
+
+----------------------------------------
+
+    PerconaFT is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License, version 3,
+    as published by the Free Software Foundation.
+
+    PerconaFT is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with PerconaFT.  If not, see <http://www.gnu.org/licenses/>.
+======= */
+
+#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved."
+
+#include "test.h"
+#include "cachetable/checkpoint.h"
+
+static TOKUTXN const null_txn = 0;
+static const char *fname = TOKU_TEST_FILENAME;
+
+/* test for_backup in ft_close */
+static void test_in_backup() {
+    int r;
+    CACHETABLE ct;
+    FT_HANDLE ft;
+    unlink(fname);
+
+    toku_cachetable_create(&ct, 0, ZERO_LSN, nullptr);
+    CHECKPOINTER cp = toku_cachetable_get_checkpointer(ct);
+
+    // TEST1 : for normal
+    r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);
+    assert_zero(r);
+    r = toku_close_ft_handle_nolsn(ft, 0);
+    assert_zero(r);
+
+    r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);
+    assert_zero(r);
+    {
+        DBT k,v;
+        toku_ft_insert(ft, toku_fill_dbt(&k, "hello", 6), toku_fill_dbt(&v, "there", 6), null_txn);
+    }
+    r = toku_close_ft_handle_nolsn(ft, 0);
+    assert_zero(r);
+
+    r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);
+    assert_zero(r);
+    ft_lookup_and_check_nodup(ft, "hello", "there");
+    r = toku_close_ft_handle_nolsn(ft, 0);
+    assert_zero(r);
+    toku_cachetable_close(&ct);
+
+    // TEST2: in fly without checkpoint test
+    toku_cachetable_create(&ct, 0, ZERO_LSN, nullptr);
+    cp = toku_cachetable_get_checkpointer(ct);
+    r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);
+    assert_zero(r);
+
+    toku_cachetable_begin_backup(ct);
+    // this key/value just in fly since we are in backing up
+    {
+        DBT k,v;
+        toku_ft_insert(ft, toku_fill_dbt(&k, "halou", 6), toku_fill_dbt(&v, "not there", 10), null_txn);
+    }
+    r = toku_close_ft_handle_nolsn(ft, 0);
+    assert_zero(r);
+
+    r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);
+    assert_zero(r);
+    ft_lookup_and_check_nodup(ft, "halou", "not there");
+
+    // because we are in backup, so the FT header is stale after cachefile&cachetable closed
+    // here has a leak for this ft evicts from memroy, but that makes sense
+    r = toku_close_ft_handle_nolsn(ft, 0);
+    assert_zero(r);
+    toku_cachetable_end_backup(ct);
+    toku_cachetable_close(&ct);
+
+    // check the in fly key/value, it shouldn't exist
+    toku_cachetable_create(&ct, 0, ZERO_LSN, nullptr);
+    r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);
+    assert_zero(r);
+    ft_lookup_and_fail_nodup(ft, (char*)"halou");
+    r = toku_close_ft_handle_nolsn(ft, 0);
+    assert_zero(r);
+    toku_cachetable_close(&ct);
+
+    // TEST3: in fly with checkpoint test
+    toku_cachetable_create(&ct, 0, ZERO_LSN, nullptr);
+    cp = toku_cachetable_get_checkpointer(ct);
+    r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);
+    assert_zero(r);
+
+    toku_cachetable_begin_backup(ct);
+    // this key/value just in fly since we are in backup
+    {
+        DBT k,v;
+        toku_ft_insert(ft, toku_fill_dbt(&k, "halou1", 7), toku_fill_dbt(&v, "not there", 10), null_txn);
+    }
+    r = toku_close_ft_handle_nolsn(ft, 0);
+    assert_zero(r);
+
+    r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);
+    assert_zero(r);
+    ft_lookup_and_check_nodup(ft, "halou1", "not there");
+
+    // because we are in backup, so the FT header is stale after cachefile&cachetable closed
+    r = toku_close_ft_handle_nolsn(ft, 0);
+    assert_zero(r);
+    toku_cachetable_end_backup(ct);
+    r = toku_checkpoint(cp, NULL, NULL, NULL, NULL, NULL, CLIENT_CHECKPOINT);
+    assert_zero(r);
+    toku_cachetable_close(&ct);
+
+    // check after checkpoint
+    toku_cachetable_create(&ct, 0, ZERO_LSN, nullptr);
+    r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);
+    assert_zero(r);
+    ft_lookup_and_check_nodup(ft, "halou1", "not there");
+    r = toku_close_ft_handle_nolsn(ft, 0);
+    assert_zero(r);
+    toku_cachetable_close(&ct);
+}
+
+int
+test_main (int argc , const char *argv[]) {
+    default_parse_args(argc, argv);
+    test_in_backup();
+    if (verbose) printf("test ok\n");
+    return 0;
+}

src/ydb.cc

@@ -1623,7 +1623,10 @@ env_checkpointing_postpone(DB_ENV * env) {
     HANDLE_PANICKED_ENV(env);
     int r = 0;
     if (!env_opened(env)) r = EINVAL;
-    else toku_checkpoint_safe_client_lock();
+    else {
+        toku_checkpoint_safe_client_lock();
+        toku_cachetable_begin_backup(env->i->cachetable);
+    }
     return r;
 }
 
@@ -1632,7 +1635,10 @@ env_checkpointing_resume(DB_ENV * env) {
     HANDLE_PANICKED_ENV(env);
     int r = 0;
     if (!env_opened(env)) r = EINVAL;
-    else toku_checkpoint_safe_client_unlock();
+    else {
+        toku_cachetable_end_backup(env->i->cachetable);
+        toku_checkpoint_safe_client_unlock();
+    }
     return r;
 }