Skip to content

Commit

Permalink
Merge 10.2 into 10.3
Browse files Browse the repository at this point in the history
  • Loading branch information
@dr-m
dr-m committed Jun 18, 2018
2 parents 63027a5 + c55de8d commit 0121d5a
Show file tree
Hide file tree
Showing 66 changed files with 2,161 additions and 905 deletions.
273 changes: 271 additions & 2 deletions extra/mariabackup/backup_copy.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ Place, Suite 330, Boston, MA 02111-1307 USA
#include "backup_mysql.h"
#include <btr0btr.h>

#define ROCKSDB_BACKUP_DIR "#rocksdb"

/* list of files to sync for --rsync mode */
static std::set<std::string> rsync_list;
/* locations of tablespaces read from .isl files */
Expand All @@ -66,6 +68,21 @@ static std::map<std::string, std::string> tablespace_locations;
/* Whether LOCK BINLOG FOR BACKUP has been issued during backup */
bool binlog_locked;

static void rocksdb_create_checkpoint();
static bool has_rocksdb_plugin();
static void copy_or_move_dir(const char *from, const char *to, bool copy, bool allow_hardlinks);
static void rocksdb_backup_checkpoint();
static void rocksdb_copy_back();

static bool is_abs_path(const char *path)
{
#ifdef _WIN32
return path[0] && path[1] == ':' && (path[2] == '/' || path[2] == '\\');
#else
return path[0] == '/';
#endif
}

/************************************************************************
Struct represents file or directory. */
struct datadir_node_t {
Expand Down Expand Up @@ -1138,7 +1155,8 @@ bool
copy_or_move_file(const char *src_file_path,
const char *dst_file_path,
const char *dst_dir,
uint thread_n)
uint thread_n,
bool copy = xtrabackup_copy_back)
{
ds_ctxt_t *datasink = ds_data; /* copy to datadir by default */
char filedir[FN_REFLEN];
Expand Down Expand Up @@ -1186,7 +1204,7 @@ copy_or_move_file(const char *src_file_path,
free(link_filepath);
}

ret = (xtrabackup_copy_back ?
ret = (copy ?
copy_file(datasink, src_file_path, dst_file_path, thread_n) :
move_file(datasink, src_file_path, dst_file_path,
dst_dir, thread_n));
Expand Down Expand Up @@ -1371,6 +1389,10 @@ bool backup_start()
return false;
}

if (has_rocksdb_plugin()) {
rocksdb_create_checkpoint();
}

// There is no need to stop slave thread before coping non-Innodb data when
// --no-lock option is used because --no-lock option requires that no DDL or
// DML to non-transaction tables can occur.
Expand Down Expand Up @@ -1456,6 +1478,10 @@ bool backup_finish()
}
}

if (has_rocksdb_plugin()) {
rocksdb_backup_checkpoint();
}

msg_ts("Backup created in directory '%s'\n", xtrabackup_target_dir);
if (mysql_binlog_position != NULL) {
msg("MySQL binlog position: %s\n", mysql_binlog_position);
Expand Down Expand Up @@ -1771,6 +1797,16 @@ copy_back()
int i_tmp;
bool is_ibdata_file;

if (strstr(node.filepath,"/" ROCKSDB_BACKUP_DIR "/")
#ifdef _WIN32
|| strstr(node.filepath,"\\" ROCKSDB_BACKUP_DIR "\\")
#endif
)
{
// copied at later step
continue;
}

/* create empty directories */
if (node.is_empty_dir) {
char path[FN_REFLEN];
Expand Down Expand Up @@ -1855,6 +1891,8 @@ copy_back()
}
}

rocksdb_copy_back();

cleanup:
if (it != NULL) {
datadir_iter_free(it);
Expand Down Expand Up @@ -2031,3 +2069,234 @@ static bool backup_files_from_datadir(const char *dir_path)
os_file_closedir(dir);
return ret;
}


static int rocksdb_remove_checkpoint_directory()
{
xb_mysql_query(mysql_connection, "set global rocksdb_remove_mariabackup_checkpoint=ON", false);
return 0;
}

static bool has_rocksdb_plugin()
{
static bool first_time = true;
static bool has_plugin= false;
if (!first_time || !xb_backup_rocksdb)
return has_plugin;

const char *query = "SELECT COUNT(*) FROM information_schema.plugins WHERE plugin_name='rocksdb'";
MYSQL_RES* result = xb_mysql_query(mysql_connection, query, true);
MYSQL_ROW row = mysql_fetch_row(result);
if (row)
has_plugin = !strcmp(row[0], "1");
mysql_free_result(result);
first_time = false;
return has_plugin;
}

static char *trim_trailing_dir_sep(char *path)
{
size_t path_len = strlen(path);
while (path_len)
{
char c = path[path_len - 1];
if (c == '/' IF_WIN(|| c == '\\', ))
path_len--;
else
break;
}
path[path_len] = 0;
return path;
}

/*
Create a file hardlink.
@return true on success, false on error.
*/
static bool make_hardlink(const char *from_path, const char *to_path)
{
DBUG_EXECUTE_IF("no_hardlinks", return false;);
char to_path_full[FN_REFLEN];
if (!is_abs_path(to_path))
{
fn_format(to_path_full, to_path, ds_data->root, "", MYF(MY_RELATIVE_PATH));
}
else
{
strncpy(to_path_full, to_path, sizeof(to_path_full));
}
#ifdef _WIN32
return CreateHardLink(to_path_full, from_path, NULL);
#else
return !link(from_path, to_path_full);
#endif
}

/*
Copies or moves a directory (non-recursively so far).
Helper function used to backup rocksdb checkpoint, or copy-back the
rocksdb files.
Has optimization that allows to use hardlinks when possible
(source and destination are directories on the same device)
*/
static void copy_or_move_dir(const char *from, const char *to, bool do_copy, bool allow_hardlinks)
{
datadir_node_t node;
datadir_node_init(&node);
datadir_iter_t *it = datadir_iter_new(from, false);

while (datadir_iter_next(it, &node))
{
char to_path[FN_REFLEN];
const char *from_path = node.filepath;
snprintf(to_path, sizeof(to_path), "%s/%s", to, base_name(from_path));
bool rc = false;
if (do_copy && allow_hardlinks)
{
rc = make_hardlink(from_path, to_path);
if (rc)
{
msg_ts("[%02u] Creating hardlink from %s to %s\n",
1, from_path, to_path);
}
else
{
allow_hardlinks = false;
}
}

if (!rc)
{
rc = (do_copy ?
copy_file(ds_data, from_path, to_path, 1) :
move_file(ds_data, from_path, node.filepath_rel,
to, 1));
}
if (!rc)
exit(EXIT_FAILURE);
}
datadir_iter_free(it);
datadir_node_free(&node);

}

/*
Obtain user level lock , to protect the checkpoint directory of the server
from being user/overwritten by different backup processes, if backups are
running in parallel.
This lock will be acquired before rocksdb checkpoint is created, held
while all files from it are being copied to their final backup destination,
and finally released after the checkpoint is removed.
*/
static void rocksdb_lock_checkpoint()
{
msg_ts("Obtaining rocksdb checkpoint lock.\n");
MYSQL_RES *res =
xb_mysql_query(mysql_connection, "SELECT GET_LOCK('mariabackup_rocksdb_checkpoint',3600)", true, true);

MYSQL_ROW r = mysql_fetch_row(res);
if (r && r[0] && strcmp(r[0], "1"))
{
msg_ts("Could not obtain rocksdb checkpont lock\n");
exit(EXIT_FAILURE);
}
}

static void rocksdb_unlock_checkpoint()
{
xb_mysql_query(mysql_connection,
"SELECT RELEASE_LOCK('mariabackup_rocksdb_checkpoint')", false, true);
}


/*
Create temporary checkpoint in $rocksdb_datadir/mariabackup-checkpoint
directory.
A (user-level) lock named 'mariabackup_rocksdb_checkpoint' will also be
acquired be this function.
*/
#define MARIADB_CHECKPOINT_DIR "mariabackup-checkpoint"
static char rocksdb_checkpoint_dir[FN_REFLEN];

static void rocksdb_create_checkpoint()
{
MYSQL_RES *result = xb_mysql_query(mysql_connection, "SELECT @@rocksdb_datadir,@@datadir", true, true);
MYSQL_ROW row = mysql_fetch_row(result);

DBUG_ASSERT(row && row[0] && row[1]);

char *rocksdbdir = row[0];
char *datadir = row[1];

if (is_abs_path(rocksdbdir))
{
snprintf(rocksdb_checkpoint_dir, sizeof(rocksdb_checkpoint_dir),
"%s/" MARIADB_CHECKPOINT_DIR, trim_trailing_dir_sep(rocksdbdir));
}
else
{
snprintf(rocksdb_checkpoint_dir, sizeof(rocksdb_checkpoint_dir),
"%s/%s/" MARIADB_CHECKPOINT_DIR, trim_trailing_dir_sep(datadir),
trim_dotslash(rocksdbdir));
}
mysql_free_result(result);

#ifdef _WIN32
for (char *p = rocksdb_checkpoint_dir; *p; p++)
if (*p == '\\') *p = '/';
#endif

rocksdb_lock_checkpoint();

if (!access(rocksdb_checkpoint_dir, 0))
{
msg_ts("Removing rocksdb checkpoint from previous backup attempt.\n");
rocksdb_remove_checkpoint_directory();
}

char query[FN_REFLEN + 32];
snprintf(query, sizeof(query), "SET GLOBAL rocksdb_create_checkpoint='%s'", rocksdb_checkpoint_dir);
xb_mysql_query(mysql_connection, query, false, true);
}

/*
Copy files from rocksdb temporary checkpoint to final destination.
remove temp.checkpoint directory (in server's datadir)
and release user level lock acquired inside rocksdb_create_checkpoint().
*/
static void rocksdb_backup_checkpoint()
{
msg_ts("Backing up rocksdb files.\n");
char rocksdb_backup_dir[FN_REFLEN];
snprintf(rocksdb_backup_dir, sizeof(rocksdb_backup_dir), "%s/" ROCKSDB_BACKUP_DIR , xtrabackup_target_dir);
bool backup_to_directory = xtrabackup_backup && xtrabackup_stream_fmt == XB_STREAM_FMT_NONE;
if (backup_to_directory)
{
if (my_mkdir(rocksdb_backup_dir, 0777, MYF(0))){
msg_ts("Can't create rocksdb backup directory %s\n", rocksdb_backup_dir);
exit(EXIT_FAILURE);
}
}
copy_or_move_dir(rocksdb_checkpoint_dir, ROCKSDB_BACKUP_DIR, true, backup_to_directory);
rocksdb_remove_checkpoint_directory();
rocksdb_unlock_checkpoint();
}

/*
Copies #rocksdb directory to the $rockdb_data_dir, on copy-back
*/
static void rocksdb_copy_back() {
if (access(ROCKSDB_BACKUP_DIR, 0))
return;
char rocksdb_home_dir[FN_REFLEN];
if (xb_rocksdb_datadir && is_abs_path(xb_rocksdb_datadir)) {
strncpy(rocksdb_home_dir, xb_rocksdb_datadir, sizeof(rocksdb_home_dir));
} else {
snprintf(rocksdb_home_dir, sizeof(rocksdb_home_dir), "%s/%s", mysql_data_home,
xb_rocksdb_datadir?trim_dotslash(xb_rocksdb_datadir): ROCKSDB_BACKUP_DIR);
}
mkdirp(rocksdb_home_dir, 0777, MYF(0));
copy_or_move_dir(ROCKSDB_BACKUP_DIR, rocksdb_home_dir, xtrabackup_copy_back, xtrabackup_copy_back);
}
19 changes: 17 additions & 2 deletions extra/mariabackup/xtrabackup.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,8 @@ char *xtrabackup_tmpdir;
char *xtrabackup_tables;
char *xtrabackup_tables_file;
char *xtrabackup_tables_exclude;
char *xb_rocksdb_datadir;
my_bool xb_backup_rocksdb = 1;

typedef std::list<regex_t> regex_list_t;
static regex_list_t regex_include_list;
Expand Down Expand Up @@ -683,7 +685,9 @@ enum options_xtrabackup
OPT_XTRA_TABLES_EXCLUDE,
OPT_XTRA_DATABASES_EXCLUDE,
OPT_PROTOCOL,
OPT_LOCK_DDL_PER_TABLE
OPT_LOCK_DDL_PER_TABLE,
OPT_ROCKSDB_DATADIR,
OPT_BACKUP_ROCKSDB
};

struct my_option xb_client_options[] =
Expand Down Expand Up @@ -1182,7 +1186,7 @@ struct my_option xb_server_options[] =
"The algorithm InnoDB uses for page checksumming. [CRC32, STRICT_CRC32, "
"INNODB, STRICT_INNODB, NONE, STRICT_NONE]", &srv_checksum_algorithm,
&srv_checksum_algorithm, &innodb_checksum_algorithm_typelib, GET_ENUM,
REQUIRED_ARG, SRV_CHECKSUM_ALGORITHM_INNODB, 0, 0, 0, 0, 0},
REQUIRED_ARG, SRV_CHECKSUM_ALGORITHM_CRC32, 0, 0, 0, 0, 0},

{"innodb_undo_directory", OPT_INNODB_UNDO_DIRECTORY,
"Directory where undo tablespace files live, this path can be absolute.",
Expand Down Expand Up @@ -1227,6 +1231,17 @@ struct my_option xb_server_options[] =
(uchar*) &opt_lock_ddl_per_table, (uchar*) &opt_lock_ddl_per_table, 0,
GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},

{"rocksdb-datadir", OPT_ROCKSDB_DATADIR, "RocksDB data directory."
"This option is only used with --copy-back or --move-back option",
&xb_rocksdb_datadir, &xb_rocksdb_datadir,
0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },

{ "backup-rocksdb", OPT_BACKUP_ROCKSDB, "Backup rocksdb data, if rocksdb plugin is installed."
"Used only with --backup option. Can be useful for partial backups, to exclude all rocksdb data",
&xb_backup_rocksdb, &xb_backup_rocksdb,
0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 },


{ 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
};

Expand Down
3 changes: 3 additions & 0 deletions extra/mariabackup/xtrabackup.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ extern char *xtrabackup_incremental_basedir;
extern char *innobase_data_home_dir;
extern char *innobase_buffer_pool_filename;
extern char *xb_plugin_dir;
extern char *xb_rocksdb_datadir;
extern my_bool xb_backup_rocksdb;

extern uint opt_protocol;
extern ds_ctxt_t *ds_meta;
extern ds_ctxt_t *ds_data;
Expand Down
Loading

0 comments on commit 0121d5a

Please sign in to comment.