Merge 10.2 into 10.3

Author: dr-m

extra/mariabackup/backup_copy.cc

@@ -58,6 +58,8 @@ Place, Suite 330, Boston, MA 02111-1307 USA
 #include "backup_mysql.h"
 #include <btr0btr.h>
 
+#define ROCKSDB_BACKUP_DIR "#rocksdb"
+
 /* list of files to sync for --rsync mode */
 static std::set<std::string> rsync_list;
 /* locations of tablespaces read from .isl files */
@@ -66,6 +68,21 @@ static std::map<std::string, std::string> tablespace_locations;
 /* Whether LOCK BINLOG FOR BACKUP has been issued during backup */
 bool binlog_locked;
 
+static void rocksdb_create_checkpoint();
+static bool has_rocksdb_plugin();
+static void copy_or_move_dir(const char *from, const char *to, bool copy, bool allow_hardlinks);
+static void rocksdb_backup_checkpoint();
+static void rocksdb_copy_back();
+
+static bool is_abs_path(const char *path)
+{
+#ifdef _WIN32
+	return path[0] && path[1] == ':' && (path[2] == '/' || path[2] == '\\');
+#else
+	return path[0] == '/';
+#endif
+}
+
 /************************************************************************
 Struct represents file or directory. */
 struct datadir_node_t {
@@ -1138,7 +1155,8 @@ bool
 copy_or_move_file(const char *src_file_path,
 		  const char *dst_file_path,
 		  const char *dst_dir,
-		  uint thread_n)
+		  uint thread_n,
+		 bool copy = xtrabackup_copy_back)
 {
 	ds_ctxt_t *datasink = ds_data;		/* copy to datadir by default */
 	char filedir[FN_REFLEN];
@@ -1186,7 +1204,7 @@ copy_or_move_file(const char *src_file_path,
 		free(link_filepath);
 	}
 
-	ret = (xtrabackup_copy_back ?
+	ret = (copy ?
 		copy_file(datasink, src_file_path, dst_file_path, thread_n) :
 		move_file(datasink, src_file_path, dst_file_path,
 			  dst_dir, thread_n));
@@ -1371,6 +1389,10 @@ bool backup_start()
 		return false;
 	}
 
+	if (has_rocksdb_plugin()) {
+		rocksdb_create_checkpoint();
+	}
+
 	// There is no need to stop slave thread before coping non-Innodb data when
 	// --no-lock option is used because --no-lock option requires that no DDL or
 	// DML to non-transaction tables can occur.
@@ -1456,6 +1478,10 @@ bool backup_finish()
 		}
 	}
 
+	if (has_rocksdb_plugin()) {
+		rocksdb_backup_checkpoint();
+	}
+
 	msg_ts("Backup created in directory '%s'\n", xtrabackup_target_dir);
 	if (mysql_binlog_position != NULL) {
 		msg("MySQL binlog position: %s\n", mysql_binlog_position);
@@ -1771,6 +1797,16 @@ copy_back()
 		int i_tmp;
 		bool is_ibdata_file;
 
+		if (strstr(node.filepath,"/" ROCKSDB_BACKUP_DIR "/")
+#ifdef _WIN32
+			|| strstr(node.filepath,"\\" ROCKSDB_BACKUP_DIR "\\")
+#endif
+		)
+		{
+			// copied at later step
+			continue;
+		}
+
 		/* create empty directories */
 		if (node.is_empty_dir) {
 			char path[FN_REFLEN];
@@ -1855,6 +1891,8 @@ copy_back()
 		}
 	}
 
+	rocksdb_copy_back();
+
 cleanup:
 	if (it != NULL) {
 		datadir_iter_free(it);
@@ -2031,3 +2069,234 @@ static bool backup_files_from_datadir(const char *dir_path)
 	os_file_closedir(dir);
 	return ret;
 }
+
+
+static int rocksdb_remove_checkpoint_directory()
+{
+	xb_mysql_query(mysql_connection, "set global rocksdb_remove_mariabackup_checkpoint=ON", false);
+	return 0;
+}
+
+static bool has_rocksdb_plugin()
+{
+	static bool first_time = true;
+	static bool has_plugin= false;
+	if (!first_time || !xb_backup_rocksdb)
+		return has_plugin;
+
+	const char *query = "SELECT COUNT(*) FROM information_schema.plugins WHERE plugin_name='rocksdb'";
+	MYSQL_RES* result = xb_mysql_query(mysql_connection, query, true);
+	MYSQL_ROW row = mysql_fetch_row(result);
+	if (row)
+		has_plugin = !strcmp(row[0], "1");
+	mysql_free_result(result);
+	first_time = false;
+	return has_plugin;
+}
+
+static char *trim_trailing_dir_sep(char *path)
+{
+	size_t path_len = strlen(path);
+	while (path_len)
+	{
+		char c = path[path_len - 1];
+		if (c == '/' IF_WIN(|| c == '\\', ))
+			path_len--;
+		else
+			break;
+	}
+	path[path_len] = 0;
+	return path;
+}
+
+/*
+Create a file hardlink.
+@return true on success, false on error.
+*/
+static bool make_hardlink(const char *from_path, const char *to_path)
+{
+	DBUG_EXECUTE_IF("no_hardlinks", return false;);
+	char to_path_full[FN_REFLEN];
+	if (!is_abs_path(to_path))
+	{
+		fn_format(to_path_full, to_path, ds_data->root, "", MYF(MY_RELATIVE_PATH));
+	}
+	else
+	{
+		strncpy(to_path_full, to_path, sizeof(to_path_full));
+	}
+#ifdef _WIN32
+	return  CreateHardLink(to_path_full, from_path, NULL);
+#else
+	return !link(from_path, to_path_full);
+#endif
+}
+
+/*
+ Copies or moves a directory (non-recursively so far).
+ Helper function used to backup rocksdb checkpoint, or copy-back the
+ rocksdb files.
+
+ Has optimization that allows to use hardlinks when possible
+ (source and destination are directories on the same device)
+*/
+static void copy_or_move_dir(const char *from, const char *to, bool do_copy, bool allow_hardlinks)
+{
+	datadir_node_t node;
+	datadir_node_init(&node);
+	datadir_iter_t *it = datadir_iter_new(from, false);
+
+	while (datadir_iter_next(it, &node))
+	{
+		char to_path[FN_REFLEN];
+		const char *from_path = node.filepath;
+		snprintf(to_path, sizeof(to_path), "%s/%s", to, base_name(from_path));
+		bool rc = false;
+		if (do_copy && allow_hardlinks)
+		{
+			rc = make_hardlink(from_path, to_path);
+			if (rc)
+			{
+				msg_ts("[%02u] Creating hardlink from %s to %s\n",
+					1, from_path, to_path);
+			}
+			else
+			{
+				allow_hardlinks = false;
+			}
+		}
+
+		if (!rc) 
+		{
+			rc = (do_copy ?
+				copy_file(ds_data, from_path, to_path, 1) :
+				move_file(ds_data, from_path, node.filepath_rel,
+					to, 1));
+		}
+		if (!rc)
+			exit(EXIT_FAILURE);
+	}
+	datadir_iter_free(it);
+	datadir_node_free(&node);
+
+}
+
+/*
+  Obtain user level lock , to protect the checkpoint directory of the server
+  from being  user/overwritten by different backup processes, if backups are
+  running in parallel.
+  
+  This lock will be acquired before rocksdb checkpoint is created,  held
+  while all files from it are being copied to their final backup destination,
+  and finally released after the checkpoint is removed.
+*/
+static void rocksdb_lock_checkpoint()
+{
+	msg_ts("Obtaining rocksdb checkpoint lock.\n");
+	MYSQL_RES *res =
+		xb_mysql_query(mysql_connection, "SELECT GET_LOCK('mariabackup_rocksdb_checkpoint',3600)", true, true);
+
+	MYSQL_ROW r = mysql_fetch_row(res);
+	if (r && r[0] && strcmp(r[0], "1"))
+	{
+		msg_ts("Could not obtain rocksdb checkpont lock\n");
+		exit(EXIT_FAILURE);
+	}
+}
+
+static void rocksdb_unlock_checkpoint()
+{
+	xb_mysql_query(mysql_connection, 
+		"SELECT RELEASE_LOCK('mariabackup_rocksdb_checkpoint')", false, true);
+}
+
+
+/*
+  Create temporary checkpoint in $rocksdb_datadir/mariabackup-checkpoint
+  directory.
+  A (user-level) lock named 'mariabackup_rocksdb_checkpoint' will also be
+  acquired be this function.
+*/
+#define MARIADB_CHECKPOINT_DIR "mariabackup-checkpoint"
+static 	char rocksdb_checkpoint_dir[FN_REFLEN];
+
+static void rocksdb_create_checkpoint()
+{
+	MYSQL_RES *result = xb_mysql_query(mysql_connection, "SELECT @@rocksdb_datadir,@@datadir", true, true);
+	MYSQL_ROW row = mysql_fetch_row(result);
+
+	DBUG_ASSERT(row && row[0] && row[1]);
+
+	char *rocksdbdir = row[0];
+	char *datadir = row[1];
+
+	if (is_abs_path(rocksdbdir))
+	{
+		snprintf(rocksdb_checkpoint_dir, sizeof(rocksdb_checkpoint_dir),
+			"%s/" MARIADB_CHECKPOINT_DIR, trim_trailing_dir_sep(rocksdbdir));
+	}
+	else 
+	{
+		snprintf(rocksdb_checkpoint_dir, sizeof(rocksdb_checkpoint_dir),
+			"%s/%s/" MARIADB_CHECKPOINT_DIR, trim_trailing_dir_sep(datadir),
+			trim_dotslash(rocksdbdir));
+	}
+	mysql_free_result(result);
+
+#ifdef _WIN32
+	for (char *p = rocksdb_checkpoint_dir; *p; p++)
+		if (*p == '\\') *p = '/';
+#endif
+
+	rocksdb_lock_checkpoint();
+
+	if (!access(rocksdb_checkpoint_dir, 0))
+	{
+		msg_ts("Removing rocksdb checkpoint from previous backup attempt.\n");
+		rocksdb_remove_checkpoint_directory();
+	}
+
+	char query[FN_REFLEN + 32];
+	snprintf(query, sizeof(query), "SET GLOBAL rocksdb_create_checkpoint='%s'", rocksdb_checkpoint_dir);
+	xb_mysql_query(mysql_connection, query, false, true);
+}
+
+/*
+  Copy files from rocksdb temporary checkpoint to final destination.
+  remove temp.checkpoint directory (in server's datadir)
+  and release user level lock acquired inside rocksdb_create_checkpoint().
+*/
+static void rocksdb_backup_checkpoint()
+{
+	msg_ts("Backing up rocksdb files.\n");
+	char rocksdb_backup_dir[FN_REFLEN];
+	snprintf(rocksdb_backup_dir, sizeof(rocksdb_backup_dir), "%s/" ROCKSDB_BACKUP_DIR , xtrabackup_target_dir);
+	bool backup_to_directory = xtrabackup_backup && xtrabackup_stream_fmt == XB_STREAM_FMT_NONE;
+	if (backup_to_directory) 
+	{
+		if (my_mkdir(rocksdb_backup_dir, 0777, MYF(0))){
+			msg_ts("Can't create rocksdb backup directory %s\n", rocksdb_backup_dir);
+			exit(EXIT_FAILURE);
+		}
+	}
+	copy_or_move_dir(rocksdb_checkpoint_dir, ROCKSDB_BACKUP_DIR, true, backup_to_directory);
+	rocksdb_remove_checkpoint_directory();
+	rocksdb_unlock_checkpoint();
+}
+
+/*
+  Copies #rocksdb directory to the $rockdb_data_dir, on copy-back
+*/
+static void rocksdb_copy_back() {
+	if (access(ROCKSDB_BACKUP_DIR, 0))
+		return;
+	char rocksdb_home_dir[FN_REFLEN];
+        if (xb_rocksdb_datadir && is_abs_path(xb_rocksdb_datadir)) {
+		strncpy(rocksdb_home_dir, xb_rocksdb_datadir, sizeof(rocksdb_home_dir));
+	} else {
+	   snprintf(rocksdb_home_dir, sizeof(rocksdb_home_dir), "%s/%s", mysql_data_home, 
+		xb_rocksdb_datadir?trim_dotslash(xb_rocksdb_datadir): ROCKSDB_BACKUP_DIR);
+	}
+	mkdirp(rocksdb_home_dir, 0777, MYF(0));
+	copy_or_move_dir(ROCKSDB_BACKUP_DIR, rocksdb_home_dir, xtrabackup_copy_back, xtrabackup_copy_back);
+}

extra/mariabackup/xtrabackup.cc

@@ -146,6 +146,8 @@ char *xtrabackup_tmpdir;
 char *xtrabackup_tables;
 char *xtrabackup_tables_file;
 char *xtrabackup_tables_exclude;
+char *xb_rocksdb_datadir;
+my_bool xb_backup_rocksdb = 1;
 
 typedef std::list<regex_t> regex_list_t;
 static regex_list_t regex_include_list;
@@ -683,7 +685,9 @@ enum options_xtrabackup
   OPT_XTRA_TABLES_EXCLUDE,
   OPT_XTRA_DATABASES_EXCLUDE,
   OPT_PROTOCOL,
-  OPT_LOCK_DDL_PER_TABLE
+  OPT_LOCK_DDL_PER_TABLE,
+  OPT_ROCKSDB_DATADIR,
+  OPT_BACKUP_ROCKSDB
 };
 
 struct my_option xb_client_options[] =
@@ -1182,7 +1186,7 @@ struct my_option xb_server_options[] =
   "The algorithm InnoDB uses for page checksumming. [CRC32, STRICT_CRC32, "
    "INNODB, STRICT_INNODB, NONE, STRICT_NONE]", &srv_checksum_algorithm,
    &srv_checksum_algorithm, &innodb_checksum_algorithm_typelib, GET_ENUM,
-   REQUIRED_ARG, SRV_CHECKSUM_ALGORITHM_INNODB, 0, 0, 0, 0, 0},
+   REQUIRED_ARG, SRV_CHECKSUM_ALGORITHM_CRC32, 0, 0, 0, 0, 0},
 
   {"innodb_undo_directory", OPT_INNODB_UNDO_DIRECTORY,
    "Directory where undo tablespace files live, this path can be absolute.",
@@ -1227,6 +1231,17 @@ struct my_option xb_server_options[] =
    (uchar*) &opt_lock_ddl_per_table, (uchar*) &opt_lock_ddl_per_table, 0,
    GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
 
+  {"rocksdb-datadir", OPT_ROCKSDB_DATADIR, "RocksDB data directory."
+   "This option is only  used with --copy-back or --move-back option",
+  &xb_rocksdb_datadir, &xb_rocksdb_datadir,
+  0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0 },
+
+  { "backup-rocksdb", OPT_BACKUP_ROCKSDB, "Backup rocksdb data, if rocksdb plugin is installed."
+   "Used only with --backup option. Can be useful for partial backups, to exclude all rocksdb data",
+   &xb_backup_rocksdb, &xb_backup_rocksdb,
+   0, GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0 },
+
+
   { 0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
 };
 

extra/mariabackup/xtrabackup.h

@@ -44,6 +44,9 @@ extern char		*xtrabackup_incremental_basedir;
 extern char		*innobase_data_home_dir;
 extern char		*innobase_buffer_pool_filename;
 extern char		*xb_plugin_dir;
+extern char		*xb_rocksdb_datadir;
+extern my_bool	xb_backup_rocksdb;
+
 extern uint		opt_protocol;
 extern ds_ctxt_t	*ds_meta;
 extern ds_ctxt_t	*ds_data;