[Issue #66] add sanity checks for incremental restore

Author: gsmolk

src/catalog.c

@@ -1902,7 +1902,7 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
 		if (file->linked)
 			len += sprintf(line+len, ",\"linked\":\"%s\"", file->linked);
 
-		if (file->n_blocks != BLOCKNUM_INVALID)
+		if (file->n_blocks > 0)
 			len += sprintf(line+len, ",\"n_blocks\":\"%i\"", file->n_blocks);
 
 		sprintf(line+len, "}\n");

src/dir.c

@@ -1021,7 +1021,7 @@ opt_externaldir_map(ConfigOption *opt, const char *arg)
  */
 void
 create_data_directories(parray *dest_files, const char *data_dir, const char *backup_dir,
-						bool extract_tablespaces, fio_location location)
+						bool extract_tablespaces, bool incremental, fio_location location)
 {
 	int			i;
 	parray		*links = NULL;
@@ -1126,7 +1126,7 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba
 					fio_mkdir(linked_path, pg_tablespace_mode, location);
 
 					/* create link to linked_path */
-					if (fio_symlink(linked_path, to_path, location) < 0)
+					if (fio_symlink(linked_path, to_path, incremental, location) < 0)
 						elog(ERROR, "Could not create symbolic link \"%s\": %s",
 							 to_path, strerror(errno));
 
@@ -1203,13 +1203,18 @@ read_tablespace_map(parray *files, const char *backup_dir)
 
 /*
  * Check that all tablespace mapping entries have correct linked directory
- * paths. Linked directories must be empty or do not exist.
+ * paths. Linked directories must be empty or do not exist, unless
+ * we are running incremental restore, then linked directories can be nonempty.
  *
  * If tablespace-mapping option is supplied, all OLDDIR entries must have
  * entries in tablespace_map file.
+ *
+ * When running incremental restore with tablespace remapping, then
+ * new tablespace directory MUST be empty, because there is no
+ * we can be sure, that files laying there belong to our instance.
  */
 void
-check_tablespace_mapping(pgBackup *backup)
+check_tablespace_mapping(pgBackup *backup, bool incremental, bool *tblspaces_are_empty)
 {
 //	char		this_backup_path[MAXPGPATH];
 	parray	   *links;
@@ -1236,6 +1241,18 @@ check_tablespace_mapping(pgBackup *backup)
 			elog(ERROR, "--tablespace-mapping option's old directory "
 				 "doesn't have an entry in tablespace_map file: \"%s\"",
 				 cell->old_dir);
+
+		/* For incremental restore, check that new directory is empty */
+		if (incremental)
+		{
+			if (!is_absolute_path(cell->new_dir))
+				elog(ERROR, "tablespace directory is not an absolute path: %s\n",
+					 cell->new_dir);
+
+			if (!dir_is_empty(cell->new_dir, FIO_DB_HOST))
+				elog(ERROR, "restore tablespace destination is not empty: \"%s\"",
+					 cell->new_dir);
+		}
 	}
 
 	/* 2 - all linked directories must be empty */
@@ -1257,8 +1274,12 @@ check_tablespace_mapping(pgBackup *backup)
 				 linked_path);
 
 		if (!dir_is_empty(linked_path, FIO_DB_HOST))
-			elog(ERROR, "restore tablespace destination is not empty: \"%s\"",
-				 linked_path);
+		{
+			if (!incremental)
+				elog(ERROR, "restore tablespace destination is not empty: \"%s\"",
+					 linked_path);
+			*tblspaces_are_empty = false;
+		}
 	}
 
 	free(tmp_file);

src/merge.c

@@ -589,7 +589,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup)
 
 	/* Create directories */
 	create_data_directories(dest_backup->files, full_database_dir,
-							dest_backup->root_dir, false, FIO_BACKUP_HOST);
+							dest_backup->root_dir, false, false, FIO_BACKUP_HOST);
 
 	/* External directories stuff */
 	if (dest_backup->external_dir_str)

src/pg_probackup.h

@@ -867,12 +867,13 @@ extern void create_data_directories(parray *dest_files,
 										const char *data_dir,
 										const char *backup_dir,
 										bool extract_tablespaces,
+										bool incremental,
 										fio_location location);
 
 extern void read_tablespace_map(parray *files, const char *backup_dir);
 extern void opt_tablespace_map(ConfigOption *opt, const char *arg);
 extern void opt_externaldir_map(ConfigOption *opt, const char *arg);
-extern void check_tablespace_mapping(pgBackup *backup);
+extern void check_tablespace_mapping(pgBackup *backup, bool incremental, bool *tblspaces_are_empty);
 extern void check_external_dir_mapping(pgBackup *backup);
 extern char *get_external_remap(char *current_dir);
 
@@ -951,6 +952,7 @@ extern bool create_empty_file(fio_location from_location, const char *to_root,
 
 extern uint16 *get_checksum_map(const char *fullpath, uint32 checksum_version,
 								int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno);
+extern pid_t check_postmaster(const char *pgdata);
 
 extern bool check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn,
 							 uint32 checksum_version, uint32 backup_version);
@@ -1048,6 +1050,7 @@ extern bool pgut_rmtree(const char *path, bool rmtopdir, bool strict);
 
 extern uint16 *fio_get_checksum_map(const char *fullpath, uint32 checksum_version,
 							int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno);
+extern pid_t fio_check_postmaster(const char *pgdata);
 
 extern int32 fio_decompress(void* dst, void const* src, size_t size, int compress_alg);
 

src/restore.c

@@ -49,6 +49,8 @@ static void pg12_recovery_config(pgBackup *backup, bool add_include);
 static void restore_chain(pgBackup *dest_backup, parray *parent_chain,
 						  parray *dbOid_exclude_list, pgRestoreParams *params,
 						  const char *pgdata_path, bool no_sync);
+static void check_incremental_compatibility(const char *pgdata, uint64 system_identifier,
+											bool lsn_based);
 
 /*
  * Iterate over backup list to find all ancestors of the broken parent_backup
@@ -111,6 +113,8 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
 	char	   *action = params->is_restore ? "Restore":"Validate";
 	parray	   *parent_chain = NULL;
 	parray	   *dbOid_exclude_list = NULL;
+	bool        pgdata_is_empty = true;
+	bool        tblspaces_are_empty = true;
 
 	if (params->is_restore)
 	{
@@ -126,16 +130,17 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
 			{
 				elog(INFO, "Running incremental restore into nonempty directory: \"%s\"",
 					 instance_config.pgdata);
+
+				check_incremental_compatibility(instance_config.pgdata,
+												instance_config.system_identifier,
+												false);
 			}
 			else
 				elog(ERROR, "Restore destination is not empty: \"%s\"",
 					 instance_config.pgdata);
-		}
-		else
-		{
-			/* if remote directory is empty then disable incremental restore */
-			if (params->incremental)
-				params->incremental = false;
+
+			/* if remote directory is empty then incremental restore may be disabled */
+			pgdata_is_empty = true;
 		}
 	}
 
@@ -340,7 +345,10 @@ do_restore_or_validate(time_t target_backup_id, pgRecoveryTarget *rt,
 	 */
 	if (params->is_restore)
 	{
-		check_tablespace_mapping(dest_backup);
+		check_tablespace_mapping(dest_backup, params->incremental, &tblspaces_are_empty);
+
+		if (pgdata_is_empty && tblspaces_are_empty)
+			params->incremental = false;
 
 		/* no point in checking external directories if their restore is not requested */
 		if (!params->skip_external_dirs)
@@ -602,7 +610,8 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
 	 * Restore dest_backup internal directories.
 	 */
 	create_data_directories(dest_files, instance_config.pgdata,
-							dest_backup->root_dir, true, FIO_DB_HOST);
+							dest_backup->root_dir, true,
+							params->incremental, FIO_DB_HOST);
 
 	/*
 	 * Restore dest_backup external directories.
@@ -660,16 +669,23 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
 		elog(INFO, "Extracting the content of destination directory for incremental restore");
 
 		/* TODO: external directorues */
+		time(&start_time);
 		if (fio_is_remote(FIO_DB_HOST))
 			fio_list_dir(pgdata_files, pgdata_path, false, true, false, false, true, 0);
 		else
 			dir_list_file(pgdata_files, pgdata_path,
 						  false, true, false, false, true, 0, FIO_LOCAL_HOST);
-
 		parray_qsort(pgdata_files, pgFileCompareRelPathWithExternalDesc);
-		elog(INFO, "Destination directory content extracted, time elapsed:");
 
-		elog(INFO, "Removing redundant files");
+		time(&end_time);
+		pretty_time_interval(difftime(end_time, start_time),
+							 pretty_time, lengthof(pretty_time));
+
+		elog(INFO, "Destination directory content extracted, time elapsed: %s",
+				pretty_time);
+
+		elog(INFO, "Removing redundant files in destination directory");
+		time(&start_time);
 		for (i = 0; i < parray_num(pgdata_files); i++)
 		{
 			pgFile	   *file = (pgFile *) parray_get(pgdata_files, i);
@@ -686,7 +702,11 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
 			}
 		}
 
-		elog(INFO, "Redundant files are removed, time elapsed:");
+		time(&end_time);
+		pretty_time_interval(difftime(end_time, start_time),
+							 pretty_time, lengthof(pretty_time));
+
+		elog(INFO, "Redundant files are removed, time elapsed: %s", pretty_time);
 
 //		use_bitmap = true;
 		/* At this point PDATA do not contain files, that also do not exists in backup filelist */
@@ -756,9 +776,9 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
 		elog(INFO, "Backup files are restored. Transfered bytes: %s, time elapsed: %s",
 			pretty_total_bytes, pretty_time);
 
-		elog(INFO, "Approximate restore efficiency ratio: %.f%% (%s/%s)",
-			((float) dest_bytes / total_bytes) * 100,
-			pretty_dest_bytes, pretty_total_bytes);
+		elog(INFO, "Restore overwriting ratio (less is better): %.f%% (%s/%s)",
+			((float) total_bytes / dest_bytes) * 100,
+			pretty_total_bytes, pretty_dest_bytes);
 	}
 	else
 		elog(ERROR, "Backup files restoring failed. Transfered bytes: %s, time elapsed: %s",
@@ -921,7 +941,9 @@ restore_files(void *arg)
 
 		if (arguments->incremental &&
 			parray_bsearch(arguments->pgdata_files, dest_file, pgFileCompareRelPathWithExternalDesc))
+		{
 			already_exists = true;
+		}
 
 		/*
 		 * Handle incremental restore case for data files.
@@ -933,7 +955,6 @@ restore_files(void *arg)
 			dest_file->is_datafile && !dest_file->is_cfs &&
 			dest_file->n_blocks > 0)
 		{
-			elog(INFO, "HELLO");
 			/* remote mode */
 			if (fio_is_remote(FIO_DB_HOST))
 				checksum_map = fio_get_checksum_map(to_fullpath, arguments->dest_backup->checksum_version,
@@ -1743,3 +1764,72 @@ get_dbOid_exclude_list(pgBackup *backup, parray *datname_list,
 
 	return dbOid_exclude_list;
 }
+
+/* check that instance has the same SYSTEM_ID, */
+void
+check_incremental_compatibility(const char *pgdata, uint64 system_identifier, bool lsn_based)
+{
+	uint64	system_id_pgdata;
+	bool    success = true;
+	pid_t   pid;
+	char    backup_label[MAXPGPATH];
+
+	/* slurp pg_control and check that system ID is the same */
+	/* check that instance is not running */
+	/* if lsn_based, check that there is no backup_label files is around AND
+	 * get redo point lsn from destination pg_control.
+
+	 * It is really important to be sure that pg_control is in cohesion with
+	 * data files content, because based on pg_control information we will
+	 * choose a backup suitable for lsn based incremental restore.
+	 */
+	/* TODO: handle timeline discrepancies */
+
+	system_id_pgdata = get_system_identifier(pgdata);
+
+	if (system_id_pgdata != instance_config.system_identifier)
+	{
+		elog(WARNING, "Backup catalog was initialized for system id %lu, "
+					"but destination directory system id is %lu",
+					system_identifier, system_id_pgdata);
+		success = false;
+	}
+
+	/* check postmaster pid */
+	if (fio_is_remote(FIO_DB_HOST))
+		pid = fio_check_postmaster(pgdata);
+	else
+		pid = check_postmaster(pgdata);
+
+	if (pid == 1) /* postmaster.pid is mangled */
+	{
+		char pid_file[MAXPGPATH];
+
+		snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", pgdata);
+		elog(WARNING, "Pid file \"%s\" is mangled, cannot determine whether postmaster is running or not",
+			pid_file);
+		success = false;
+	}
+	else if (pid > 1)
+	{
+		elog(WARNING, "Postmaster with pid %u is running in destination directory \"%s\"",
+			pid, pgdata);
+		success = false;
+	}
+
+	if (lsn_based)
+	{
+		snprintf(backup_label, MAXPGPATH, "%s/backup_label", pgdata);
+		if (fio_access(backup_label, F_OK, FIO_DB_HOST) == 0)
+		{
+			elog(WARNING, "Destination directory contains \"backup_control\" file. "
+				"It does not mean that you should delete this file, only that "
+				"lsn-based incremental restore is dangerous to use in this case. "
+				"Consider to use checksum-based incremental restore");
+			success = false;
+		}
+	}
+
+	if (!success)
+		elog(ERROR, "Incremental restore is impossible");
+}

src/util.c

@@ -549,3 +549,50 @@ datapagemap_print_debug(datapagemap_t *map)
 
 	pg_free(iter);
 }
+
+/*
+ * return pid of postmaster process running in given pgdata.
+ * return 0 if there is none.
+ */
+pid_t
+check_postmaster(const char *pgdata)
+{
+	FILE  *fp;
+	pid_t  pid;
+	char   pid_file[MAXPGPATH];
+
+	snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", pgdata);
+
+	fp = fopen(pid_file, "r");
+	if (fp == NULL)
+	{
+		/* No pid file, acceptable*/
+		if (errno == ENOENT)
+			return 0;
+		else
+			elog(ERROR, "Cannot open file \"%s\": %s",
+				pid_file, strerror(errno));
+	}
+
+	if (fscanf(fp, "%i", &pid) != 1)
+	{
+		/* something is wrong with the file content */
+		pid = 1;
+	}
+
+	if (pid > 1)
+	{
+		if (kill(pid, 0) != 0)
+		{
+			/* process no longer exists */
+			if (errno == ESRCH)
+				pid = 0;
+			else
+				elog(ERROR, "Failed to send signal 0 to a process %d: %s",
+						pid, strerror(errno));
+		}
+	}
+
+	fclose(fp);
+	return pid;
+}