[Issue #201] fix restore of files from versions older than 2.3.0

Author: gsmolk

src/data.c

@@ -861,14 +861,25 @@ backup_non_data_file(pgFile *file, pgFile *prev_file,
  * Apply changed blocks to destination file from every backup in parent chain.
  */
 size_t
-restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char *to_fullpath)
+restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out,
+						const char *to_fullpath, bool use_bitmap)
 {
-	int    i;
 	size_t total_write_len = 0;
 	char  *in_buf = pgut_malloc(STDIO_BUFSIZE);
+	int    backup_seq = 0;
+
+	// FULL -> INCR -> DEST
+	//  2       1       0
+	if (use_bitmap)
+		/* start with dest backup  */
+		backup_seq = 0;
+	else
+		/* start with full backup */
+		backup_seq = parray_num(parent_chain) - 1;
 
 //	for (i = parray_num(parent_chain) - 1; i >= 0; i--)
-	for (i = 0; i < parray_num(parent_chain); i++)
+//	for (i = 0; i < parray_num(parent_chain); i++)
+	while (backup_seq >= 0 && backup_seq < parray_num(parent_chain))
 	{
 		char     from_root[MAXPGPATH];
 		char     from_fullpath[MAXPGPATH];
@@ -877,7 +888,12 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char
 		pgFile **res_file = NULL;
 		pgFile  *tmp_file = NULL;
 
-		pgBackup   *backup = (pgBackup *) parray_get(parent_chain, i);
+		pgBackup   *backup = (pgBackup *) parray_get(parent_chain, backup_seq);
+
+		if (use_bitmap)
+			backup_seq++;
+		else
+			backup_seq--;
 
 		/* lookup file in intermediate backup */
 		res_file =  parray_bsearch(backup->files, dest_file, pgFileCompareRelPathWithExternal);
@@ -925,7 +941,7 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char
 		total_write_len += restore_data_file_internal(in, out, tmp_file,
 					  parse_program_version(backup->program_version),
 					  from_fullpath, to_fullpath, dest_file->n_blocks,
-					  &(dest_file)->pagemap);
+					  use_bitmap ? &(dest_file)->pagemap : NULL);
 
 		if (fclose(in) != 0)
 			elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath,
@@ -938,6 +954,11 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, const char
 	return total_write_len;
 }
 
+/* Restore block from "in" file to "out" file.
+ * If "nblocks" is greater than zero, then skip restoring blocks,
+ * whose position if greater than "nblocks".
+ * If map is NULL, then page bitmap cannot be used for restore optimization
+ */
 size_t
 restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version,
 					  const char *from_fullpath, const char *to_fullpath, int nblocks,
@@ -1050,7 +1071,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers
 			elog(ERROR, "Size of a blknum %i exceed BLCKSZ", blknum);
 
 		/* if this page is marked as already restored, then skip it */
-		if (datapagemap_is_set(map, blknum))
+		if (map && datapagemap_is_set(map, blknum))
 		{
 			/* skip to the next page */
 			if (fseek(in, MAXALIGN(compressed_size), SEEK_CUR) != 0)
@@ -1115,7 +1136,8 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers
 		write_len += BLCKSZ;
 		cur_pos = write_pos + BLCKSZ; /* update current write position */
 
-		datapagemap_add(map, blknum);
+		if (map)
+			datapagemap_add(map, blknum);
 	}
 
 	elog(VERBOSE, "Copied file \"%s\": %lu bytes", from_fullpath, write_len);
@@ -1191,6 +1213,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
 		 * full copy of destination file.
 		 * Full copy is latest possible destination file with size equal or
 		 * greater than zero.
+		 * TODO: rewrite to use parent_link of dest backup.
 		 */
 		for (i = 1; i < parray_num(parent_chain); i++)
 		{

src/merge.c

@@ -28,6 +28,7 @@ typedef struct
 //	size_t		in_place_merge_bytes;
 	bool		compression_match;
 	bool		program_version_match;
+	bool        use_bitmap;
 
 	/*
 	 * Return value from the thread.
@@ -47,7 +48,7 @@ get_external_index(const char *key, const parray *list);
 static void
 merge_data_file(parray *parent_chain, pgBackup *full_backup,
 				pgBackup *dest_backup, pgFile *dest_file,
-				pgFile *tmp_file, const char *to_root);
+				pgFile *tmp_file, const char *to_root, bool use_bitmap);
 
 static void
 merge_non_data_file(parray *parent_chain, pgBackup *full_backup,
@@ -439,6 +440,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup)
 				*dest_externals = NULL;
 
 	parray		*result_filelist = NULL;
+	bool        use_bitmap = true;
 //	size_t 		total_in_place_merge_bytes = 0;
 
 	pthread_t	*threads = NULL;
@@ -601,6 +603,9 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup)
 	if (full_externals && dest_externals)
 		reorder_external_dirs(full_backup, full_externals, dest_externals);
 
+	if (parse_program_version(dest_backup->program_version) < 20300)
+		use_bitmap = false;
+
 	/* Setup threads */
 	for (i = 0; i < parray_num(dest_backup->files); i++)
 	{
@@ -639,6 +644,7 @@ merge_chain(parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup)
 
 		arg->compression_match = compression_match;
 		arg->program_version_match = program_version_match;
+		arg->use_bitmap = use_bitmap;
 		/* By default there are some error */
 		arg->ret = 1;
 
@@ -1028,7 +1034,8 @@ merge_files(void *arg)
 							arguments->full_backup,
 							arguments->dest_backup,
 							dest_file, tmp_file,
-							arguments->full_database_dir);
+							arguments->full_database_dir,
+							arguments->use_bitmap);
 		else
 			merge_non_data_file(arguments->parent_chain,
 								arguments->full_backup,
@@ -1128,7 +1135,7 @@ reorder_external_dirs(pgBackup *to_backup, parray *to_external,
 void
 merge_data_file(parray *parent_chain, pgBackup *full_backup,
 				pgBackup *dest_backup, pgFile *dest_file, pgFile *tmp_file,
-				const char *full_database_dir)
+				const char *full_database_dir, bool use_bitmap)
 {
 	FILE   *out = NULL;
 	char   *buffer = pgut_malloc(STDIO_BUFSIZE);
@@ -1154,7 +1161,7 @@ merge_data_file(parray *parent_chain, pgBackup *full_backup,
 	setvbuf(out, buffer, _IOFBF, STDIO_BUFSIZE);
 
 	/* restore file into temp file */
-	tmp_file->size = restore_data_file(parent_chain, dest_file, out, to_fullpath_tmp1);
+	tmp_file->size = restore_data_file(parent_chain, dest_file, out, to_fullpath_tmp1, use_bitmap);
 	fclose(out);
 	pg_free(buffer);
 

src/pg_probackup.h

@@ -935,7 +935,7 @@ extern void backup_non_data_file_internal(const char *from_fullpath,
 										  bool missing_ok);
 
 extern size_t restore_data_file(parray *parent_chain, pgFile *dest_file,
-								  FILE *out, const char *to_fullpath);
+								  FILE *out, const char *to_fullpath, bool use_bitmap);
 extern size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version,
 										 const char *from_fullpath, const char *to_fullpath, int nblocks,
 										 datapagemap_t *map);

src/restore.c

@@ -27,6 +27,7 @@ typedef struct
 	bool		skip_external_dirs;
 	const char *to_root;
 	size_t		restored_bytes;
+	bool        use_bitmap;
 
 	/*
 	 * Return value from the thread.
@@ -501,6 +502,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
 	pthread_t  *threads;
 	restore_files_arg *threads_args;
 	bool		restore_isok = true;
+	bool        use_bitmap = true;
 
 	/* fancy reporting */
 	char		pretty_dest_bytes[20];
@@ -560,6 +562,13 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
 		parray_qsort(backup->files, pgFileCompareRelPathWithExternal);
 	}
 
+	/* If dest backup version is older than 2.3.0, then bitmap optimization
+	 * is impossible to use, because bitmap restore rely on pgFile.n_blocks,
+	 * which is not always available in old backups.
+	 */
+	if (parse_program_version(dest_backup->program_version) < 20300)
+		use_bitmap = false;
+
 	/*
 	 * Restore dest_backup internal directories.
 	 */
@@ -645,6 +654,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
 		arg->dbOid_exclude_list = dbOid_exclude_list;
 		arg->skip_external_dirs = params->skip_external_dirs;
 		arg->to_root = pgdata_path;
+		arg->use_bitmap = use_bitmap;
 		threads_args[i].restored_bytes = 0;
 		/* By default there are some error */
 		threads_args[i].ret = 1;
@@ -859,7 +869,8 @@ restore_files(void *arg)
 				setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE);
 			/* Destination file is data file */
 			arguments->restored_bytes += restore_data_file(arguments->parent_chain,
-															dest_file, out, to_fullpath);
+															dest_file, out, to_fullpath,
+															arguments->use_bitmap);
 		}
 		else
 		{
@@ -976,6 +987,9 @@ create_recovery_conf(time_t backup_id,
 		elog(ERROR, "cannot open file \"%s\": %s", path,
 			strerror(errno));
 
+	if (fio_chmod(path, FILE_PERMISSION, FIO_DB_HOST) == -1)
+		elog(ERROR, "Cannot change mode of \"%s\": %s", path, strerror(errno));
+
 #if PG_VERSION_NUM >= 120000
 	fio_fprintf(fp, "# probackup_recovery.conf generated by pg_probackup %s\n",
 				PROGRAM_VERSION);