[Issue #66] support external directories, handle timeline mismatch

Author: gsmolk

src/catalog.c

@@ -1048,7 +1048,7 @@ catalog_get_timelines(InstanceConfig *instance)
 			TimeLineHistoryEntry *tln;
 
 			sscanf(file->name, "%08X.history", &tli);
-			timelines = read_timeline_history(arclog_path, tli);
+			timelines = read_timeline_history(arclog_path, tli, true);
 
 			if (!tlinfo || tlinfo->tli != tli)
 			{

src/data.c

@@ -933,7 +933,8 @@ restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out,
 					  from_fullpath, to_fullpath, dest_file->n_blocks,
 					  use_bitmap ? &(dest_file)->pagemap : NULL,
 					  checksum_map, backup->checksum_version,
-					  backup->start_lsn <= horizonLsn ? lsn_map : NULL);
+					  /* shiftmap can be used only if backup state precedes the shift */
+					  backup->stop_lsn <= horizonLsn ? lsn_map : NULL);
 
 		if (fclose(in) != 0)
 			elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath,
@@ -1026,7 +1027,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers
 		 * special value PageIsTruncated is encountered.
 		 * It was inefficient.
 		 *
-		 * Nowadays every backup type has n_blocks, so instead
+		 * Nowadays every backup type has n_blocks, so instead of
 		 * writing and then truncating redundant data, writing
 		 * is not happening in the first place.
 		 * TODO: remove in 3.0.0
@@ -1097,7 +1098,9 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers
 		}
 
 		/* Incremental restore
-		 * TODO: move to another function
+		 * TODO: move to separate function,
+		 * TODO: move BackupPageHeader headers to some separate storage,
+		 * so they can be accessed without reading through whole file.
 		 */
 		if (checksum_map && checksum_map[blknum].checksum != 0)
 		{
@@ -1110,7 +1113,7 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers
 				char uncompressed_buf[BLCKSZ];
 				fio_decompress(uncompressed_buf, page.data, compressed_size, file->compress_alg);
 
-				/* If checksumms are enabled, then we can trust checksumm in header */
+				/* If checksums are enabled, then we can trust checksum in header */
 				if (checksum_version)
 					page_crc = ((PageHeader) uncompressed_buf)->pd_checksum;
 				else
@@ -1143,6 +1146,8 @@ restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_vers
 					datapagemap_add(map, blknum);
 				continue;
 			}
+
+//			elog(INFO, "Replace blknum %u in file %s", blknum, to_fullpath);
 		}
 
 		/*
@@ -1313,7 +1318,7 @@ restore_non_data_file(parray *parent_chain, pgBackup *dest_backup,
 
 		if (file_crc == tmp_file->crc)
 		{
-			elog(VERBOSE, "Remote nondata file \"%s\" is unchanged, skip restore",
+			elog(VERBOSE, "Remote nonedata file \"%s\" is unchanged, skip restore",
 				to_fullpath);
 			return 0;
 		}
@@ -1820,7 +1825,7 @@ check_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn,
 
 /* read local data file and construct map with block checksums */
 PageState *get_checksum_map(const char *fullpath, uint32 checksum_version,
-						 int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno)
+							int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno)
 {
 	PageState  *checksum_map = NULL;
 	FILE       *in = NULL;

src/dir.c

@@ -123,11 +123,11 @@ typedef struct TablespaceCreatedList
 
 static int pgCompareString(const void *str1, const void *str2);
 
-static char dir_check_file(pgFile *file);
+static char dir_check_file(pgFile *file, bool backup_logs);
 
 static void dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
-								   bool exclude, bool follow_symlink, bool skip_hidden,
-								   int external_dir_num, fio_location location);
+								   bool exclude, bool follow_symlink, bool backup_logs,
+								   bool skip_hidden, int external_dir_num, fio_location location);
 static void opt_path_map(ConfigOption *opt, const char *arg,
 						 TablespaceList *list, const char *type);
 
@@ -561,17 +561,6 @@ dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink
 			return;
 	}
 
-	/* setup exclusion list for file search */
-	if (!backup_logs)
-	{
-		int			i;
-
-		for (i = 0; pgdata_exclude_dir[i]; i++);		/* find first empty slot */
-
-		/* Set 'pg_log' in first empty slot */
-		pgdata_exclude_dir[i] = PG_LOG_DIR;
-	}
-
 	if (!S_ISDIR(file->mode))
 	{
 		if (external_dir_num > 0)
@@ -585,7 +574,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink
 		parray_append(files, file);
 
 	dir_list_file_internal(files, file, root, exclude, follow_symlink,
-						   skip_hidden, external_dir_num, location);
+						   backup_logs, skip_hidden, external_dir_num, location);
 
 	if (!add_root)
 		pgFileFree(file);
@@ -609,7 +598,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink
  * - datafiles
  */
 static char
-dir_check_file(pgFile *file)
+dir_check_file(pgFile *file, bool backup_logs)
 {
 	int			i;
 	int			sscanf_res;
@@ -623,7 +612,7 @@ dir_check_file(pgFile *file)
 		if (!exclusive_backup)
 		{
 			for (i = 0; pgdata_exclude_files_non_exclusive[i]; i++)
-				if (strcmp(file->name,
+				if (strcmp(file->rel_path,
 						   pgdata_exclude_files_non_exclusive[i]) == 0)
 				{
 					/* Skip */
@@ -633,7 +622,7 @@ dir_check_file(pgFile *file)
 		}
 
 		for (i = 0; pgdata_exclude_files[i]; i++)
-			if (strcmp(file->name, pgdata_exclude_files[i]) == 0)
+			if (strcmp(file->rel_path, pgdata_exclude_files[i]) == 0)
 			{
 				/* Skip */
 				elog(VERBOSE, "Excluding file: %s", file->name);
@@ -660,6 +649,16 @@ dir_check_file(pgFile *file)
 				return CHECK_EXCLUDE_FALSE;
 			}
 		}
+
+		if (!backup_logs)
+		{
+			if (strcmp(file->rel_path, PG_LOG_DIR) == 0)
+			{
+				/* Skip */
+				elog(VERBOSE, "Excluding directory content: %s", file->rel_path);
+				return CHECK_EXCLUDE_FALSE;
+			}
+		}
 	}
 
 	/*
@@ -807,8 +806,8 @@ dir_check_file(pgFile *file)
  */
 static void
 dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
-					   bool exclude, bool follow_symlink, bool skip_hidden,
-					   int external_dir_num, fio_location location)
+					   bool exclude, bool follow_symlink, bool backup_logs,
+					   bool skip_hidden, int external_dir_num, fio_location location)
 {
 	DIR			  *dir;
 	struct dirent *dent;
@@ -874,7 +873,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
 
 		if (exclude)
 		{
-			check_res = dir_check_file(file);
+			check_res = dir_check_file(file, backup_logs);
 			if (check_res == CHECK_FALSE)
 			{
 				/* Skip */
@@ -897,7 +896,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
 		 */
 		if (S_ISDIR(file->mode))
 			dir_list_file_internal(files, file, child, exclude, follow_symlink,
-								   skip_hidden, external_dir_num, location);
+								   backup_logs, skip_hidden, external_dir_num, location);
 	}
 
 	if (errno && errno != ENOENT)
@@ -1288,7 +1287,7 @@ check_tablespace_mapping(pgBackup *backup, bool incremental, bool *tblspaces_are
 }
 
 void
-check_external_dir_mapping(pgBackup *backup)
+check_external_dir_mapping(pgBackup *backup, bool incremental)
 {
 	TablespaceListCell *cell;
 	parray *external_dirs_to_restore;
@@ -1341,7 +1340,7 @@ check_external_dir_mapping(pgBackup *backup)
 		char	    *external_dir = (char *) parray_get(external_dirs_to_restore,
 														i);
 
-		if (!dir_is_empty(external_dir, FIO_DB_HOST))
+		if (!incremental && !dir_is_empty(external_dir, FIO_DB_HOST))
 			elog(ERROR, "External directory is not empty: \"%s\"",
 				 external_dir);
 	}

src/pg_probackup.h

@@ -740,7 +740,8 @@ extern parray *get_dbOid_exclude_list(pgBackup *backup, parray *datname_list,
 										PartialRestoreType partial_restore_type);
 
 extern parray *get_backup_filelist(pgBackup *backup, bool strict);
-extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetTLI);
+extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict);
+extern bool tliIsPartOfHistory(const parray *timelines, TimeLineID tli);
 
 /* in merge.c */
 extern void do_merge(time_t backup_id);
@@ -888,7 +889,7 @@ extern void read_tablespace_map(parray *files, const char *backup_dir);
 extern void opt_tablespace_map(ConfigOption *opt, const char *arg);
 extern void opt_externaldir_map(ConfigOption *opt, const char *arg);
 extern void check_tablespace_mapping(pgBackup *backup, bool incremental, bool *tblspaces_are_empty);
-extern void check_external_dir_mapping(pgBackup *backup);
+extern void check_external_dir_mapping(pgBackup *backup, bool incremental);
 extern char *get_external_remap(char *current_dir);
 
 extern void print_database_map(FILE *out, parray *database_list);