[Issue #205] now fio_dir_list honors the "--backup-pg-log" flag and correctly handles exclusive backups

Author: gsmolk

src/archive.c

@@ -944,7 +944,7 @@ setup_push_filelist(const char *archive_status_dir, const char *first_file,
 
 	/* get list of files from archive_status */
 	status_files = parray_new();
-	dir_list_file(status_files, archive_status_dir, false, false, false, 0, FIO_DB_HOST);
+	dir_list_file(status_files, archive_status_dir, false, false, false, false, 0, FIO_DB_HOST);
 	parray_qsort(status_files, pgFileCompareName);
 
 	for (i = 0; i < parray_num(status_files); i++)

src/backup.c

@@ -80,7 +80,7 @@ static void backup_cleanup(bool fatal, void *userdata);
 
 static void *backup_files(void *arg);
 
-static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync);
+static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs);
 
 static void pg_start_backup(const char *label, bool smooth, pgBackup *backup,
 							PGNodeInfo *nodeInfo, PGconn *backup_conn, PGconn *master_conn);
@@ -129,7 +129,7 @@ backup_stopbackup_callback(bool fatal, void *userdata)
  * Move files from 'pgdata' to a subdirectory in 'backup_path'.
  */
 static void
-do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
+do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs)
 {
 	int			i;
 	char		database_path[MAXPGPATH];
@@ -333,10 +333,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 	/* list files with the logical path. omit $PGDATA */
 	if (fio_is_remote(FIO_DB_HOST))
 		fio_list_dir(backup_files_list, instance_config.pgdata,
-					 true, true, false, 0);
+					 true, true, false, backup_logs, 0);
 	else
 		dir_list_file(backup_files_list, instance_config.pgdata,
-					  true, true, false, 0, FIO_LOCAL_HOST);
+					  true, true, false, backup_logs, 0, FIO_LOCAL_HOST);
 
 	/*
 	 * Get database_map (name to oid) for use in partial restore feature.
@@ -356,10 +356,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 			 * 0 value is not external dir */
 			if (fio_is_remote(FIO_DB_HOST))
 				fio_list_dir(backup_files_list, parray_get(external_dirs, i),
-						  false, true, false, i+1);
+						  false, true, false, false, i+1);
 			else
 				dir_list_file(backup_files_list, parray_get(external_dirs, i),
-							  false, true, false, i+1, FIO_LOCAL_HOST);
+							  false, true, false, false, i+1, FIO_LOCAL_HOST);
 		}
 	}
 
@@ -615,7 +615,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 		/* Scan backup PG_XLOG_DIR */
 		xlog_files_list = parray_new();
 		join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR);
-		dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, 0,
+		dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, false, 0,
 					  FIO_BACKUP_HOST);
 
 		/* TODO: Drop streamed WAL segments greater than stop_lsn */
@@ -801,8 +801,8 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo)
  * Entry point of pg_probackup BACKUP subcommand.
  */
 int
-do_backup(time_t start_time, bool no_validate,
-			pgSetBackupParams *set_backup_params, bool no_sync)
+do_backup(time_t start_time, pgSetBackupParams *set_backup_params,
+			bool no_validate, bool no_sync, bool backup_logs)
 {
 	PGconn		*backup_conn = NULL;
 	PGNodeInfo	nodeInfo;
@@ -901,7 +901,7 @@ do_backup(time_t start_time, bool no_validate,
 		add_note(&current, set_backup_params->note);
 
 	/* backup data */
-	do_backup_instance(backup_conn, &nodeInfo, no_sync);
+	do_backup_instance(backup_conn, &nodeInfo, no_sync, backup_logs);
 	pgut_atexit_pop(backup_cleanup, NULL);
 
 	/* compute size of wal files of this backup stored in the archive */

src/catalog.c

@@ -878,7 +878,7 @@ catalog_get_timelines(InstanceConfig *instance)
 
 	/* read all xlog files that belong to this archive */
 	sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance->name);
-	dir_list_file(xlog_files_list, arclog_path, false, false, false, 0, FIO_BACKUP_HOST);
+	dir_list_file(xlog_files_list, arclog_path, false, false, false, false, 0, FIO_BACKUP_HOST);
 	parray_qsort(xlog_files_list, pgFileCompareName);
 
 	timelineinfos = parray_new();

src/checkdb.c

@@ -207,8 +207,8 @@ do_block_validation(char *pgdata, uint32 checksum_version)
 	files_list = parray_new();
 
 	/* list files with the logical path. omit $PGDATA */
-	dir_list_file(files_list, pgdata,
-				  true, true, false, 0, FIO_DB_HOST);
+	dir_list_file(files_list, pgdata, true, true,
+				  false, false, 0, FIO_DB_HOST);
 
 	/*
 	 * Sort pathname ascending.

src/delete.c

@@ -750,7 +750,7 @@ delete_backup_files(pgBackup *backup)
 
 	/* list files to be deleted */
 	files = parray_new();
-	dir_list_file(files, backup->root_dir, false, true, true, 0, FIO_BACKUP_HOST);
+	dir_list_file(files, backup->root_dir, false, false, true, false, 0, FIO_LOCAL_HOST);
 
 	/* delete leaf node first */
 	parray_qsort(files, pgFileCompareRelPathWithExternalDesc);
@@ -966,7 +966,6 @@ do_delete_instance(void)
 {
 	parray		*backup_list;
 	int 		i;
-	int 		rc;
 	char		instance_config_path[MAXPGPATH];
 
 

src/dir.c

@@ -513,7 +513,7 @@ db_map_entry_free(void *entry)
  */
 void
 dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink,
-			  bool add_root, int external_dir_num, fio_location location)
+			  bool add_root, bool backup_logs, int external_dir_num, fio_location location)
 {
 	pgFile	   *file;
 
@@ -527,6 +527,17 @@ dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink
 			return;
 	}
 
+	/* setup exclusion list for file search */
+	if (!backup_logs)
+	{
+		int			i;
+
+		for (i = 0; pgdata_exclude_dir[i]; i++);		/* find first empty slot */
+
+		/* Set 'pg_log' in first empty slot */
+		pgdata_exclude_dir[i] = PG_LOG_DIR;
+	}
+
 	if (!S_ISDIR(file->mode))
 	{
 		if (external_dir_num > 0)

src/merge.c

@@ -1055,7 +1055,7 @@ remove_dir_with_files(const char *path)
 	int			i;
 	char 		full_path[MAXPGPATH];
 
-	dir_list_file(files, path, true, true, true, 0, FIO_LOCAL_HOST);
+	dir_list_file(files, path, false, false, true, false, 0, FIO_LOCAL_HOST);
 	parray_qsort(files, pgFileCompareRelPathWithExternalDesc);
 	for (i = 0; i < parray_num(files); i++)
 	{

src/pg_probackup.c

@@ -668,17 +668,6 @@ main(int argc, char *argv[])
 	if (instance_config.conn_opt.pguser != NULL)
 		dbuser = pstrdup(instance_config.conn_opt.pguser);
 
-	/* setup exclusion list for file search */
-	if (!backup_logs)
-	{
-		int			i;
-
-		for (i = 0; pgdata_exclude_dir[i]; i++);		/* find first empty slot */
-
-		/* Set 'pg_log' in first empty slot */
-		pgdata_exclude_dir[i] = PG_LOG_DIR;
-	}
-
 	if (backup_subcmd == VALIDATE_CMD || backup_subcmd == RESTORE_CMD)
 	{
 		/*
@@ -802,7 +791,7 @@ main(int argc, char *argv[])
 					elog(ERROR, "required parameter not specified: BACKUP_MODE "
 						 "(-b, --backup-mode)");
 
-				return do_backup(start_time, no_validate, set_backup_params, no_sync);
+				return do_backup(start_time, set_backup_params, no_validate, no_sync, backup_logs);
 			}
 		case RESTORE_CMD:
 			return do_restore_or_validate(current.backup_id,

src/pg_probackup.h

@@ -694,8 +694,8 @@ extern char** commands_args;
 extern const char *pgdata_exclude_dir[];
 
 /* in backup.c */
-extern int do_backup(time_t start_time, bool no_validate,
-					 pgSetBackupParams *set_backup_params, bool no_sync);
+extern int do_backup(time_t start_time, pgSetBackupParams *set_backup_params,
+					 bool no_validate, bool no_sync, bool backup_logs);
 extern void do_checkdb(bool need_amcheck, ConnectionOptions conn_opt,
 				  char *pgdata);
 extern BackupMode parse_backup_mode(const char *value);
@@ -859,7 +859,7 @@ extern const char* deparse_compress_alg(int alg);
 
 /* in dir.c */
 extern void dir_list_file(parray *files, const char *root, bool exclude,
-						  bool follow_symlink, bool add_root,
+						  bool follow_symlink, bool add_root, bool backup_logs,
 						  int external_dir_num, fio_location location);
 
 extern void create_data_directories(parray *dest_files,
@@ -1035,8 +1035,8 @@ extern int fio_send_file_gz(const char *from_fullpath, const char *to_fullpath,
 extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FILE* out,
 														pgFile *file, char **errormsg);
 
-extern void fio_list_dir(parray *files, const char *root, bool exclude,
-						 bool follow_symlink, bool add_root, int external_dir_num);
+extern void fio_list_dir(parray *files, const char *root, bool exclude, bool follow_symlink,
+						 bool add_root, bool backup_logs, int external_dir_num);
 
 /* return codes for fio_send_pages() and fio_send_file() */
 #define SEND_OK       (0)

src/utils/file.c

@@ -42,23 +42,24 @@ typedef struct
 	bool exclude;
 	bool follow_symlink;
 	bool add_root;
+	bool backup_logs;
+	bool exclusive_backup;
 	int  external_dir_num;
 } fio_list_dir_request;
 
 typedef struct
 {
-	mode_t	mode;
-	size_t	size;
+	mode_t  mode;
+	size_t  size;
 	time_t  mtime;
-	bool	is_datafile;
-	bool	is_database;
-
-	Oid		tblspcOid;
-	Oid		dbOid;
-	Oid		relOid;
+	bool    is_datafile;
+	bool    is_database;
+	Oid     tblspcOid;
+	Oid     dbOid;
+	Oid     relOid;
 	ForkName   forkName;
-	int		segno;
-	int		external_dir_num;
+	int     segno;
+	int     external_dir_num;
 	int     linked_len;
 } fio_pgFile;
 
@@ -2065,7 +2066,7 @@ static void fio_send_file_impl(int out, char const* path)
 
 /* Compile the array of files located on remote machine in directory root */
 void fio_list_dir(parray *files, const char *root, bool exclude,
-				  bool follow_symlink, bool add_root, int external_dir_num)
+				  bool follow_symlink, bool add_root, bool backup_logs, int external_dir_num)
 {
 	fio_header hdr;
 	fio_list_dir_request req;
@@ -2076,6 +2077,8 @@ void fio_list_dir(parray *files, const char *root, bool exclude,
 	req.exclude = exclude;
 	req.follow_symlink = follow_symlink;
 	req.add_root = add_root;
+	req.backup_logs = backup_logs;
+	req.exclusive_backup = exclusive_backup;
 	req.external_dir_num = external_dir_num;
 
 	hdr.cop = FIO_LIST_DIR;
@@ -2162,12 +2165,15 @@ static void fio_list_dir_impl(int out, char* buf)
 
 	/*
 	 * Disable logging into console any messages with exception of ERROR messages,
-	 * to avoid sending messages to main process, because it may screw his FIO message parsing.
+	 * because currently we have no mechanism to notify the main process
+	 * about then message been sent.
+	 * TODO: correctly send elog messages from agent to main process.
 	 */
 	instance_config.logger.log_level_console = ERROR;
+	exclusive_backup = req->exclusive_backup;
 
 	dir_list_file(file_files, req->path, req->exclude, req->follow_symlink,
-				req->add_root, req->external_dir_num, FIO_LOCAL_HOST);
+				req->add_root, req->backup_logs, req->external_dir_num, FIO_LOCAL_HOST);
 
 	/* send information about files to the main process */
 	for (i = 0; i < parray_num(file_files); i++)
@@ -2352,7 +2358,6 @@ void fio_communicate(int in, int out)
 			SYS_CHECK(ftruncate(fd[hdr.handle], hdr.arg));
 			break;
 		  case FIO_LIST_DIR:
-			// buf contain fio_send_request header and bitmap.
 			fio_list_dir_impl(out, buf);
 			break;
 		  case FIO_SEND_PAGES: