Merge branch 'issue_205' into issue_201

Author: gsmolk

src/archive.c

@@ -944,7 +944,7 @@ setup_push_filelist(const char *archive_status_dir, const char *first_file,
 
 	/* get list of files from archive_status */
 	status_files = parray_new();
-	dir_list_file(status_files, archive_status_dir, false, false, false, 0, FIO_DB_HOST);
+	dir_list_file(status_files, archive_status_dir, false, false, false, false, 0, FIO_DB_HOST);
 	parray_qsort(status_files, pgFileCompareName);
 
 	for (i = 0; i < parray_num(status_files); i++)
@@ -1387,15 +1387,16 @@ get_wal_file(const char *filename, const char *from_fullpath,
 	 */
 	if (fio_is_remote(FIO_BACKUP_HOST))
 	{
+		char *errmsg = NULL;
 		/* get file via ssh */
 #ifdef HAVE_LIBZ
 		/* If requested file is regular WAL segment, then try to open it with '.gz' suffix... */
 		if (IsXLogFileName(filename))
-			rc = fio_send_file_gz(from_fullpath_gz, to_fullpath, out, thread_num);
+			rc = fio_send_file_gz(from_fullpath_gz, to_fullpath, out, NULL, &errmsg);
 		if (rc == FILE_MISSING)
 #endif
 			/* ... failing that, use uncompressed */
-			rc = fio_send_file(from_fullpath, to_fullpath, out, thread_num);
+			rc = fio_send_file(from_fullpath, to_fullpath, out, NULL, &errmsg);
 
 		/* When not in prefetch mode, try to use partial file */
 		if (rc == FILE_MISSING && !prefetch_mode && IsXLogFileName(filename))
@@ -1405,18 +1406,27 @@ get_wal_file(const char *filename, const char *from_fullpath,
 #ifdef HAVE_LIBZ
 			/* '.gz.partial' goes first ... */
 			snprintf(from_partial, sizeof(from_partial), "%s.gz.partial", from_fullpath);
-			rc = fio_send_file_gz(from_partial, to_fullpath, out, thread_num);
+			rc = fio_send_file_gz(from_partial, to_fullpath, out, NULL, &errmsg);
 			if (rc == FILE_MISSING)
 #endif
 			{
 				/* ... failing that, use '.partial' */
 				snprintf(from_partial, sizeof(from_partial), "%s.partial", from_fullpath);
-				rc = fio_send_file(from_partial, to_fullpath, out, thread_num);
+				rc = fio_send_file(from_partial, to_fullpath, out, NULL, &errmsg);
 			}
 
 			if (rc == SEND_OK)
 				src_partial = true;
 		}
+
+		if (rc == WRITE_FAILED)
+			elog(WARNING, "Thread [%d]: Cannot write to file '%s': %s",
+					thread_num, to_fullpath, strerror(errno));
+
+		if (errmsg)
+			elog(WARNING, "Thread [%d]: %s", thread_num, errmsg);
+
+		pg_free(errmsg);
 	}
 	else
 	{

src/backup.c

@@ -80,7 +80,7 @@ static void backup_cleanup(bool fatal, void *userdata);
 
 static void *backup_files(void *arg);
 
-static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync);
+static void do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs);
 
 static void pg_start_backup(const char *label, bool smooth, pgBackup *backup,
 							PGNodeInfo *nodeInfo, PGconn *backup_conn, PGconn *master_conn);
@@ -129,7 +129,7 @@ backup_stopbackup_callback(bool fatal, void *userdata)
  * Move files from 'pgdata' to a subdirectory in 'backup_path'.
  */
 static void
-do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
+do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs)
 {
 	int			i;
 	char		database_path[MAXPGPATH];
@@ -331,8 +331,12 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 	backup_files_list = parray_new();
 
 	/* list files with the logical path. omit $PGDATA */
-	dir_list_file(backup_files_list, instance_config.pgdata,
-				  true, true, false, 0, FIO_DB_HOST);
+	if (fio_is_remote(FIO_DB_HOST))
+		fio_list_dir(backup_files_list, instance_config.pgdata,
+					 true, true, false, backup_logs, 0);
+	else
+		dir_list_file(backup_files_list, instance_config.pgdata,
+					  true, true, false, backup_logs, 0, FIO_LOCAL_HOST);
 
 	/*
 	 * Get database_map (name to oid) for use in partial restore feature.
@@ -345,11 +349,19 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 	 * from external directory option
 	 */
 	if (external_dirs)
+	{
 		for (i = 0; i < parray_num(external_dirs); i++)
+		{
 			/* External dirs numeration starts with 1.
 			 * 0 value is not external dir */
-			dir_list_file(backup_files_list, parray_get(external_dirs, i),
-						  false, true, false, i+1, FIO_DB_HOST);
+			if (fio_is_remote(FIO_DB_HOST))
+				fio_list_dir(backup_files_list, parray_get(external_dirs, i),
+						  false, true, false, false, i+1);
+			else
+				dir_list_file(backup_files_list, parray_get(external_dirs, i),
+							  false, true, false, false, i+1, FIO_LOCAL_HOST);
+		}
+	}
 
 	/* close ssh session in main thread */
 	fio_disconnect();
@@ -551,19 +563,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 		elog(ERROR, "Data files transferring failed, time elapsed: %s",
 			pretty_time);
 
-	/* Remove disappeared during backup files from backup_list */
-	for (i = 0; i < parray_num(backup_files_list); i++)
-	{
-		pgFile	   *tmp_file = (pgFile *) parray_get(backup_files_list, i);
-
-		if (tmp_file->write_size == FILE_NOT_FOUND)
-		{
-			pgFileFree(tmp_file);
-			parray_remove(backup_files_list, i);
-			i--;
-		}
-	}
-
 	/* clean previous backup file list */
 	if (prev_backup_filelist)
 	{
@@ -616,7 +615,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 		/* Scan backup PG_XLOG_DIR */
 		xlog_files_list = parray_new();
 		join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR);
-		dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, 0,
+		dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, false, 0,
 					  FIO_BACKUP_HOST);
 
 		/* TODO: Drop streamed WAL segments greater than stop_lsn */
@@ -802,8 +801,8 @@ pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo)
  * Entry point of pg_probackup BACKUP subcommand.
  */
 int
-do_backup(time_t start_time, bool no_validate,
-			pgSetBackupParams *set_backup_params, bool no_sync)
+do_backup(time_t start_time, pgSetBackupParams *set_backup_params,
+			bool no_validate, bool no_sync, bool backup_logs)
 {
 	PGconn		*backup_conn = NULL;
 	PGNodeInfo	nodeInfo;
@@ -902,7 +901,7 @@ do_backup(time_t start_time, bool no_validate,
 		add_note(&current, set_backup_params->note);
 
 	/* backup data */
-	do_backup_instance(backup_conn, &nodeInfo, no_sync);
+	do_backup_instance(backup_conn, &nodeInfo, no_sync, backup_logs);
 	pgut_atexit_pop(backup_cleanup, NULL);
 
 	/* compute size of wal files of this backup stored in the archive */
@@ -2146,6 +2145,12 @@ backup_files(void *arg)
 								 current.backup_mode, current.parent_backup, true);
 		}
 
+		/* No point in storing empty, missing or not changed files */
+		if (file->write_size <= 0)
+			unlink(to_fullpath);
+//			elog(ERROR, "Cannot remove file \"%s\": %s", to_fullpath,
+//					strerror(errno));
+
 		if (file->write_size == FILE_NOT_FOUND)
 			continue;
 
@@ -2188,11 +2193,8 @@ parse_filelist_filenames(parray *files, const char *root)
 	while (i < parray_num(files))
 	{
 		pgFile	   *file = (pgFile *) parray_get(files, i);
-//		char	   *relative;
 		int 		sscanf_result;
 
-//		relative = GetRelativePath(file->rel_path, root);
-
 		if (S_ISREG(file->mode) &&
 			path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path))
 		{

src/catalog.c

@@ -878,7 +878,7 @@ catalog_get_timelines(InstanceConfig *instance)
 
 	/* read all xlog files that belong to this archive */
 	sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance->name);
-	dir_list_file(xlog_files_list, arclog_path, false, false, false, 0, FIO_BACKUP_HOST);
+	dir_list_file(xlog_files_list, arclog_path, false, false, false, false, 0, FIO_BACKUP_HOST);
 	parray_qsort(xlog_files_list, pgFileCompareName);
 
 	timelineinfos = parray_new();
@@ -1772,6 +1772,7 @@ pgBackupWriteControl(FILE *out, pgBackup *backup)
 
 /*
  * Save the backup content into BACKUP_CONTROL_FILE.
+ * TODO: honor the strict flag
  */
 void
 write_backup(pgBackup *backup, bool strict)
@@ -1856,6 +1857,10 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
 		char      line[BLCKSZ];
 		pgFile   *file = (pgFile *) parray_get(files, i);
 
+		/* Ignore disappeared file */
+		if (file->write_size == FILE_NOT_FOUND)
+			continue;
+
 		if (S_ISDIR(file->mode))
 		{
 			backup_size_on_disk += 4096;

src/checkdb.c

@@ -207,8 +207,8 @@ do_block_validation(char *pgdata, uint32 checksum_version)
 	files_list = parray_new();
 
 	/* list files with the logical path. omit $PGDATA */
-	dir_list_file(files_list, pgdata,
-				  true, true, false, 0, FIO_DB_HOST);
+	dir_list_file(files_list, pgdata, true, true,
+				  false, false, 0, FIO_DB_HOST);
 
 	/*
 	 * Sort pathname ascending.