[Issue #205] Avoid opening remote files via fio_open when copying files during backup

Author: gsmolk

src/archive.c

@@ -1387,15 +1387,16 @@ get_wal_file(const char *filename, const char *from_fullpath,
 	 */
 	if (fio_is_remote(FIO_BACKUP_HOST))
 	{
+		char *errmsg = NULL;
 		/* get file via ssh */
 #ifdef HAVE_LIBZ
 		/* If requested file is regular WAL segment, then try to open it with '.gz' suffix... */
 		if (IsXLogFileName(filename))
-			rc = fio_send_file_gz(from_fullpath_gz, to_fullpath, out, thread_num);
+			rc = fio_send_file_gz(from_fullpath_gz, to_fullpath, out, NULL, &errmsg);
 		if (rc == FILE_MISSING)
 #endif
 			/* ... failing that, use uncompressed */
-			rc = fio_send_file(from_fullpath, to_fullpath, out, thread_num);
+			rc = fio_send_file(from_fullpath, to_fullpath, out, NULL, &errmsg);
 
 		/* When not in prefetch mode, try to use partial file */
 		if (rc == FILE_MISSING && !prefetch_mode && IsXLogFileName(filename))
@@ -1405,18 +1406,27 @@ get_wal_file(const char *filename, const char *from_fullpath,
 #ifdef HAVE_LIBZ
 			/* '.gz.partial' goes first ... */
 			snprintf(from_partial, sizeof(from_partial), "%s.gz.partial", from_fullpath);
-			rc = fio_send_file_gz(from_partial, to_fullpath, out, thread_num);
+			rc = fio_send_file_gz(from_partial, to_fullpath, out, NULL, &errmsg);
 			if (rc == FILE_MISSING)
 #endif
 			{
 				/* ... failing that, use '.partial' */
 				snprintf(from_partial, sizeof(from_partial), "%s.partial", from_fullpath);
-				rc = fio_send_file(from_partial, to_fullpath, out, thread_num);
+				rc = fio_send_file(from_partial, to_fullpath, out, NULL, &errmsg);
 			}
 
 			if (rc == SEND_OK)
 				src_partial = true;
 		}
+
+		if (rc == WRITE_FAILED)
+			elog(WARNING, "Thread [%d]: Cannot write to file '%s': %s",
+					thread_num, to_fullpath, strerror(errno));
+
+		if (errmsg)
+			elog(WARNING, "Thread [%d]: %s", thread_num, errmsg);
+
+		pg_free(errmsg);
 	}
 	else
 	{

src/backup.c

@@ -550,19 +550,6 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 		elog(ERROR, "Data files transferring failed, time elapsed: %s",
 			pretty_time);
 
-	/* Remove disappeared during backup files from backup_list */
-	for (i = 0; i < parray_num(backup_files_list); i++)
-	{
-		pgFile	   *tmp_file = (pgFile *) parray_get(backup_files_list, i);
-
-		if (tmp_file->write_size == FILE_NOT_FOUND)
-		{
-			pgFileFree(tmp_file);
-			parray_remove(backup_files_list, i);
-			i--;
-		}
-	}
-
 	/* clean previous backup file list */
 	if (prev_backup_filelist)
 	{
@@ -2150,6 +2137,12 @@ backup_files(void *arg)
 								 current.backup_mode, current.parent_backup, true);
 		}
 
+		/* No point in storing empty, missing or not changed files */
+		if (file->write_size <= 0)
+			unlink(to_fullpath);
+//			elog(ERROR, "Cannot remove file \"%s\": %s", to_fullpath,
+//					strerror(errno));
+
 		if (file->write_size == FILE_NOT_FOUND)
 			continue;
 
@@ -2192,11 +2185,8 @@ parse_filelist_filenames(parray *files, const char *root)
 	while (i < parray_num(files))
 	{
 		pgFile	   *file = (pgFile *) parray_get(files, i);
-//		char	   *relative;
 		int 		sscanf_result;
 
-//		relative = GetRelativePath(file->rel_path, root);
-
 		if (S_ISREG(file->mode) &&
 			path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path))
 		{

src/catalog.c

@@ -1856,6 +1856,10 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
 		char      line[BLCKSZ];
 		pgFile   *file = (pgFile *) parray_get(files, i);
 
+		/* Ignore disappeared file */
+		if (file->write_size == FILE_NOT_FOUND)
+			continue;
+
 		if (S_ISDIR(file->mode))
 		{
 			backup_size_on_disk += 4096;