Merge branch 'release_2_4' into issue_205

Author: gsmolk

doc/pgprobackup.xml

@@ -4055,10 +4055,10 @@ pg_probackup archive-push -B <replaceable>backup_dir</replaceable> --instance <r
         corruption.
       </para>
       <para>
-        To speed up archiving, you can specify the <option>-j</option> option
-        to run <command>archive-push</command> on multiple threads.
-        If you provide the <option>--batch-size</option> option, WAL files
-        will be copied in batches of the specified size.
+        To speed up archiving, you can specify the <option>--batch-size</option> option
+        to copy WAL segments in batches of the specified size.
+        If <option>--batch-size</option> option is used, then you can also specify
+        the <option>-j</option> option to copy the batch of WAL segments on multiple threads.
       </para>
       <para>
         WAL segments copied to the archive are synced to disk unless
@@ -4096,10 +4096,10 @@ pg_probackup archive-get -B <replaceable>backup_dir</replaceable> --instance <re
       </para>
 
       <para>
-        To speed up recovery, you can specify the <option>-j</option> option
-        to run <command>archive-get</command> on multiple threads.
-        If you provide the <option>--batch-size</option> option, WAL segments
-        will be copied in batches of the specified size.
+        To speed up recovery, you can specify the <option>--batch-size</option> option
+        to copy WAL segments in batches of the specified size.
+        If <option>--batch-size</option> option is used, then you can also specify
+        the <option>-j</option> option to copy the batch of WAL segments on multiple threads.
       </para>
 
       <para>

src/archive.c

@@ -1109,7 +1109,7 @@ do_archive_get(InstanceConfig *instance, const char *prefetch_dir_arg,
 			{
 				/* discard prefetch */
 //				n_fetched = 0;
-				rmtree(prefetch_dir, false);
+				pgut_rmtree(prefetch_dir, false, false);
 			}
 		}
 		else

src/backup.c

@@ -191,7 +191,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
 		{
 			/* try to setup multi-timeline backup chain */
 			elog(WARNING, "Valid backup on current timeline %u is not found, "
-						"try to look up on previous timelines",
+						"trying to look up on previous timelines",
 						current.tli);
 
 			tli_list = catalog_get_timelines(&instance_config);

src/delete.c

@@ -985,7 +985,7 @@ do_delete_instance(void)
 	parray_free(backup_list);
 
 	/* Delete all wal files. */
-	rmtree(arclog_path, false);
+	pgut_rmtree(arclog_path, false, true);
 
 	/* Delete backup instance config file */
 	join_path_components(instance_config_path, backup_instance_path, BACKUP_CATALOG_CONF_FILE);

src/pg_probackup.h

@@ -1038,6 +1038,8 @@ extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FIL
 extern void fio_list_dir(parray *files, const char *root, bool exclude, bool follow_symlink,
 						 bool add_root, bool backup_logs, int external_dir_num);
 
+extern bool pgut_rmtree(const char *path, bool rmtopdir, bool strict);
+
 /* return codes for fio_send_pages() and fio_send_file() */
 #define SEND_OK       (0)
 #define FILE_MISSING (-1)

src/utils/pgut.c

@@ -43,6 +43,9 @@ static void on_interrupt(void);
 static void on_cleanup(void);
 static pqsigfunc oldhandler = NULL;
 
+static char ** pgut_pgfnames(const char *path, bool strict);
+static void pgut_pgfnames_cleanup(char **filenames);
+
 void discard_response(PGconn *conn);
 
 void
@@ -1062,3 +1065,140 @@ discard_response(PGconn *conn)
 			PQclear(res);
 	} while (res);
 }
+
+/*
+ * pgfnames
+ *
+ * return a list of the names of objects in the argument directory.  Caller
+ * must call pgfnames_cleanup later to free the memory allocated by this
+ * function.
+ */
+char **
+pgut_pgfnames(const char *path, bool strict)
+{
+	DIR		   *dir;
+	struct dirent *file;
+	char	  **filenames;
+	int			numnames = 0;
+	int			fnsize = 200;	/* enough for many small dbs */
+
+	dir = opendir(path);
+	if (dir == NULL)
+	{
+		elog(strict ? ERROR : WARNING, "could not open directory \"%s\": %m", path);
+		return NULL;
+	}
+
+	filenames = (char **) palloc(fnsize * sizeof(char *));
+
+	while (errno = 0, (file = readdir(dir)) != NULL)
+	{
+		if (strcmp(file->d_name, ".") != 0 && strcmp(file->d_name, "..") != 0)
+		{
+			if (numnames + 1 >= fnsize)
+			{
+				fnsize *= 2;
+				filenames = (char **) repalloc(filenames,
+											   fnsize * sizeof(char *));
+			}
+			filenames[numnames++] = pstrdup(file->d_name);
+		}
+	}
+
+	if (errno)
+	{
+		elog(strict ? ERROR : WARNING, "could not read directory \"%s\": %m", path);
+		return NULL;
+	}
+
+	filenames[numnames] = NULL;
+
+	if (closedir(dir))
+	{
+		elog(strict ? ERROR : WARNING, "could not close directory \"%s\": %m", path);
+		return NULL;
+	}
+
+	return filenames;
+}
+
+/*
+ *	pgfnames_cleanup
+ *
+ *	deallocate memory used for filenames
+ */
+void
+pgut_pgfnames_cleanup(char **filenames)
+{
+	char	  **fn;
+
+	for (fn = filenames; *fn; fn++)
+		pfree(*fn);
+
+	pfree(filenames);
+}
+
+/* Shamelessly stolen from commom/rmtree.c */
+bool
+pgut_rmtree(const char *path, bool rmtopdir, bool strict)
+{
+	bool		result = true;
+	char		pathbuf[MAXPGPATH];
+	char	  **filenames;
+	char	  **filename;
+	struct stat statbuf;
+
+	/*
+	 * we copy all the names out of the directory before we start modifying
+	 * it.
+	 */
+	filenames = pgut_pgfnames(path, strict);
+
+	if (filenames == NULL)
+		return false;
+
+	/* now we have the names we can start removing things */
+	for (filename = filenames; *filename; filename++)
+	{
+		snprintf(pathbuf, MAXPGPATH, "%s/%s", path, *filename);
+
+		if (lstat(pathbuf, &statbuf) != 0)
+		{
+			elog(strict ? ERROR : WARNING, "could not stat file or directory \"%s\": %m", pathbuf);
+			result = false;
+			break;
+		}
+
+		if (S_ISDIR(statbuf.st_mode))
+		{
+			/* call ourselves recursively for a directory */
+			if (!pgut_rmtree(pathbuf, true, strict))
+			{
+				result = false;
+				break;
+			}
+		}
+		else
+		{
+			if (unlink(pathbuf) != 0)
+			{
+				elog(strict ? ERROR : WARNING, "could not remove file or directory \"%s\": %m", pathbuf);
+				result = false;
+				break;
+			}
+		}
+	}
+
+	if (rmtopdir)
+	{
+		if (rmdir(path) != 0)
+		{
+			elog(strict ? ERROR : WARNING, "could not remove file or directory \"%s\": %m", path);
+			result = false;
+		}
+	}
+
+	pgut_pgfnames_cleanup(filenames);
+
+	return result;
+}