Merge branch 'master' into issue_120

Author: gsmolk

README.md

@@ -7,6 +7,7 @@ The utility is compatible with:
 
 As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data:
 * Incremental backup: page-level incremental backup allows you to save disk space, speed up backup and restore. With three different incremental modes, you can plan the backup strategy in accordance with your data flow.
+* Merge: using this feature allows you to implement "incrementally updated backups" strategy, eliminating the need to to do periodical full backups.
 * Validation: automatic data consistency checks and on-demand backup validation without actual data recovery
 * Verification: on-demand verification of PostgreSQL instance with the `checkdb` command.
 * Retention: managing WAL archive and backups in accordance with retention policy. You can configure retention policy based on recovery time or the number of backups to keep, as well as specify `time to live` (TTL) for a particular backup. Expired backups can be merged or deleted.

doc/pgprobackup.xml

@@ -2243,6 +2243,13 @@ BACKUP INSTANCE 'node'
                 <literal>MERGING</literal> — the backup is being merged.
               </para>
             </listitem>
+            <listitem>
+              <para>
+                <literal>MERGED</literal> — the backup data files were
+                successfully merged, but its metadata is in the process
+                of been updated. Only full backup can have this status.
+              </para>
+            </listitem>
             <listitem>
               <para>
                 <literal>DELETING</literal> — the backup files are being deleted.
@@ -3147,7 +3154,9 @@ pg_probackup merge -B <replaceable>backup_dir</replaceable> --instance <replacea
     <para>
       This command merges the specified incremental backup to its
       parent full backup, together with all incremental backups
-      between them. Once the merge is complete, the incremental
+      between them. If the specified backup ID belong to the full backup,
+      then it will be merged with the closest incremental backup.
+      Once the merge is complete, the incremental
       backups are removed as redundant. Thus, the merge operation is
       virtually equivalent to retaking a full backup and removing all
       the outdated backups, but it allows to save much time,
@@ -3166,7 +3175,8 @@ pg_probackup show -B <replaceable>backup_dir</replaceable> --instance <replaceab
 </programlisting>
     <para>
       If the merge is still in progress, the backup status is
-      displayed as <literal>MERGING</literal>. The merge is idempotent, so you can
+      displayed as <literal>MERGING</literal> or, at the final stage,
+      <literal>MERGED</literal>. The merge is idempotent, so you can
       restart the merge if it was interrupted.
     </para>
   </refsect2>
@@ -3412,6 +3422,7 @@ pg_probackup backup -B <replaceable>backup_dir</replaceable> -b <replaceable>bac
 [--no-validate] [--skip-block-validation]
 [-w --no-password] [-W --password]
 [--archive-timeout=<replaceable>timeout</replaceable>] [--external-dirs=<replaceable>external_directory_path</replaceable>]
+[--no-sync]
 [<replaceable>connection_options</replaceable>] [<replaceable>compression_options</replaceable>] [<replaceable>remote_options</replaceable>]
 [<replaceable>retention_options</replaceable>] [<replaceable>pinning_options</replaceable>] [<replaceable>logging_options</replaceable>]
 </programlisting>
@@ -3564,6 +3575,18 @@ pg_probackup backup -B <replaceable>backup_dir</replaceable> -b <replaceable>bac
       </para>
       </listitem>
       </varlistentry>
+
+      <varlistentry>
+<term><option>--no-sync</option></term>
+      <listitem>
+      <para>
+        Do not sync backed up files to disk. You can use this flag to speed
+        up backup process. Using this flag can result in data
+        corruption in case of operating system or hardware crash.
+        Corruption can be detected by backup validation.
+      </para>
+      </listitem>
+      </varlistentry>
       </variablelist>
       </para>
 
@@ -3590,9 +3613,11 @@ pg_probackup restore -B <replaceable>backup_dir</replaceable> --instance <replac
 [--help] [-D <replaceable>data_dir</replaceable>] [-i <replaceable>backup_id</replaceable>]
 [-j <replaceable>num_threads</replaceable>] [--progress]
 [-T <replaceable>OLDDIR</replaceable>=<replaceable>NEWDIR</replaceable>] [--external-mapping=<replaceable>OLDDIR</replaceable>=<replaceable>NEWDIR</replaceable>] [--skip-external-dirs]
-[-R | --restore-as-replica] [--no-validate] [--skip-block-validation] [--force]
+[-R | --restore-as-replica] [--no-validate] [--skip-block-validation]
+[--force] [--no-sync]
 [--restore-command=<replaceable>cmdline</replaceable>]
 [--primary-conninfo=<replaceable>primary_conninfo</replaceable>]
+[-S | --primary-slot-name=<replaceable>slotname</replaceable>]
 [<replaceable>recovery_target_options</replaceable>] [<replaceable>logging_options</replaceable>] [<replaceable>remote_options</replaceable>]
 [<replaceable>partial_restore_options</replaceable>] [<replaceable>remote_wal_archive_options</replaceable>]
 </programlisting>
@@ -3637,14 +3662,27 @@ pg_probackup restore -B <replaceable>backup_dir</replaceable> --instance <replac
         Sets the
         <ulink url="https://postgrespro.com/docs/postgresql/current/runtime-config-replication.html#GUC-PRIMARY-CONNINFO">primary_conninfo</ulink>
         parameter to the specified value.
-        This option will be ignored if used without the <option>-R</option> flag.
+        This option will be ignored unless the <option>-R</option> flag if specified.
       </para>
       <para>
         Example: <literal>--primary-conninfo='host=192.168.1.50 port=5432 user=foo password=foopass'</literal>
       </para>
       </listitem>
       </varlistentry>
 
+      <varlistentry>
+<term><option>-S</option></term>
+<term><option>--primary-slot-name=<replaceable>slot_name</replaceable></option></term>
+      <listitem>
+      <para>
+        Sets the
+        <ulink url="https://postgrespro.com/docs/postgresql/current/runtime-config-replication#GUC-PRIMARY-SLOT-NAME">primary_slot_name</ulink>
+        parameter to the specified value.
+        This option will be ignored unless the <option>-R</option> flag if specified.
+      </para>
+      </listitem>
+      </varlistentry>
+
       <varlistentry>
 <term><option>-T <replaceable>OLDDIR</replaceable>=<replaceable>NEWDIR</replaceable></option></term>
 <term><option>--tablespace-mapping=<replaceable>OLDDIR</replaceable>=<replaceable>NEWDIR</replaceable></option></term>
@@ -3729,6 +3767,17 @@ pg_probackup restore -B <replaceable>backup_dir</replaceable> --instance <replac
       </para>
       </listitem>
       </varlistentry>
+
+      <varlistentry>
+<term><option>--no-sync</option></term>
+      <listitem>
+      <para>
+        Do not sync restored files to disk. You can use this flag to speed
+        up restore process. Using this flag can result in data
+        corruption in case of operating system or hardware crash.
+      </para>
+      </listitem>
+      </varlistentry>
     </variablelist>
     </para>
       <para>
@@ -3855,7 +3904,9 @@ pg_probackup merge -B <replaceable>backup_dir</replaceable> --instance <replacea
       <para>
         Merges the specified incremental backup to its parent full
         backup, together with all incremental backups between them, if
-        any. As a result, the full backup takes in all the merged
+        any. If the specified backup ID belong to the full backup,
+        then it will be merged with the closest incremental backup.
+        As a result, the full backup takes in all the merged
         data, and the incremental backups are removed as redundant.
       </para>
       <para>

src/archive.c

@@ -479,7 +479,7 @@ get_wal_file(const char *from_path, const char *to_path)
 		}
 	}
 
-	if (fio_flush(out) != 0 || fio_close(out) != 0)
+	if (fio_close(out) != 0)
 	{
 		errno_temp = errno;
 		fio_unlink(to_path_temp, FIO_DB_HOST);
@@ -594,11 +594,11 @@ fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed)
 	else
 #endif
 	{
-		crc2 = pgFileGetCRC(path2, true, true, NULL, FIO_BACKUP_HOST);
+		crc2 = fio_get_crc32(path2, FIO_BACKUP_HOST);
 	}
 
 	/* Get checksum of original file */
-	crc1 = pgFileGetCRC(path1, true, true, NULL, FIO_DB_HOST);
+	crc1 = fio_get_crc32(path1, FIO_DB_HOST);
 
 	return EQ_CRC32C(crc1, crc2);
 }

src/backup.c

@@ -575,6 +575,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 	{
 		parray     *xlog_files_list;
 		char		pg_xlog_path[MAXPGPATH];
+		char		wal_full_path[MAXPGPATH];
 
 		/* Scan backup PG_XLOG_DIR */
 		xlog_files_list = parray_new();
@@ -586,11 +587,13 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
 		for (i = 0; i < parray_num(xlog_files_list); i++)
 		{
 			pgFile	   *file = (pgFile *) parray_get(xlog_files_list, i);
+
+			join_path_components(wal_full_path, pg_xlog_path, file->rel_path);
+
 			if (S_ISREG(file->mode))
 			{
-				file->crc = pgFileGetCRC(file->path, true, false,
-										 &file->read_size, FIO_BACKUP_HOST);
-				file->write_size = file->read_size;
+				file->crc = pgFileGetCRC(wal_full_path, true, false);
+				file->write_size = file->size;
 			}
 			/* Remove file path root prefix*/
 			if (strstr(file->path, database_path) == file->path)
@@ -1090,7 +1093,7 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup,
 
 	PQclear(res);
 
-	if (current.backup_mode == BACKUP_MODE_DIFF_PAGE &&
+	if ((!stream_wal || current.backup_mode == BACKUP_MODE_DIFF_PAGE) &&
 		!backup->from_replica &&
 		!(nodeInfo->server_version < 90600 &&
 		  !nodeInfo->is_superuser))
@@ -1102,17 +1105,14 @@ pg_start_backup(const char *label, bool smooth, pgBackup *backup,
 		 */
 		pg_switch_wal(conn);
 
-	if (current.backup_mode == BACKUP_MODE_DIFF_PAGE)
-		/* In PAGE mode wait for current segment... */
+	/* In PAGE mode or in ARCHIVE wal-mode wait for current segment */
+	if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||!stream_wal)
+		/*
+		 * Do not wait start_lsn for stream backup.
+		 * Because WAL streaming will start after pg_start_backup() in stream
+		 * mode.
+		 */
 		wait_wal_lsn(backup->start_lsn, true, backup->tli, false, true, ERROR, false);
-	/*
-	 * Do not wait start_lsn for stream backup.
-	 * Because WAL streaming will start after pg_start_backup() in stream
-	 * mode.
-	 */
-	else if (!stream_wal)
-		/* ...for others wait for previous segment */
-		wait_wal_lsn(backup->start_lsn, true, backup->tli, true, true, ERROR, false);
 }
 
 /*
@@ -1805,10 +1805,11 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn,
 			{
 				file = pgFileNew(backup_label, PG_BACKUP_LABEL_FILE, true, 0,
 								 FIO_BACKUP_HOST);
-				file->crc = pgFileGetCRC(file->path, true, false,
-										 &file->read_size, FIO_BACKUP_HOST);
-				file->write_size = file->read_size;
-				file->uncompressed_size = file->read_size;
+
+				file->crc = pgFileGetCRC(backup_label, true, false);
+
+				file->write_size = file->size;
+				file->uncompressed_size = file->size;
 				free(file->path);
 				file->path = strdup(PG_BACKUP_LABEL_FILE);
 				parray_append(backup_files_list, file);
@@ -1854,9 +1855,8 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn,
 								 FIO_BACKUP_HOST);
 				if (S_ISREG(file->mode))
 				{
-					file->crc = pgFileGetCRC(file->path, true, false,
-											 &file->read_size, FIO_BACKUP_HOST);
-					file->write_size = file->read_size;
+					file->crc = pgFileGetCRC(tablespace_map, true, false);
+					file->write_size = file->size;
 				}
 				free(file->path);
 				file->path = strdup(PG_TABLESPACE_MAP_FILE);