Skip to content

Commit

Permalink
Exclude directories during backup that are cleaned, recreated, or zer…
Browse files Browse the repository at this point in the history 
…oed by PostgreSQL at startup.

These include (depending on the version where they were introduced): pgsql_tmp, pg_dynshmem, pg_notify, pg_replslot, pg_serial, pg_snapshots, pg_stat_tmp, pg_subtrans. The postgresql.auto.conf.tmp file is now excluded in addition to files that were already excluded: backup_label.old, postmaster.opts, postmaster.pid, recovery.conf, recovery.done.
  • Loading branch information
@dwsteele
dwsteele committed Aug 16, 2016
1 parent 3320559 commit f1412ba
Show file tree
Hide file tree
Showing 13 changed files with 446 additions and 112 deletions.
4 changes: 4 additions & 0 deletions doc/xml/release.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,10 @@
</release-bug-list>

<release-feature-list>
<release-item>
<p>Exclude directories during backup that are cleaned, recreated, or zeroed by <postgres/> at startup. These include (depending on the version where they were introduced): <path>pgsql_tmp</path>, <path>pg_dynshmem</path>, <path>pg_notify</path>, <path>pg_replslot</path>, <path>pg_serial</path>, <path>pg_snapshots</path>, <path>pg_stat_tmp</path>, <path>pg_subtrans</path>. The <file>postgresql.auto.conf.tmp</file> file is now excluded in addition to files that were already excluded: <file>backup_label.old</file>, <file>postmaster.opts</file>, <file>postmaster.pid</file>, <file>recovery.conf</file>, <file>recovery.done</file>.</p>
</release-item>

<release-item>
<p>Experimental support for non-exclusive backups in <postgres/> 9.6 beta4. Changes to the control/catalog/WAL versions in subsequent betas may break compatibility but <backrest/> will be updated with each release to keep pace.</p>
</release-item>
Expand Down
2 changes: 1 addition & 1 deletion lib/pgBackRest/Backup.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -666,7 +666,7 @@ sub process
}

# Build the manifest
$oBackupManifest->build($self->{oFile}, optionGet(OPTION_DB_PATH), $oLastManifest, optionGet(OPTION_ONLINE),
$oBackupManifest->build($self->{oFile}, $strDbVersion, optionGet(OPTION_DB_PATH), $oLastManifest, optionGet(OPTION_ONLINE),
$oTablespaceMap, $oDatabaseMap);
&log(TEST, TEST_MANIFEST_BUILD);

Expand Down
95 changes: 86 additions & 9 deletions lib/pgBackRest/Manifest.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,20 @@ use constant MANIFEST_SUBKEY_USER => 'user';
####################################################################################################################################
use constant DB_PATH_GLOBAL => 'global';
push @EXPORT, qw(DB_PATH_GLOBAL);
use constant DB_PATH_PGDYNSHMEM => 'pg_dynshmem';
push @EXPORT, qw(DB_PATH_PGDYNSHMEM);
use constant DB_PATH_PGNOTIFY => 'pg_notify';
push @EXPORT, qw(DB_PATH_PGNOTIFY);
use constant DB_PATH_PGREPLSLOT => 'pg_replslot';
push @EXPORT, qw(DB_PATH_PGREPLSLOT);
use constant DB_PATH_PGSERIAL => 'pg_serial';
push @EXPORT, qw(DB_PATH_PGSERIAL);
use constant DB_PATH_PGSNAPSHOTS => 'pg_snapshots';
push @EXPORT, qw(DB_PATH_PGSNAPSHOTS);
use constant DB_PATH_PGSTATTMP => 'pg_stat_tmp';
push @EXPORT, qw(DB_PATH_PGSTATTMP);
use constant DB_PATH_PGSUBTRANS => 'pg_subtrans';
push @EXPORT, qw(DB_PATH_PGSUBTRANS);
use constant DB_PATH_PGTBLSPC => 'pg_tblspc';
push @EXPORT, qw(DB_PATH_PGTBLSPC);
use constant DB_PATH_PGXLOG => 'pg_xlog';
Expand All @@ -167,6 +181,8 @@ use constant DB_FILE_PGCONTROL => DB_PATH_G
push @EXPORT, qw(DB_FILE_PGCONTROL);
use constant DB_FILE_PGVERSION => 'PG_VERSION';
push @EXPORT, qw(DB_FILE_PGVERSION);
use constant DB_FILE_POSTGRESQLAUTOCONFTMP => 'postgresql.auto.conf.tmp';
push @EXPORT, qw(DB_FILE_POSTGRESQLAUTOCONFTMP);
use constant DB_FILE_POSTMASTEROPTS => 'postmaster.opts';
push @EXPORT, qw(DB_FILE_POSTMASTEROPTS);
use constant DB_FILE_POSTMASTERPID => 'postmaster.pid';
Expand All @@ -178,16 +194,45 @@ use constant DB_FILE_RECOVERYDONE => 'recovery
use constant DB_FILE_TABLESPACEMAP => 'tablespace_map';
push @EXPORT, qw(DB_FILE_TABLESPACEMAP);

use constant DB_FILE_PREFIX_TMP => 'pgsql_tmp';
push @EXPORT, qw(DB_FILE_PREFIX_TMP);

####################################################################################################################################
# Manifest locations for important files/paths
####################################################################################################################################
use constant MANIFEST_PATH_PGDYNSHMEM => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGDYNSHMEM;
push @EXPORT, qw(MANIFEST_PATH_PGDYNSHMEM);
use constant MANIFEST_PATH_PGNOTIFY => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGNOTIFY;
push @EXPORT, qw(MANIFEST_PATH_PGNOTIFY);
use constant MANIFEST_PATH_PGREPLSLOT => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGREPLSLOT;
push @EXPORT, qw(MANIFEST_PATH_PGREPLSLOT);
use constant MANIFEST_PATH_PGSERIAL => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSERIAL;
push @EXPORT, qw(MANIFEST_PATH_PGSERIAL);
use constant MANIFEST_PATH_PGSNAPSHOTS => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSNAPSHOTS;
push @EXPORT, qw(MANIFEST_PATH_PGSNAPSHOTS);
use constant MANIFEST_PATH_PGSTATTMP => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSTATTMP;
push @EXPORT, qw(MANIFEST_PATH_PGSTATTMP);
use constant MANIFEST_PATH_PGSUBTRANS => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSUBTRANS;
push @EXPORT, qw(MANIFEST_PATH_PGSUBTRANS);
use constant MANIFEST_PATH_PGXLOG => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGXLOG;
push @EXPORT, qw(MANIFEST_PATH_PGXLOG);

use constant MANIFEST_FILE_BACKUPLABEL => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_BACKUPLABEL;
push @EXPORT, qw(MANIFEST_FILE_BACKUPLABEL);
use constant MANIFEST_FILE_BACKUPLABELOLD => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_BACKUPLABELOLD;
push @EXPORT, qw(MANIFEST_FILE_BACKUPLABELOLD);
use constant MANIFEST_FILE_PGCONTROL => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_PGCONTROL;
push @EXPORT, qw(MANIFEST_FILE_PGCONTROL);
use constant MANIFEST_FILE_POSTGRESQLAUTOCONFTMP => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_POSTGRESQLAUTOCONFTMP;
push @EXPORT, qw(MANIFEST_FILE_PGCONTROL);
use constant MANIFEST_FILE_POSTMASTEROPTS => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_POSTMASTEROPTS;
push @EXPORT, qw(MANIFEST_FILE_POSTMASTEROPTS);
use constant MANIFEST_FILE_POSTMASTERPID => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_POSTMASTERPID;
push @EXPORT, qw(MANIFEST_FILE_POSTMASTERPID);
use constant MANIFEST_FILE_RECOVERYCONF => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_RECOVERYCONF;
push @EXPORT, qw(MANIFEST_FILE_RECOVERYCONF);
use constant MANIFEST_FILE_RECOVERYDONE => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_RECOVERYDONE;
push @EXPORT, qw(MANIFEST_FILE_RECOVERYDONE);
use constant MANIFEST_FILE_TABLESPACEMAP => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_TABLESPACEMAP;
push @EXPORT, qw(MANIFEST_FILE_TABLESPACEMAP);

Expand Down Expand Up @@ -472,6 +517,7 @@ sub build
(
$strOperation,
$oFile,
$strDbVersion,
$strPath,
$oLastManifest,
$bOnline,
Expand All @@ -486,6 +532,7 @@ sub build
(
__PACKAGE__ . '->build', \@_,
{name => 'oFile'},
{name => 'strDbVersion'},
{name => 'strPath'},
{name => 'oLastManifest', required => false},
{name => 'bOnline'},
Expand Down Expand Up @@ -594,14 +641,44 @@ sub build
$strManifestType = MANIFEST_VALUE_PATH;
}

# Skip certain files during backup
if ($strFile =~ ('^' . MANIFEST_PATH_PGXLOG . '.*\/') && $bOnline || # pg_xlog/ - this will be reconstructed
$strLevel eq MANIFEST_TARGET_PGDATA &&
($strName eq DB_FILE_BACKUPLABELOLD || # backup_label.old - old backup labels are not useful
$strName eq DB_FILE_POSTMASTEROPTS || # postmaster.opts - not useful for backup
$strName eq DB_FILE_POSTMASTERPID || # postmaster.pid - to avoid confusing postgres after restore
$strName eq DB_FILE_RECOVERYCONF || # recovery.conf - doesn't make sense to backup this file
$strName eq DB_FILE_RECOVERYDONE)) # recovery.done - doesn't make sense to backup this file
# Skip pg_xlog/* when doing an online backup. WAL will be restored from the archive or stored in pg_xlog at the end of the
# backup if the archive-copy option is set.
next if ($strFile =~ ('^' . MANIFEST_PATH_PGXLOG . '\/') && $bOnline);

# Skip all directories and files that start with pgsql_tmp. The files are removed when the server is restarted and the
# directories are recreated. Since temp files cannnot be created on the replica it makes sense to delete the directories
# and let the server recreate them when they are needed.
next if $strName =~ ('(^|\/)' . DB_FILE_PREFIX_TMP);

# Skip pg_dynshmem/* since these files cannot be reused on recovery
next if $strFile =~ ('^' . MANIFEST_PATH_PGDYNSHMEM . '\/') && $strDbVersion >= PG_VERSION_94;

# Skip pg_notify/* since these files cannot be reused on recovery
next if $strFile =~ ('^' . MANIFEST_PATH_PGNOTIFY . '\/') && $strDbVersion >= PG_VERSION_90;

# Skip pg_replslot/* since these files cannot be reused on recovery
next if $strFile =~ ('^' . MANIFEST_PATH_PGREPLSLOT . '\/') && $strDbVersion >= PG_VERSION_94;

# Skip pg_serial/* since these files are reset
next if $strFile =~ ('^' . MANIFEST_PATH_PGSERIAL . '\/') && $strDbVersion >= PG_VERSION_91;

# Skip pg_snapshots/* since these files cannot be reused on recovery
next if $strFile =~ ('^' . MANIFEST_PATH_PGSNAPSHOTS . '\/') && $strDbVersion >= PG_VERSION_92;

# Skip temporary statistics in pg_stat_tmp even when stats_temp_directory is set because PGSS_TEXT_FILE is always created
# there.
next if $strFile =~ ('^' . MANIFEST_PATH_PGSTATTMP . '\/') && $strDbVersion >= PG_VERSION_84;

# Skip pg_subtrans/* since these files are reset
next if $strFile =~ ('^' . MANIFEST_PATH_PGSUBTRANS . '\/');

# Skip ignored files
if ($strFile eq MANIFEST_FILE_POSTGRESQLAUTOCONFTMP || # postgresql.auto.conf.tmp - temp file for safe writes
$strFile eq MANIFEST_FILE_BACKUPLABELOLD || # backup_label.old - old backup labels are not useful
$strFile eq MANIFEST_FILE_POSTMASTEROPTS || # postmaster.opts - not useful for backup
$strFile eq MANIFEST_FILE_POSTMASTERPID || # postmaster.pid - to avoid confusing postgres after restore
$strFile eq MANIFEST_FILE_RECOVERYCONF || # recovery.conf - doesn't make sense to backup this file
$strFile eq MANIFEST_FILE_RECOVERYDONE) # recovery.done - doesn't make sense to backup this file
{
next;
}
Expand Down Expand Up @@ -703,7 +780,7 @@ sub build

$strPath = dirname("${strPath}/${strName}");

$self->build($oFile, $strLinkDestination, undef, $bOnline, $oTablespaceMapRef, $oDatabaseMapRef,
$self->build($oFile, $strDbVersion, $strLinkDestination, undef, $bOnline, $oTablespaceMapRef, $oDatabaseMapRef,
$strFile, $bTablespace, $strPath, $strFilter, $strLinkDestination);
}
}
Expand Down
Loading

0 comments on commit f1412ba

Please sign in to comment.