Skip to content

Commit

Permalink
Added the --repo-sync option to allow directory syncs in the reposito…
Browse files Browse the repository at this point in the history 
…ry to be disabled.

This is for file systems that do not support them, e.g. NTFS
  • Loading branch information
@dwsteele
dwsteele committed Dec 10, 2016
1 parent 36378c4 commit f818901
Show file tree
Hide file tree
Showing 43 changed files with 2,231 additions and 1,282 deletions.
9 changes: 9 additions & 0 deletions doc/xml/reference.xml
View file
Expand Up @@ -219,6 +219,15 @@

<example>/backup/db/backrest</example>
</config-key>

<!-- CONFIG - GENERAL SECTION - REPO-SYNC KEY -->
<config-key id="repo-sync" name="Repository Sync">
<summary>Sync directories in repository.</summary>

<text>Syncs directories when writing to the repository. Not all file systems support directory syncs (e.g., NTFS) so this option allows them to be disabled.</text>

<example>n</example>
</config-key>
</config-key-list>
</config-section>

Expand Down
4 changes: 4 additions & 0 deletions doc/xml/release.xml
View file
Expand Up @@ -156,6 +156,10 @@
<p>Added the <br-option>--repo-link</br-option> option to allow internal symlinks to be suppressed when the repository is located on a filesystem that does not support symlinks. This does not affect any <backrest/> functionality, but the convenience link <path>latest</path> will not be created and neither will internal tablespace symlinks, which will affect the ability to bring up clusters in-place manually using filesystem snapshots.</p>
</release-item>

<release-item>
<p>Added the <br-option>--repo-sync</br-option> option to allow directory syncs in the repository to be disabled for file systems that do not support them, e.g. NTFS.</p>
</release-item>

<release-item>
<p>Added a predictable log entry to signal that a command has completed successfully. For example a backup ends successfully with: <code>INFO: backup command end: completed successfully</code>.</p>
</release-item>
Expand Down
29 changes: 18 additions & 11 deletions lib/pgBackRest/Archive.pm
View file
Expand Up @@ -746,8 +746,9 @@ sub push
if ($bArchiveFile)
{
my ($strDbVersion, $ullDbSysId) = $self->walInfo($strSourceFile);
($strArchiveId, $strChecksum) = $self->pushCheck($oFile, substr(basename($strSourceFile), 0, 24), $bPartial,
$strSourceFile, $strDbVersion, $ullDbSysId);
($strArchiveId, $strChecksum) = $self->pushCheck(
$oFile, substr(basename($strSourceFile), 0, 24), $bPartial, $strSourceFile, $strDbVersion, $ullDbSysId,
optionGet(OPTION_REPO_SYNC));
}
else
{
Expand All @@ -774,7 +775,8 @@ sub push
undef, undef, undef, # Unused params
true, # Create path if it does not exist
undef, undef, # User and group
$bArchiveFile); # Append checksum if archive file
$bArchiveFile, # Append checksum if archive file
$bAsync ? true : optionGet(OPTION_REPO_SYNC)); # Sync if spool, else check repo sync option
}

# Return from function and log return values if any
Expand All @@ -800,7 +802,8 @@ sub pushCheck
$bPartial,
$strWalFile,
$strDbVersion,
$ullDbSysId
$ullDbSysId,
$bPathSync,
) =
logDebugParam
(
Expand All @@ -810,7 +813,8 @@ sub pushCheck
{name => 'bPartial'},
{name => 'strWalFile', required => false},
{name => 'strDbVersion'},
{name => 'ullDbSysId'}
{name => 'ullDbSysId'},
{name => 'bPathSync'},
);

# Set operation and debug strings
Expand All @@ -821,15 +825,16 @@ sub pushCheck
{
# Execute the command
($strArchiveId, $strChecksum) = $oFile->{oProtocol}->cmdExecute(
OP_ARCHIVE_PUSH_CHECK, [$strWalSegment, $bPartial, undef, $strDbVersion, $ullDbSysId], true);
OP_ARCHIVE_PUSH_CHECK, [$strWalSegment, $bPartial, undef, $strDbVersion, $ullDbSysId, $bPathSync], true);
}
else
{
# Create the archive path if it does not exist
$oFile->pathCreate(PATH_BACKUP_ARCHIVE, undef, undef, true, true);

# If the info file exists check db version and system-id, else create it with the db version and system-id passed and the history
$strArchiveId = (new pgBackRest::ArchiveInfo($oFile->pathGet(PATH_BACKUP_ARCHIVE)))->check($strDbVersion, $ullDbSysId);
# If the info file exists check db version and system-id, else create it
$strArchiveId = (new pgBackRest::ArchiveInfo($oFile->pathGet(PATH_BACKUP_ARCHIVE)))->check(
$strDbVersion, $ullDbSysId, $bPathSync);

# Check if the WAL segment already exists in the archive
$strChecksum = $self->walFileName($oFile, $strArchiveId, $strWalSegment, $bPartial);
Expand Down Expand Up @@ -996,8 +1001,9 @@ sub xfer
if ($bArchiveFile)
{
my ($strDbVersion, $ullDbSysId) = $self->walInfo($strArchiveFile);
($strArchiveId, $strChecksum) = $self->pushCheck($oFile, substr(basename($strArchiveFile), 0, 24), $bPartial,
$strArchiveFile, $strDbVersion, $ullDbSysId);
($strArchiveId, $strChecksum) = $self->pushCheck(
$oFile, substr(basename($strArchiveFile), 0, 24), $bPartial, $strArchiveFile, $strDbVersion, $ullDbSysId,
optionGet(OPTION_REPO_SYNC));
}
else
{
Expand All @@ -1019,7 +1025,8 @@ sub xfer
undef, undef, undef, # Unused params
true, # Create path if it does not exist
undef, undef, # Unused params
true); # Append checksum
true, # Append checksum
optionGet(OPTION_REPO_SYNC)); # Sync path if set

# If appended checksum does not equal copy checksum
if (defined($strAppendedChecksum) && $strAppendedChecksum ne $strCopyChecksum)
Expand Down
12 changes: 10 additions & 2 deletions lib/pgBackRest/ArchiveInfo.pm
View file
Expand Up @@ -108,13 +108,15 @@ sub check
(
$strOperation,
$strDbVersion,
$ullDbSysId
$ullDbSysId,
$bPathSync,
) =
logDebugParam
(
__PACKAGE__ . '->check', \@_,
{name => 'strDbVersion'},
{name => 'ullDbSysId'}
{name => 'ullDbSysId'},
{name => 'bPathSync', default => false},
);

my $bSave = false;
Expand Down Expand Up @@ -162,6 +164,12 @@ sub check
if ($bSave)
{
$self->save();

# Sync path if requested
if ($bPathSync)
{
filePathSync($self->{strArchiveClusterPath});
}
}

# Return from function and log return values if any
Expand Down
18 changes: 15 additions & 3 deletions lib/pgBackRest/Backup.pm
View file
Expand Up @@ -273,7 +273,7 @@ sub processManifest
# Create paths
foreach my $strPath ($oBackupManifest->keys(MANIFEST_SECTION_TARGET_PATH))
{
$oFileMaster->pathCreate(PATH_BACKUP_TMP, $strPath);
$oFileMaster->pathCreate(PATH_BACKUP_TMP, $strPath, undef, true);
}

if (optionGet(OPTION_REPO_LINK))
Expand Down Expand Up @@ -448,7 +448,7 @@ sub process
my $bHardLink = optionGet(OPTION_HARDLINK);

# Create the cluster backup and history path
$oFileLocal->pathCreate(PATH_BACKUP_CLUSTER, PATH_BACKUP_HISTORY, undef, true, true);
$oFileLocal->pathCreate(PATH_BACKUP_CLUSTER, PATH_BACKUP_HISTORY, undef, true, true, optionGet(OPTION_REPO_SYNC));

# Load or build backup.info
my $oBackupInfo = new pgBackRest::BackupInfo($oFileLocal->pathGet(PATH_BACKUP_CLUSTER));
Expand Down Expand Up @@ -891,6 +891,12 @@ sub process
# Final save of the backup manifest
$oBackupManifest->save();

# Sync all paths in the backup tmp path
if (optionGet(OPTION_REPO_SYNC))
{
$oFileLocal->pathSync(PATH_BACKUP_TMP, undef, true);
}

&log(INFO, "new backup label = ${strBackupLabel}");

# Make a compressed copy of the manifest for history
Expand All @@ -905,7 +911,7 @@ sub process
# Copy manifest to history
$oFileLocal->move(PATH_BACKUP_CLUSTER, "${strBackupLabel}/" . FILE_MANIFEST . '.gz',
PATH_BACKUP_CLUSTER, PATH_BACKUP_HISTORY . qw{/} . substr($strBackupLabel, 0, 4) .
"/${strBackupLabel}.manifest.gz", true);
"/${strBackupLabel}.manifest.gz", true, optionGet(OPTION_REPO_SYNC));

# Create a link to the most recent backup
$oFileLocal->remove(PATH_BACKUP_CLUSTER, LINK_LATEST);
Expand All @@ -918,6 +924,12 @@ sub process
# Save backup info
$oBackupInfo->add($oBackupManifest);

# Sync the cluster path
if (optionGet(OPTION_REPO_SYNC))
{
$oFileLocal->pathSync(PATH_BACKUP_CLUSTER);
}

# Return from function and log return values if any
return logDebugReturn($strOperation);
}
Expand Down
2 changes: 1 addition & 1 deletion lib/pgBackRest/Common/Lock.pm
View file
Expand Up @@ -49,7 +49,7 @@ sub lockFileName
####################################################################################################################################
sub lockPathCreate
{
filePathCreate(optionGet(OPTION_LOCK_PATH), '770');
filePathCreate(optionGet(OPTION_LOCK_PATH), '770', true);
}

####################################################################################################################################
Expand Down
19 changes: 19 additions & 0 deletions lib/pgBackRest/Config/Config.pm
View file
Expand Up @@ -18,6 +18,7 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::FileCommon;
use pgBackRest::Protocol::Common;
use pgBackRest::Version;

Expand Down Expand Up @@ -258,6 +259,8 @@ use constant OPTION_COMPRESS_LEVEL_NETWORK => 'compress
push @EXPORT, qw(OPTION_COMPRESS_LEVEL_NETWORK);
use constant OPTION_NEUTRAL_UMASK => 'neutral-umask';
push @EXPORT, qw(OPTION_NEUTRAL_UMASK);
use constant OPTION_REPO_SYNC => 'repo-sync';
push @EXPORT, qw(OPTION_REPO_SYNC);
use constant OPTION_PROTOCOL_TIMEOUT => 'protocol-timeout';
push @EXPORT, qw(OPTION_PROTOCOL_TIMEOUT);
use constant OPTION_PROCESS_MAX => 'process-max';
Expand Down Expand Up @@ -441,6 +444,8 @@ use constant OPTION_DEFAULT_DB_TIMEOUT_MIN => WAIT_TIME
use constant OPTION_DEFAULT_DB_TIMEOUT_MAX => 86400 * 7;
push @EXPORT, qw(OPTION_DEFAULT_DB_TIMEOUT_MAX);

use constant OPTION_DEFAULT_REPO_SYNC => true;

use constant OPTION_DEFAULT_PROTOCOL_TIMEOUT => OPTION_DEFAULT_DB_TIMEOUT + 30;
push @EXPORT, qw(OPTION_DEFAULT_PROTOCOL_TIMEOUT);
use constant OPTION_DEFAULT_PROTOCOL_TIMEOUT_MIN => OPTION_DEFAULT_DB_TIMEOUT_MIN;
Expand Down Expand Up @@ -1187,6 +1192,20 @@ my %oOptionRule =
},
},

&OPTION_REPO_SYNC =>
{
&OPTION_RULE_SECTION => CONFIG_SECTION_GLOBAL,
&OPTION_RULE_TYPE => OPTION_TYPE_BOOLEAN,
&OPTION_RULE_DEFAULT => OPTION_DEFAULT_REPO_SYNC,
&OPTION_RULE_NEGATE => true,
&OPTION_RULE_COMMAND =>
{
&CMD_ARCHIVE_PUSH => true,
&CMD_BACKUP => true,
&CMD_STANZA_CREATE => true,
},
},

&OPTION_PROTOCOL_TIMEOUT =>
{
&OPTION_RULE_SECTION => CONFIG_SECTION_GLOBAL,
Expand Down
15 changes: 15 additions & 0 deletions lib/pgBackRest/Config/ConfigHelpData.pm
View file
Expand Up @@ -601,6 +601,18 @@ my $oConfigHelpData =
"need, though of course requirements will likely change over time as your database evolves."
},

# REPO-SYNC Option Help
#---------------------------------------------------------------------------------------------------------------------------
'repo-sync' =>
{
section => 'general',
summary =>
"Sync directories in repository.",
description =>
"Syncs directories when writing to the repository. Not all file systems support directory syncs (e.g., NTFS) so " .
"this option allows them to be disabled."
},

# RESUME Option Help
#---------------------------------------------------------------------------------------------------------------------------
'resume' =>
Expand Down Expand Up @@ -848,6 +860,7 @@ my $oConfigHelpData =
'neutral-umask' => 'section',
'protocol-timeout' => 'section',
'repo-path' => 'section',
'repo-sync' => 'section',
'spool-path' => 'section',
'stanza' => 'default'
}
Expand Down Expand Up @@ -917,6 +930,7 @@ my $oConfigHelpData =
'protocol-timeout' => 'section',
'repo-link' => 'section',
'repo-path' => 'section',
'repo-sync' => 'section',
'resume' => 'section',
'retention-archive' => 'section',
'retention-archive-type' => 'section',
Expand Down Expand Up @@ -1286,6 +1300,7 @@ my $oConfigHelpData =
'neutral-umask' => 'section',
'protocol-timeout' => 'section',
'repo-path' => 'section',
'repo-sync' => 'section',
'stanza' => 'default'
}
},
Expand Down

0 comments on commit f818901

Please sign in to comment.