Skip to content
Permalink
Browse files

The local command for restore is implemented entirely in C.

This is just the part of restore run by the local helper processes, not the entire command.

Even so, various optimizations in the code (like pipelining and optimizations for zero-length files) should make the restore command faster on object stores.
  • Loading branch information...
dwsteele committed May 20, 2019
1 parent a839830 commit 1bc84c647490d6cf78975b5dde5b1937016d24c6
@@ -14,6 +14,12 @@
<release-list>
<release date="XXXX-XX-XX" version="2.15dev" title="UNDER DEVELOPMENT">
<release-core-list>
<release-improvement-list>
<release-item>
<p>The <cmd>local</cmd> command for restore is implemented entirely in C.</p>
</release-item>
</release-improvement-list>

<release-development-list>
<release-item>
<release-item-contributor-list>
@@ -172,7 +172,8 @@ sub outputRead
# Raise the error if a warning is not requested
if (!$bWarnOnError)
{
confess &log(ERROR, $strError, $hResult->{err}, $bSuppressLog);
confess &log(
ERROR, $strError . (defined($hResult->{errStack}) ? "\n$hResult->{errStack}" : ''), $hResult->{err}, $bSuppressLog);
}

&log(WARN, $strError, $hResult->{err});
@@ -54,7 +54,6 @@ sub init
my $hCommandMap =
{
&OP_BACKUP_FILE => sub {backupFile(@{shift()})},
&OP_RESTORE_FILE => sub {restoreFile(@{shift()})},

# To be run after each command to keep the remote alive
&OP_POST => sub {protocolKeepAlive()},
@@ -330,6 +330,15 @@ sub process
eval
{
$hJob->{rResult} = $hLocal->{oLocal}->outputRead(true, undef, undef, true);

# Create a result array when the result is not already an array. The Perl locals always return an array but the C
# locals only do so when needed.
if (ref($hJob->{rResult}) ne 'ARRAY')
{
my @resultArray = (${$hJob->{rResult}});
$hJob->{rResult} = \@resultArray;
}

return true;
}
or do
@@ -25,156 +25,6 @@ use pgBackRest::Storage::Filter::Gzip;
use pgBackRest::Storage::Filter::Sha;
use pgBackRest::Storage::Helper;

####################################################################################################################################
# restoreFile
#
# Restores a single file.
####################################################################################################################################
sub restoreFile
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strDbFile,
$lSize,
$lModificationTime,
$strChecksum,
$bZero,
$bForce, # Force flag
$strRepoFile,
$strReference,
$strMode,
$strUser,
$strGroup,
$lCopyTimeStart, # Backup start time - used for size/timestamp deltas
$bDelta, # Is restore a delta?
$strBackupPath, # Backup path
$bSourceCompressed, # Is the source compressed?
$strCipherPass, # Passphrase to decrypt the repo file (undefined if repo not encrypted)
) =
logDebugParam
(
__PACKAGE__ . '::restoreFile', \@_,
{name => 'strDbFile', trace => true},
{name => 'lSize', trace => true},
{name => 'lModificationTime', trace => true},
{name => 'strChecksum', required => false, trace => true},
{name => 'bZero', required => false, default => false, trace => true},
{name => 'bForce', trace => true},
{name => 'strRepoFile', trace => true},
{name => 'strReference', required => false, trace => true},
{name => 'strMode', trace => true},
{name => 'strUser', trace => true},
{name => 'strGroup', trace => true},
{name => 'lCopyTimeStart', trace => true},
{name => 'bDelta', trace => true},
{name => 'strBackupPath', trace => true},
{name => 'bSourceCompressed', trace => true},
{name => 'strCipherPass', required => false, trace => true},
);

# Does the file need to be copied?
my $oStorageDb = storageDb();
my $bCopy = true;

# Zero file if requested
if ($bZero)
{
$bCopy = false;

my $oDestinationFileIo = $oStorageDb->openWrite(
$strDbFile, {strMode => $strMode, strUser => $strUser, strGroup => $strGroup, lTimestamp => $lModificationTime});
$oDestinationFileIo->open();

# Now truncate to the original size. This will create a sparse file which is very efficient for this use case.
truncate($oDestinationFileIo->handle(), $lSize);

$oDestinationFileIo->close();
}
# Perform delta if requested
elsif ($bDelta)
{
my $oStat = $oStorageDb->info($strDbFile, {bIgnoreMissing => true});

# Do the delta if the file exists and is not a link or the link destination exists
if (defined($oStat) &&
(!S_ISLNK($oStat->mode) ||
$oStorageDb->exists(
$oStorageDb->pathAbsolute(dirname($strDbFile), $oStorageDb->{oDriver}->linkDestination($strDbFile)))))
{
# If force then use size/timestamp delta
if ($bForce)
{
# Make sure that timestamp/size are equal and that timestamp is before the copy start time of the backup
if (defined($oStat) && $oStat->size == $lSize &&
$oStat->mtime == $lModificationTime && $oStat->mtime < $lCopyTimeStart)
{
$bCopy = false;
}
}
else
{
my ($strActualChecksum, $lActualSize) = $oStorageDb->hashSize($strDbFile);

if ($lActualSize == $lSize && ($lSize == 0 || $strActualChecksum eq $strChecksum))
{
# Even if hash is the same set the time back to backup time. This helps with unit testing, but also
# presents a pristine version of the database after restore.
utime($lModificationTime, $lModificationTime, $strDbFile)
or confess &log(ERROR, "unable to set time for ${strDbFile}");

$bCopy = false;
}
}
}
}

# Copy file from repository to database
if ($bCopy)
{
# Add sha filter
my $rhyFilter = [{strClass => STORAGE_FILTER_SHA}];

# Add compression
if ($bSourceCompressed)
{
unshift(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS}]});
}

# Open destination file
my $oDestinationFileIo = $oStorageDb->openWrite(
$strDbFile,
{strMode => $strMode, strUser => $strUser, strGroup => $strGroup, lTimestamp => $lModificationTime,
rhyFilter => $rhyFilter});

# Copy file
storageRepo()->copy(
storageRepo()->openRead(
STORAGE_REPO_BACKUP . qw(/) . (defined($strReference) ? $strReference : $strBackupPath) .
"/${strRepoFile}" . ($bSourceCompressed ? qw{.} . COMPRESS_EXT : ''),
{bProtocolCompress => !$bSourceCompressed && $lSize != 0, strCipherPass => $strCipherPass}),
$oDestinationFileIo);

# Validate checksum
if ($oDestinationFileIo->result(COMMON_IO_HANDLE) != 0 && $oDestinationFileIo->result(STORAGE_FILTER_SHA) ne $strChecksum)
{
confess &log(ERROR,
"error restoring ${strDbFile}: actual checksum '" . $oDestinationFileIo->digest() .
"' does not match expected checksum ${strChecksum}", ERROR_CHECKSUM);
}
}

# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'bCopy', value => $bCopy, trace => true}
);
}

push @EXPORT, qw(restoreFile);

####################################################################################################################################
# restoreLog
#
@@ -55,6 +55,8 @@ SRCS = \
command/command.c \
command/control/control.c \
command/local/local.c \
command/restore/file.c \
command/restore/protocol.c \
command/remote/remote.c \
common/compress/gzip/common.c \
common/compress/gzip/compress.c \
@@ -223,12 +225,18 @@ command/help/help.o: command/help/help.c build.auto.h common/assert.h common/deb
command/info/info.o: command/info/info.c build.auto.h command/archive/common.h command/info/info.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h perl/exec.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
$(CC) $(CFLAGS) $(CMAKE) -c command/info/info.c -o command/info/info.o

command/local/local.o: command/local/local.c build.auto.h command/archive/get/protocol.h command/archive/push/protocol.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleRead.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/protocol.h protocol/client.h protocol/command.h protocol/helper.h protocol/server.h
command/local/local.o: command/local/local.c build.auto.h command/archive/get/protocol.h command/archive/push/protocol.h command/restore/protocol.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleRead.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/protocol.h protocol/client.h protocol/command.h protocol/helper.h protocol/server.h
$(CC) $(CFLAGS) $(CMAKE) -c command/local/local.c -o command/local/local.o

command/remote/remote.o: command/remote/remote.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleRead.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/protocol.h protocol/client.h protocol/command.h protocol/helper.h protocol/server.h storage/remote/protocol.h
$(CC) $(CFLAGS) $(CMAKE) -c command/remote/remote.c -o command/remote/remote.o

command/restore/file.o: command/restore/file.c build.auto.h command/restore/file.h common/assert.h common/compress/gzip/common.h common/compress/gzip/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h storage/helper.h storage/info.h storage/posix/common.h storage/read.h storage/storage.h storage/write.h
$(CC) $(CFLAGS) $(CMAKE) -c command/restore/file.c -o command/restore/file.o

command/restore/protocol.o: command/restore/protocol.c build.auto.h command/restore/file.h command/restore/protocol.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
$(CC) $(CFLAGS) $(CMAKE) -c command/restore/protocol.c -o command/restore/protocol.o

common/compress/gzip/common.o: common/compress/gzip/common.c build.auto.h common/assert.h common/compress/gzip/common.h common/debug.h common/error.auto.h common/error.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/convert.h
$(CC) $(CFLAGS) $(CMAKE) -c common/compress/gzip/common.c -o common/compress/gzip/common.o

@@ -5,6 +5,7 @@ Local Command

#include "command/archive/get/protocol.h"
#include "command/archive/push/protocol.h"
#include "command/restore/protocol.h"
#include "common/debug.h"
#include "common/io/handleRead.h"
#include "common/io/handleWrite.h"
@@ -33,6 +34,7 @@ cmdLocal(int handleRead, int handleWrite)
ProtocolServer *server = protocolServerNew(name, PROTOCOL_SERVICE_LOCAL_STR, read, write);
protocolServerHandlerAdd(server, archiveGetProtocol);
protocolServerHandlerAdd(server, archivePushProtocol);
protocolServerHandlerAdd(server, restoreProtocol);
protocolServerProcess(server);
}
MEM_CONTEXT_TEMP_END();

0 comments on commit 1bc84c6

Please sign in to comment.
You can’t perform that action at this time.