The local command for restore is implemented entirely in C.

This is just the part of restore run by the local helper processes, not the entire command. Even so, various optimizations in the code (like pipelining and optimizations for zero-length files) should make the restore command faster on object stores.
pgbackrest · May 20, 2019 · 1bc84c6 · 1bc84c6
1 parent a839830
commit 1bc84c6
Show file tree

Hide file tree

Showing 15 changed files with 597 additions and 290 deletions.
diff --git a/doc/xml/release.xml b/doc/xml/release.xml
@@ -14,6 +14,12 @@
     <release-list>
         <release date="XXXX-XX-XX" version="2.15dev" title="UNDER DEVELOPMENT">
             <release-core-list>
+                <release-improvement-list>
+                    <release-item>
+                        <p>The <cmd>local</cmd> command for restore is implemented entirely in C.</p>
+                    </release-item>
+                </release-improvement-list>
+
                 <release-development-list>
                     <release-item>
                         <release-item-contributor-list>

diff --git a/lib/pgBackRest/Protocol/Base/Master.pm b/lib/pgBackRest/Protocol/Base/Master.pm
@@ -172,7 +172,8 @@ sub outputRead
         # Raise the error if a warning is not requested
         if (!$bWarnOnError)
         {
-            confess &log(ERROR, $strError, $hResult->{err}, $bSuppressLog);
+            confess &log(
+                ERROR, $strError . (defined($hResult->{errStack}) ? "\n$hResult->{errStack}" : ''), $hResult->{err}, $bSuppressLog);
         }
 
         &log(WARN, $strError, $hResult->{err});

diff --git a/lib/pgBackRest/Protocol/Local/Minion.pm b/lib/pgBackRest/Protocol/Local/Minion.pm
@@ -54,7 +54,6 @@ sub init
     my $hCommandMap =
     {
         &OP_BACKUP_FILE => sub {backupFile(@{shift()})},
-        &OP_RESTORE_FILE => sub {restoreFile(@{shift()})},
 
         # To be run after each command to keep the remote alive
         &OP_POST => sub {protocolKeepAlive()},

diff --git a/lib/pgBackRest/Protocol/Local/Process.pm b/lib/pgBackRest/Protocol/Local/Process.pm
@@ -330,6 +330,15 @@ sub process
             eval
             {
                 $hJob->{rResult} = $hLocal->{oLocal}->outputRead(true, undef, undef, true);
+
+                # Create a result array when the result is not already an array.  The Perl locals always return an array but the C
+                # locals only do so when needed.
+                if (ref($hJob->{rResult}) ne 'ARRAY')
+                {
+                    my @resultArray = (${$hJob->{rResult}});
+                    $hJob->{rResult} = \@resultArray;
+                }
+
                 return true;
             }
             or do

diff --git a/lib/pgBackRest/RestoreFile.pm b/lib/pgBackRest/RestoreFile.pm
@@ -25,156 +25,6 @@ use pgBackRest::Storage::Filter::Gzip;
 use pgBackRest::Storage::Filter::Sha;
 use pgBackRest::Storage::Helper;
 
-####################################################################################################################################
-# restoreFile
-#
-# Restores a single file.
-####################################################################################################################################
-sub restoreFile
-{
-    # Assign function parameters, defaults, and log debug info
-    my
-    (
-        $strOperation,
-        $strDbFile,
-        $lSize,
-        $lModificationTime,
-        $strChecksum,
-        $bZero,
-        $bForce,                                    # Force flag
-        $strRepoFile,
-        $strReference,
-        $strMode,
-        $strUser,
-        $strGroup,
-        $lCopyTimeStart,                            # Backup start time - used for size/timestamp deltas
-        $bDelta,                                    # Is restore a delta?
-        $strBackupPath,                             # Backup path
-        $bSourceCompressed,                         # Is the source compressed?
-        $strCipherPass,                             # Passphrase to decrypt the repo file (undefined if repo not encrypted)
-    ) =
-        logDebugParam
-        (
-            __PACKAGE__ . '::restoreFile', \@_,
-            {name => 'strDbFile', trace => true},
-            {name => 'lSize', trace => true},
-            {name => 'lModificationTime', trace => true},
-            {name => 'strChecksum', required => false, trace => true},
-            {name => 'bZero', required => false, default => false, trace => true},
-            {name => 'bForce', trace => true},
-            {name => 'strRepoFile', trace => true},
-            {name => 'strReference', required => false, trace => true},
-            {name => 'strMode', trace => true},
-            {name => 'strUser', trace => true},
-            {name => 'strGroup', trace => true},
-            {name => 'lCopyTimeStart', trace => true},
-            {name => 'bDelta', trace => true},
-            {name => 'strBackupPath', trace => true},
-            {name => 'bSourceCompressed', trace => true},
-            {name => 'strCipherPass', required => false, trace => true},
-        );
-
-    # Does the file need to be copied?
-    my $oStorageDb = storageDb();
-    my $bCopy = true;
-
-    # Zero file if requested
-    if ($bZero)
-    {
-        $bCopy = false;
-
-        my $oDestinationFileIo = $oStorageDb->openWrite(
-            $strDbFile, {strMode => $strMode, strUser => $strUser, strGroup => $strGroup, lTimestamp => $lModificationTime});
-        $oDestinationFileIo->open();
-
-        # Now truncate to the original size.  This will create a sparse file which is very efficient for this use case.
-        truncate($oDestinationFileIo->handle(), $lSize);
-
-        $oDestinationFileIo->close();
-    }
-    # Perform delta if requested
-    elsif ($bDelta)
-    {
-        my $oStat = $oStorageDb->info($strDbFile, {bIgnoreMissing => true});
-
-        # Do the delta if the file exists and is not a link or the link destination exists
-        if (defined($oStat) &&
-            (!S_ISLNK($oStat->mode) ||
-                $oStorageDb->exists(
-                    $oStorageDb->pathAbsolute(dirname($strDbFile), $oStorageDb->{oDriver}->linkDestination($strDbFile)))))
-        {
-            # If force then use size/timestamp delta
-            if ($bForce)
-            {
-                # Make sure that timestamp/size are equal and that timestamp is before the copy start time of the backup
-                if (defined($oStat) && $oStat->size == $lSize &&
-                    $oStat->mtime == $lModificationTime && $oStat->mtime < $lCopyTimeStart)
-                {
-                    $bCopy = false;
-                }
-            }
-            else
-            {
-                my ($strActualChecksum, $lActualSize) = $oStorageDb->hashSize($strDbFile);
-
-                if ($lActualSize == $lSize && ($lSize == 0 || $strActualChecksum eq $strChecksum))
-                {
-                    # Even if hash is the same set the time back to backup time.  This helps with unit testing, but also
-                    # presents a pristine version of the database after restore.
-                    utime($lModificationTime, $lModificationTime, $strDbFile)
-                        or confess &log(ERROR, "unable to set time for ${strDbFile}");
-
-                    $bCopy = false;
-                }
-            }
-        }
-    }
-
-    # Copy file from repository to database
-    if ($bCopy)
-    {
-        # Add sha filter
-        my $rhyFilter = [{strClass => STORAGE_FILTER_SHA}];
-
-        # Add compression
-        if ($bSourceCompressed)
-        {
-            unshift(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS}]});
-        }
-
-        # Open destination file
-        my $oDestinationFileIo = $oStorageDb->openWrite(
-            $strDbFile,
-            {strMode => $strMode, strUser => $strUser, strGroup => $strGroup, lTimestamp => $lModificationTime,
-                rhyFilter => $rhyFilter});
-
-        # Copy file
-        storageRepo()->copy(
-            storageRepo()->openRead(
-                STORAGE_REPO_BACKUP . qw(/) . (defined($strReference) ? $strReference : $strBackupPath) .
-                    "/${strRepoFile}" . ($bSourceCompressed ? qw{.} . COMPRESS_EXT : ''),
-                {bProtocolCompress => !$bSourceCompressed && $lSize != 0, strCipherPass => $strCipherPass}),
-            $oDestinationFileIo);
-
-        # Validate checksum
-        if ($oDestinationFileIo->result(COMMON_IO_HANDLE) != 0 && $oDestinationFileIo->result(STORAGE_FILTER_SHA) ne $strChecksum)
-        {
-            confess &log(ERROR,
-                "error restoring ${strDbFile}: actual checksum '" . $oDestinationFileIo->digest() .
-                "' does not match expected checksum ${strChecksum}", ERROR_CHECKSUM);
-        }
-    }
-
-    # Return from function and log return values if any
-    return logDebugReturn
-    (
-        $strOperation,
-        {name => 'bCopy', value => $bCopy, trace => true}
-    );
-}
-
-push @EXPORT, qw(restoreFile);
-
 ####################################################################################################################################
 # restoreLog
 #

diff --git a/src/Makefile.in b/src/Makefile.in
@@ -55,6 +55,8 @@ SRCS = \
 	command/command.c \
 	command/control/control.c \
 	command/local/local.c \
+	command/restore/file.c \
+	command/restore/protocol.c \
 	command/remote/remote.c \
 	common/compress/gzip/common.c \
 	common/compress/gzip/compress.c \
@@ -223,12 +225,18 @@ command/help/help.o: command/help/help.c build.auto.h common/assert.h common/deb
 command/info/info.o: command/info/info.c build.auto.h command/archive/common.h command/info/info.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h perl/exec.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
 	$(CC) $(CFLAGS) $(CMAKE) -c command/info/info.c -o command/info/info.o
 
-command/local/local.o: command/local/local.c build.auto.h command/archive/get/protocol.h command/archive/push/protocol.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleRead.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/protocol.h protocol/client.h protocol/command.h protocol/helper.h protocol/server.h
+command/local/local.o: command/local/local.c build.auto.h command/archive/get/protocol.h command/archive/push/protocol.h command/restore/protocol.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleRead.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/protocol.h protocol/client.h protocol/command.h protocol/helper.h protocol/server.h
 	$(CC) $(CFLAGS) $(CMAKE) -c command/local/local.c -o command/local/local.o
 
 command/remote/remote.o: command/remote/remote.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleRead.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/protocol.h protocol/client.h protocol/command.h protocol/helper.h protocol/server.h storage/remote/protocol.h
 	$(CC) $(CFLAGS) $(CMAKE) -c command/remote/remote.c -o command/remote/remote.o
 
+command/restore/file.o: command/restore/file.c build.auto.h command/restore/file.h common/assert.h common/compress/gzip/common.h common/compress/gzip/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h storage/helper.h storage/info.h storage/posix/common.h storage/read.h storage/storage.h storage/write.h
+	$(CC) $(CFLAGS) $(CMAKE) -c command/restore/file.c -o command/restore/file.o
+
+command/restore/protocol.o: command/restore/protocol.c build.auto.h command/restore/file.h command/restore/protocol.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
+	$(CC) $(CFLAGS) $(CMAKE) -c command/restore/protocol.c -o command/restore/protocol.o
+
 common/compress/gzip/common.o: common/compress/gzip/common.c build.auto.h common/assert.h common/compress/gzip/common.h common/debug.h common/error.auto.h common/error.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/convert.h
 	$(CC) $(CFLAGS) $(CMAKE) -c common/compress/gzip/common.c -o common/compress/gzip/common.o
 

diff --git a/src/command/local/local.c b/src/command/local/local.c
@@ -5,6 +5,7 @@ Local Command
 
 #include "command/archive/get/protocol.h"
 #include "command/archive/push/protocol.h"
+#include "command/restore/protocol.h"
 #include "common/debug.h"
 #include "common/io/handleRead.h"
 #include "common/io/handleWrite.h"
@@ -33,6 +34,7 @@ cmdLocal(int handleRead, int handleWrite)
         ProtocolServer *server = protocolServerNew(name, PROTOCOL_SERVICE_LOCAL_STR, read, write);
         protocolServerHandlerAdd(server, archiveGetProtocol);
         protocolServerHandlerAdd(server, archivePushProtocol);
+        protocolServerHandlerAdd(server, restoreProtocol);
         protocolServerProcess(server);
     }
     MEM_CONTEXT_TEMP_END();