diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2d27fabdba..cc9de1b705 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,13 +32,13 @@ jobs: - param: test --vm=d10 --param=no-performance --param=c-only # All integration tests for 32-bit - - param: test --vm=d10 --param=module=mock --param=module=real + - param: test --vm=d10 --param=module=mock --param=module=integration # Debian/Ubuntu documentation - param: doc --vm=u20 # All integration tests - - param: test --vm=u22 --param=build-package --param=module=mock --param=module=real + - param: test --vm=u22 --param=build-package --param=module=mock --param=module=integration # All unit tests with coverage, backtrace and alternate timezone - param: test --vm=u22 --param=c-only --param=no-valgrind --param=tz=America/New_York @@ -53,7 +53,7 @@ jobs: - param: doc --vm=rh8 # All integration tests - - param: test --vm=rh7 --param=module=mock --param=module=real + - param: test --vm=rh7 --param=module=mock --param=module=integration steps: - name: Checkout Code diff --git a/doc/lib/pgBackRestDoc/Common/DocExecute.pm b/doc/lib/pgBackRestDoc/Common/DocExecute.pm index 91f360b783..695166d348 100644 --- a/doc/lib/pgBackRestDoc/Common/DocExecute.pm +++ b/doc/lib/pgBackRestDoc/Common/DocExecute.pm @@ -16,11 +16,11 @@ use File::Basename qw(dirname); use Storable qw(dclone); use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostTest; -use pgBackRestTest::Common::HostGroupTest; use pgBackRestDoc::Common::DocManifest; use pgBackRestDoc::Common::Exception; +use pgBackRestDoc::Common::Host; +use pgBackRestDoc::Common::HostGroup; use pgBackRestDoc::Common::Ini; use pgBackRestDoc::Common::Log; use pgBackRestDoc::Common::String; @@ -1050,7 +1050,7 @@ sub sectionChildProcess $strOption =~ s/\{\[host\-repo\-path\]\}/${strHostRepoPath}/g; } - my $oHost = new pgBackRestTest::Common::HostTest( + my $oHost = new pgBackRestDoc::Common::Host( $$hCacheKey{name}, "doc-$$hCacheKey{name}", $strImage, $strHostUser, defined($strMount) ? [$strMount] : undef, $strOption, $$hCacheKey{param}, $$hCacheKey{'update-hosts'}); diff --git a/test/lib/pgBackRestTest/Common/HostTest.pm b/doc/lib/pgBackRestDoc/Common/Host.pm similarity index 99% rename from test/lib/pgBackRestTest/Common/HostTest.pm rename to doc/lib/pgBackRestDoc/Common/Host.pm index d7d76a4e04..20606976b2 100644 --- a/test/lib/pgBackRestTest/Common/HostTest.pm +++ b/doc/lib/pgBackRestDoc/Common/Host.pm @@ -1,7 +1,7 @@ #################################################################################################################################### -# HostTest.pm - Encapsulate a docker host for testing +# HostTest.pm - Encapsulate a docker host #################################################################################################################################### -package pgBackRestTest::Common::HostTest; +package pgBackRestDoc::Common::Host; #################################################################################################################################### # Perl includes diff --git a/test/lib/pgBackRestTest/Common/HostGroupTest.pm b/doc/lib/pgBackRestDoc/Common/HostGroup.pm similarity index 96% rename from test/lib/pgBackRestTest/Common/HostGroupTest.pm rename to doc/lib/pgBackRestDoc/Common/HostGroup.pm index 345a54e04b..e49bc44639 100644 --- a/test/lib/pgBackRestTest/Common/HostGroupTest.pm +++ b/doc/lib/pgBackRestDoc/Common/HostGroup.pm @@ -1,7 +1,7 @@ #################################################################################################################################### -# HostGroupTest.pm - Encapsulate a group of docker containers for testing +# HostGroupTest.pm - Encapsulate a group of docker containers #################################################################################################################################### -package pgBackRestTest::Common::HostGroupTest; +package pgBackRestDoc::Common::HostGroup; #################################################################################################################################### # Perl includes @@ -177,7 +177,7 @@ sub hostGroupGet { if (!defined($oHostGroup)) { - $oHostGroup = new pgBackRestTest::Common::HostGroupTest(); + $oHostGroup = new pgBackRestDoc::Common::HostGroup(); } return $oHostGroup; diff --git a/src/build/common/exec.c b/src/build/common/exec.c index ebe27d1954..eb1e52d294 100644 --- a/src/build/common/exec.c +++ b/src/build/common/exec.c @@ -9,10 +9,11 @@ Execute Process Extensions /**********************************************************************************************************************************/ static String * -execProcess(Exec *const this) +execProcess(Exec *const this, const ExecOneParam param) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(EXEC, this); + FUNCTION_LOG_PARAM(INT, param.resultExpect); FUNCTION_LOG_END(); String *const result = strNew(); @@ -35,7 +36,7 @@ execProcess(Exec *const this) // If the process exited normally but without a success status if (WIFEXITED(processStatus)) { - if (WEXITSTATUS(processStatus) != 0) + if (WEXITSTATUS(processStatus) != param.resultExpect) execCheckStatusError(this, processStatus, strTrim(result)); } // Else if the process did not exit normally then it must have been a signal @@ -53,7 +54,8 @@ execOne(const String *const command, const ExecOneParam param) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STRING, command); - (void)param; + FUNCTION_LOG_PARAM(STRING, param.shell); + FUNCTION_LOG_PARAM(INT, param.resultExpect); FUNCTION_LOG_END(); String *result = NULL; @@ -61,23 +63,23 @@ execOne(const String *const command, const ExecOneParam param) MEM_CONTEXT_TEMP_BEGIN() { const StringList *const shellList = strLstNewSplitZ(param.shell != NULL ? param.shell : STRDEF("sh -c"), " "); - StringList *const param = strLstNew(); + StringList *const paramList = strLstNew(); ASSERT(strLstSize(shellList) != 0); for (unsigned int shellIdx = 1; shellIdx < strLstSize(shellList); shellIdx++) - strLstAdd(param, strLstGet(shellList, shellIdx)); + strLstAdd(paramList, strLstGet(shellList, shellIdx)); - strLstAddFmt(param, "%s 2>&1", strZ(command)); - strLstAddZ(param, "2>&1"); + strLstAddFmt(paramList, "%s 2>&1", strZ(command)); + strLstAddZ(paramList, "2>&1"); - Exec *const exec = execNew(strLstGet(shellList, 0), param, command, ioTimeoutMs()); + Exec *const exec = execNew(strLstGet(shellList, 0), paramList, command, ioTimeoutMs()); execOpen(exec); MEM_CONTEXT_PRIOR_BEGIN() { - result = execProcess(exec); + result = execProcess(exec, param); } MEM_CONTEXT_PRIOR_END(); } diff --git a/src/build/common/exec.h b/src/build/common/exec.h index cf53a43770..94171dcd0a 100644 --- a/src/build/common/exec.h +++ b/src/build/common/exec.h @@ -17,6 +17,7 @@ typedef struct ExecOneParam { VAR_PARAM_HEADER; const String *shell; // Shell command to use for exec (default is sh -c) + int resultExpect; // Expected result, if not 0 } ExecOneParam; #define execOneP(command, ...) \ diff --git a/test/certificate/pgbackrest-test-server.cnf b/test/certificate/pgbackrest-test-server.cnf index 0349097535..7443b43b55 100644 --- a/test/certificate/pgbackrest-test-server.cnf +++ b/test/certificate/pgbackrest-test-server.cnf @@ -37,6 +37,6 @@ DNS.4 = 127.0.0.1 IP.1 = 127.0.0.1 # Used in integration tests -DNS.5 = db-primary -DNS.6 = db-standby -DNS.7 = backup +DNS.5 = pg1 +DNS.6 = pg2 +DNS.7 = repo diff --git a/test/certificate/pgbackrest-test-server.crt b/test/certificate/pgbackrest-test-server.crt index 95040f2beb..01c9042654 100644 --- a/test/certificate/pgbackrest-test-server.crt +++ b/test/certificate/pgbackrest-test-server.crt @@ -1,8 +1,8 @@ -----BEGIN CERTIFICATE----- -MIIGAjCCA+qgAwIBAgIUW0gPWoZD5DqjIWIP3PliYA0IAOQwDQYJKoZIhvcNAQEL +MIIF8jCCA9qgAwIBAgIUJCya0E5vFzyH2AgiM3HSAHmpZ1QwDQYJKoZIhvcNAQEL BQAwXDELMAkGA1UEBhMCVVMxDDAKBgNVBAgMA0FsbDEMMAoGA1UEBwwDQWxsMRMw EQYDVQQKDApwZ0JhY2tSZXN0MRwwGgYDVQQDDBN0ZXN0LnBnYmFja3Jlc3Qub3Jn -MCAXDTIxMDgyNjEyMjkwM1oYDzIyOTUwNjEwMTIyOTAzWjB6MQswCQYDVQQGEwJV +MCAXDTI0MDMwNDAxMzgzMFoYDzIyOTcxMjE3MDEzODMwWjB6MQswCQYDVQQGEwJV UzEMMAoGA1UECAwDQWxsMQwwCgYDVQQHDANBbGwxEzARBgNVBAoMCnBnQmFja1Jl c3QxHDAaBgNVBAsME1VuaXQgVGVzdGluZyBEb21haW4xHDAaBgNVBAMME3Rlc3Qu cGdiYWNrcmVzdC5vcmcwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDD @@ -16,20 +16,19 @@ dCdyXuSftGFx0JxvmDhl9qFGarv1BKgwO83j7sy3IREte1K21JaIHNBVWP+NwU0N 4Z4OMqnpnnnGiyi0xnfJVqOXghu5BLWl9MuOntZ0nnzLmFD7w795uNRgjE6jmRmF FlX+PGqhHsZr0wZsBDsE9xO4i2l8aqJZx1hT5l3LIXC+lei/qo2gJi3nyePuz4UB t53sTNEdrZndFUaRyq/aJfkR13J0eaoqKn5BRRHhw8tRef6S84e0kQ6ABYNRGHQN -V+GswPl1fV37114FTBnz2Bi/GSQSs8vWjw49HHKK5wIDAQABo4GbMIGYMAkGA1Ud -EwQCMAAwCwYDVR0PBAQDAgXgMH4GA1UdEQR3MHWCE3Rlc3QucGdiYWNrcmVzdC5v +V+GswPl1fV37114FTBnz2Bi/GSQSs8vWjw49HHKK5wIDAQABo4GLMIGIMAkGA1Ud +EwQCMAAwCwYDVR0PBAQDAgXgMG4GA1UdEQRnMGWCE3Rlc3QucGdiYWNrcmVzdC5v cmeCFSoudGVzdC5wZ2JhY2tyZXN0Lm9yZ4IWKi50ZXN0Mi5wZ2JhY2tyZXN0Lm9y -Z4IJMTI3LjAuMC4xhwR/AAABggpkYi1wcmltYXJ5ggpkYi1zdGFuZGJ5ggZiYWNr -dXAwDQYJKoZIhvcNAQELBQADggIBAMbSq1/hjvQZJ2PFE/VVz9OcA7vlewq632eE -P5BalSJJgLVEsv1AxPx8VT08xfFQHQtEcCg/PFqT3RQ5yb1kHfa6glJkjYIdKQbn -lv9OVc/iutQwKPwk32QQjSgQFb/m0tXv9SlQ+gNTdkK4UKffXPj5rpgwaSiVwuLF -d+3TUpJihS48LLRC27kcL5Ur69/fu0ZD7xZSoCr/n8MUq4f9LwOhBqq+h64wM9cV -V79iPWmEJXoNAJrPYmK+XNhcro071c4m+HR4CCNikjxz/GUUf/NGHWT3pL0Ildku -X3dHmsNRVT/wLqi2v2oa6zr9FfVzjDAdCfnvTLOJ6H6dmofzQUFJBSWfhqGNDR8U -oblwirM2sjaOUjnkBS6Cb26yHSClStI+GZvS0KZfSVd2Qbe4YmtQMTNl/hdZGK3z -ZoqV++idVR+A0NQP8xR4VWqQdq0BR5eQOXDA4wtqvivqlIXpbJvqh1kBHPU9cAF+ -g/t3Wa7EomwLazRaV9djLUpon6wGwScKJGzv+vyQSgXN1tQG9tLV4NCFUKDueUUZ -U/j1t64KF9hp5NU2A16zLp6V5GPIJhufXOYa66AFjV8c880eLd5YlkfzgyYwReOx -7vHkiLylbx2tc6aYUqdwjpMwnkxTsn52BBVxDvXToBIRdq/ea/LnZ/yhpnaac/Um -bJOTMee+ +Z4IJMTI3LjAuMC4xhwR/AAABggNwZzGCA3BnMoIEcmVwbzANBgkqhkiG9w0BAQsF +AAOCAgEAX0MEXH6ANllRhdQz6neQ7SVG48Aj1lEAGeOhfpoNKzuyBcRjVw7+NyNN +IwlPKSSBDpnxaWQ5rCLtBtXod6yPMGKTRlFHwFFzfOps6nlRQjPsA848d6daLBvj +unpUQx4NFGPZJSs6z5z4BlT/+5mJVHC9qsrZBtkndYpRWo37xbVhRqP0+FSTbzrx +Gj2td3PoqQBgA/AmSKIpwagGzw7cSor1r4uEjkVxxyOMRbjuuASHMHUM7MtQV3YR +rz9UspvGfoKBdUkzMoqKRwxZWuh+uAoM+1GWXBjqlN6uAdQxpV2wZ75iRJp3Y4Bk +/CkXTLZ83lARGLqS/E5EFfg7Z9Bre2f5fHzV8C/h6WGpvCt/GlZqTx8fix/mMPT2 +CFq+FcSmvF5JsIMxUnpvTw1hcTDNRPnOkFKnO1bjf9+jGCwzDUoGReYpb5veFSxh +IFkQ3oyw3/6v11aPstXSADTvRTFMyRklqu5NIHkMVQCPwQCAE3346KpEsT5HO2T7 +X57sP05alMESjUv1sR260yYC3GUr0dbmf8gthVDhH1SsP3Drn/A3l70xGASnJA46 +LCZwAJZ0KI6G/ok17lTZe9Hjwn2DkmVf1CKD7gXjmroYQI9O+etUtD/g+B2AISTG +SP60NakteOBWqmcSirV5npKh/SZR8Il5oaLS+3HerhgwvNDJ078= -----END CERTIFICATE----- diff --git a/test/ci.pl b/test/ci.pl index 5553243001..824402c776 100755 --- a/test/ci.pl +++ b/test/ci.pl @@ -183,7 +183,7 @@ sub processEnd # Build list of packages that need to be installed my $strPackage = "make gcc ccache python3-pip git rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config" . - " uncrustify libssh2-1-dev"; + " uncrustify libssh2-1-dev valgrind"; # Add lcov when testing coverage if (vmCoverageC($strVm)) @@ -194,7 +194,7 @@ sub processEnd # Extra packages required when testing without containers if ($strVm eq VM_NONE) { - $strPackage .= " valgrind liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev"; + $strPackage .= " liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev"; } # Else packages needed for integration tests on containers else diff --git a/test/define.yaml b/test/define.yaml index 324648ebba..9334c6316a 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -608,7 +608,9 @@ unit: # ---------------------------------------------------------------------------------------------------------------------------- - name: sftp total: 19 - harness: libSsh2 + harness: + name: libSsh2 + integration: false harness: name: fd shim: @@ -635,7 +637,9 @@ unit: # ---------------------------------------------------------------------------------------------------------------------------- - name: client total: 1 - harness: pq + harness: + name: pq + integration: false coverage: - postgres/client @@ -881,6 +885,7 @@ unit: total: 12 harness: name: backup + integration: false shim: command/backup/backup: function: @@ -1004,13 +1009,14 @@ unit: integration: # ******************************************************************************************************************************** - - name: real + - name: integration db: true test: # ---------------------------------------------------------------------------------------------------------------------------- - name: all total: 1 + harness: host # ********************************************************************************************************************************** # Performance tests diff --git a/test/lib/pgBackRestTest/Common/CoverageTest.pm b/test/lib/pgBackRestTest/Common/CoverageTest.pm index 13c860c7ff..4a681aab27 100644 --- a/test/lib/pgBackRestTest/Common/CoverageTest.pm +++ b/test/lib/pgBackRestTest/Common/CoverageTest.pm @@ -26,7 +26,31 @@ use pgBackRestTest::Common::ContainerTest; use pgBackRestTest::Common::DefineTest; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::ListTest; -use pgBackRestTest::Common::RunTest; + +#################################################################################################################################### +# testRunName +# +# Create module/test names by upper-casing the first letter and then inserting capitals after each -. +#################################################################################################################################### +sub testRunName +{ + my $strName = shift; + my $bInitCapFirst = shift; + + $bInitCapFirst = defined($bInitCapFirst) ? $bInitCapFirst : true; + my $bFirst = true; + + my @stryName = split('\-', $strName); + $strName = undef; + + foreach my $strPart (@stryName) + { + $strName .= ($bFirst && $bInitCapFirst) || !$bFirst ? ucfirst($strPart) : $strPart; + $bFirst = false; + } + + return $strName; +} #################################################################################################################################### # Generate an lcov configuration file diff --git a/test/lib/pgBackRestTest/Common/FileTest.pm b/test/lib/pgBackRestTest/Common/FileTest.pm deleted file mode 100644 index 6fddeddd22..0000000000 --- a/test/lib/pgBackRestTest/Common/FileTest.pm +++ /dev/null @@ -1,277 +0,0 @@ -#################################################################################################################################### -# CommonTest.pm - Common globals used for testing -#################################################################################################################################### -package pgBackRestTest::Common::FileTest; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Cwd qw(abs_path cwd); -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); -use File::Copy qw(move); -use File::Path qw(remove_tree); -use IO::Select; -use IPC::Open3; -use POSIX ':sys_wait_h'; -use Symbol 'gensym'; - -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::Common::String; - -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageBase; -use pgBackRestTest::Common::VmTest; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Host::HostDbCommonTest; -use pgBackRestTest::Env::Host::HostDbTest; -use pgBackRestTest::Env::Host::HostS3Test; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# testLinkCreate -# -# Create a symlink -#################################################################################################################################### -sub testLinkCreate -{ - my $strLink = shift; - my $strDestination = shift; - - # Create the file - symlink($strDestination, $strLink) - or confess "unable to link ${strLink} to ${strDestination}"; -} - -push(@EXPORT, qw(testLinkCreate)); - -#################################################################################################################################### -# testPathMode -# -# Set mode of an existing path. -#################################################################################################################################### -sub testPathMode -{ - my $strPath = shift; - my $strMode = shift; - - # Set the mode - chmod(oct($strMode), $strPath) - or confess 'unable to set mode ${strMode} for ${strPath}'; -} - -push(@EXPORT, qw(testPathMode)); - -#################################################################################################################################### -# testPathRemove -# -# Remove a path and all subpaths. -#################################################################################################################################### -sub testPathRemove -{ - my $strPath = shift; - my $bSuppressError = shift; - - executeTest('rm -rf ' . $strPath, {bSuppressError => $bSuppressError}); -} - -push(@EXPORT, qw(testPathRemove)); - -#################################################################################################################################### -# testFileCreate -# -# Create a file specifying content, mode, and time. -#################################################################################################################################### -sub testFileCreate -{ - my $strFile = shift; - my $strContent = shift; - my $lTime = shift; - my $strMode = shift; - - # Open the file and save strContent to it - my $hFile = shift; - - open($hFile, '>', $strFile) - or confess "unable to open ${strFile} for writing"; - - if (defined($strContent) && $strContent ne '') - { - syswrite($hFile, $strContent) - or confess "unable to write to ${strFile}: $!"; - } - - close($hFile); - - # Set the time - if (defined($lTime)) - { - utime($lTime, $lTime, $strFile) - or confess 'unable to set time ${lTime} for ${strPath}'; - } - - # Set the mode - chmod(oct(defined($strMode) ? $strMode : '0600'), $strFile) - or confess 'unable to set mode ${strMode} for ${strFile}'; -} - -push(@EXPORT, qw(testFileCreate)); - -#################################################################################################################################### -# testFileRemove -# -# Remove a file. -#################################################################################################################################### -sub testFileRemove -{ - my $strFile = shift; - - unlink($strFile) - or confess "unable to remove ${strFile}: $!"; -} - -push(@EXPORT, qw(testFileRemove)); - -#################################################################################################################################### -# forceStorageMode - force mode on a file or path -#################################################################################################################################### -sub forceStorageMode -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oStorage, - $strPath, - $strMode, - $bRecurse - ) = - logDebugParam - ( - __PACKAGE__ . '::forceStorageMode', \@_, - {name => 'oStorage'}, - {name => 'strPath'}, - {name => 'strMode'}, - {name => 'bRecurse', optional => true, default => false}, - ); - - # Mode commands are ignored on object storage - if ($oStorage->type() ne STORAGE_OBJECT) - { - executeTest('chmod ' . ($bRecurse ? '-R ' : '') . "${strMode} ${strPath}"); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -push(@EXPORT, qw(forceStorageMode)); - -#################################################################################################################################### -# forceStorageMove - force move a directory or file -#################################################################################################################################### -sub forceStorageMove -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oStorage, - $strSourcePath, - $strDestinationPath, - $bRecurse, - ) = - logDebugParam - ( - __PACKAGE__ . '->forceStorageMove', \@_, - {name => 'oStorage'}, - {name => 'strSourcePath'}, - {name => 'strDestinationPath'}, - {name => 'bRecurse', optional => true, default => true}, - ); - - # If object storage then use storage commands to remove - if ($oStorage->type() eq STORAGE_OBJECT) - { - if ($bRecurse) - { - my $rhManifest = $oStorage->manifest($strSourcePath); - - foreach my $strName (sort(keys(%{$rhManifest}))) - { - if ($rhManifest->{$strName}{type} eq 'f') - { - $oStorage->put( - "${strDestinationPath}/${strName}", ${$oStorage->get("${strSourcePath}/${strName}", {bRaw => true})}, - {bRaw => true}); - } - } - - $oStorage->pathRemove($strSourcePath, {bRecurse => true}); - } - else - { - $oStorage->put($strDestinationPath, ${$oStorage->get($strSourcePath, {bRaw => true})}, {bRaw => true}); - $oStorage->remove($strSourcePath); - } - } - # Else remove using filesystem commands - else - { - executeTest("mv ${strSourcePath} ${strDestinationPath}"); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -push(@EXPORT, qw(forceStorageMove)); - -#################################################################################################################################### -# forceStorageRemove - force remove a file or path from storage -#################################################################################################################################### -sub forceStorageRemove -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oStorage, - $strPath, - $bRecurse - ) = - logDebugParam - ( - __PACKAGE__ . '->forceStorageRemove', \@_, - {name => 'oStorage'}, - {name => 'strPath'}, - {name => 'bRecurse', optional => true, default => false}, - ); - - # If object storage then use storage commands to remove - if ($oStorage->type() eq STORAGE_OBJECT) - { - $oStorage->pathRemove($strPath, {bRecurse => true}); - } - else - { - executeTest('rm -f' . ($bRecurse ? 'r ' : ' ') . $strPath); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -push(@EXPORT, qw(forceStorageRemove)); - -1; diff --git a/test/lib/pgBackRestTest/Common/JobTest.pm b/test/lib/pgBackRestTest/Common/JobTest.pm index 0a02b5de53..55fd291d37 100644 --- a/test/lib/pgBackRestTest/Common/JobTest.pm +++ b/test/lib/pgBackRestTest/Common/JobTest.pm @@ -30,7 +30,6 @@ use pgBackRestTest::Common::DbVersion; use pgBackRestTest::Common::DefineTest; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::ListTest; -use pgBackRestTest::Common::RunTest; use pgBackRestTest::Common::VmTest; #################################################################################################################################### @@ -115,7 +114,9 @@ sub new $self->{iTry} = 0; # Setup the path where unit test will be built - $self->{strUnitPath} = "$self->{strTestPath}/unit-$self->{iVmIdx}/$self->{oTest}->{&TEST_VM}"; + $self->{strUnitPath} = + "$self->{strTestPath}/unit-$self->{iVmIdx}/" . + ($self->{oTest}->{&TEST_TYPE} eq TESTDEF_INTEGRATION ? 'none' : $self->{oTest}->{&TEST_VM}); $self->{strDataPath} = "$self->{strTestPath}/data-$self->{iVmIdx}"; $self->{strRepoPath} = "$self->{strTestPath}/repo"; @@ -246,58 +247,32 @@ sub run if (!$self->{bDryRun} || $self->{bVmOut}) { - my $strCommand = undef; # Command to run test - - # If testing with C harness - if ($self->{oTest}->{&TEST_C}) - { - # Create command - # ------------------------------------------------------------------------------------------------------------------ - # Build filename for valgrind suppressions - my $strValgrindSuppress = $self->{strRepoPath} . '/test/src/valgrind.suppress.' . $self->{oTest}->{&TEST_VM}; - - $strCommand = - ($self->{oTest}->{&TEST_VM} ne VM_NONE ? "docker exec -i -u ${\TEST_USER} ${strImage} bash -l -c '" : '') . - " \\\n" . - $self->{strTestPath} . '/build/' . $self->{oTest}->{&TEST_VM} . '/test/src/test-pgbackrest' . - ' --repo-path=' . $self->{strTestPath} . '/repo' . ' --test-path=' . $self->{strTestPath} . - " --log-level=$self->{strLogLevel}" . ' --vm=' . $self->{oTest}->{&TEST_VM} . - ' --vm-id=' . $self->{iVmIdx} . ($self->{bProfile} ? ' --profile' : '') . - ($self->{bLogTimestamp} ? '' : ' --no-log-timestamp') . - ($self->{strTimeZone} ? " --tz='$self->{strTimeZone}'" : '') . - ($self->{bBackTraceUnit} ? '' : ' --no-back-trace') . ($bCoverage ? '' : ' --no-coverage') . ' test ' . - $self->{oTest}->{&TEST_MODULE} . '/' . $self->{oTest}->{&TEST_NAME} . " && \\\n" . - # Allow stderr to be copied to stderr and stdout - "exec 3>&1 && \\\n" . - # Test with valgrind when requested - ($self->{bValgrindUnit} && $self->{oTest}->{&TEST_TYPE} ne TESTDEF_PERFORMANCE ? - 'valgrind -q --gen-suppressions=all' . - ($self->{oStorageTest}->exists($strValgrindSuppress) ? " --suppressions=${strValgrindSuppress}" : '') . - " --exit-on-first-error=yes --leak-check=full --leak-resolution=high --error-exitcode=25" . ' ' : '') . - "$self->{strUnitPath}/build/test-unit 2>&1 1>&3 | tee /dev/stderr" . - ($self->{oTest}->{&TEST_VM} ne VM_NONE ? "'" : ''); - } - else - { - $strCommand = - ($self->{oTest}->{&TEST_CONTAINER} ? 'docker exec -i -u ' . TEST_USER . " ${strImage} " : '') . - abs_path($0) . - " --test-path=${strVmTestPath}" . - " --vm=$self->{oTest}->{&TEST_VM}" . - " --vm-id=$self->{iVmIdx}" . - " --module=" . $self->{oTest}->{&TEST_MODULE} . - ' --test=' . $self->{oTest}->{&TEST_NAME} . - $strCommandRunParam . - (defined($self->{oTest}->{&TEST_DB}) ? ' --pg-version=' . $self->{oTest}->{&TEST_DB} : '') . - ($self->{strLogLevel} ne lc(INFO) ? " --log-level=$self->{strLogLevel}" : '') . - ($self->{strLogLevelTestFile} ne lc(TRACE) ? " --log-level-test-file=$self->{strLogLevelTestFile}" : '') . + my $bValgrind = $self->{bValgrindUnit} && $self->{oTest}->{&TEST_TYPE} ne TESTDEF_PERFORMANCE; + my $strValgrindSuppress = + $self->{strRepoPath} . '/test/src/valgrind.suppress.' . + ($self->{oTest}->{&TEST_TYPE} eq TESTDEF_INTEGRATION ? VM_NONE : $self->{oTest}->{&TEST_VM}); + my $strVm = $self->{oTest}->{&TEST_TYPE} eq TESTDEF_INTEGRATION ? VM_NONE : $self->{oTest}->{&TEST_VM}; + + my $strCommand = + ($strVm ne VM_NONE ? "docker exec -i -u ${\TEST_USER} ${strImage} bash -l -c '\\\n" : '') . + $self->{strTestPath} . "/build/${strVm}/test/src/test-pgbackrest" . + ' --repo-path=' . $self->{strTestPath} . '/repo' . ' --test-path=' . $self->{strTestPath} . + " --log-level=$self->{strLogLevel}" . ' --vm=' . $self->{oTest}->{&TEST_VM} . + ' --vm-id=' . $self->{iVmIdx} . ($self->{bProfile} ? ' --profile' : '') . ($self->{bLogTimestamp} ? '' : ' --no-log-timestamp') . - ' --psql-bin=' . $self->{oTest}->{&TEST_PGSQL_BIN} . ($self->{strTimeZone} ? " --tz='$self->{strTimeZone}'" : '') . - ($self->{bDryRun} ? ' --dry-run' : '') . - ($self->{bDryRun} ? ' --vm-out' : '') . - ($self->{bNoCleanup} ? " --no-cleanup" : ''); - } + (defined($self->{oTest}->{&TEST_DB}) ? ' --pg-version=' . $self->{oTest}->{&TEST_DB} : '') . + ($self->{bBackTraceUnit} ? '' : ' --no-back-trace') . ($bCoverage ? '' : ' --no-coverage') . ' test ' . + $self->{oTest}->{&TEST_MODULE} . '/' . $self->{oTest}->{&TEST_NAME} . " && \\\n" . + # Allow stderr to be copied to stderr and stdout + "exec 3>&1 && \\\n" . + # Test with valgrind when requested + ($bValgrind ? + 'valgrind -q --gen-suppressions=all' . + ($self->{oStorageTest}->exists($strValgrindSuppress) ? " --suppressions=${strValgrindSuppress}" : '') . + " --exit-on-first-error=yes --leak-check=full --leak-resolution=high --error-exitcode=25" . ' ' : '') . + "$self->{strUnitPath}/build/test-unit 2>&1 1>&3 | tee /dev/stderr" . + ($strVm ne VM_NONE ? "'" : ''); my $oExec = new pgBackRestTest::Common::ExecuteTest( $strCommand, {bSuppressError => true, bShowOutputAsync => $self->{bShowOutputAsync}}); diff --git a/test/lib/pgBackRestTest/Common/RunTest.pm b/test/lib/pgBackRestTest/Common/RunTest.pm deleted file mode 100644 index 2c1ce7e4e9..0000000000 --- a/test/lib/pgBackRestTest/Common/RunTest.pm +++ /dev/null @@ -1,426 +0,0 @@ -#################################################################################################################################### -# RunTest.pm - All tests are inherited from this object -#################################################################################################################################### -package pgBackRestTest::Common::RunTest; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::Common::String; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::BuildTest; -use pgBackRestTest::Common::DefineTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::Storage; -use pgBackRestTest::Common::StoragePosix; -use pgBackRestTest::Common::VmTest; -use pgBackRestTest::Common::Wait; - -#################################################################################################################################### -# Constant to use when bogus data is required -#################################################################################################################################### -use constant BOGUS => 'bogus'; - push @EXPORT, qw(BOGUS); - -#################################################################################################################################### -# The current test run that is executing. Only a single run should ever occur in a process to prevent various cleanup issues from -# affecting the next run. Of course multiple subtests can be executed in a single run. -#################################################################################################################################### -my $oTestRun; -my $oStorage; - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Create the class hash - my $self = {}; - bless $self, $class; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '->new'); - - # Initialize run counter - $self->{iRun} = 0; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# initModule -# -# Empty init sub in case the ancestor class does not declare one. -#################################################################################################################################### -sub initModule {} - -#################################################################################################################################### -# initTest -# -# Empty init sub in case the ancestor class does not declare one. -#################################################################################################################################### -sub initTest {} - -#################################################################################################################################### -# cleanTest -# -# Delete all files in test directory. -#################################################################################################################################### -sub cleanTest -{ - my $self = shift; - - executeTest('rm -rf ' . $self->testPath() . '/*'); -} - -#################################################################################################################################### -# cleanModule -# -# Empty final sub in case the ancestor class does not declare one. -#################################################################################################################################### -sub cleanModule {} - -#################################################################################################################################### -# process -#################################################################################################################################### -sub process -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - ( - my $strOperation, - $self->{strVm}, - $self->{iVmId}, - $self->{strBasePath}, - $self->{strTestPath}, - $self->{strBackRestExe}, - $self->{strBackRestExeHelper}, - $self->{strPgBinPath}, - $self->{strPgVersion}, - $self->{strModule}, - $self->{strModuleTest}, - $self->{iyModuleTestRun}, - $self->{bOutput}, - $self->{bDryRun}, - $self->{bCleanup}, - $self->{strLogLevelTestFile}, - $self->{strPgUser}, - $self->{strGroup}, - ) = - logDebugParam - ( - __PACKAGE__ . '->process', \@_, - {name => 'strVm'}, - {name => 'iVmId'}, - {name => 'strBasePath'}, - {name => 'strTestPath'}, - {name => 'strBackRestExe'}, - {name => 'strBackRestExeHelper'}, - {name => 'strPgBinPath', required => false}, - {name => 'strPgVersion', required => false}, - {name => 'strModule'}, - {name => 'strModuleTest'}, - {name => 'iModuleTestRun', required => false}, - {name => 'bOutput'}, - {name => 'bDryRun'}, - {name => 'bCleanup'}, - {name => 'strLogLevelTestFile'}, - {name => 'strPgUser'}, - {name => 'strGroup'}, - ); - - # Init will only be run on first test, clean/init on subsequent tests - $self->{bFirstTest} = true; - - # Initialize test storage - $oStorage = new pgBackRestTest::Common::Storage( - $self->testPath(), new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false})); - - # Init, run, and clean the test(s) - $self->initModule(); - $self->run(); - $self->cleanModule(); - - # Make sure the correct number of tests ran - my $hModuleTest = testDefModuleTest($self->{strModule}, $self->{strModuleTest}); - - if ($hModuleTest->{&TESTDEF_TOTAL} != $self->runCurrent()) - { - confess &log(ASSERT, "expected $hModuleTest->{&TESTDEF_TOTAL} tests to run but $self->{iRun} ran"); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# begin -#################################################################################################################################### -sub begin -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDescription, - ) = - logDebugParam - ( - __PACKAGE__ . '->begin', \@_, - {name => 'strDescription'}, - ); - - # Increment the run counter; - $self->{iRun}++; - - # Return if this test should not be run - if (@{$self->{iyModuleTestRun}} != 0 && !grep(/^$self->{iRun}$/i, @{$self->{iyModuleTestRun}})) - { - return false; - } - - # Output information about test to run - &log(INFO, 'run ' . sprintf('%03d', $self->runCurrent()) . ' - ' . $strDescription); - - if ($self->isDryRun()) - { - return false; - } - - if (!$self->{bFirstTest}) - { - $self->cleanTest(); - } - - $self->initTest(); - $self->{bFirstTest} = false; - - return true; -} - -#################################################################################################################################### -# testResult -#################################################################################################################################### -sub testResult -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $fnSub, - $strExpected, - $strDescription, - $iWaitSeconds, - ) = - logDebugParam - ( - __PACKAGE__ . '::testResult', \@_, - {name => 'fnSub', trace => true}, - {name => 'strExpected', required => false, trace => true}, - {name => 'strDescription', trace => true}, - {name => 'iWaitSeconds', optional => true, default => 0, trace => true}, - ); - - &log(INFO, ' ' . $strDescription); - my $strActual; - my $bWarnValid = true; - - my $oWait = waitInit($iWaitSeconds); - my $bDone = false; - - # Clear the cache for this test - logFileCacheClear(); - - my @stryResult; - - do - { - eval - { - @stryResult = ref($fnSub) eq 'CODE' ? $fnSub->() : $fnSub; - - if (@stryResult <= 1) - { - $strActual = ${logDebugBuild($stryResult[0])}; - } - else - { - $strActual = ${logDebugBuild(\@stryResult)}; - } - - return true; - } - or do - { - if (!isException(\$EVAL_ERROR)) - { - confess "unexpected standard Perl exception" . (defined($EVAL_ERROR) ? ": ${EVAL_ERROR}" : ''); - } - - confess &logException($EVAL_ERROR); - }; - - if ($strActual ne (defined($strExpected) ? $strExpected : "[undef]")) - { - if (!waitMore($oWait)) - { - confess - "expected:\n" . (defined($strExpected) ? "\"${strExpected}\"" : '[undef]') . - "\nbut actual was:\n" . (defined($strActual) ? "\"${strActual}\"" : '[undef]'); - } - } - else - { - $bDone = true; - } - } while (!$bDone); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'result', value => \@stryResult, trace => true} - ); -} - -#################################################################################################################################### -# testRunName -# -# Create module/test names by upper-casing the first letter and then inserting capitals after each -. -#################################################################################################################################### -sub testRunName -{ - my $strName = shift; - my $bInitCapFirst = shift; - - $bInitCapFirst = defined($bInitCapFirst) ? $bInitCapFirst : true; - my $bFirst = true; - - my @stryName = split('\-', $strName); - $strName = undef; - - foreach my $strPart (@stryName) - { - $strName .= ($bFirst && $bInitCapFirst) || !$bFirst ? ucfirst($strPart) : $strPart; - $bFirst = false; - } - - return $strName; -} - -push @EXPORT, qw(testRunName); - -#################################################################################################################################### -# testRun -#################################################################################################################################### -sub testRun -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strModule, - $strModuleTest, - ) = - logDebugParam - ( - __PACKAGE__ . '::testRun', \@_, - {name => 'strModule', trace => true}, - {name => 'strModuleTest', trace => true}, - ); - - # Error if the test run is already defined - only one run per process is allowed - if (defined($oTestRun)) - { - confess &log(ASSERT, 'a test run has already been created in this process'); - } - - my $strModuleName = - 'pgBackRestTest::Module::' . testRunName($strModule) . '::' . testRunName($strModule) . testRunName($strModuleTest) . - 'Test'; - - $oTestRun = eval("require ${strModuleName}; ${strModuleName}->import(); return new ${strModuleName}();") - or do {confess $EVAL_ERROR}; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'oRun', value => $oTestRun, trace => true} - ); -} - -push @EXPORT, qw(testRun); - -#################################################################################################################################### -# testRunGet -#################################################################################################################################### -sub testRunGet -{ - return $oTestRun; -} - -push @EXPORT, qw(testRunGet); - -#################################################################################################################################### -# storageTest - get the storage for the current test -#################################################################################################################################### -sub storageTest -{ - return $oStorage; -} - -push(@EXPORT, qw(storageTest)); - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub archBits {return vmArchBits(shift->{strVm})} -sub backrestExe {return shift->{strBackRestExe}} -sub backrestExeHelper {return shift->{strBackRestExeHelper}} -sub basePath {return shift->{strBasePath}} -sub dataPath {return shift->basePath() . '/test/data'} -sub doCleanup {return shift->{bCleanup}} -sub logLevelTestFile {return shift->{strLogLevelTestFile}} -sub group {return shift->{strGroup}} -sub isDryRun {return shift->{bDryRun}} -sub module {return shift->{strModule}} -sub moduleTest {return shift->{strModuleTest}} -sub pgBinPath {return shift->{strPgBinPath}} -sub pgUser {return shift->{strPgUser}} -sub pgVersion {return shift->{strPgVersion}} -sub runCurrent {return shift->{iRun}} -sub stanza {return 'db'} -sub testPath {return shift->{strTestPath}} -sub vm {return shift->{strVm}} -sub vmId {return shift->{iVmId}} - -1; diff --git a/test/lib/pgBackRestTest/Common/StorageRepo.pm b/test/lib/pgBackRestTest/Common/StorageRepo.pm deleted file mode 100644 index 7469342d2d..0000000000 --- a/test/lib/pgBackRestTest/Common/StorageRepo.pm +++ /dev/null @@ -1,623 +0,0 @@ -#################################################################################################################################### -# C Storage Interface -#################################################################################################################################### -package pgBackRestTest::Common::StorageRepo; -use parent 'pgBackRestTest::Common::StorageBase'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Digest::SHA qw(sha1_hex); -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); -use Fcntl qw(:mode); -use File::stat qw{lstat}; -use JSON::PP; - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::Io::Handle; -use pgBackRestTest::Common::Io::Process; -use pgBackRestTest::Common::StorageBase; - -#################################################################################################################################### -# Temp file extension -#################################################################################################################################### -use constant STORAGE_TEMP_EXT => PROJECT_EXE . '.tmp'; - push @EXPORT, qw(STORAGE_TEMP_EXT); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Create the class hash - my $self = {}; - bless $self, $class; - - # Assign function parameters, defaults, and log debug info - ( - my $strOperation, - $self->{strCommand}, - $self->{strType}, - $self->{lBufferMax}, - $self->{iTimeoutIo}, - $self->{iRepo}, - $self->{strDefaultPathMode}, - $self->{strDefaultFileMode}, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'strCommand'}, - {name => 'strType'}, - {name => 'lBufferMax'}, - {name => 'iTimeoutIo'}, - {name => 'iRepo'}, - {name => 'strDefaultPathMode', optional => true, default => '0750'}, - {name => 'strDefaultFileMode', optional => true, default => '0640'}, - ); - - # Create JSON object - $self->{oJSON} = JSON::PP->new()->allow_nonref(); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# Escape characteres that have special meaning on the command line -#################################################################################################################################### -sub escape -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strValue, - ) = - logDebugParam - ( - __PACKAGE__ . '->escape', \@_, - {name => 'strValue', trace => true}, - ); - - $strValue =~ s/\\/\\\\/g; - $strValue =~ s/\/\\\>/g; - $strValue =~ s/\!/\\\!/g; - $strValue =~ s/\*/\\\*/g; - $strValue =~ s/\(/\\\(/g; - $strValue =~ s/\)/\\\)/g; - $strValue =~ s/\&/\\\&/g; - $strValue =~ s/\'/\\\'/g; - $strValue =~ s/\;/\\\;/g; - $strValue =~ s/\?/\\\?/g; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strValue', value => $strValue}, - ); -} - -#################################################################################################################################### -# Execute command and return the output -#################################################################################################################################### -sub exec -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strCommand, - ) = - logDebugParam - ( - __PACKAGE__ . '->exec', \@_, - {name => 'strCommand'}, - ); - - $strCommand = "$self->{strCommand} ${strCommand}"; - my $oBuffer = new pgBackRestTest::Common::Io::Buffered( - new pgBackRestTest::Common::Io::Handle($strCommand), $self->{iTimeoutIo}, $self->{lBufferMax}); - my $oProcess = new pgBackRestTest::Common::Io::Process($oBuffer, $strCommand); - - my $tResult; - - while (!$oBuffer->eof()) - { - $oBuffer->read(\$tResult, $self->{lBufferMax}, false); - } - - $oProcess->close(); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'tResult', value => $tResult}, - {name => 'iExitStatus', value => $oProcess->exitStatus()}, - ); -} - -#################################################################################################################################### -# Create storage -#################################################################################################################################### -sub create -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '->create'); - - $self->exec("--repo=$self->{iRepo} repo-create"); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# Check if file exists (not a path) -#################################################################################################################################### -sub exists -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFileExp, - ) = - logDebugParam - ( - __PACKAGE__ . '->exists', \@_, - {name => 'strFileExp'}, - ); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bExists', value => $self->info($strFileExp, {bIgnoreMissing => true})->{type} eq 'f'} - ); -} - -#################################################################################################################################### -# Read a buffer from storage all at once -#################################################################################################################################### -sub get -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $xFile, - $strCipherPass, - $bRaw, - ) = - logDebugParam - ( - __PACKAGE__ . '->get', \@_, - {name => 'xFile', required => false}, - {name => 'strCipherPass', optional => true, redact => true}, - {name => 'bRaw', optional => true, default => false}, - ); - - # If openRead() was called first set values from that call - my $strFile = $xFile; - my $bIgnoreMissing = false; - - if (ref($xFile)) - { - $strFile = $xFile->{strFile}; - $bIgnoreMissing = $xFile->{bIgnoreMissing}; - $strCipherPass = $xFile->{strCipherPass}; - } - - # Check invalid params - if ($bRaw && defined($strCipherPass)) - { - confess &log(ERROR, 'bRaw and strCipherPass cannot both be set'); - } - - # Get file - my ($tResult, $iExitStatus) = $self->exec( - (defined($strCipherPass) ? ' --cipher-pass=' . $self->escape($strCipherPass) : '') . ($bRaw ? ' --raw' : '') . - ($bIgnoreMissing ? ' --ignore-missing' : '') . " --repo=$self->{iRepo} repo-get " . $self->escape($strFile)); - - # Error if missing an not ignored - if ($iExitStatus == 1 && !$bIgnoreMissing) - { - confess &log(ERROR, "unable to open '${strFile}'", ERROR_FILE_OPEN); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'rtContent', value => $iExitStatus == 0 ? \$tResult : undef, trace => true}, - ); -} - -#################################################################################################################################### -# Get information for path/file -#################################################################################################################################### -sub info -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPathFileExp, - $bIgnoreMissing, - ) = - logDebugParam - ( - __PACKAGE__ . '->info', \@_, - {name => 'strPathFileExp'}, - {name => 'bIgnoreMissing', optional => true, default => false}, - ); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'rhInfo', value => $self->manifest($strPathFileExp, {bRecurse => false})->{'.'}, trace => true} - ); -} - -#################################################################################################################################### -# List all files/paths in path -#################################################################################################################################### -sub list -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPathExp, - $strExpression, - $strSortOrder, - $bIgnoreMissing, - ) = - logDebugParam - ( - __PACKAGE__ . '->list', \@_, - {name => 'strPathExp', required => false}, - {name => 'strExpression', optional => true}, - {name => 'strSortOrder', optional => true, default => 'forward'}, - {name => 'bIgnoreMissing', optional => true, default => false}, - ); - - # Get file list - my $rstryFileList = []; - my $rhManifest = $self->manifest($strPathExp, {bRecurse => false}); - - foreach my $strKey ($strSortOrder eq 'reverse' ? sort {$b cmp $a} keys(%{$rhManifest}) : sort keys(%{$rhManifest})) - { - next if $strKey eq '.'; - next if defined($strExpression) && $strKey !~ $strExpression; - - push(@{$rstryFileList}, $strKey); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'stryFileList', value => $rstryFileList} - ); -} - -#################################################################################################################################### -# Build path/file/link manifest starting with base path and including all subpaths -#################################################################################################################################### -sub manifest -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPathExp, - $bRecurse, - ) = - logDebugParam - ( - __PACKAGE__ . '->manifest', \@_, - {name => 'strPathExp'}, - {name => 'bRecurse', optional => true, default => true}, - ); - - my $rhManifest = $self->{oJSON}->decode( - $self->exec( - "--output=json" . ($bRecurse ? ' --recurse' : '') . " --repo=$self->{iRepo} repo-ls " . $self->escape($strPathExp))); - - # Transform the manifest to the old format - foreach my $strKey (keys(%{$rhManifest})) - { - if ($rhManifest->{$strKey}{type} eq 'file') - { - $rhManifest->{$strKey}{type} = 'f'; - - if (defined($rhManifest->{$strKey}{time})) - { - $rhManifest->{$strKey}{modified_time} = $rhManifest->{$strKey}{time}; - delete($rhManifest->{$strKey}{time}); - } - } - elsif ($rhManifest->{$strKey}{type} eq 'path') - { - $rhManifest->{$strKey}{type} = 'd'; - } - elsif ($rhManifest->{$strKey}{type} eq 'link') - { - $rhManifest->{$strKey}{type} = 'l'; - } - elsif ($rhManifest->{$strKey}{type} eq 'special') - { - $rhManifest->{$strKey}{type} = 's'; - } - else - { - confess "invalid file type '$rhManifest->{type}'"; - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'rhManifest', value => $rhManifest, trace => true} - ); -} - -#################################################################################################################################### -# Open file for reading -#################################################################################################################################### -sub openRead -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFile, - $bIgnoreMissing, - $strCipherPass, - ) = - logDebugParam - ( - __PACKAGE__ . '->openRead', \@_, - {name => 'strFile'}, - {name => 'bIgnoreMissing', optional => true, default => false}, - {name => 'strCipherPass', optional => true, redact => true}, - ); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'rhFileIo', value => {strFile => $strFile, bIgnoreMissing => $bIgnoreMissing, strCipherPass => $strCipherPass}, - trace => true}, - ); -} - -#################################################################################################################################### -# Remove path and all files below it -#################################################################################################################################### -sub pathRemove -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPath, - $bRecurse, - ) = - logDebugParam - ( - __PACKAGE__ . '->pathRemove', \@_, - {name => 'strPath'}, - {name => 'bRecurse', optional => true, default => false}, - ); - - $self->exec("--repo=$self->{iRepo} repo-rm " . ($bRecurse ? '--recurse ' : '') . $self->escape($strPath)); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# put - writes a buffer out to storage all at once -#################################################################################################################################### -sub put -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFile, - $tContent, - $strCipherPass, - $bRaw, - ) = - logDebugParam - ( - __PACKAGE__ . '->put', \@_, - {name => 'strFile'}, - {name => 'tContent', required => false}, - {name => 'strCipherPass', optional => true, redact => true}, - {name => 'bRaw', optional => true, default => false}, - ); - - # Check invalid params - if ($bRaw && defined($strCipherPass)) - { - confess &log(ERROR, 'bRaw and strCipherPass cannot both be set'); - } - - # Put file - my $strCommand = - "$self->{strCommand}" . (defined($strCipherPass) ? ' --cipher-pass=' . $self->escape($strCipherPass) : '') . - ($bRaw ? ' --raw' : '') . " --repo=$self->{iRepo} repo-put " . $self->escape($strFile); - - my $oBuffer = new pgBackRestTest::Common::Io::Buffered( - new pgBackRestTest::Common::Io::Handle($strCommand), $self->{iTimeoutIo}, $self->{lBufferMax}); - my $oProcess = new pgBackRestTest::Common::Io::Process($oBuffer, $strCommand); - - if (defined($tContent)) - { - $oBuffer->write(\$tContent); - } - - close($oBuffer->handleWrite()); - - my $tResult; - - while (!$oBuffer->eof()) - { - $oBuffer->read(\$tResult, $self->{lBufferMax}, false); - } - - close($oBuffer->handleRead()); - $oProcess->close(); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# Remove file -#################################################################################################################################### -sub remove -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFile, - ) = - logDebugParam - ( - __PACKAGE__ . '->remove', \@_, - {name => 'xFileExp'}, - ); - - $self->exec("--repo=$self->{iRepo} repo-rm " . $self->escape($strFile)); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# Cache storage so it can be retrieved quickly -#################################################################################################################################### -my $oRepoStorage; - -#################################################################################################################################### -# storageRepoCommandSet -#################################################################################################################################### -my $strStorageRepoCommand; -my $strStorageRepoType; - -sub storageRepoCommandSet -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strCommand, - $strStorageType, - ) = - logDebugParam - ( - __PACKAGE__ . '::storageRepoCommandSet', \@_, - {name => 'strCommand'}, - {name => 'strStorageType'}, - ); - - $strStorageRepoCommand = $strCommand; - $strStorageRepoType = $strStorageType; - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -push @EXPORT, qw(storageRepoCommandSet); - -#################################################################################################################################### -# storageRepo - get repository storage -#################################################################################################################################### -sub storageRepo -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strStanza, - $iRepo, - ) = - logDebugParam - ( - __PACKAGE__ . '::storageRepo', \@_, - {name => 'strStanza', optional => true, trace => true}, - {name => 'iRepo', optional => true, default => 1, trace => true}, - ); - - # Create storage if not defined - if (!defined($oRepoStorage->{$iRepo})) - { - $oRepoStorage->{$iRepo} = new pgBackRestTest::Common::StorageRepo( - $strStorageRepoCommand, $strStorageRepoType, 64 * 1024, 60, $iRepo); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'oStorageRepo', value => $oRepoStorage->{$iRepo}, trace => true}, - ); -} - -push @EXPORT, qw(storageRepo); - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub capability {shift->type() eq STORAGE_POSIX} -sub type {shift->{strType}} - -1; diff --git a/test/lib/pgBackRestTest/Env/ArchiveInfo.pm b/test/lib/pgBackRestTest/Env/ArchiveInfo.pm deleted file mode 100644 index 42257b9ea7..0000000000 --- a/test/lib/pgBackRestTest/Env/ArchiveInfo.pm +++ /dev/null @@ -1,478 +0,0 @@ -#################################################################################################################################### -# ARCHIVE INFO MODULE -# -# The archive.info file is created when archiving begins. It is located under the stanza directory. The file contains information -# regarding the stanza database version, database WAL segment system id and other information to ensure that archiving is being -# performed on the proper database. -#################################################################################################################################### -package pgBackRestTest::Env::ArchiveInfo; -use parent 'pgBackRestDoc::Common::Ini'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname basename); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; - -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::StorageBase; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Env::InfoCommon; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# File/path constants -#################################################################################################################################### -use constant ARCHIVE_INFO_FILE => 'archive.info'; - push @EXPORT, qw(ARCHIVE_INFO_FILE); - -#################################################################################################################################### -# RegEx constants -#################################################################################################################################### -use constant REGEX_ARCHIVE_DIR_DB_VERSION => '^[0-9]+(\.[0-9]+)*-[0-9]+$'; - push @EXPORT, qw(REGEX_ARCHIVE_DIR_DB_VERSION); -use constant REGEX_ARCHIVE_DIR_WAL => '^[0-F]{16}$'; - push @EXPORT, qw(REGEX_ARCHIVE_DIR_WAL); - -#################################################################################################################################### -# WAL segment size -#################################################################################################################################### -use constant PG_WAL_SEGMENT_SIZE => 16777216; - push @EXPORT, qw(PG_WAL_SEGMENT_SIZE); - -#################################################################################################################################### -# Archive info constants -#################################################################################################################################### -use constant INFO_ARCHIVE_SECTION_DB => INFO_BACKUP_SECTION_DB; - push @EXPORT, qw(INFO_ARCHIVE_SECTION_DB); -use constant INFO_ARCHIVE_SECTION_DB_HISTORY => INFO_BACKUP_SECTION_DB_HISTORY; - push @EXPORT, qw(INFO_ARCHIVE_SECTION_DB_HISTORY); - -use constant INFO_ARCHIVE_KEY_DB_VERSION => MANIFEST_KEY_DB_VERSION; - push @EXPORT, qw(INFO_ARCHIVE_KEY_DB_VERSION); -use constant INFO_ARCHIVE_KEY_DB_ID => MANIFEST_KEY_DB_ID; - push @EXPORT, qw(INFO_ARCHIVE_KEY_DB_ID); -use constant INFO_ARCHIVE_KEY_DB_SYSTEM_ID => MANIFEST_KEY_SYSTEM_ID; - push @EXPORT, qw(INFO_ARCHIVE_KEY_DB_SYSTEM_ID); - -#################################################################################################################################### -# Global variables -#################################################################################################################################### -my $strArchiveInfoMissingMsg = - ARCHIVE_INFO_FILE . " does not exist but is required to push/get WAL segments\n" . - "HINT: is archive_command configured in postgresql.conf?\n" . - "HINT: has a stanza-create been performed?\n" . - "HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme."; - -#################################################################################################################################### -# CONSTRUCTOR -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strArchiveClusterPath, # Archive cluster path - $bRequired, # Is archive info required? - $bLoad, # Should the file attempt to be loaded? - $bIgnoreMissing, # Don't error on missing files - $strCipherPassSub, # Passphrase to encrypt the subsequent archive files if repo is encrypted - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'strArchiveClusterPath'}, - {name => 'bRequired', default => true}, - {name => 'bLoad', optional => true, default => true}, - {name => 'bIgnoreMissing', optional => true, default => false}, - {name => 'strCipherPassSub', optional => true}, - ); - - # Build the archive info path/file name - my $strArchiveInfoFile = "${strArchiveClusterPath}/" . ARCHIVE_INFO_FILE; - my $self = {}; - my $iResult = 0; - my $strResultMessage; - - # Init object and store variables - eval - { - $self = $class->SUPER::new( - storageRepo(), $strArchiveInfoFile, - {bLoad => $bLoad, bIgnoreMissing => $bIgnoreMissing, strCipherPassSub => $strCipherPassSub}); - return true; - } - or do - { - # Capture error information - $iResult = exceptionCode($EVAL_ERROR); - $strResultMessage = exceptionMessage($EVAL_ERROR); - }; - - if ($iResult != 0) - { - # If the file does not exist but is required to exist, then error - # The archive info is only allowed not to exist when running a stanza-create on a new install - if ($iResult == ERROR_FILE_MISSING) - { - if ($bRequired) - { - confess &log(ERROR, $strArchiveInfoMissingMsg, ERROR_FILE_MISSING); - } - } - elsif ($iResult == ERROR_CRYPTO && $strResultMessage =~ "^unable to flush") - { - confess &log(ERROR, "unable to parse '$strArchiveInfoFile'\nHINT: is or was the repo encrypted?", $iResult); - } - else - { - confess $EVAL_ERROR; - } - } - - $self->{strArchiveClusterPath} = $strArchiveClusterPath; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# check -# -# Check archive info file and make sure it is compatible with the current version of the database for the stanza. If the file does -# not exist an error will occur. -#################################################################################################################################### -sub check -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $ullDbSysId, - $bRequired, - ) = - logDebugParam - ( - __PACKAGE__ . '->check', \@_, - {name => 'strDbVersion'}, - {name => 'ullDbSysId'}, - {name => 'bRequired', default => true}, - ); - - # ??? remove bRequired after stanza-upgrade - if ($bRequired) - { - # Confirm the info file exists with the DB section - $self->confirmExists(); - } - - my $strError = undef; - - if (!$self->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef, $strDbVersion)) - { - $strError = "WAL segment version ${strDbVersion} does not match archive version " . - $self->get(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION); - } - - if (!$self->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_SYSTEM_ID, undef, $ullDbSysId)) - { - $strError = (defined($strError) ? ($strError . "\n") : "") . - "WAL segment system-id ${ullDbSysId} does not match archive system-id " . - $self->get(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_SYSTEM_ID); - } - - if (defined($strError)) - { - confess &log(ERROR, "${strError}\nHINT: are you archiving to the correct stanza?", ERROR_ARCHIVE_MISMATCH); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strArchiveId', value => $self->archiveId()} - ); -} - -#################################################################################################################################### -# archiveId -# -# Get the archive id which is a combination of the DB version and the db-id setting (e.g. 9.4-1) -#################################################################################################################################### -sub archiveId -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $ullDbSysId, - ) = logDebugParam - ( - __PACKAGE__ . '->archiveId', \@_, - {name => 'strDbVersion', optional => true}, - {name => 'ullDbSysId', optional => true}, - ); - - my $strArchiveId = undef; - - # If neither optional version and system-id are passed then set the archive id to the current one - if (!defined($strDbVersion) && !defined($ullDbSysId)) - { - $strArchiveId = $self->get(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION) . "-" . - $self->get(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_ID); - } - # If both the optional version and system-id are passed - elsif (defined($strDbVersion) && defined($ullDbSysId)) - { - # Get the newest archiveId for the version/system-id passed - $strArchiveId = ($self->archiveIdList($strDbVersion, $ullDbSysId))[0]; - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strArchiveId', value => $strArchiveId} - ); -} - -#################################################################################################################################### -# archiveIdList -# -# Get a sorted list of the archive ids for the db-version and db-system-id passed. -#################################################################################################################################### -sub archiveIdList -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $ullDbSysId, - ) = logDebugParam - ( - __PACKAGE__ . '->archiveIdList', \@_, - {name => 'strDbVersion'}, - {name => 'ullDbSysId'}, - ); - - my @stryArchiveId; - - # Get the version and system-id for all known databases - my $hDbList = $self->dbHistoryList(); - - foreach my $iDbHistoryId (sort {$a <=> $b} keys %$hDbList) - { - # If the version and system-id match then construct the archive id so that the constructed array has the newest match first - if (($hDbList->{$iDbHistoryId}{&INFO_DB_VERSION} eq $strDbVersion) && - ($hDbList->{$iDbHistoryId}{&INFO_SYSTEM_ID} eq $ullDbSysId)) - { - unshift(@stryArchiveId, $strDbVersion . "-" . $iDbHistoryId); - } - } - - # If the archive id has still not been found, then error - if (@stryArchiveId == 0) - { - confess &log( - ERROR, "unable to retrieve the archive id for database version '$strDbVersion' and system-id '$ullDbSysId'", - ERROR_ARCHIVE_MISMATCH); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'stryArchiveId', value => \@stryArchiveId} - ); -} - -#################################################################################################################################### -# create -# -# Creates the archive.info file. WARNING - this function should only be called from stanza-create or tests. -#################################################################################################################################### -sub create -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $ullDbSysId, - $bSave, - ) = - logDebugParam - ( - __PACKAGE__ . '->create', \@_, - {name => 'strDbVersion'}, - {name => 'ullDbSysId'}, - {name => 'bSave', default => true}, - ); - - # Fill db section and db history section - $self->dbSectionSet($strDbVersion, $ullDbSysId, $self->dbHistoryIdGet(false)); - - if ($bSave) - { - $self->save(); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# dbHistoryIdGet -# -# Get the db history ID -#################################################################################################################################### -sub dbHistoryIdGet -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $bFileRequired, - ) = - logDebugParam - ( - __PACKAGE__ . '->dbHistoryIdGet', \@_, - {name => 'bFileRequired', default => true}, - ); - - # Confirm the info file exists if it is required - if ($bFileRequired) - { - $self->confirmExists(); - } - - # If the DB section does not exist, initialize the history to one, else return the latest ID - my $iDbHistoryId = (!$self->test(INFO_ARCHIVE_SECTION_DB)) - ? 1 : $self->numericGet(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_ID); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'iDbHistoryId', value => $iDbHistoryId} - ); -} - -#################################################################################################################################### -# dbHistoryList -# -# Get the data from the db history section. -#################################################################################################################################### -sub dbHistoryList -{ - my $self = shift; - my - ( - $strOperation, - ) = logDebugParam - ( - __PACKAGE__ . '->dbHistoryList', - ); - - my %hDbHash; - - foreach my $iHistoryId ($self->keys(INFO_ARCHIVE_SECTION_DB_HISTORY)) - { - $hDbHash{$iHistoryId}{&INFO_DB_VERSION} = - $self->get(INFO_ARCHIVE_SECTION_DB_HISTORY, $iHistoryId, INFO_ARCHIVE_KEY_DB_VERSION); - $hDbHash{$iHistoryId}{&INFO_SYSTEM_ID} = - $self->get(INFO_ARCHIVE_SECTION_DB_HISTORY, $iHistoryId, INFO_ARCHIVE_KEY_DB_ID); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'hDbHash', value => \%hDbHash} - ); -} - -#################################################################################################################################### -# dbSectionSet -# -# Set the db and db:history sections. -#################################################################################################################################### -sub dbSectionSet -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $ullDbSysId, - $iDbHistoryId, - ) = - logDebugParam - ( - __PACKAGE__ . '->dbSectionSet', \@_, - {name => 'strDbVersion', trace => true}, - {name => 'ullDbSysId', trace => true}, - {name => 'iDbHistoryId', trace => true} - ); - - # Fill db section - $self->numericSet(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_SYSTEM_ID, undef, $ullDbSysId); - # Force the version to a string since newer versions of JSON::PP lose track of the fact that it is one - $self->set(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef, $strDbVersion . ''); - $self->numericSet(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_ID, undef, $iDbHistoryId); - - # Fill db history - $self->numericSet(INFO_ARCHIVE_SECTION_DB_HISTORY, $iDbHistoryId, INFO_ARCHIVE_KEY_DB_ID, $ullDbSysId); - # Force the version to a string since newer versions of JSON::PP lose track of the fact that it is one - $self->set(INFO_ARCHIVE_SECTION_DB_HISTORY, $iDbHistoryId, INFO_ARCHIVE_KEY_DB_VERSION, $strDbVersion . ''); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# confirmExists -# -# Ensure that the archive.info file and the db section exist. -#################################################################################################################################### -sub confirmExists -{ - my $self = shift; - - # Confirm the file exists and the DB section is filled out - if (!$self->test(INFO_ARCHIVE_SECTION_DB) || !$self->{bExists}) - { - confess &log(ERROR, $strArchiveInfoMissingMsg, ERROR_FILE_MISSING); - } -} - -1; diff --git a/test/lib/pgBackRestTest/Env/BackupInfo.pm b/test/lib/pgBackRestTest/Env/BackupInfo.pm deleted file mode 100644 index 3b8b4061ac..0000000000 --- a/test/lib/pgBackRestTest/Env/BackupInfo.pm +++ /dev/null @@ -1,898 +0,0 @@ -#################################################################################################################################### -# BACKUP INFO MODULE -#################################################################################################################################### -package pgBackRestTest::Env::BackupInfo; -use parent 'pgBackRestDoc::Common::Ini'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname basename); -use File::stat; - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; - -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Env::ArchiveInfo; -use pgBackRestTest::Env::InfoCommon; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# Backup type constants -#################################################################################################################################### -use constant CFGOPTVAL_BACKUP_TYPE_FULL => 'full'; - push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_FULL); -use constant CFGOPTVAL_BACKUP_TYPE_DIFF => 'diff'; - push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_DIFF); -use constant CFGOPTVAL_BACKUP_TYPE_INCR => 'incr'; - push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_INCR); - -#################################################################################################################################### -# File/path constants -#################################################################################################################################### -use constant FILE_BACKUP_INFO => 'backup.info'; - push @EXPORT, qw(FILE_BACKUP_INFO); - -#################################################################################################################################### -# Backup info constants -#################################################################################################################################### -use constant INFO_BACKUP_SECTION_BACKUP => MANIFEST_SECTION_BACKUP; - push @EXPORT, qw(INFO_BACKUP_SECTION_BACKUP); -use constant INFO_BACKUP_SECTION_BACKUP_CURRENT => INFO_BACKUP_SECTION_BACKUP . ':current'; - push @EXPORT, qw(INFO_BACKUP_SECTION_BACKUP_CURRENT); - -use constant INFO_BACKUP_KEY_ARCHIVE_CHECK => MANIFEST_KEY_ARCHIVE_CHECK; - push @EXPORT, qw(INFO_BACKUP_KEY_ARCHIVE_CHECK); -use constant INFO_BACKUP_KEY_ARCHIVE_COPY => MANIFEST_KEY_ARCHIVE_COPY; - push @EXPORT, qw(INFO_BACKUP_KEY_ARCHIVE_COPY); -use constant INFO_BACKUP_KEY_ARCHIVE_START => MANIFEST_KEY_ARCHIVE_START; - push @EXPORT, qw(INFO_BACKUP_KEY_ARCHIVE_START); -use constant INFO_BACKUP_KEY_ARCHIVE_STOP => MANIFEST_KEY_ARCHIVE_STOP; - push @EXPORT, qw(INFO_BACKUP_KEY_ARCHIVE_STOP); -use constant INFO_BACKUP_KEY_BACKUP_STANDBY => MANIFEST_KEY_BACKUP_STANDBY; - push @EXPORT, qw(INFO_BACKUP_KEY_BACKUP_STANDBY); -use constant INFO_BACKUP_KEY_BACKUP_REPO_SIZE => 'backup-info-repo-size'; - push @EXPORT, qw(INFO_BACKUP_KEY_BACKUP_REPO_SIZE); -use constant INFO_BACKUP_KEY_BACKUP_REPO_SIZE_DELTA => 'backup-info-repo-size-delta'; - push @EXPORT, qw(INFO_BACKUP_KEY_BACKUP_REPO_SIZE_DELTA); -use constant INFO_BACKUP_KEY_BACKUP_SIZE => 'backup-info-size'; - push @EXPORT, qw(INFO_BACKUP_KEY_BACKUP_SIZE); -use constant INFO_BACKUP_KEY_BACKUP_SIZE_DELTA => 'backup-info-size-delta'; - push @EXPORT, qw(INFO_BACKUP_KEY_BACKUP_SIZE_DELTA); -use constant INFO_BACKUP_KEY_CATALOG => MANIFEST_KEY_CATALOG; - push @EXPORT, qw(INFO_BACKUP_KEY_CATALOG); -use constant INFO_BACKUP_KEY_CONTROL => MANIFEST_KEY_CONTROL; - push @EXPORT, qw(INFO_BACKUP_KEY_CONTROL); -use constant INFO_BACKUP_KEY_COMPRESS => MANIFEST_KEY_COMPRESS; - push @EXPORT, qw(INFO_BACKUP_KEY_COMPRESS); -use constant INFO_BACKUP_KEY_CHECKSUM_PAGE => MANIFEST_KEY_CHECKSUM_PAGE; - push @EXPORT, qw(INFO_BACKUP_KEY_CHECKSUM_PAGE); -use constant INFO_BACKUP_KEY_DB_VERSION => MANIFEST_KEY_DB_VERSION; - push @EXPORT, qw(INFO_BACKUP_KEY_DB_VERSION); -use constant INFO_BACKUP_KEY_FORMAT => INI_KEY_FORMAT; - push @EXPORT, qw(INFO_BACKUP_KEY_FORMAT); -use constant INFO_BACKUP_KEY_HARDLINK => MANIFEST_KEY_HARDLINK; - push @EXPORT, qw(INFO_BACKUP_KEY_HARDLINK); -use constant INFO_BACKUP_KEY_HISTORY_ID => MANIFEST_KEY_DB_ID; - push @EXPORT, qw(INFO_BACKUP_KEY_HISTORY_ID); -use constant INFO_BACKUP_KEY_LABEL => MANIFEST_KEY_LABEL; - push @EXPORT, qw(INFO_BACKUP_KEY_LABEL); -use constant INFO_BACKUP_KEY_PRIOR => MANIFEST_KEY_PRIOR; - push @EXPORT, qw(INFO_BACKUP_KEY_PRIOR); -use constant INFO_BACKUP_KEY_REFERENCE => 'backup-reference'; - push @EXPORT, qw(INFO_BACKUP_KEY_REFERENCE); -use constant INFO_BACKUP_KEY_ONLINE => MANIFEST_KEY_ONLINE; - push @EXPORT, qw(INFO_BACKUP_KEY_ONLINE); -use constant INFO_BACKUP_KEY_SYSTEM_ID => MANIFEST_KEY_SYSTEM_ID; - push @EXPORT, qw(INFO_BACKUP_KEY_SYSTEM_ID); -use constant INFO_BACKUP_KEY_TIMESTAMP_START => MANIFEST_KEY_TIMESTAMP_START; - push @EXPORT, qw(INFO_BACKUP_KEY_TIMESTAMP_START); -use constant INFO_BACKUP_KEY_TIMESTAMP_STOP => MANIFEST_KEY_TIMESTAMP_STOP; - push @EXPORT, qw(INFO_BACKUP_KEY_TIMESTAMP_STOP); -use constant INFO_BACKUP_KEY_TYPE => MANIFEST_KEY_TYPE; - push @EXPORT, qw(INFO_BACKUP_KEY_TYPE); -use constant INFO_BACKUP_KEY_VERSION => INI_KEY_VERSION; - push @EXPORT, qw(INFO_BACKUP_KEY_VERSION); - -#################################################################################################################################### -# Global variables -#################################################################################################################################### -my $strBackupInfoMissingMsg = - FILE_BACKUP_INFO . " does not exist and is required to perform a backup.\n" . - "HINT: has a stanza-create been performed?"; - -#################################################################################################################################### -# CONSTRUCTOR -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strBackupClusterPath, - $bRequired, - $oStorage, - $bLoad, # Should the file attemp to be loaded? - $bIgnoreMissing, # Don't error on missing files - $strCipherPassSub, # Generated passphrase to encrypt manifest files if the repo is encrypted - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'strBackupClusterPath'}, - {name => 'bRequired', default => true}, - {name => 'oStorage', optional => true, default => storageRepo()}, - {name => 'bLoad', optional => true, default => true}, - {name => 'bIgnoreMissing', optional => true, default => false}, - {name => 'strCipherPassSub', optional => true}, - ); - - # Build the backup info path/file name - my $strBackupInfoFile = "${strBackupClusterPath}/" . FILE_BACKUP_INFO; - my $self = {}; - my $iResult = 0; - my $strResultMessage; - - # Init object and store variables - eval - { - $self = $class->SUPER::new( - $oStorage, $strBackupInfoFile, - {bLoad => $bLoad, bIgnoreMissing => $bIgnoreMissing, strCipherPassSub => $strCipherPassSub}); - return true; - } - or do - { - # Capture error information - $iResult = exceptionCode($EVAL_ERROR); - $strResultMessage = exceptionMessage($EVAL_ERROR); - }; - - if ($iResult != 0) - { - # If the backup info file does not exist and is required, then throw an error - # The backup info is only allowed not to exist when running a stanza-create on a new install - if ($iResult == ERROR_FILE_MISSING) - { - if ($bRequired) - { - confess &log(ERROR, "${strBackupClusterPath}/$strBackupInfoMissingMsg", ERROR_FILE_MISSING); - } - } - elsif ($iResult == ERROR_CRYPTO && $strResultMessage =~ "^unable to flush") - { - confess &log(ERROR, "unable to parse '$strBackupInfoFile'\nHINT: is or was the repo encrypted?", $iResult); - } - else - { - confess $EVAL_ERROR; - } - } - - $self->{strBackupClusterPath} = $strBackupClusterPath; - $self->{oStorage} = $oStorage; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# check -# -# Check db info and make sure it matches what is already in the repository. Return the db-id if everything matches. -#################################################################################################################################### -sub check -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $iControlVersion, - $iCatalogVersion, - $ullDbSysId, - $bRequired, - ) = - logDebugParam - ( - __PACKAGE__ . '->check', \@_, - {name => 'strDbVersion', trace => true}, - {name => 'iControlVersion', trace => true}, - {name => 'iCatalogVersion', trace => true}, - {name => 'ullDbSysId', trace => true}, - {name => 'bRequired', default => true}, - ); - - # Confirm the info file exists with the DB section - if ($bRequired) - { - $self->confirmExists(); - } - - if (!$self->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_SYSTEM_ID, undef, $ullDbSysId) || - !$self->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION, undef, $strDbVersion)) - { - confess &log(ERROR, "database version = ${strDbVersion}, system-id ${ullDbSysId} does not match backup version = " . - $self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION) . ", system-id = " . - $self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_SYSTEM_ID) . "\n" . - "HINT: is this the correct stanza?", ERROR_BACKUP_MISMATCH); - } - - if (!$self->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CATALOG, undef, $iCatalogVersion) || - !$self->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CONTROL, undef, $iControlVersion)) - { - confess &log(ERROR, "database control-version = ${iControlVersion}, catalog-version ${iCatalogVersion}" . - " does not match backup control-version = " . - $self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CONTROL) . ", catalog-version = " . - $self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CATALOG) . "\n" . - "HINT: this may be a symptom of database or repository corruption!", ERROR_BACKUP_MISMATCH); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'iDbHistoryId', value => $self->numericGet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_HISTORY_ID)} - ); -} - -#################################################################################################################################### -# add -# -# Add a backup to the info file. -#################################################################################################################################### -sub add -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oBackupManifest, - $bSave, - $bRequired, - ) = - logDebugParam - ( - __PACKAGE__ . '->add', \@_, - {name => 'oBackupManifest', trace => true}, - {name => 'bSave', default => true, trace => true}, - {name => 'bRequired', default => true, trace => true}, - ); - - # Confirm the info file exists with the DB section - if ($bRequired) - { - $self->confirmExists(); - } - - # Get the backup label - my $strBackupLabel = $oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL); - - # Calculate backup sizes and references - my $lBackupSize = 0; - my $lBackupSizeDelta = 0; - my $lBackupRepoSize = 0; - my $lBackupRepoSizeDelta = 0; - my $oReferenceHash = undef; - - foreach my $strFileKey ($oBackupManifest->keys(MANIFEST_SECTION_TARGET_FILE)) - { - my $lFileSize = - $oBackupManifest->get(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_SIZE); - my $lRepoSize = - $oBackupManifest->get(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE, false, $lFileSize); - my $strFileReference = - $oBackupManifest->get(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REFERENCE, false); - - # Temporary until compressed size is back in - $lBackupSize += $lFileSize; - $lBackupRepoSize += $lRepoSize; - - if (defined($strFileReference)) - { - $$oReferenceHash{$strFileReference} = true; - } - else - { - $lBackupSizeDelta += $lFileSize; - $lBackupRepoSizeDelta += $lRepoSize; - } - } - - # Set backup size info - $self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_BACKUP_SIZE, $lBackupSize); - $self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_BACKUP_SIZE_DELTA, $lBackupSizeDelta); - $self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_BACKUP_REPO_SIZE, $lBackupRepoSize); - $self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_BACKUP_REPO_SIZE_DELTA, - $lBackupRepoSizeDelta); - - $self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_ARCHIVE_CHECK, - $oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_CHECK)); - $self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_ARCHIVE_COPY, - $oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_COPY)); - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_ARCHIVE_START, - $oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_START, undef, false)); - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_ARCHIVE_STOP, - $oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_STOP, undef, false)); - $self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_BACKUP_STANDBY, - $oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_BACKUP_STANDBY)); - $self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_CHECKSUM_PAGE, - $oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_CHECKSUM_PAGE)); - $self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_COMPRESS, - $oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS)); - $self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_FORMAT, - $oBackupManifest->numericGet(INI_SECTION_BACKREST, INI_KEY_FORMAT)); - $self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_HARDLINK, - $oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_HARDLINK)); - $self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_ONLINE, - $oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ONLINE)); - $self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_TIMESTAMP_START, - $oBackupManifest->numericGet(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_START)); - $self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_TIMESTAMP_STOP, - $oBackupManifest->numericGet(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP)); - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_TYPE, - $oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE)); - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_VERSION, - $oBackupManifest->get(INI_SECTION_BACKREST, INI_KEY_VERSION)); - - if ($bRequired) - { - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_HISTORY_ID, - $self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_HISTORY_ID)); - } - # If we are reconstructing, then the history id must be taken from the manifest - else - { - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_HISTORY_ID, - $oBackupManifest->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_ID)); - } - - if (!$oBackupManifest->test(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE, undef, CFGOPTVAL_BACKUP_TYPE_FULL)) - { - my @stryReference = sort(keys(%$oReferenceHash)); - - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_PRIOR, - $oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR)); - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_REFERENCE, - \@stryReference); - } - - if ($bSave) - { - $self->save(); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# current -# -# Test if a backup is current. -#################################################################################################################################### -sub current -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strBackup - ) = - logDebugParam - ( - __PACKAGE__ . '->current', \@_, - {name => 'strBackup'} - ); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bTest', value => $self->test(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackup)} - ); -} - -#################################################################################################################################### -# list -# -# Get backup keys. -#################################################################################################################################### -sub list -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFilter, - $strOrder - ) = - logDebugParam - ( - __PACKAGE__ . '->list', \@_, - {name => 'strFilter', required => false}, - {name => 'strOrder', default => 'forward'} - ); - - # List of backups - my @stryBackup; - - # Iterate through the backups and filter - for my $strBackup ($self->keys(INFO_BACKUP_SECTION_BACKUP_CURRENT)) - { - if (!defined($strFilter) || $strBackup =~ $strFilter) - { - if ($strOrder eq 'reverse') - { - unshift(@stryBackup, $strBackup) - } - else - { - push(@stryBackup, $strBackup) - } - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'stryBackup', value => \@stryBackup} - ); -} - -#################################################################################################################################### -# backupArchiveDbHistoryId -# -# Gets the backup.info db-id for the archiveId passed. -#################################################################################################################################### -sub backupArchiveDbHistoryId -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strArchiveId, - $strPathBackupArchive, - ) = - logDebugParam - ( - __PACKAGE__ . '->backupArchiveDbHistoryId', \@_, - {name => 'strArchiveId'}, - {name => 'strPathBackupArchive'}, - ); - - # List of backups associated with the db-id provided - my @stryArchiveBackup; - - # Build the db list from the history in the backup info and archive info file - my $oArchiveInfo = new pgBackRestTest::Env::ArchiveInfo($strPathBackupArchive, true); - my $hDbListArchive = $oArchiveInfo->dbHistoryList(); - my $hDbListBackup = $self->dbHistoryList(); - my $iDbHistoryId = undef; - - # Get the db-version and db-id (history id) from the archiveId - my ($strDbVersionArchive, $iDbIdArchive) = split("-", $strArchiveId); - - # Get the DB system ID to map back to the backup info if it exists in the archive info file - if (exists($hDbListArchive->{$iDbIdArchive})) - { - my $ullDbSysIdArchive = $$hDbListArchive{$iDbIdArchive}{&INFO_SYSTEM_ID}; - - # Get the db-id from backup info history that corresponds to the archive db-version and db-system-id - # Sort from newest (highest db-id) to oldest - foreach my $iDbIdBackup (sort {$b <=> $a} keys %{$hDbListBackup}) - { - if ($$hDbListBackup{$iDbIdBackup}{&INFO_SYSTEM_ID} == $ullDbSysIdArchive && - $$hDbListBackup{$iDbIdBackup}{&INFO_DB_VERSION} eq $strDbVersionArchive) - { - $iDbHistoryId = $iDbIdBackup; - last; - } - } - } - - # If the database is not found in the backup.info history list - if (!defined($iDbHistoryId)) - { - # Check to see that the current DB sections match for the archive and backup info files - if (!($oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef, - ($self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION)))) || - !($oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_SYSTEM_ID, undef, - ($self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_SYSTEM_ID))))) - { - # This should never happen unless the backup.info file is corrupt - confess &log(ASSERT, "the archive and backup database sections do not match", ERROR_FILE_INVALID); - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'iDbHistoryId', value => $iDbHistoryId} - ); -} - -#################################################################################################################################### -# listByArchiveId -# -# Filters a list of backups by the archiveId passed. -#################################################################################################################################### -sub listByArchiveId -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strArchiveId, - $strPathBackupArchive, - $stryBackup, - $strOrder, - ) = - logDebugParam - ( - __PACKAGE__ . '->listByArchiveId', \@_, - {name => 'strArchiveId'}, - {name => 'strPathBackupArchive'}, - {name => 'stryBackup'}, - {name => 'strOrder', default => 'forward'} - ); - - # List of backups associated with the db-id provided - my @stryArchiveBackup; - - my $iDbHistoryId = $self->backupArchiveDbHistoryId($strArchiveId, $strPathBackupArchive); - - # If history found, then build list of backups associated with the archive id passed, else return empty array - if (defined($iDbHistoryId)) - { - # Iterate through the backups and filter - foreach my $strBackup (@$stryBackup) - { - # From the backup.info current backup section, get the db-id for the backup and if it is the same, add to the list - if ($self->test(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackup, INFO_BACKUP_KEY_HISTORY_ID, $iDbHistoryId)) - { - if ($strOrder eq 'reverse') - { - unshift(@stryArchiveBackup, $strBackup) - } - else - { - push(@stryArchiveBackup, $strBackup) - } - } - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'stryArchiveBackup', value => \@stryArchiveBackup} - ); -} - -#################################################################################################################################### -# last -# -# Find the last backup depending on the type. -#################################################################################################################################### -sub last -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strType - ) = - logDebugParam - ( - __PACKAGE__ . '->last', \@_, - {name => 'strType'} - ); - - my $strFilter = backupRegExpGet(true, $strType ne CFGOPTVAL_BACKUP_TYPE_FULL, $strType eq CFGOPTVAL_BACKUP_TYPE_INCR); - my $strBackup = ($self->list($strFilter, 'reverse'))[0]; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strBackup', value => $strBackup} - ); -} - -#################################################################################################################################### -# delete -# -# Delete a backup from the info file. -#################################################################################################################################### -sub delete -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strBackupLabel - ) = - logDebugParam - ( - __PACKAGE__ . '->delete', \@_, - {name => 'strBackupLabel'} - ); - - $self->remove(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# create -# -# Create the info file. WARNING - this file should only be called from stanza-create or test modules. -#################################################################################################################################### -sub create -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $ullDbSysId, - $iControlVersion, - $iCatalogVersion, - $bSave, - ) = - logDebugParam - ( - __PACKAGE__ . '->create', \@_, - {name => 'strDbVersion'}, - {name => 'ullDbSysId'}, - {name => 'iControlVersion'}, - {name => 'iCatalogVersion'}, - {name => 'bSave', default => true}, - ); - - # Fill db section and db history section - $self->dbSectionSet($strDbVersion, $iControlVersion, $iCatalogVersion, $ullDbSysId, $self->dbHistoryIdGet(false)); - - if ($bSave) - { - $self->save(); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# dbHistoryIdGet -# -# Get the db history ID -#################################################################################################################################### -sub dbHistoryIdGet -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $bFileRequired, - ) = logDebugParam - ( - __PACKAGE__ . '->dbHistoryIdGet', \@_, - {name => 'bFileRequired', default => true}, - ); - - # Confirm the info file exists if it is required - if ($bFileRequired) - { - $self->confirmExists(); - } - - # If the DB section does not exist, initialize the history to one, else return the latest ID - my $iDbHistoryId = (!$self->test(INFO_BACKUP_SECTION_DB)) - ? 1 : $self->numericGet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_HISTORY_ID); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'iDbHistoryId', value => $iDbHistoryId} - ); -} - -#################################################################################################################################### -# dbHistoryList -# -# Get the data from the db history section. -#################################################################################################################################### -sub dbHistoryList -{ - my $self = shift; - my - ( - $strOperation, - ) = logDebugParam - ( - __PACKAGE__ . '->dbHistoryList', - ); - - my %hDbHash; - - foreach my $iHistoryId ($self->keys(INFO_BACKUP_SECTION_DB_HISTORY)) - { - $hDbHash{$iHistoryId}{&INFO_DB_VERSION} = - $self->get(INFO_BACKUP_SECTION_DB_HISTORY, $iHistoryId, INFO_BACKUP_KEY_DB_VERSION); - $hDbHash{$iHistoryId}{&INFO_SYSTEM_ID} = - $self->get(INFO_BACKUP_SECTION_DB_HISTORY, $iHistoryId, INFO_BACKUP_KEY_SYSTEM_ID); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'hDbHash', value => \%hDbHash} - ); -} - -#################################################################################################################################### -# dbSectionSet -# -# Set the db and db:history sections. -#################################################################################################################################### -sub dbSectionSet -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $iControlVersion, - $iCatalogVersion, - $ullDbSysId, - $iDbHistoryId, - ) = - logDebugParam - ( - __PACKAGE__ . '->dbSectionSet', \@_, - {name => 'strDbVersion', trace => true}, - {name => 'iControlVersion', trace => true}, - {name => 'iCatalogVersion', trace => true}, - {name => 'ullDbSysId', trace => true}, - {name => 'iDbHistoryId', trace => true}, - ); - - # Fill db section - $self->numericSet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CATALOG, undef, $iCatalogVersion); - $self->numericSet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CONTROL, undef, $iControlVersion); - $self->numericSet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_SYSTEM_ID, undef, $ullDbSysId); - # Force the version to a string since newer versions of JSON::PP lose track of the fact that it is one - $self->set(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION, undef, $strDbVersion . ''); - $self->numericSet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_HISTORY_ID, undef, $iDbHistoryId); - - # Fill db history - $self->numericSet(INFO_BACKUP_SECTION_DB_HISTORY, $iDbHistoryId, INFO_BACKUP_KEY_CATALOG, $iCatalogVersion); - $self->numericSet(INFO_BACKUP_SECTION_DB_HISTORY, $iDbHistoryId, INFO_BACKUP_KEY_CONTROL, $iControlVersion); - $self->numericSet(INFO_BACKUP_SECTION_DB_HISTORY, $iDbHistoryId, INFO_BACKUP_KEY_SYSTEM_ID, $ullDbSysId); - $self->set(INFO_BACKUP_SECTION_DB_HISTORY, $iDbHistoryId, INFO_BACKUP_KEY_DB_VERSION, $strDbVersion . ''); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# confirmDb -# -# Ensure that the backup is associated with the database passed. -# NOTE: The backup must exist in the backup:current section. -#################################################################################################################################### -sub confirmDb -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strBackup, - $strDbVersion, - $ullDbSysId, - ) = - logDebugParam - ( - __PACKAGE__ . '->confirmDb', \@_, - {name => 'strBackup', trace => true}, - {name => 'strDbVersion', trace => true}, - {name => 'ullDbSysId', trace => true}, - ); - - my $bConfirmDb = undef; - - # Get the db-id associated with the backup - my $iDbHistoryId = $self->get(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackup, INFO_BACKUP_KEY_HISTORY_ID); - - # Get the version and system-id for all known databases - my $hDbList = $self->dbHistoryList(); - - # If the db-id for the backup exists in the list - if (exists $hDbList->{$iDbHistoryId}) - { - # If the version and system-id match then database is confirmed for the backup - if (($hDbList->{$iDbHistoryId}{&INFO_DB_VERSION} eq $strDbVersion) && - ($hDbList->{$iDbHistoryId}{&INFO_SYSTEM_ID} eq $ullDbSysId)) - { - $bConfirmDb = true; - } - else - { - $bConfirmDb = false; - } - } - # If not, the backup.info file must be corrupt - else - { - confess &log(ERROR, "backup info file is missing database history information for an existing backup", ERROR_FILE_INVALID); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bConfirmDb', value => $bConfirmDb} - ); -} - -#################################################################################################################################### -# confirmExists -# -# Ensure that the backup.info file and the db section exist. -#################################################################################################################################### -sub confirmExists -{ - my $self = shift; - - # Confirm the file exists and the DB section is filled out - if (!$self->test(INFO_BACKUP_SECTION_DB) || !$self->{bExists}) - { - confess &log(ERROR, $self->{strBackupClusterPath} . "/" . $strBackupInfoMissingMsg, ERROR_FILE_MISSING); - } -} - -1; diff --git a/test/lib/pgBackRestTest/Env/ExpireEnvTest.pm b/test/lib/pgBackRestTest/Env/ExpireEnvTest.pm deleted file mode 100644 index 541bc8a2dc..0000000000 --- a/test/lib/pgBackRestTest/Env/ExpireEnvTest.pm +++ /dev/null @@ -1,639 +0,0 @@ -#################################################################################################################################### -# ExpireCommonTest.pm - Common code for expire tests -#################################################################################################################################### -package pgBackRestTest::Env::ExpireEnvTest; -use parent 'pgBackRestTest::Env::HostEnvTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Fcntl qw(O_RDONLY); -use File::Basename qw(basename); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::FileTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Env::ArchiveInfo; -use pgBackRestTest::Env::BackupInfo; -use pgBackRestTest::Env::HostEnvTest; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Create the class hash - my $self = {}; - bless $self, $class; - - # Assign function parameters, defaults, and log debug info - ( - my $strOperation, - $self->{oHostBackup}, - $self->{strBackRestExe}, - $self->{oStorageRepo}, - $self->{strPgPath}, - $self->{oRunTest}, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oHostBackup', required => false, trace => true}, - {name => 'strBackRestExe', trace => true}, - {name => 'oStorageRepo', trace => true}, - {name => 'strPgPath', trace => true}, - {name => 'oRunTest', required => false, trace => true}, - ); - - $self->{strVm} = $self->{oRunTest}->vm(); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# get into from pg_control -#################################################################################################################################### -my $oPgControlVersionHash = -{ - # iControlVersion => {iCatalogVersion => strDbVersion} - 942 => - { - 201409291 => PG_VERSION_94, - 201510051 => PG_VERSION_95, - }, - 960 => - { - 201608131 => PG_VERSION_96, - }, - 1002 => - { - 201707211 => PG_VERSION_10, - }, - 1100 => - { - 201809051 => PG_VERSION_11, - }, - 1201 => - { - 201909212 => PG_VERSION_12, - }, - 1300 => - { - 202007201 => PG_VERSION_13, - }, -}; - -sub info -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbPath - ) = - logDebugParam - ( - __PACKAGE__ . '->info', \@_, - {name => 'strDbPath', default => $self->{strPgPath}} - ); - - # Open the control file and read system id and versions - #----------------------------------------------------------------------------------------------------------------------- - my $strControlFile = "${strDbPath}/" . DB_FILE_PGCONTROL; - my $hFile; - my $tBlock; - - sysopen($hFile, $strControlFile, O_RDONLY) - or confess &log(ERROR, "unable to open ${strControlFile}", ERROR_FILE_OPEN); - - # Read system identifier - sysread($hFile, $tBlock, 8) == 8 - or confess &log(ERROR, "unable to read database system identifier"); - - $self->{info}{$strDbPath}{ullDbSysId} = unpack('Q', $tBlock); - - # Read control version - sysread($hFile, $tBlock, 4) == 4 - or confess &log(ERROR, "unable to read control version"); - - $self->{info}{$strDbPath}{iDbControlVersion} = unpack('L', $tBlock); - - # Read catalog version - sysread($hFile, $tBlock, 4) == 4 - or confess &log(ERROR, "unable to read catalog version"); - - $self->{info}{$strDbPath}{iDbCatalogVersion} = unpack('L', $tBlock); - - # Close the control file - close($hFile); - - # Get PostgreSQL version - $self->{info}{$strDbPath}{strDbVersion} = - $oPgControlVersionHash->{$self->{info}{$strDbPath}{iDbControlVersion}} - {$self->{info}{$strDbPath}{iDbCatalogVersion}}; - - if (!defined($self->{info}{$strDbPath}{strDbVersion})) - { - confess &log( - ERROR, - 'unexpected control version = ' . $self->{info}{$strDbPath}{iDbControlVersion} . - ' and catalog version = ' . $self->{info}{$strDbPath}{iDbCatalogVersion} . "\n" . - 'HINT: is this version of PostgreSQL supported?'); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strDbVersion', value => $self->{info}{$strDbPath}{strDbVersion}}, - {name => 'iDbControlVersion', value => $self->{info}{$strDbPath}{iDbControlVersion}}, - {name => 'iDbCatalogVersion', value => $self->{info}{$strDbPath}{iDbCatalogVersion}}, - {name => 'ullDbSysId', value => $self->{info}{$strDbPath}{ullDbSysId}} - ); -} - -#################################################################################################################################### -# stanzaSet - set the local stanza object -#################################################################################################################################### -sub stanzaSet -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strStanza, - $strDbVersion, - $bStanzaUpgrade, - ) = - logDebugParam - ( - __PACKAGE__ . '->stanzaSet', \@_, - {name => 'strStanza'}, - {name => 'strDbVersion'}, - {name => 'bStanzaUpgrade'}, - ); - - # Assign variables - my $oStanza = {}; - my $oArchiveInfo = {}; - my $oBackupInfo = {}; - my $iArchiveDbId = 1; - my $iBackupDbId = 1; - - # If we're not upgrading, then create the info files - if (!$bStanzaUpgrade) - { - $oArchiveInfo = - new pgBackRestTest::Env::ArchiveInfo($self->{oHostBackup}->repoArchivePath(), false, - {bIgnoreMissing => true, strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_ARCHIVE : undef}); - $oBackupInfo = - new pgBackRestTest::Env::BackupInfo($self->{oHostBackup}->repoBackupPath(), false, - {bIgnoreMissing => true, strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_MANIFEST : undef}); - } - # Else get the info data from disk - else - { - $oArchiveInfo = - new pgBackRestTest::Env::ArchiveInfo($self->{oHostBackup}->repoArchivePath(), - {strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_ARCHIVE : undef}); - $oBackupInfo = - new pgBackRestTest::Env::BackupInfo($self->{oHostBackup}->repoBackupPath(), - {strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_MANIFEST : undef}); - } - - # Get the database info for the stanza - (my $strVersion, $$oStanza{iControlVersion}, $$oStanza{iCatalogVersion}, $$oStanza{ullDbSysId}) = $self->info(); - $$oStanza{strDbVersion} = $strDbVersion; - - if ($bStanzaUpgrade) - { - $iArchiveDbId = $oArchiveInfo->dbHistoryIdGet() + 1; - $iBackupDbId = $oBackupInfo->dbHistoryIdGet() + 1; - } - - $oArchiveInfo->dbSectionSet($$oStanza{strDbVersion}, $$oStanza{ullDbSysId}, $iArchiveDbId); - $oArchiveInfo->save(); - - $oBackupInfo->dbSectionSet($$oStanza{strDbVersion}, $$oStanza{iControlVersion}, $$oStanza{iCatalogVersion}, - $$oStanza{ullDbSysId}, $iBackupDbId); - $oBackupInfo->save(); - - # Get the archive and directory paths for the stanza - $$oStanza{strArchiveClusterPath} = $self->{oHostBackup}->repoArchivePath($oArchiveInfo->archiveId()); - $$oStanza{strBackupClusterPath} = $self->{oHostBackup}->repoBackupPath(); - - $self->{oStanzaHash}{$strStanza} = $oStanza; - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# stanzaCreate -#################################################################################################################################### -sub stanzaCreate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strStanza, - $strDbVersion, - ) = - logDebugParam - ( - __PACKAGE__ . '->stanzaCreate', \@_, - {name => 'strStanza'}, - {name => 'strDbVersion'}, - ); - - my $strDbVersionTemp = $strDbVersion; - $strDbVersionTemp =~ s/\.//; - - # Create the test path for pg_control - storageTest()->pathCreate(($self->{strPgPath} . '/' . DB_PATH_GLOBAL), {bIgnoreExists => true}); - - # Generate pg_control for stanza-create - $self->controlGenerate($self->{strPgPath}, $strDbVersion); - executeTest('chmod 600 ' . $self->{strPgPath} . '/' . DB_FILE_PGCONTROL); - - # Create the stanza and set the local stanza object - $self->stanzaSet($strStanza, $strDbVersion, false); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# stanzaUpgrade -#################################################################################################################################### -sub stanzaUpgrade -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strStanza, - $strDbVersion, - ) = - logDebugParam - ( - __PACKAGE__ . '->stanzaUpgrade', \@_, - {name => 'strStanza'}, - {name => 'strDbVersion'}, - ); - - my $strDbVersionTemp = $strDbVersion; - $strDbVersionTemp =~ s/\.//; - - # Remove pg_control - storageTest()->remove($self->{strPgPath} . '/' . DB_FILE_PGCONTROL); - - # Copy pg_control for stanza-upgrade - $self->controlGenerate($self->{strPgPath}, $strDbVersion); - executeTest('chmod 600 ' . $self->{strPgPath} . '/' . DB_FILE_PGCONTROL); - - $self->stanzaSet($strStanza, $strDbVersion, true); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} -#################################################################################################################################### -# backupCreate -#################################################################################################################################### -sub backupCreate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strStanza, - $strType, - $lTimestamp, - $iArchiveBackupTotal, - $iArchiveBetweenTotal - ) = - logDebugParam - ( - __PACKAGE__ . '->backupCreate', \@_, - {name => 'strStanza'}, - {name => 'strType'}, - {name => 'lTimestamp'}, - {name => 'iArchiveBackupTotal', default => 3}, - {name => 'iArchiveBetweenTotal', default => 3} - ); - - my $oStanza = $self->{oStanzaHash}{$strStanza}; - - my ($strArchiveStart, $strArchiveStop); - - if ($iArchiveBackupTotal != -1) - { - ($strArchiveStart, $strArchiveStop) = $self->archiveCreate($strStanza, $iArchiveBackupTotal); - } - - # Create the manifest - my $oLastManifest = $strType ne CFGOPTVAL_BACKUP_TYPE_FULL ? $$oStanza{oManifest} : undef; - - my $strBackupLabel = - backupLabelFormat($strType, - defined($oLastManifest) ? $oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL) : undef, - $lTimestamp); - - my $strBackupClusterSetPath = "$$oStanza{strBackupClusterPath}/${strBackupLabel}"; - - &log(INFO, "create backup ${strBackupLabel}"); - - # Get passphrase (returns undefined if repo not encrypted) to access the manifest - my $strCipherPassManifest = - (new pgBackRestTest::Env::BackupInfo($self->{oHostBackup}->repoBackupPath()))->cipherPassSub(); - my $strCipherPassBackupSet; - - # If repo is encrypted then get passphrase for accessing the backup files from the last manifest if it exists provide one - if (defined($strCipherPassManifest)) - { - $strCipherPassBackupSet = (defined($oLastManifest)) ? $oLastManifest->cipherPassSub() : - ENCRYPTION_KEY_BACKUPSET; - } - - my $strManifestFile = "$$oStanza{strBackupClusterPath}/${strBackupLabel}/" . FILE_MANIFEST; - - my $oManifest = new pgBackRestTest::Env::Manifest($strManifestFile, {bLoad => false, strDbVersion => PG_VERSION_94, - iDbCatalogVersion => $self->dbCatalogVersion(PG_VERSION_94), - strCipherPass => $strCipherPassManifest, strCipherPassSub => $strCipherPassBackupSet}); - - # Store information about the backup into the backup section - $oManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL, undef, $strBackupLabel); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_CHECK, undef, true); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_COPY, undef, false); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_BACKUP_STANDBY, undef, false); - $oManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_START, undef, $strArchiveStart); - $oManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_STOP, undef, $strArchiveStop); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, 'backup-bundle', undef, true); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_CHECKSUM_PAGE, undef, true); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS, undef, true); - $oManifest->numericSet(INI_SECTION_BACKREST, INI_KEY_FORMAT, undef, REPOSITORY_FORMAT); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_HARDLINK, undef, false); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ONLINE, undef, true); - $oManifest->numericSet(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_START, undef, $lTimestamp); - $oManifest->numericSet(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP, undef, $lTimestamp); - $oManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE, undef, $strType); - $oManifest->set(INI_SECTION_BACKREST, INI_KEY_VERSION, undef, PROJECT_VERSION); - - if ($strType ne CFGOPTVAL_BACKUP_TYPE_FULL) - { - if (!defined($oLastManifest)) - { - confess &log(ERROR, "oLastManifest must be defined when strType = ${strType}"); - } - - # Set backup-prior - if ($strType eq CFGOPTVAL_BACKUP_TYPE_INCR) - { - # If this is an incremental backup, then it is always based on the prior backup so use the label from the last backup - $oManifest->set( - MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR, undef, - $oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL)); - } - else - { - # If it is a differential then backup-prior must be set to the newest full backup so get the full backup label from - # the prior label - $oManifest->set( - MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR, undef, - substr($oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL), 0, 16)); - } - } - - $oManifest->save(); - $$oStanza{oManifest} = $oManifest; - - # Add the backup to info - my $oBackupInfo = new pgBackRestTest::Env::BackupInfo($$oStanza{strBackupClusterPath}, false); - - $oBackupInfo->check($$oStanza{strDbVersion}, $$oStanza{iControlVersion}, $$oStanza{iCatalogVersion}, $$oStanza{ullDbSysId}); - $oBackupInfo->add($oManifest); - - # Create the backup description string - if (defined($$oStanza{strBackupDescription})) - { - $$oStanza{strBackupDescription} .= "\n"; - } - - $$oStanza{strBackupDescription} .= - "* ${strType} backup: label = ${strBackupLabel}" . - (defined($oLastManifest) ? ', prior = ' . $oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL) : '') . - (defined($strArchiveStart) ? ", start = ${strArchiveStart}, stop = ${strArchiveStop}" : ', not online'); - - if ($iArchiveBetweenTotal != -1) - { - $self->archiveCreate($strStanza, $iArchiveBetweenTotal); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# archiveNext -#################################################################################################################################### -sub archiveNext -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strArchive, - $bSkipFF - ) = - logDebugParam - ( - __PACKAGE__ . '->archiveNext', \@_, - {name => 'strArchive', trace => true}, - {name => 'bSkipFF', trace => true} - ); - - # Break archive log into components - my $lTimeline = hex(substr($strArchive, 0, 8)); - my $lMajor = hex(substr($strArchive, 8, 8)); - my $lMinor = hex(substr($strArchive, 16, 8)); - - # Increment the minor component (and major when needed) - $lMinor += 1; - - if ($bSkipFF && $lMinor == 255 || !$bSkipFF && $lMinor == 256) - { - $lMajor += 1; - $lMinor = 0; - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strArchiveNext', value => uc(sprintf("%08x%08x%08x", $lTimeline, $lMajor, $lMinor)), trace => true} - ); -} - -#################################################################################################################################### -# archiveCreate -#################################################################################################################################### -sub archiveCreate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strStanza, - $iArchiveTotal - ) = - logDebugParam - ( - __PACKAGE__ . '->archiveCreate', \@_, - {name => 'strStanza'}, - {name => 'iArchiveTotal'} - ); - - my $oStanza = $self->{oStanzaHash}{$strStanza}; - my $iArchiveIdx = 0; - - my $strArchive = defined($$oStanza{strArchiveLast}) ? $self->archiveNext($$oStanza{strArchiveLast}, false) : - '000000010000000000000000'; - - # Get passphrase (returns undefined if repo not encrypted) to access the archive files - my $strCipherPass = - (new pgBackRestTest::Env::ArchiveInfo($self->{oHostBackup}->repoArchivePath()))->cipherPassSub(); - - push(my @stryArchive, $strArchive); - - do - { - my $strPath = "$$oStanza{strArchiveClusterPath}/" . substr($strArchive, 0, 16); - my $strFile = "${strPath}/${strArchive}-0000000000000000000000000000000000000000" . ($iArchiveIdx % 2 == 0 ? '.gz' : ''); - - storageRepo()->put($strFile, 'ARCHIVE', {strCipherPass => $strCipherPass}); - - $iArchiveIdx++; - - if ($iArchiveIdx < $iArchiveTotal) - { - $strArchive = $self->archiveNext($strArchive, false); - } - } - while ($iArchiveIdx < $iArchiveTotal); - - push(@stryArchive, $strArchive); - $$oStanza{strArchiveLast} = $strArchive; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'stryArchive', value => \@stryArchive} - ); -} - -#################################################################################################################################### -# process -#################################################################################################################################### -sub process -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strStanza, - $iExpireFull, - $iExpireDiff, - $strExpireArchiveType, - $iExpireArchive, - $strDescription - ) = - logDebugParam - ( - __PACKAGE__ . '->process', \@_, - {name => 'strStanza'}, - {name => 'iExpireFull', required => false}, - {name => 'iExpireDiff', required => false}, - {name => 'strExpireArchiveType'}, - {name => 'iExpireArchive', required => false}, - {name => 'strDescription'} - ); - - my $oStanza = $self->{oStanzaHash}{$strStanza}; - - undef($$oStanza{strBackupDescription}); - - my $strCommand = - $self->{strBackRestExe} . ' --config="' . $self->{oHostBackup}->backrestConfig() . '"' . ' --stanza=' . $strStanza . - ' --log-level-console=' . lc(DETAIL); - - if (defined($iExpireFull)) - { - $strCommand .= ' --repo1-retention-full=' . $iExpireFull; - } - - if (defined($iExpireDiff)) - { - $strCommand .= ' --repo1-retention-diff=' . $iExpireDiff; - } - - if (defined($strExpireArchiveType)) - { - if (defined($iExpireArchive)) - { - $strCommand .= ' --repo1-retention-archive-type=' . $strExpireArchiveType . - ' --repo1-retention-archive=' . $iExpireArchive; - } - else - { - $strCommand .= ' --repo1-retention-archive-type=' . $strExpireArchiveType; - } - } - - $strCommand .= ' expire'; - - $self->{oHostBackup}->executeSimple($strCommand, {strComment => $strDescription}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostAzureTest.pm b/test/lib/pgBackRestTest/Env/Host/HostAzureTest.pm deleted file mode 100644 index d882c2275c..0000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostAzureTest.pm +++ /dev/null @@ -1,82 +0,0 @@ -#################################################################################################################################### -# Azure Test Host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostAzureTest; -use parent 'pgBackRestTest::Common::HostTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Cwd qw(abs_path); -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); -use Storable qw(dclone); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# Azure defaults -#################################################################################################################################### -use constant HOST_AZURE_ACCOUNT => 'azaccount'; - push @EXPORT, qw(HOST_AZURE_ACCOUNT); -use constant HOST_AZURE_KEY => 'YXpLZXk='; - push @EXPORT, qw(HOST_AZURE_KEY); -use constant HOST_AZURE_CONTAINER => 'azcontainer'; - push @EXPORT, qw(HOST_AZURE_CONTAINER); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - ); - - # Create the host - my $strProjectPath = dirname(dirname(abs_path($0))); - my $strFakeCertPath = "${strProjectPath}/doc/resource/fake-cert"; - - my $self = $class->SUPER::new( - HOST_AZURE, 'test-' . testRunGet()->vmId() . '-' . HOST_AZURE, 'mcr.microsoft.com/azure-storage/azurite', 'root', - ["${strFakeCertPath}/s3-server.crt:/root/public.crt:ro", "${strFakeCertPath}/s3-server.key:/root/private.key:ro"], - '-e AZURITE_ACCOUNTS="' . HOST_AZURE_ACCOUNT . ':' . HOST_AZURE_KEY . '"', - 'azurite-blob --blobPort 443 --blobHost 0.0.0.0 --cert=/root/public.crt --key=/root/private.key -d debug.log" - " --disableProductStyleUrl', - false); - bless $self, $class; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm b/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm deleted file mode 100644 index 6e0e0e2afa..0000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm +++ /dev/null @@ -1,2321 +0,0 @@ -#################################################################################################################################### -# HostBackupTest.pm - Backup host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostBackupTest; -use parent 'pgBackRestTest::Env::Host::HostBaseTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Exporter qw(import); - our @EXPORT = qw(); -use Fcntl ':mode'; -use File::Basename qw(dirname); -use File::stat qw{lstat}; -use Storable qw(dclone); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::Common::String; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::StorageBase; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Env::ArchiveInfo; -use pgBackRestTest::Env::BackupInfo; -use pgBackRestTest::Env::Host::HostAzureTest; -use pgBackRestTest::Env::Host::HostGcsTest; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Host::HostS3Test; -use pgBackRestTest::Env::Host::HostSftpTest; -use pgBackRestTest::Env::Manifest; -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::VmTest; - -#################################################################################################################################### -# Error constants -#################################################################################################################################### -use constant ERROR_REPO_INVALID => 103; -push @EXPORT, qw(ERROR_REPO_INVALID); - -#################################################################################################################################### -# Latest backup link constant -#################################################################################################################################### -use constant LINK_LATEST => 'latest'; - push @EXPORT, qw(LINK_LATEST); - -#################################################################################################################################### -# Host defaults -#################################################################################################################################### -use constant HOST_PATH_LOCK => 'lock'; - push @EXPORT, qw(HOST_PATH_LOCK); -use constant HOST_PATH_LOG => 'log'; - push @EXPORT, qw(HOST_PATH_LOG); -use constant HOST_PATH_REPO => 'repo'; - -use constant HOST_PROTOCOL_TIMEOUT => 10; - push @EXPORT, qw(HOST_PROTOCOL_TIMEOUT); - -#################################################################################################################################### -# Configuration constants -#################################################################################################################################### -use constant CFGDEF_SECTION_GLOBAL => 'global'; - push @EXPORT, qw(CFGDEF_SECTION_GLOBAL); -use constant CFGDEF_SECTION_STANZA => 'stanza'; - push @EXPORT, qw(CFGDEF_SECTION_STANZA); - -use constant CFGOPTVAL_BACKUP_TYPE_FULL => 'full'; - push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_FULL); -use constant CFGOPTVAL_BACKUP_TYPE_DIFF => 'diff'; - push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_DIFF); -use constant CFGOPTVAL_BACKUP_TYPE_INCR => 'incr'; - push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_INCR); - -use constant CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC => 'aes-256-cbc'; - push @EXPORT, qw(CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC); - -use constant AZURE => 'azure'; - push @EXPORT, qw(AZURE); -use constant CIFS => 'cifs'; - push @EXPORT, qw(CIFS); -use constant GCS => 'gcs'; - push @EXPORT, qw(GCS); -use constant POSIX => STORAGE_POSIX; - push @EXPORT, qw(POSIX); -use constant S3 => 's3'; - push @EXPORT, qw(S3); -use constant SFTP => 'sftp'; - push @EXPORT, qw(SFTP); - -use constant CFGOPTVAL_RESTORE_TYPE_DEFAULT => 'default'; - push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_DEFAULT); -use constant CFGOPTVAL_RESTORE_TYPE_IMMEDIATE => 'immediate'; - push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_IMMEDIATE); -use constant CFGOPTVAL_RESTORE_TYPE_NAME => 'name'; - push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_NAME); -use constant CFGOPTVAL_RESTORE_TYPE_PRESERVE => 'preserve'; - push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_PRESERVE); -use constant CFGOPTVAL_RESTORE_TYPE_STANDBY => 'standby'; - push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_STANDBY); -use constant CFGOPTVAL_RESTORE_TYPE_TIME => 'time'; - push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_TIME); -use constant CFGOPTVAL_RESTORE_TYPE_XID => 'xid'; - push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_XID); - -use constant NONE => 'none'; - push @EXPORT, qw(NONE); -use constant BZ2 => 'bz2'; - push @EXPORT, qw(BZ2); -use constant GZ => 'gz'; - push @EXPORT, qw(GZ); -use constant LZ4 => 'lz4'; - push @EXPORT, qw(LZ4); -use constant ZST => 'zst'; - push @EXPORT, qw(ZST); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oParam', required => false, trace => true}, - ); - - # If params are not passed - my $oHostGroup = hostGroupGet(); - - my ($strName, $strImage, $strUser); - - if (!defined($$oParam{strName}) || $$oParam{strName} eq HOST_BACKUP) - { - $strName = HOST_BACKUP; - $strImage = containerRepo() . ':' . testRunGet()->vm() . '-test'; - } - else - { - $strName = $$oParam{strName}; - $strImage = $$oParam{strImage}; - } - - $strUser = testRunGet()->pgUser(); - - # Create the host - my $self = $class->SUPER::new($strName, {strImage => $strImage, strUser => $strUser, bTls => $oParam->{bTls}}); - bless $self, $class; - - # If repo is on local filesystem then set the repo-path locally - if ($oParam->{bRepoLocal} || $oParam->{strBackupDestination} eq HOST_SFTP) - { - $self->{strRepoPath} = $self->testRunGet()->testPath() . "/$$oParam{strBackupDestination}/" . HOST_PATH_REPO; - } - # Else on KV store and repo will be in root - else - { - $self->{strRepoPath} = '/'; - } - - # If there is a repo2 it will always be posix on the repo host - $self->{strRepo2Path} = $self->testRunGet()->testPath() . "/$$oParam{strBackupDestination}/" . HOST_PATH_REPO . "2"; - - # Set log/lock paths - $self->{strLogPath} = $self->testPath() . '/' . HOST_PATH_LOG; - storageTest()->pathCreate($self->{strLogPath}, {strMode => '0770'}); - $self->{strLockPath} = $self->testPath() . '/' . HOST_PATH_LOCK; - - # Set conf file - $self->{strBackRestConfig} = $self->testPath() . '/' . PROJECT_CONF; - - # Set synthetic - $self->{bSynthetic} = defined($$oParam{bSynthetic}) && $$oParam{bSynthetic} ? true : false; - - # Set the backup destination - $self->{strBackupDestination} = $$oParam{strBackupDestination}; - - # Default hardlink to false - $self->{bHardLink} = false; - - # By default there is no bogus host - $self->{bBogusHost} = false; - - # Create a placeholder hash for file munging - $self->{hInfoFile} = {}; - - # Set whether repo should be encrypted or not - $self->{bRepoEncrypt} = defined($$oParam{bRepoEncrypt}) ? $$oParam{bRepoEncrypt} : false; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# timestampFileFormat -#################################################################################################################################### -sub timestampFileFormat -{ - my $strFormat = shift; - my $lTime = shift; - - return timestampFormat(defined($strFormat) ? $strFormat : '%4d%02d%02d-%02d%02d%02d', $lTime); -} - -push @EXPORT, qw(timestampFileFormat); - -#################################################################################################################################### -# backupLabelFormat -# -# Format the label for a backup. -#################################################################################################################################### -sub backupLabelFormat -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strType, - $strBackupLabelLast, - $lTimestampStart - ) = - logDebugParam - ( - __PACKAGE__ . '::backupLabelFormat', \@_, - {name => 'strType', trace => true}, - {name => 'strBackupLabelLast', required => false, trace => true}, - {name => 'lTimestampTart', trace => true} - ); - - # Full backup label - my $strBackupLabel; - - if ($strType eq CFGOPTVAL_BACKUP_TYPE_FULL) - { - # Last backup label must not be defined - if (defined($strBackupLabelLast)) - { - confess &log(ASSERT, "strBackupLabelLast must not be defined when strType = '${strType}'"); - } - - # Format the timestamp and add the full indicator - $strBackupLabel = timestampFileFormat(undef, $lTimestampStart) . 'F'; - } - # Else diff or incr label - else - { - # Last backup label must be defined - if (!defined($strBackupLabelLast)) - { - confess &log(ASSERT, "strBackupLabelLast must be defined when strType = '${strType}'"); - } - - # Get the full backup portion of the last backup label - $strBackupLabel = substr($strBackupLabelLast, 0, 16); - - # Format the timestamp - $strBackupLabel .= '_' . timestampFileFormat(undef, $lTimestampStart); - - # Add the diff indicator - if ($strType eq CFGOPTVAL_BACKUP_TYPE_DIFF) - { - $strBackupLabel .= 'D'; - } - # Else incr indicator - else - { - $strBackupLabel .= 'I'; - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strBackupLabel', value => $strBackupLabel, trace => true} - ); -} - -push @EXPORT, qw(backupLabelFormat); - -#################################################################################################################################### -# backupBegin -#################################################################################################################################### -sub backupBegin -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strType, - $strComment, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->backupBegin', \@_, - {name => 'strType', trace => true}, - {name => 'strComment', trace => true}, - {name => 'oParam', required => false, trace => true}, - ); - - # Set defaults - my $oExpectedManifest = defined($$oParam{oExpectedManifest}) ? $$oParam{oExpectedManifest} : undef; - - $strComment = - "${strType} backup" . (defined($strComment) ? " - ${strComment}" : '') . - ' (' . $self->nameGet() . ' host)'; - - &log(INFO, " $strComment"); - - # Execute the backup command - my $oExecuteBackup = $self->execute( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - (defined($oExpectedManifest) ? " --no-online" : '') . - (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . - (defined($$oParam{bStandby}) && $$oParam{bStandby} ? " --backup-standby" : '') . - (defined($oParam->{strRepoType}) ? " --repo1-type=$oParam->{strRepoType}" : '') . - (defined($oParam->{iRepo}) ? ' --repo=' . $oParam->{iRepo} : '') . - ($strType ne 'incr' ? " --type=${strType}" : '') . - ' --stanza=' . (defined($oParam->{strStanza}) ? $oParam->{strStanza} : $self->stanza()) . ' backup', - {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, bLogOutput => $self->synthetic()}); - - $oExecuteBackup->begin(); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'oExecuteBackup', value => $oExecuteBackup, trace => true}, - ); -} - -#################################################################################################################################### -# backupEnd -#################################################################################################################################### -sub backupEnd -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strType, - $oExecuteBackup, - $oParam, - $bManifestCompare, - ) = - logDebugParam - ( - __PACKAGE__ . '->backupEnd', \@_, - {name => 'strType', trace => true}, - {name => 'oExecuteBackup', trace => true}, - {name => 'oParam', required => false, trace => true}, - {name => 'bManifestCompare', required => false, default => true, trace => true}, - ); - - # Set defaults - my $oExpectedManifest = defined($$oParam{oExpectedManifest}) ? dclone($$oParam{oExpectedManifest}) : undef; - - my $iExitStatus = $oExecuteBackup->end(); - - return if ($oExecuteBackup->{iExpectedExitStatus} != 0); - - # If an alternate stanza was specified - if (defined($oParam->{strStanza})) - { - confess &log(ASSERT, - 'if an alternate stanza is specified it must generate an error - the remaining code will not be aware of the stanza'); - } - - my $strBackup = $self->backupLast($oParam->{iRepo}); - - # Only compare backups that are in repo1. There is not a lot of value in comparing backups in other repos and it would require a - # lot of changes to the test harness. - if (!defined($oParam->{iRepo}) || $oParam->{iRepo} == 1) - { - # If a real backup then load the expected manifest from the actual manifest. An expected manifest can't be generated - # perfectly because a running database is always in flux. Even so, it allows us to test many things. - if (!$self->synthetic()) - { - $oExpectedManifest = iniParse( - ${storageRepo()->get( - storageRepo()->openRead( - 'backup/' . $self->stanza() . "/${strBackup}/" . FILE_MANIFEST, - {strCipherPass => $self->cipherPassManifest()}))}); - } - - # Make sure tablespace links are correct - if ($self->hasLink()) - { - if (($strType eq CFGOPTVAL_BACKUP_TYPE_FULL || $self->hardLink()) && - !$oExpectedManifest->{&MANIFEST_SECTION_BACKUP}{'backup-bundle'}) - { - my $hTablespaceManifest = storageTest()->manifest( - $self->repoBackupPath("${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC)); - - # Remove . and .. - delete($hTablespaceManifest->{'.'}); - delete($hTablespaceManifest->{'..'}); - - # Iterate file links - for my $strFile (sort(keys(%{$hTablespaceManifest}))) - { - # Make sure the link is in the expected manifest - my $hManifestTarget = - $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGTBLSPC . "/${strFile}"}; - - if (!defined($hManifestTarget) || $hManifestTarget->{&MANIFEST_SUBKEY_TYPE} ne MANIFEST_VALUE_LINK || - $hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID} ne $strFile) - { - confess &log(ERROR, "'${strFile}' is not in expected manifest as a link with the correct tablespace id"); - } - - # Make sure the link really is a link - if ($hTablespaceManifest->{$strFile}{type} ne 'l') - { - confess &log(ERROR, "'${strFile}' in tablespace directory is not a link"); - } - - # Make sure the link destination is correct - my $strLinkDestination = '../../' . MANIFEST_TARGET_PGTBLSPC . "/${strFile}"; - - if ($hTablespaceManifest->{$strFile}{link_destination} ne $strLinkDestination) - { - confess &log(ERROR, - "'${strFile}' link should reference '${strLinkDestination}' but actually references " . - "'$hTablespaceManifest->{$strFile}{link_destination}'"); - } - } - - # Iterate manifest targets - for my $strTarget (sort(keys(%{$oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}}))) - { - my $hManifestTarget = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}; - my $strTablespaceId = $hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID}; - - # Make sure the target exists as a link on disk - if ($hManifestTarget->{&MANIFEST_SUBKEY_TYPE} eq MANIFEST_VALUE_LINK && defined($strTablespaceId) && - !defined($hTablespaceManifest->{$strTablespaceId})) - { - confess &log(ERROR, - "target '${strTarget}' does not have a link at '" . DB_PATH_PGTBLSPC. "/${strTablespaceId}'"); - } - } - } - # Else there should not be a tablespace directory at all. This is only valid for storage that supports links. - elsif (storageRepo()->capability(STORAGE_CAPABILITY_LINK) && - storageTest()->pathExists( - $self->repoBackupPath("${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC))) - { - confess &log(ERROR, 'backup must be full or hard-linked to have ' . DB_PATH_PGTBLSPC . ' directory'); - } - } - - # Check that latest link exists unless repo links are disabled - my $strLatestLink = $self->repoBackupPath(LINK_LATEST); - my $bLatestLinkExists = storageRepo()->exists($strLatestLink); - - if ((!defined($oParam->{strRepoType}) || $oParam->{strRepoType} eq POSIX) && $self->hasLink()) - { - my $strLatestLinkDestination = readlink($strLatestLink); - - if ($strLatestLinkDestination ne $strBackup) - { - confess &log(ERROR, "'" . LINK_LATEST . "' link should be '${strBackup}' but is '${strLatestLinkDestination}"); - } - } - elsif ($bLatestLinkExists) - { - confess &log(ERROR, "'" . LINK_LATEST . "' link should not exist"); - } - - # Only do compare for synthetic backups since for real backups the expected manifest *is* the actual manifest. - if ($self->synthetic()) - { - # Compare only if expected to do so - if ($bManifestCompare) - { - # Set backup type in the expected manifest - ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TYPE} = $strType; - - $self->backupCompare($strBackup, $oExpectedManifest); - } - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strBackup', value => $strBackup, trace => true}, - ); -} - -#################################################################################################################################### -# backup -#################################################################################################################################### -sub backup -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strType, - $strComment, - $oParam, - $bManifestCompare, - ) = - logDebugParam - ( - __PACKAGE__ . '->backup', \@_, - {name => 'strType'}, - {name => 'strComment'}, - {name => 'oParam', required => false}, - {name => 'bManifestCompare', required => false, default => true}, - ); - - my $oExecuteBackup = $self->backupBegin($strType, $strComment, $oParam); - my $strBackup = $self->backupEnd($strType, $oExecuteBackup, $oParam, $bManifestCompare); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strBackup', value => $strBackup}, - ); -} - -#################################################################################################################################### -# backupCompare -#################################################################################################################################### -sub backupCompare -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strBackup, - $oExpectedManifest, - ) = - logDebugParam - ( - __PACKAGE__ . '->backupCompare', \@_, - {name => 'strBackup', trace => true}, - {name => 'oExpectedManifest', trace => true}, - ); - - ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL} = $strBackup; - - my $oActualManifest = new pgBackRestTest::Env::Manifest( - $self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), {strCipherPass => $self->cipherPassManifest()}); - - ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{'backup-reference'} = - $oActualManifest->get(MANIFEST_SECTION_BACKUP, 'backup-reference'); - ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START} = - $oActualManifest->get(MANIFEST_SECTION_BACKUP, &MANIFEST_KEY_TIMESTAMP_START); - ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_STOP} = - $oActualManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP); - ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_COPY_START} = - $oActualManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START); - ${$oExpectedManifest}{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM} = - $oActualManifest->get(INI_SECTION_BACKREST, INI_KEY_CHECKSUM); - ${$oExpectedManifest}{&INI_SECTION_BACKREST}{&INI_KEY_FORMAT} = REPOSITORY_FORMAT + 0; - - if (defined($oExpectedManifest->{&INI_SECTION_CIPHER}) && - defined($oExpectedManifest->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS}) && - $oActualManifest->test(INI_SECTION_CIPHER, INI_KEY_CIPHER_PASS)) - { - $oExpectedManifest->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS} = - $oActualManifest->get(INI_SECTION_CIPHER, INI_KEY_CIPHER_PASS); - } - - # Update the expected manifest with whether the --delta option was used or not to perform the backup. - $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA} = - $oActualManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_DELTA) ? INI_TRUE : INI_FALSE; - - my $strSectionPath = $oActualManifest->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH); - - foreach my $strFileKey ($oActualManifest->keys(MANIFEST_SECTION_TARGET_FILE)) - { - # Remove repo checksum - $oActualManifest->remove(&MANIFEST_SECTION_TARGET_FILE, $strFileKey, 'rck'); - - # Determine repo size if compression or encryption is enabled - my $strCompressType = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE}; - - if ($strCompressType ne NONE || - (defined($oExpectedManifest->{&INI_SECTION_CIPHER}) && - defined($oExpectedManifest->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS}))) - { - - my $lRepoSize = - $oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REFERENCE) ? - $oActualManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE, false) : - (storageRepo()->info( - $self->repoBackupPath("${strBackup}/${strFileKey}") . - ($strCompressType eq NONE ? '' : ".${strCompressType}")))->{size}; - - if (defined($lRepoSize) && - $lRepoSize != $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_SIZE}) - { - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_REPO_SIZE} = $lRepoSize; - } - } - - # If the backup does not have page checksums then no need to compare - if (!$oExpectedManifest->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_CHECKSUM_PAGE}) - { - delete($oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_CHECKSUM_PAGE}); - delete($oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR}); - } - # Else make sure things that should have checks do have checks - elsif ($oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_CHECKSUM_PAGE) != - isChecksumPage($strFileKey)) - { - confess - "check-page actual for ${strFileKey} is " . - ($oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, - MANIFEST_SUBKEY_CHECKSUM_PAGE) ? 'set' : '[undef]') . - ' but isChecksumPage() says it should be ' . - (isChecksumPage($strFileKey) ? 'set' : 'undef') . '.'; - } - } - - $self->manifestDefault($oExpectedManifest); - - my $strTestPath = $self->testPath(); - - storageTest()->put("${strTestPath}/actual.manifest", iniRender($oActualManifest->{oContent})); - storageTest()->put("${strTestPath}/expected.manifest", iniRender($oExpectedManifest)); - - executeTest("diff ${strTestPath}/expected.manifest ${strTestPath}/actual.manifest"); - - storageTest()->remove("${strTestPath}/expected.manifest"); - storageTest()->remove("${strTestPath}/actual.manifest"); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# manifestDefault -#################################################################################################################################### -sub manifestDefault -{ - my $self = shift; - my $oExpectedManifest = shift; - - # Defaults for subkeys that tend to repeat - my $strDefaultUser = $oExpectedManifest->{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_USER}; - my $strDefaultGroup = $oExpectedManifest->{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_GROUP}; - my $strDefaultPathMode = $oExpectedManifest->{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_MODE}; - my $strDefaultFileMode = sprintf('%04o', oct($strDefaultPathMode) & (S_IRUSR | S_IWUSR | S_IRGRP)); - - # Remove subkeys that match the defaults - foreach my $strSection (&MANIFEST_SECTION_TARGET_FILE, &MANIFEST_SECTION_TARGET_PATH, &MANIFEST_SECTION_TARGET_LINK) - { - next if !defined($oExpectedManifest->{$strSection}); - - foreach my $strFile (keys(%{$oExpectedManifest->{$strSection}})) - { - if (defined($oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_USER}) && - $oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_USER} eq $strDefaultUser) - { - delete($oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_USER}); - } - - if (defined($oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_GROUP}) && - $oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_GROUP} eq $strDefaultGroup) - { - delete($oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_GROUP}); - } - - if (defined($oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_MODE}) && - $oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_MODE} eq - ($strSection eq MANIFEST_SECTION_TARGET_PATH ? $strDefaultPathMode : $strDefaultFileMode)) - { - delete($oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_MODE}); - } - } - } - - # Write defaults - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE . ':default'}{&MANIFEST_SUBKEY_USER} = $strDefaultUser; - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE . ':default'}{&MANIFEST_SUBKEY_GROUP} = $strDefaultGroup; - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE . ':default'}{&MANIFEST_SUBKEY_MODE} = $strDefaultFileMode; - - if (defined($oExpectedManifest->{&MANIFEST_SECTION_TARGET_LINK})) - { - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_LINK . ':default'}{&MANIFEST_SUBKEY_USER} = $strDefaultUser; - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_LINK . ':default'}{&MANIFEST_SUBKEY_GROUP} = $strDefaultGroup; - } - - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_PATH . ':default'}{&MANIFEST_SUBKEY_USER} = $strDefaultUser; - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_PATH . ':default'}{&MANIFEST_SUBKEY_GROUP} = $strDefaultGroup; - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_PATH . ':default'}{&MANIFEST_SUBKEY_MODE} = $strDefaultPathMode; -} - -#################################################################################################################################### -# backupLast -#################################################################################################################################### -sub backupLast -{ - my $self = shift; - my $iRepo = shift; - - my @stryBackup = storageRepo({iRepo => $iRepo})->list( - $self->repoBackupPath(undef, $iRepo), - {strExpression => '[0-9]{8}-[0-9]{6}F(_[0-9]{8}-[0-9]{6}(D|I)){0,1}', strSortOrder => 'reverse'}); - - if (!defined($stryBackup[0])) - { - confess 'no backup was found: ' . join(@stryBackup, ', '); - } - - return $stryBackup[0]; -} - -#################################################################################################################################### -# check -#################################################################################################################################### -sub check -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strComment, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->check', \@_, - {name => 'strComment'}, - {name => 'oParam', required => false}, - ); - - $strComment = - 'check ' . $self->stanza() . ' - ' . $strComment . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " $strComment"); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - (defined($$oParam{iTimeout}) ? " --archive-timeout=$$oParam{iTimeout}" : '') . - (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . - (!defined($oParam->{bStanza}) || $oParam->{bStanza} ? ' --stanza=' . $self->stanza() : '') . ' check', - {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, bLogOutput => $self->synthetic()}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# expire -#################################################################################################################################### -sub expire -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->check', \@_, - {name => 'oParam', required => false}, - ); - - my $strComment = - 'expire' . - (defined($$oParam{iRetentionFull}) ? " full=$$oParam{iRetentionFull}" : '') . - (defined($$oParam{iRetentionDiff}) ? " diff=$$oParam{iRetentionDiff}" : '') . - (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " ${strComment}"); - - # Determine whether or not to expect an error - my $oHostGroup = hostGroupGet(); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - (defined($$oParam{iRetentionFull}) ? " --repo1-retention-full=$$oParam{iRetentionFull}" : '') . - (defined($$oParam{iRetentionDiff}) ? " --repo1-retention-diff=$$oParam{iRetentionDiff}" : '') . - (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . - ' --repo=' . (defined($oParam->{iRepo}) ? $oParam->{iRepo} : '1') . - ' --stanza=' . $self->stanza() . ' expire', - {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, bLogOutput => $self->synthetic()}); -} - -#################################################################################################################################### -# info -#################################################################################################################################### -sub info -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strComment, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->info', \@_, - {name => 'strComment'}, - {name => 'oParam', required => false}, - ); - - $strComment = - 'info' . (defined($$oParam{strStanza}) ? " $$oParam{strStanza} stanza" : ' all stanzas') . ' - ' . $strComment . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " $strComment"); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - ' --log-level-console=warn' . - (defined($$oParam{strStanza}) ? " --stanza=$$oParam{strStanza}" : '') . - (defined($$oParam{strOutput}) ? " --output=$$oParam{strOutput}" : '') . ' info', - {strComment => $strComment, bLogOutput => $self->synthetic()}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# stanzaCreate -#################################################################################################################################### -sub stanzaCreate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strComment, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->stanzaCreate', \@_, - {name => 'strComment'}, - {name => 'oParam', required => false}, - ); - - $strComment = - 'stanza-create ' . $self->stanza() . ' - ' . $strComment . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " $strComment"); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - ' --stanza=' . $self->stanza() . - (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . - ' stanza-create', - {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, bLogOutput => $self->synthetic()}); - - if (storageRepo()->exists('backup/' . $self->stanza() . qw{/} . FILE_BACKUP_INFO)) - { - # Get the passphrase for accessing the manifest file - $self->{strCipherPassManifest} = (new pgBackRestTest::Env::BackupInfo($self->repoBackupPath()))->cipherPassSub(); - } - - if (storageRepo()->exists('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE)) - { - - # Get the passphrase for accessing the archived files - $self->{strCipherPassArchive} = - (new pgBackRestTest::Env::ArchiveInfo($self->repoArchivePath()))->cipherPassSub(); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# stanzaUpgrade -#################################################################################################################################### -sub stanzaUpgrade -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strComment, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->stanzaUpgrade', \@_, - {name => 'strComment'}, - {name => 'oParam', required => false}, - ); - - $strComment = - 'stanza-upgrade ' . $self->stanza() . ' - ' . $strComment . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " $strComment"); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - ' --stanza=' . $self->stanza() . - (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . - ' stanza-upgrade', - {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, bLogOutput => $self->synthetic()}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# stanzaDelete -#################################################################################################################################### -sub stanzaDelete -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strComment, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->stanzaDelete', \@_, - {name => 'strComment'}, - {name => 'oParam', required => false}, - ); - - $strComment = - 'stanza-delete ' . $self->stanza() . ' - ' . $strComment . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " $strComment"); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - ' --repo=' . (defined($oParam->{iRepo}) ? $oParam->{iRepo} : '1') . - ' --stanza=' . $self->stanza() . - (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . - ' stanza-delete', - {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, bLogOutput => $self->synthetic()}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# start -#################################################################################################################################### -sub start -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->start', \@_, - {name => 'oParam', required => false}, - ); - - my $strComment = - 'start' . (defined($$oParam{strStanza}) ? " $$oParam{strStanza} stanza" : ' all stanzas') . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " $strComment"); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - (defined($$oParam{strStanza}) ? " --stanza=$$oParam{strStanza}" : '') . ' start', - {strComment => $strComment, bLogOutput => $self->synthetic()}); -} - -#################################################################################################################################### -# stop -#################################################################################################################################### -sub stop -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->stop', \@_, - {name => 'oParam', required => false}, - ); - - my $strComment = - 'stop' . (defined($$oParam{strStanza}) ? " $$oParam{strStanza} stanza" : ' all stanzas') . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " $strComment"); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - (defined($$oParam{strStanza}) ? " --stanza=$$oParam{strStanza}" : '') . - (defined($$oParam{bForce}) && $$oParam{bForce} ? ' --force' : '') . ' stop', - {strComment => $strComment, bLogOutput => $self->synthetic()}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# configCreate -#################################################################################################################################### -sub configCreate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->stop', \@_, - {name => 'oParam', required => false}, - ); - - my %oParamHash; - my $strStanza = $self->stanza(); - my $oHostGroup = hostGroupGet(); - my $oHostBackup = $oHostGroup->hostGet($self->backupDestination()); - my $oHostDbPrimary = $oHostGroup->hostGet(HOST_DB_PRIMARY); - my $oHostDbStandby = $oHostGroup->hostGet(HOST_DB_STANDBY, true); - - my $bArchiveAsync = defined($$oParam{bArchiveAsync}) ? $$oParam{bArchiveAsync} : false; - - my $iRepoTotal = defined($oParam->{iRepoTotal}) ? $oParam->{iRepoTotal} : 1; - - if ($iRepoTotal < 1 || $iRepoTotal > 2) - { - confess "invalid repo total ${iRepoTotal}"; - } - - # General options - # ------------------------------------------------------------------------------------------------------------------------------ - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'beta'} = 'y'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'job-retry'} = 0; - - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-level-console'} = lc(DETAIL); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-level-file'} = testRunGet()->logLevelTestFile(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-level-stderr'} = lc(OFF); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-subprocess'} = - testRunGet()->logLevelTestFile() eq lc(OFF) ? 'n' : 'y'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-timestamp'} = 'n'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'buffer-size'} = '64k'; - - if ($oParam->{bBundle}) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-bundle'} = 'y'; - # Set bundle size/limit smaller for testing and because FakeGCS does not do multi-part upload - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-bundle-size'} = '1MiB'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-bundle-limit'} = '64KiB'; - } - - if ($oParam->{bBlockIncr}) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-block'} = 'y'; - } - - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-path'} = $self->logPath(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'lock-path'} = $self->lockPath(); - - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'protocol-timeout'} = 60; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'db-timeout'} = 45; - - # Set to make sure that changing the default works and to speed compression for testing - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'compress-level'} = 3; - - # Only set network compress level if there is more than one host - if ($oHostBackup != $oHostDbPrimary) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'compress-level-network'} = 1; - } - - if (defined($oParam->{strCompressType}) && $oParam->{strCompressType} ne 'gz') - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'compress-type'} = $oParam->{strCompressType}; - } - - if ($self->isHostBackup()) - { - if ($self->repoEncrypt()) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-cipher-type'} = CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-cipher-pass'} = 'x'; - } - - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-path'} = $self->repoPath(); - - # S3 settings - if ($oParam->{strStorage} eq S3) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-type'} = S3; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-key'} = HOST_S3_ACCESS_KEY; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-key-secret'} = HOST_S3_ACCESS_SECRET_KEY; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-bucket'} = HOST_S3_BUCKET; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-endpoint'} = HOST_S3_ENDPOINT; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-region'} = HOST_S3_REGION; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-verify-ssl'} = 'n'; - } - elsif ($oParam->{strStorage} eq AZURE) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-type'} = AZURE; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-azure-account'} = HOST_AZURE_ACCOUNT; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-azure-key'} = HOST_AZURE_KEY; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-azure-container'} = HOST_AZURE_CONTAINER; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-azure-host'} = HOST_AZURE; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-azure-verify-tls'} = 'n'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-azure-uri-style'} = 'path'; - } - elsif ($oParam->{strStorage} eq GCS) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-type'} = GCS; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-gcs-bucket'} = HOST_GCS_BUCKET; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-gcs-key-type'} = HOST_GCS_KEY_TYPE; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-gcs-key'} = HOST_GCS_KEY; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-gcs-endpoint'} = HOST_GCS . ':' . HOST_GCS_PORT; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-storage-verify-tls'} = 'n'; - } - - if ($iRepoTotal == 2) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-path'} = $self->repo2Path(); - } - - if (defined($$oParam{bHardlink}) && $$oParam{bHardlink}) - { - $self->{bHardLink} = true; - $oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'repo1-s3-hardlink'} = 'y'; - } - - $oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'archive-copy'} = 'y'; - $oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'start-fast'} = 'y'; - } - - # Host specific options - # ------------------------------------------------------------------------------------------------------------------------------ - - # If this is the backup host - if ($self->isHostBackup()) - { - my $oHostDb1 = $oHostDbPrimary; - my $oHostDb2 = $oHostDbStandby; - - if ($self->nameTest(HOST_DB_STANDBY)) - { - $oHostDb1 = $oHostDbStandby; - $oHostDb2 = $oHostDbPrimary; - } - - if ($self->nameTest(HOST_BACKUP)) - { - $oParamHash{$strStanza}{'pg1-host'} = $oHostDb1->nameGet(); - $oParamHash{$strStanza}{'pg1-host-user'} = $oHostDb1->userGet(); - $oParamHash{$strStanza}{'pg1-host-cmd'} = $oHostDb1->backrestExe(); - $oParamHash{$strStanza}{'pg1-host-config'} = $oHostDb1->backrestConfig(); - - if ($oParam->{bTls}) - { - $oParamHash{$strStanza}{'pg1-host-type'} = 'tls'; - $oParamHash{$strStanza}{'pg1-host-cert-file'} = testRunGet()->basePath() . HOST_CLIENT_CERT; - $oParamHash{$strStanza}{'pg1-host-key-file'} = testRunGet()->basePath() . HOST_CLIENT_KEY; - } - - # Port can't be configured for a synthetic host - if (!$self->synthetic()) - { - $oParamHash{$strStanza}{'pg1-port'} = $oHostDb1->pgPort(); - } - } - - $oParamHash{$strStanza}{'pg1-path'} = $oHostDb1->dbBasePath(); - - if (defined($oHostDb2)) - { - # Add an invalid replica to simulate more than one replica. A warning should be thrown when a stanza is created and a - # valid replica should be chosen. Only do this for SSH since TLS takes longer to timeout. - if (!$oParam->{bTls}) - { - $oParamHash{$strStanza}{"pg2-host"} = BOGUS; - $oParamHash{$strStanza}{"pg2-host-user"} = $oHostDb2->userGet(); - $oParamHash{$strStanza}{"pg2-host-cmd"} = $oHostDb2->backrestExe(); - $oParamHash{$strStanza}{"pg2-host-config"} = $oHostDb2->backrestConfig(); - $oParamHash{$strStanza}{"pg2-path"} = $oHostDb2->dbBasePath(); - } - - # Set a flag so we know there's a bogus host - $self->{bBogusHost} = true; - - # Set a valid replica to a higher index to ensure skipping indexes does not make a difference - $oParamHash{$strStanza}{"pg256-host"} = $oHostDb2->nameGet(); - $oParamHash{$strStanza}{"pg256-host-user"} = $oHostDb2->userGet(); - $oParamHash{$strStanza}{"pg256-host-cmd"} = $oHostDb2->backrestExe(); - $oParamHash{$strStanza}{"pg256-host-config"} = $oHostDb2->backrestConfig(); - $oParamHash{$strStanza}{"pg256-path"} = $oHostDb2->dbBasePath(); - - if ($oParam->{bTls}) - { - $oParamHash{$strStanza}{'pg256-host-type'} = 'tls'; - $oParamHash{$strStanza}{'pg256-host-cert-file'} = testRunGet()->basePath() . HOST_CLIENT_CERT; - $oParamHash{$strStanza}{'pg256-host-key-file'} = testRunGet()->basePath() . HOST_CLIENT_KEY; - } - - # Only test explicit ports on the backup server. This is so locally configured ports are also tested. - if (!$self->synthetic() && $self->nameTest(HOST_BACKUP)) - { - $oParamHash{$strStanza}{"pg256-port"} = $oHostDb2->pgPort(); - } - } - } - elsif ($oParam->{strStorage} eq SFTP) - { - my $oHostDb1 = $oHostDbPrimary; - my $oHostDb2 = $oHostDbStandby; - - if ($self->nameTest(HOST_DB_STANDBY)) - { - $oHostDb1 = $oHostDbStandby; - $oHostDb2 = $oHostDbPrimary; - } - - # Set a flag so we know there's a bogus host - $self->{bBogusHost} = true; - - # Set a valid replica to a higher index to ensure skipping indexes does not make a difference - $oParamHash{$strStanza}{"pg256-host"} = $oHostDb2->nameGet(); - $oParamHash{$strStanza}{"pg256-host-user"} = $oHostDb2->userGet(); - $oParamHash{$strStanza}{"pg256-host-cmd"} = $oHostDb2->backrestExe(); - $oParamHash{$strStanza}{"pg256-host-config"} = $oHostDb2->backrestConfig(); - $oParamHash{$strStanza}{"pg256-path"} = $oHostDb2->dbBasePath(); - $oParamHash{$strStanza}{"pg256-port"} = $oHostDb2->pgPort(); - } - - # If this is a database host - if ($self->isHostDb()) - { - $oParamHash{$strStanza}{'pg1-path'} = $self->dbBasePath(); - - if (!$self->synthetic()) - { - $oParamHash{$strStanza}{'pg1-socket-path'} = $self->pgSocketPath(); - $oParamHash{$strStanza}{'pg1-port'} = $self->pgPort(); - } - - if ($bArchiveAsync) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL . ':archive-push'}{'archive-async'} = 'y'; - } - - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'spool-path'} = $self->spoolPath(); - - # If the backup host is remote - if (!$self->isHostBackup()) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-user'} = $oHostBackup->userGet(); - - if ($oHostBackup->nameGet() eq HOST_SFTP) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-type'} = "sftp"; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-sftp-host'} = HOST_SFTP; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-sftp-host-key-hash-type'} = "sha1"; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-sftp-host-user'} = TEST_USER; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-sftp-private-key-file'} = testRunGet()->basePath() . SSH_PRIVATE_KEY; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-sftp-public-key-file'} = testRunGet()->basePath() . SSH_PUBLIC_KEY; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-sftp-host-key-check-type'} = "none"; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-path'} = $self->repoPath(); - - # At what count do we hit diminishing returns - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'process-max'} = 8; - - $oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'start-fast'} = 'y'; - } - else - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host'} = $oHostBackup->nameGet(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-cmd'} = $oHostBackup->backrestExe(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-config'} = $oHostBackup->backrestConfig(); - } - - if ($oParam->{bTls}) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-type'} = 'tls'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-cert-file'} = testRunGet()->basePath() . HOST_CLIENT_CERT; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-key-file'} = testRunGet()->basePath() . HOST_CLIENT_KEY; - } - - if ($iRepoTotal == 2) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host'} = $oHostBackup->nameGet(); - $oParam->{bTls} ? $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-type'} = 'tls' : undef; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-user'} = $oHostBackup->userGet(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-cmd'} = $oHostBackup->backrestExe(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-config'} = $oHostBackup->backrestConfig(); - - if ($oParam->{bTls}) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-type'} = 'tls'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-cert-file'} = testRunGet()->basePath() . HOST_CLIENT_CERT; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-key-file'} = testRunGet()->basePath() . HOST_CLIENT_KEY; - } - } - - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-path'} = $self->logPath(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'lock-path'} = $self->lockPath(); - } - } - - # Write out the configuration file - storageTest()->put($self->backrestConfig(), iniRender(\%oParamHash, true)); -} - -#################################################################################################################################### -# configUpdate - update configuration with new options -#################################################################################################################################### -sub configUpdate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $hParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->configUpdate', \@_, - {name => 'hParam'}, - ); - - # Load db config file - my $oConfig = iniParse(${storageTest()->get($self->backrestConfig())}, {bRelaxed => true}); - - # Load params - foreach my $strSection (keys(%{$hParam})) - { - foreach my $strKey (keys(%{$hParam->{$strSection}})) - { - if (defined($hParam->{$strSection}{$strKey})) - { - $oConfig->{$strSection}{$strKey} = $hParam->{$strSection}{$strKey}; - } - else - { - delete($oConfig->{$strSection}{$strKey}); - } - } - } - - storageTest()->put($self->backrestConfig(), iniRender($oConfig, true)); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# manifestMunge -# -# Allows for munging of the manifest while making it appear to be valid. This is used to create various error conditions that should -# be caught by the unit tests. -#################################################################################################################################### -sub manifestMunge -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strBackup, - $hParam, - $bCache, - ) = - logDebugParam - ( - __PACKAGE__ . '->manifestMunge', \@_, - {name => 'strBackup'}, - {name => '$hParam'}, - {name => 'bCache', default => true}, - ); - - $self->infoMunge($self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), $hParam, $bCache, true); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# manifestRestore -#################################################################################################################################### -sub manifestRestore -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strBackup, - $bSave, - ) = - logDebugParam - ( - __PACKAGE__ . '->manifestRestore', \@_, - {name => 'strBackup'}, - {name => 'bSave', default => true}, - ); - - $self->infoRestore($self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), $bSave); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# infoMunge -# -# With the file name specified (e.g. /repo/archive/db/archive.info) copy the current values from the file into the global hash and -# update the file with the new values passed. Later, using infoRestore, the global variable will be used to restore the file to its -# original state. -#################################################################################################################################### -sub infoMunge -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFileName, - $hParam, - $bCache, - $bManifest, - ) = - logDebugParam - ( - __PACKAGE__ . '->infoMunge', \@_, - {name => 'strFileName'}, - {name => 'hParam'}, - {name => 'bCache', default => true}, - {name => 'bManifest', default => false}, - ); - - # If the original file content does not exist then load it - if (!defined($self->{hInfoFile}{$strFileName})) - { - $self->{hInfoFile}{$strFileName} = new pgBackRestDoc::Common::Ini( - storageRepo(), $strFileName, - {strCipherPass => !$bManifest ? undef : $self->cipherPassManifest()}); - } - - # Make a copy of the original file contents - my $oMungeIni = new pgBackRestDoc::Common::Ini( - storageRepo(), $strFileName, - {bLoad => false, strContent => iniRender($self->{hInfoFile}{$strFileName}->{oContent}), - strCipherPass => !$bManifest ? undef : $self->cipherPassManifest()}); - - # Load params - foreach my $strSection (keys(%{$hParam})) - { - foreach my $strKey (keys(%{$hParam->{$strSection}})) - { - if (ref($hParam->{$strSection}{$strKey}) eq 'HASH') - { - foreach my $strSubKey (keys(%{$hParam->{$strSection}{$strKey}})) - { - # Munge the copy with the new parameter values - if (defined($hParam->{$strSection}{$strKey}{$strSubKey})) - { - $oMungeIni->set($strSection, $strKey, $strSubKey, $hParam->{$strSection}{$strKey}{$strSubKey}); - } - else - { - $oMungeIni->remove($strSection, $strKey, $strSubKey); - } - } - } - else - { - # Munge the copy with the new parameter values - if (defined($hParam->{$strSection}{$strKey})) - { - $oMungeIni->set($strSection, $strKey, undef, $hParam->{$strSection}{$strKey}); - } - else - { - $oMungeIni->remove($strSection, $strKey); - } - } - } - } - - # Save the munged data to the file - $oMungeIni->save(); - - # Clear the cache is requested - if (!$bCache) - { - delete($self->{hInfoFile}{$strFileName}); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# infoRestore -# -# With the file name specified (e.g. /repo/archive/db/archive.info) use the original file contents in the global hash to restore the -# file to its original state after modifying the values with infoMunge. -#################################################################################################################################### -sub infoRestore -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFileName, - $bSave, - ) = - logDebugParam - ( - __PACKAGE__ . '->infoRestore', \@_, - {name => 'strFileName'}, - {name => 'bSave', default => true}, - ); - - # If the original file content exists in the global hash, then save it to the file - if (defined($self->{hInfoFile}{$strFileName})) - { - if ($bSave) - { - # Save the munged data to the file - $self->{hInfoFile}{$strFileName}->{bModified} = true; - $self->{hInfoFile}{$strFileName}->save(); - } - } - else - { - confess &log(ASSERT, "There is no original data cached for $strFileName. infoMunge must be called first."); - } - - # Remove the element from the hash - delete($self->{hInfoFile}{$strFileName}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# configRecovery -#################################################################################################################################### -sub configRecovery -{ - my $self = shift; - my $oHostBackup = shift; - my $oRecoveryHashRef = shift; - - # Get stanza - my $strStanza = $self->stanza(); - - # Load db config file - my $oConfig = iniParse(${storageTest->get($self->backrestConfig())}, {bRelaxed => true}); - - # Rewrite recovery options - my @stryRecoveryOption; - - foreach my $strOption (sort(keys(%$oRecoveryHashRef))) - { - push (@stryRecoveryOption, "${strOption}=${$oRecoveryHashRef}{$strOption}"); - } - - if (@stryRecoveryOption) - { - $oConfig->{$strStanza}{'recovery-option'} = \@stryRecoveryOption; - } - - # Save db config file - storageTest()->put($self->backrestConfig(), iniRender($oConfig, true)); -} - -#################################################################################################################################### -# configRemap -#################################################################################################################################### -sub configRemap -{ - my $self = shift; - my $oRemapHashRef = shift; - my $oManifestRef = shift; - - # Get stanza name - my $strStanza = $self->stanza(); - - # Load db config file - my $oConfig = iniParse(${storageTest()->get($self->backrestConfig())}, {bRelaxed => true}); - - # Load backup config file - my $oRemoteConfig; - my $oHostBackup = - !$self->standby() && !$self->nameTest($self->backupDestination()) ? - hostGroupGet()->hostGet($self->backupDestination()) : undef; - - if (defined($oHostBackup)) - { - $oRemoteConfig = iniParse(${storageTest()->get($oHostBackup->backrestConfig())}, {bRelaxed => true}); - } - - # Rewrite recovery section - delete($oConfig->{"${strStanza}:restore"}{'tablespace-map'}); - my @stryTablespaceMap; - - foreach my $strRemap (sort(keys(%$oRemapHashRef))) - { - my $strRemapPath = ${$oRemapHashRef}{$strRemap}; - - if ($strRemap eq MANIFEST_TARGET_PGDATA) - { - $oConfig->{$strStanza}{'pg1-path'} = $strRemapPath; - - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_PATH} = $strRemapPath; - - if (defined($oHostBackup)) - { - $oRemoteConfig->{$strStanza}{'pg1-path'} = $strRemapPath; - } - } - else - { - my $strTablespaceOid = (split('\/', $strRemap))[1]; - push (@stryTablespaceMap, "${strTablespaceOid}=${strRemapPath}"); - - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strRemap}{&MANIFEST_SUBKEY_PATH} = $strRemapPath; - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{MANIFEST_TARGET_PGDATA . "/${strRemap}"}{destination} = $strRemapPath; - } - } - - if (@stryTablespaceMap) - { - $oConfig->{"${strStanza}:restore"}{'tablespace-map'} = \@stryTablespaceMap; - } - - # Save db config file - storageTest()->put($self->backrestConfig(), iniRender($oConfig, true)); - - # Save backup config file (but not if this is the standby which is not the source of backups) - if (defined($oHostBackup)) - { - storageTest()->put($oHostBackup->backrestConfig(), iniRender($oRemoteConfig, true)); - } -} - -#################################################################################################################################### -# restore -#################################################################################################################################### -sub restore -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strComment, - $strBackup, - $rhExpectedManifest, - $rhRemapHash, - $bDelta, - $bForce, - $strType, - $strTarget, - $bTargetExclusive, - $strTargetAction, - $strTargetTimeline, - $rhRecoveryHash, - $iExpectedExitStatus, - $strOptionalParam, - $bTablespace, - $strUser, - $strBackupExpected, - $iRepo, - ) = - logDebugParam - ( - __PACKAGE__ . '->restore', \@_, - {name => 'strComment', required => false}, - {name => 'strBackup'}, - {name => 'rhExpectedManifest', optional => true}, - {name => 'rhRemapHash', optional => true}, - {name => 'bDelta', optional => true, default => false}, - {name => 'bForce', optional => true, default => false}, - {name => 'strType', optional => true}, - {name => 'strTarget', optional => true}, - {name => 'bTargetExclusive', optional => true, default => false}, - {name => 'strTargetAction', optional => true}, - {name => 'strTargetTimeline', optional => true}, - {name => 'rhRecoveryHash', optional => true}, - {name => 'iExpectedExitStatus', optional => true}, - {name => 'strOptionalParam', optional => true}, - {name => 'bTablespace', optional => true}, - {name => 'strUser', optional => true}, - {name => 'strBackupExpected', optional => true}, - {name => 'iRepo', optional => true}, - ); - - # Build link map options - my $strLinkMap; - - foreach my $strTarget (sort(keys(%{$self->{hLinkRemap}}))) - { - $strLinkMap .= " --link-map=\"${strTarget}=${$self->{hLinkRemap}}{$strTarget}\""; - } - - $strComment = 'restore' . - ($bDelta ? ' delta' : '') . - ($bForce ? ', force' : '') . - ($strBackup ne 'latest' ? ", backup '${strBackup}'" : '') . - # This does not output 'default' for synthetic tests to make expect logs match up (may change later) - ($strType ? ", type '${strType}'" : (defined($rhExpectedManifest) ? '' : ", type 'default'")) . - ($strTarget ? ", target '${strTarget}'" : '') . - ($strTargetTimeline ? ", timeline '${strTargetTimeline}'" : '') . - ($bTargetExclusive ? ', exclusive' : '') . - (defined($strTargetAction) && $strTargetAction ne 'pause' ? ", target-action=${strTargetAction}" : '') . - (defined($rhRemapHash) ? ', remap' : '') . - (defined($iExpectedExitStatus) ? ", expect exit ${iExpectedExitStatus}" : '') . - (defined($strComment) ? " - ${strComment}" : '') . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " ${strComment}"); - - # Get the backup host - my $oHostGroup = hostGroupGet(); - my $oHostBackup = defined($oHostGroup->hostGet(HOST_BACKUP, true)) ? $oHostGroup->hostGet(HOST_BACKUP) : $self; - - # If the repo was not passed, then use repo1 as the repo for getting the expected manifest/backup - my $iRepoDefault = !defined($iRepo) ? 1 : $iRepo; - - # Load the expected manifest if it was not defined - my $oExpectedManifest = undef; - - # If an expected backup is defined, then the strBackup should be the default to allow the restore process to select the backup - # - which should be the backup passed as strBackupExpected. If it is not defined, then set it based on the strBackup passed. - if (!defined($strBackupExpected)) - { - $strBackupExpected = $strBackup eq 'latest' ? $oHostBackup->backupLast($iRepoDefault) : $strBackup; - } - - if (!defined($rhExpectedManifest)) - { - # Load the manifest from the backup expected to be chosen/processed by restore - my $oExpectedManifest = new pgBackRestTest::Env::Manifest( - $self->repoBackupPath($strBackupExpected . qw{/} . FILE_MANIFEST, $iRepoDefault), - {strCipherPass => $oHostBackup->cipherPassManifest(), oStorage => storageRepo({iRepo => $iRepoDefault})}); - - $rhExpectedManifest = $oExpectedManifest->{oContent}; - - # Remap links in the expected manifest - foreach my $strTarget (sort(keys(%{$self->{hLinkRemap}}))) - { - my $strDestination = ${$self->{hLinkRemap}}{$strTarget}; - my $strTarget = 'pg_data/' . $strTarget; - my $strTargetPath = $strDestination; - - # If this link is to a file then the specified path must be split into file and path parts - if ($oExpectedManifest->isTargetFile($strTarget)) - { - $strTargetPath = dirname($strTargetPath); - - # Error when the path is not deep enough to be valid - if (!defined($strTargetPath)) - { - confess &log(ERROR, "${strDestination} is not long enough to be target for ${strTarget}"); - } - - # Set the file part - $oExpectedManifest->set( - MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_FILE, - substr($strDestination, length($strTargetPath) + 1)); - - # Set the link target - $oExpectedManifest->set( - MANIFEST_SECTION_TARGET_LINK, $strTarget, MANIFEST_SUBKEY_DESTINATION, $strDestination); - } - else - { - # Set the link target - $oExpectedManifest->set(MANIFEST_SECTION_TARGET_LINK, $strTarget, MANIFEST_SUBKEY_DESTINATION, $strTargetPath); - } - - # Set the target path - $oExpectedManifest->set(MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_PATH, $strTargetPath); - } - } - - # Get the backup host - if (defined($rhRemapHash)) - { - $self->configRemap($rhRemapHash, $rhExpectedManifest); - } - - if (defined($rhRecoveryHash)) - { - $self->configRecovery($oHostBackup, $rhRecoveryHash); - } - - # Create the restore command - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - ($bDelta ? ' --delta' : '') . - ($bForce ? ' --force' : '') . - ($strBackup ne 'latest' ? " --set=${strBackup}" : '') . - (defined($strOptionalParam) ? " ${strOptionalParam} " : '') . - (defined($strType) && $strType ne CFGOPTVAL_RESTORE_TYPE_DEFAULT ? " --type=${strType}" : '') . - (defined($strTarget) ? " --target=\"${strTarget}\"" : '') . - (defined($strTargetTimeline) ? " --target-timeline=\"${strTargetTimeline}\"" : '') . - ($bTargetExclusive ? ' --target-exclusive' : '') . - (defined($strLinkMap) ? $strLinkMap : '') . - ($self->synthetic() ? '' : ' --link-all') . - (defined($strTargetAction) && $strTargetAction ne 'pause' ? " --target-action=${strTargetAction}" : '') . - (defined($iRepo) ? " --repo=${iRepo}" : '') . - " --stanza=" . $self->stanza() . ' restore', - {strComment => $strComment, iExpectedExitStatus => $iExpectedExitStatus, bLogOutput => $self->synthetic()}, - $strUser); - - if (!defined($iExpectedExitStatus)) - { - # Only compare restores in repo1. There is not a lot of value in comparing restores in other repos and it would require a - # lot of changes to the Perl test harness. - if ($iRepoDefault == 1) - { - $self->restoreCompare($strBackupExpected, dclone($rhExpectedManifest), $bTablespace); - } - } -} - -#################################################################################################################################### -# restoreCompare -#################################################################################################################################### -sub restoreCompare -{ - my $self = shift; - my $strBackup = shift; - my $oExpectedManifestRef = shift; - my $bTablespace = shift; - - my $strTestPath = $self->testPath(); - - # Get the backup host - my $oHostGroup = hostGroupGet(); - my $oHostBackup = defined($oHostGroup->hostGet(HOST_BACKUP, true)) ? $oHostGroup->hostGet(HOST_BACKUP) : $self; - - # Load the last manifest if it exists - my $oLastManifest = undef; - - if (defined(${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR})) - { - my $oExpectedManifest = - new pgBackRestTest::Env::Manifest( - $self->repoBackupPath( - ($strBackup eq 'latest' ? $oHostBackup->backupLast() : $strBackup) . '/' . FILE_MANIFEST), - {strCipherPass => $oHostBackup->cipherPassManifest()}); - - # Get the --delta option from the backup manifest so the actual manifest can be built the same way for comparison - $$oExpectedManifestRef{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA} = - $oExpectedManifest->get(MANIFEST_SECTION_BACKUP_OPTION, &MANIFEST_KEY_DELTA); - - $oLastManifest = - new pgBackRestTest::Env::Manifest( - $self->repoBackupPath( - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR} . qw{/} . FILE_MANIFEST), - {strCipherPass => $oHostBackup->cipherPassManifest()}); - } - - # Generate the tablespace map for real backups - my $oTablespaceMap = undef; - - if (!$self->synthetic()) - { - # Tablespace_map file is not restored in versions >= 9.5 because it interferes with internal remapping features. - if (${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} >= PG_VERSION_95) - { - delete(${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{MANIFEST_TARGET_PGDATA . '/tablespace_map'}); - } - - foreach my $strTarget (keys(%{${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}})) - { - if (defined(${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID})) - { - my $iTablespaceId = - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID}; - - $$oTablespaceMap{$iTablespaceId} = - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_NAME}; - } - } - } - - # Generate the actual manifest - my $strDbClusterPath = - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_PATH}; - - if (defined($bTablespace) && !$bTablespace) - { - foreach my $strTarget (keys(%{${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}})) - { - if ($$oExpectedManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TYPE} eq - MANIFEST_VALUE_LINK && - defined($$oExpectedManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID})) - { - my $strRemapPath; - my $iTablespaceName = - $$oExpectedManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_NAME}; - - $strRemapPath = "../../tablespace/${iTablespaceName}"; - - $$oExpectedManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_PATH} = $strRemapPath; - $$oExpectedManifestRef{&MANIFEST_SECTION_TARGET_LINK}{MANIFEST_TARGET_PGDATA . "/${strTarget}"} - {&MANIFEST_SUBKEY_DESTINATION} = $strRemapPath; - } - } - } - - my $oActualManifest = new pgBackRestTest::Env::Manifest( - "${strTestPath}/" . FILE_MANIFEST, - {bLoad => false, strDbVersion => $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION}, - iDbCatalogVersion => $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG}, - oStorage => storageTest()}); - - # Build the actual manifest using the delta setting that was actually used by the latest backup if one exists - $oActualManifest->build(storageTest(), $strDbClusterPath, $oLastManifest, false, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA}, $oTablespaceMap); - $oActualManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_DELTA, undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS_TYPE, undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE}); - - my $strSectionPath = $oActualManifest->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH); - - foreach my $strName ($oActualManifest->keys(MANIFEST_SECTION_TARGET_FILE)) - { - # Remove repo checksum - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{'rck'}); - - # When bundling zero-length files will not have a reference - if ($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-bundle'} && - $oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_SIZE} == 0) - { - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE); - } - - # If synthetic match checksum errors since they can't be verified here - if ($self->synthetic) - { - my $bChecksumPage = $oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM_PAGE}; - - if (defined($bChecksumPage)) - { - $oActualManifest->boolSet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE, $bChecksumPage); - - if (!$bChecksumPage && - defined($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR})) - { - $oActualManifest->set( - MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR, - $oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR}); - } - } - } - # Else if page checksums are enabled make sure the correct files are being checksummed - else - { - if ($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_CHECKSUM_PAGE}) - { - if (defined($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM_PAGE}) != - isChecksumPage($strName)) - { - confess - "check-page actual for ${strName} is " . - ($oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, - MANIFEST_SUBKEY_CHECKSUM_PAGE) ? 'set' : '[undef]') . - ' but isChecksumPage() says it should be ' . - (isChecksumPage($strName) ? 'set' : '[undef]') . '.'; - } - - # Because the page checksum flag is copied to incr and diff from the previous backup but further processing is not - # done, they can't be expected to match so delete them. - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM_PAGE}); - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE); - } - } - - if (!$self->synthetic()) - { - $oActualManifest->set( - MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strName}{size}); - } - - # Remove repo-size, bn*, bi* from the manifest - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_REPO_SIZE}); - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, "bni"); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{"bni"}); - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, "bno"); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{"bno"}); - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, "bi"); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{"bi"}); - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, "bic"); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{"bic"}); - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, "bim"); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{"bim"}); - - if ($oActualManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) != 0) - { - my $oStat = lstat($oActualManifest->dbPathGet($strSectionPath, $strName)); - - # When performing a selective restore, the files for the database(s) that are not restored are still copied but as empty - # sparse files (blocks == 0). If the file is not a sparse file or is a link, then get the actual checksum for comparison - if ($oStat->blocks > 0 || S_ISLNK($oStat->mode)) - { - my ($strHash) = storageTest()->hashSize($oActualManifest->dbPathGet($strSectionPath, $strName)); - - $oActualManifest->set( - MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM, $strHash); - - # If the delta option was set, it is possible that the checksum on the file changed from the last manifest. If so, - # then the file was expected to be copied by the backup and therefore the reference would have been removed. - if ($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA}) - { - # If the actual checksum and last manifest checksum don't match, remove the reference - if (defined($oLastManifest) && - $oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM) && - $strHash ne $oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM)) - { - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE); - } - } - } - else - { - # If there is a sparse file, remove the checksum and reference since they may or may not match. In this case, it is - # not important to check them since it is known that the file was intentionally not restored. - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM); - delete(${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM}); - - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE); - delete(${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_REFERENCE}); - } - } - } - - # If PostgreSQL >= 12 don't compare postgresql.auto.conf since it will have recovery settings written into it - if (${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} >= PG_VERSION_12) - { - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{'pg_data/postgresql.auto.conf'}); - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, 'pg_data/postgresql.auto.conf'); - } - - # If the link section is empty then delete it and the default section - if (keys(%{${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_LINK}}) == 0) - { - delete($$oExpectedManifestRef{&MANIFEST_SECTION_TARGET_LINK}); - delete($$oExpectedManifestRef{&MANIFEST_SECTION_TARGET_LINK . ':default'}); - } - - # Set actual to expected for settings that always change from backup to backup - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_CHECK, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ARCHIVE_CHECK}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_COPY, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ARCHIVE_COPY}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_BACKUP_STANDBY, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BACKUP_STANDBY}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_BACKUP_STANDBY, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BACKUP_STANDBY}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_BUFFER_SIZE, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BUFFER_SIZE}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS_LEVEL, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_LEVEL}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS_LEVEL_NETWORK, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_LEVEL_NETWORK}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_HARDLINK, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_HARDLINK}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ONLINE, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ONLINE}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_PROCESS_MAX, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX}); - $oActualManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_DELTA, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA}); - - $oActualManifest->set(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_VERSION, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION}); - $oActualManifest->numericSet(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_CONTROL, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CONTROL}); - $oActualManifest->numericSet(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_CATALOG, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG}); - $oActualManifest->numericSet(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_SYSTEM_ID, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_SYSTEM_ID}); - $oActualManifest->numericSet(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_ID, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_ID}); - - $oActualManifest->set(INI_SECTION_BACKREST, INI_KEY_VERSION, undef, - ${$oExpectedManifestRef}{&INI_SECTION_BACKREST}{&INI_KEY_VERSION}); - - # Copy passphrase if one exists - if (defined($oExpectedManifestRef->{&INI_SECTION_CIPHER}) && - defined($oExpectedManifestRef->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS})) - { - $oActualManifest->set(INI_SECTION_CIPHER, INI_KEY_CIPHER_PASS, undef, - $oExpectedManifestRef->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS}); - } - - # This option won't be set in the actual manifest - delete($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_CHECKSUM_PAGE}); - - if ($self->synthetic()) - { - $oActualManifest->remove(MANIFEST_SECTION_BACKUP); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}); - } - else - { - $oActualManifest->set( - INI_SECTION_BACKREST, INI_KEY_CHECKSUM, undef, $oExpectedManifestRef->{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM}); - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL, undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL}); - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, 'backup-reference', undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-reference'}); - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START, undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_COPY_START}); - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_START, undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START}); - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP, undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_STOP}); - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE, undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TYPE}); - - if (defined($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-bundle'})) - { - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, 'backup-bundle', undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-bundle'}); - } - - if (defined($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-bundle-raw'})) - { - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, 'backup-bundle-raw', undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-bundle-raw'}); - } - - # Delete block incr headers since old Perl manifest code will not generate them - delete($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-block-incr'}); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-block-incr-size'}); - - $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LSN_START, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LSN_START}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LSN_STOP, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LSN_STOP}); - - if (defined(${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_ARCHIVE_START})) - { - $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_START, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_ARCHIVE_START}); - } - - if (${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_ARCHIVE_STOP}) - { - $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_STOP, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_ARCHIVE_STOP}); - } - } - - # Check that archive status exists in the manifest for an online backup - my $strArchiveStatusPath = MANIFEST_TARGET_PGDATA . qw{/} . $oActualManifest->walPath() . qw{/} . DB_PATH_ARCHIVESTATUS; - - if ($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ONLINE} && - !defined($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_PATH}{$strArchiveStatusPath})) - { - confess &log(ERROR, "${strArchiveStatusPath} expected for online backup", ERROR_ASSERT); - } - - # Delete the list of DBs - delete($$oExpectedManifestRef{&MANIFEST_SECTION_DB}); - - # Only update defaults if the expect manifest is synthetic. If loaded from a file the defaults will already be correct. - if ($self->synthetic()) - { - $self->manifestDefault($oExpectedManifestRef); - } - - # Newer Perls will change this variable to a number whenever a numeric comparison is performed. It is expected to be a string so - # make sure it is one before saving. - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} .= ''; - - storageTest()->put("${strTestPath}/actual.manifest", iniRender($oActualManifest->{oContent})); - storageTest()->put("${strTestPath}/expected.manifest", iniRender($oExpectedManifestRef)); - - executeTest("diff ${strTestPath}/expected.manifest ${strTestPath}/actual.manifest"); - - storageTest()->remove("${strTestPath}/expected.manifest"); - storageTest()->remove("${strTestPath}/actual.manifest"); -} - -#################################################################################################################################### -# Get repo backup/archive path -#################################################################################################################################### -sub repoSubPath -{ - my $self = shift; - my $strSubPath = shift; - my $strPath = shift; - my $iRepo = shift; - - my $strRepoPath = $self->repoPath(); - - if (defined($iRepo) && $iRepo == 2) - { - $strRepoPath = $self->repo2Path(); - } - - return - ($strRepoPath eq '/' ? '' : $strRepoPath) . "/${strSubPath}/" . $self->stanza() . - (defined($strPath) ? "/${strPath}" : ''); -} - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub backrestConfig {return shift->{strBackRestConfig}} -sub backupDestination {return shift->{strBackupDestination}} -sub backrestExe {return testRunGet()->backrestExe()} -sub bogusHost {return shift->{bBogusHost}} -sub hardLink {return shift->{bHardLink}} -sub hasLink {storageRepo()->capability(STORAGE_CAPABILITY_LINK)} -sub isFS {storageRepo()->type() ne STORAGE_OBJECT} -sub isHostBackup {my $self = shift; return $self->backupDestination() eq $self->nameGet()} -sub isHostDbPrimary {return shift->nameGet() eq HOST_DB_PRIMARY} -sub isHostDbStandby {return shift->nameGet() eq HOST_DB_STANDBY} -sub isHostDb {my $self = shift; return $self->isHostDbPrimary() || $self->isHostDbStandby()} -sub lockPath {return shift->{strLockPath}} -sub logPath {return shift->{strLogPath}} -sub repoArchivePath {return shift->repoSubPath('archive', shift)} -sub repoBackupPath {return shift->repoSubPath('backup', shift, shift)} -sub repoPath {return shift->{strRepoPath}} -sub repo2Path {return shift->{strRepo2Path}} -sub repoEncrypt {return shift->{bRepoEncrypt}} -sub stanza {return testRunGet()->stanza()} -sub synthetic {return shift->{bSynthetic}} -sub cipherPassManifest {return shift->{strCipherPassManifest}} -sub cipherPassArchive {return shift->{strCipherPassArchive}} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostBaseTest.pm b/test/lib/pgBackRestTest/Env/Host/HostBaseTest.pm deleted file mode 100644 index 229055bbe8..0000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostBaseTest.pm +++ /dev/null @@ -1,139 +0,0 @@ -#################################################################################################################################### -# HostBackupTest.pm - Backup host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostBaseTest; -use parent 'pgBackRestTest::Common::HostTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Cwd qw(abs_path); -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); - -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::JobTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::VmTest; - -#################################################################################################################################### -# Host constants -#################################################################################################################################### -use constant HOST_BASE => 'base'; - push @EXPORT, qw(HOST_BASE); -use constant HOST_DB_PRIMARY => 'db-primary'; - push @EXPORT, qw(HOST_DB_PRIMARY); -use constant HOST_DB_STANDBY => 'db-standby'; - push @EXPORT, qw(HOST_DB_STANDBY); -use constant HOST_BACKUP => 'backup'; - push @EXPORT, qw(HOST_BACKUP); -use constant HOST_GCS => 'gcs'; - push @EXPORT, qw(HOST_GCS); -use constant HOST_AZURE => 'azure'; - push @EXPORT, qw(HOST_AZURE); -use constant HOST_S3 => 's3-server'; - push @EXPORT, qw(HOST_S3); -use constant HOST_SFTP => 'sftp-srvr'; - push @EXPORT, qw(HOST_SFTP); - -#################################################################################################################################### -# CA/cert/key constants -#################################################################################################################################### -use constant HOST_CERT_PATH => '/test/certificate/'; - -use constant HOST_CLIENT_CERT => HOST_CERT_PATH . 'pgbackrest-test-client.crt'; - push @EXPORT, qw(HOST_CLIENT_CERT); -use constant HOST_CLIENT_KEY => HOST_CERT_PATH . 'pgbackrest-test-client.key'; - push @EXPORT, qw(HOST_CLIENT_KEY); - -use constant HOST_SERVER_CA => HOST_CERT_PATH . 'pgbackrest-test-ca.crt'; - push @EXPORT, qw(HOST_SERVER_CA); -use constant HOST_SERVER_CERT => HOST_CERT_PATH . 'pgbackrest-test-server.crt'; -use constant HOST_SERVER_KEY => HOST_CERT_PATH . 'pgbackrest-test-server.key'; - -#################################################################################################################################### -# SFTP key constants -#################################################################################################################################### -use constant SSH_KEY_PATH => '/test/certificate/ssh/'; - -use constant SSH_PRIVATE_KEY => SSH_KEY_PATH . 'id_rsa'; - push @EXPORT, qw(SSH_PRIVATE_KEY); -use constant SSH_PUBLIC_KEY => SSH_KEY_PATH . 'id_rsa.pub'; - push @EXPORT, qw(SSH_PUBLIC_KEY); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strName, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'strName', default => HOST_BASE, trace => true}, - {name => 'oParam', required => false, trace => true}, - ); - - my $strTestPath = testRunGet()->testPath() . ($strName eq HOST_BASE ? '' : "/${strName}"); - storageTest()->pathCreate($strTestPath, {strMode => '0770'}); - - # Make sure keys have the correct permissions - if (chmod(0600, testRunGet()->basePath() . HOST_SERVER_KEY, testRunGet()->basePath() . HOST_CLIENT_KEY) != 2) - { - confess "unable to set mode on keys"; - } - - # Create the host - my $strProjectPath = dirname(dirname(abs_path($0))); - my $strBinPath = dirname(dirname($strTestPath)) . '/bin/' . testRunGet()->vm() . '/' . PROJECT_EXE; - my $strContainer = 'test-' . testRunGet()->vmId() . "-$strName"; - - my $self = $class->SUPER::new( - $strName, $strContainer, $$oParam{strImage}, $$oParam{strUser}, - ["${strProjectPath}:${strProjectPath}", "${strTestPath}:${strTestPath}", "${strBinPath}:${strBinPath}:ro"], undef, - $oParam->{bTls} ? - 'server --log-level-console=debug --tls-server-ca-file=' . testRunGet()->basePath() . HOST_SERVER_CA . - ' --tls-server-cert-file=' . testRunGet()->basePath() . HOST_SERVER_CERT . ' --tls-server-key-file=' . - testRunGet()->basePath() . HOST_SERVER_KEY . ' --tls-server-auth=pgbackrest-client=* --tls-server-address=0.0.0.0' : - undef, - undef, $oParam->{bTls} ? testRunGet()->backrestExe() : undef); - bless $self, $class; - - # Set test path - $self->{strTestPath} = $strTestPath; - - # Set permissions on the test path - $self->executeSimple('chown -R ' . $self->userGet() . ':'. TEST_GROUP . ' ' . $self->testPath(), undef, 'root'); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub testPath {return shift->{strTestPath}} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm b/test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm deleted file mode 100644 index 9418f35ce5..0000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm +++ /dev/null @@ -1,217 +0,0 @@ -#################################################################################################################################### -# HostDbTest.pm - Database host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostDbCommonTest; -use parent 'pgBackRestTest::Env::Host::HostBackupTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); -use Storable qw(dclone); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::Common::String; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# Test WAL size -#################################################################################################################################### -use constant PG_WAL_SIZE_TEST => 16777216; - -#################################################################################################################################### -# Host defaults -#################################################################################################################################### -use constant HOST_PATH_SPOOL => 'spool'; -use constant HOST_PATH_DB => 'db'; -use constant HOST_PATH_DB_BASE => 'base'; - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oParam', required => false, trace => true}, - ); - - # Get host group - my $oHostGroup = hostGroupGet(); - - # Is standby? - my $bStandby = defined($$oParam{bStandby}) && $$oParam{bStandby} ? true : false; - - my $self = $class->SUPER::new( - { - strName => $bStandby ? HOST_DB_STANDBY : HOST_DB_PRIMARY, - strImage => $$oParam{strImage}, - bTls => $oParam->{bTls}, - strBackupDestination => $$oParam{strBackupDestination}, - bSynthetic => $$oParam{bSynthetic}, - bRepoLocal => $oParam->{bRepoLocal}, - bRepoEncrypt => $oParam->{bRepoEncrypt}, - }); - bless $self, $class; - - # Set parameters - $self->{bStandby} = $bStandby; - - $self->{strDbPath} = $self->testPath() . '/' . HOST_PATH_DB; - $self->{strDbBasePath} = $self->dbPath() . '/' . HOST_PATH_DB_BASE; - $self->{strTablespacePath} = $self->dbPath() . '/tablespace'; - - storageTest()->pathCreate($self->dbBasePath(), {strMode => '0700', bCreateParent => true}); - - $self->{strSpoolPath} = $self->testPath() . '/' . HOST_PATH_SPOOL; - storageTest()->pathCreate($self->spoolPath()); - - # Initialize linkRemap Hashes - $self->{hLinkRemap} = {}; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# archivePush -#################################################################################################################################### -sub archivePush -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strWalPath, - $strArchiveTestFile, - $iArchiveNo, - $iExpectedError, - $bAsync, - $strOptionalParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->archivePush', \@_, - {name => 'strWalPath'}, - {name => 'strArchiveTestFile', required => false}, - {name => 'iArchiveNo', required => false}, - {name => 'iExpectedError', required => false}, - {name => 'bAsync', default => true}, - {name => 'strOptionalParam', required => false}, - ); - - my $strSourceFile; - - if (defined($strArchiveTestFile)) - { - $strSourceFile = "${strWalPath}/" . uc(sprintf('0000000100000001%08x', $iArchiveNo)); - - storageTest()->copy($strArchiveTestFile, storageTest()->openWrite($strSourceFile, {bPathCreate => true})); - - storageTest()->pathCreate("${strWalPath}/archive_status/", {bIgnoreExists => true, bCreateParent => true}); - storageTest()->put("${strWalPath}/archive_status/" . uc(sprintf('0000000100000001%08x', $iArchiveNo)) . '.ready'); - } - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - ' --log-level-console=warn --archive-push-queue-max=' . int(2 * PG_WAL_SIZE_TEST) . - ' --stanza=' . $self->stanza() . - ($bAsync ? '' : ' --no-archive-async') . - " archive-push" . (defined($strSourceFile) ? " ${strSourceFile}" : '') . - (defined($strOptionalParam) ? " ${strOptionalParam}" : ''), - {iExpectedExitStatus => $iExpectedError, bLogOutput => $self->synthetic()}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# linkRemap -#################################################################################################################################### -sub linkRemap -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strTarget, - $strDestination - ) = - logDebugParam - ( - __PACKAGE__ . '->linkRemap', \@_, - {name => 'strTarget'}, - {name => 'strDestination'}, - ); - - ${$self->{hLinkRemap}}{$strTarget} = $strDestination; - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub dbPath {return shift->{strDbPath};} - -sub dbBasePath -{ - my $self = shift; - my $iIndex = shift; - - return $self->{strDbBasePath} . (defined($iIndex) ? "-${iIndex}" : ''); -} - -sub spoolPath {return shift->{strSpoolPath}} -sub standby {return shift->{bStandby}} - -sub tablespacePath -{ - my $self = shift; - my $iTablespace = shift; - my $iIndex = shift; - - return - $self->{strTablespacePath} . - (defined($iTablespace) ? "/ts${iTablespace}" . - (defined($iIndex) ? "-${iIndex}" : '') : ''); -} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm b/test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm deleted file mode 100644 index ed6af2de32..0000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm +++ /dev/null @@ -1,717 +0,0 @@ -#################################################################################################################################### -# HostDbTest.pm - Database host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostDbSyntheticTest; -use parent 'pgBackRestTest::Env::Host::HostDbCommonTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Exporter qw(import); - our @EXPORT = qw(); -use Fcntl ':mode'; -use File::Basename qw(basename dirname); -use File::stat; - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::Common::String; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::FileTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Host::HostDbCommonTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oParam', required => false, trace => true}, - ); - - my $self = $class->SUPER::new( - { - strImage => containerRepo() . ':' . testRunGet()->vm() . "-test", - bTls => $oParam->{bTls}, - strBackupDestination => $$oParam{strBackupDestination}, - bSynthetic => true, - bStandby => $$oParam{bStandby}, - bRepoLocal => $oParam->{bRepoLocal}, - bRepoEncrypt => $oParam->{bRepoEncrypt}, - }); - bless $self, $class; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# dbFileCreate -# -# Create a file specifying content, mode, and time. -#################################################################################################################################### -sub dbFileCreate -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strFile = shift; - my $strContent = shift; - my $lTime = shift; - my $strMode = shift; - - # Check that strTarget is a valid - my $strPath = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_PATH}; - - if (!defined($strPath)) - { - confess &log(ERROR, "${strTarget} not a valid target: \n" . Dumper(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET})); - } - - # Get tablespace path if this is a tablespace - my $strPgPath; - - if (index($strTarget, DB_PATH_PGTBLSPC . '/') == 0) - { - my $iCatalog = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG}; - - $strPgPath = 'PG_' . ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} . "_${iCatalog}"; - } - - # Create actual file location - my $strPathFile = $strPath . - (defined($strPgPath) ? "/${strPgPath}" : '') . "/${strFile}"; - - if (index($strPathFile, '/') != 0) - { - $strPathFile = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_PATH} . '/' . - (defined(dirname($strPathFile)) ? dirname($strPathFile) : '') . "/${strPathFile}"; - } - - # Create the file - testFileCreate($strPathFile, $strContent, $lTime, $strMode); - - # Return path to created file - return $strPathFile; -} - -#################################################################################################################################### -# dbFileRemove -#################################################################################################################################### -sub dbFileRemove -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strFile = shift; - my $bIgnoreMissing = shift; - - # Get actual path location - my $strDbFile = $self->manifestDbPathGet($oManifestRef, $strTarget, $strFile); - - # Remove the file - if (!(defined($bIgnoreMissing) && $bIgnoreMissing && !(-e $strDbFile))) - { - testFileRemove($strDbFile); - } - - return $strDbFile; -} - -#################################################################################################################################### -# dbLinkCreate -# -# Create a file specifying content, mode, and time. -#################################################################################################################################### -sub dbLinkCreate -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strFile = shift; - my $strDestination = shift; - - # Create actual file location - my $strDbFile = $self->manifestDbPathGet($oManifestRef, $strTarget, $strFile); - - # Create the file - testLinkCreate($strDbFile, $strDestination); - - # Return path to created file - return $strDbFile; -} - -#################################################################################################################################### -# manifestDbPathGet -# -# Get the db path based on the target and file passed. -#################################################################################################################################### -sub manifestDbPathGet -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strFile = shift; - - # Determine the manifest key - my $strDbPath = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_PATH}; - - # If target is a tablespace - if (defined(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID})) - { - my $iCatalog = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG}; - - $strDbPath .= '/PG_' . ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} . "_${iCatalog}"; - } - - $strDbPath .= defined($strFile) ? "/${strFile}" : ''; - - return $strDbPath; -} - -#################################################################################################################################### -# manifestFileCreate -# -# Create a file specifying content, mode, and time and add it to the manifest. -#################################################################################################################################### -sub manifestFileCreate -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strFile = shift; - my $strContent = shift; - my $strChecksum = shift; - my $lTime = shift; - my $strMode = shift; - my $bPrimary = shift; - my $strChecksumPageError = shift; - - # Determine the manifest key - my $strManifestKey = $self->manifestKeyGet($oManifestRef, $strTarget, $strFile); - - # Create the file - my $strPathFile = $self->dbFileCreate($oManifestRef, $strTarget, $strFile, $strContent, $lTime, $strMode); - - # Stat the file - my $oStat = lstat($strPathFile); - - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_MODE} = - sprintf('%04o', S_IMODE($oStat->mode)); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_TIMESTAMP} = $oStat->mtime; - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_SIZE} = $oStat->size; - delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_REFERENCE}); - - my $bChecksumPage = defined($strChecksumPageError) ? false : (isChecksumPage($strManifestKey) ? true : undef); - - if (defined($bChecksumPage)) - { - $oManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_CHECKSUM_PAGE} = - $bChecksumPage ? JSON::PP::true : JSON::PP::false; - - if (!$bChecksumPage && $strChecksumPageError ne '0') - { - my @iyChecksumPageError = eval($strChecksumPageError); - - $oManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR} = - \@iyChecksumPageError; - } - else - { - delete($oManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR}); - } - } - - if (defined($strChecksum)) - { - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{checksum} = $strChecksum; - } -} - -#################################################################################################################################### -# manifestFileRemove -# -# Remove a file from disk and (optionally) the manifest. -#################################################################################################################################### -sub manifestFileRemove -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strFile = shift; - - # Determine the manifest key - my $strManifestKey = $self->manifestKeyGet($oManifestRef, $strTarget, $strFile); - - # Remove the file - $self->dbFileRemove($oManifestRef, $strTarget, $strFile, true); - - # Remove from manifest - delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}); -} - -#################################################################################################################################### -# manifestKeyGet -# -# Get the manifest key based on the target and file/path/link passed. -#################################################################################################################################### -sub manifestKeyGet -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strFile = shift; - - # Determine the manifest key - my $strManifestKey = $strTarget; - - # If target is a tablespace - if (defined(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID})) - { - my $iCatalog = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG}; - - $strManifestKey .= '/PG_' . ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} . "_${iCatalog}"; - } - - $strManifestKey .= (defined($strFile) ? "/$strFile" : ''); - - return $strManifestKey; -} - -#################################################################################################################################### -# manifestLinkCreate -# -# Create a link and add it to the manifest. -#################################################################################################################################### -sub manifestLinkCreate -{ - my $self = shift; - my $oManifestRef = shift; - my $strPath = shift; - my $strFile = shift; - my $strDestination = shift; - my $bPrimary = shift; - - # Determine the manifest key - my $strManifestKey = $self->manifestKeyGet($oManifestRef, $strPath, $strFile); - - # Load target - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_PATH} = $strDestination; - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_TYPE} = MANIFEST_VALUE_LINK; - - # Create the link - my $strDbFile = $self->dbLinkCreate($oManifestRef, $strPath, $strFile, $strDestination); - - # Stat the link - my $oStat = lstat($strDbFile); - - # Check for errors in stat - if (!defined($oStat)) - { - confess 'unable to stat ${strDbFile}'; - } - - # Load file into manifest - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strManifestKey}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strManifestKey}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strManifestKey}{&MANIFEST_SUBKEY_DESTINATION} = $strDestination; - - # Stat what the link is pointing to - my $strDestinationFile = $strDestination; - - if (index($strDestinationFile, '/') != 0) - { - $strDestinationFile = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_PATH} . '/' . - (defined(dirname($strPath)) ? dirname($strPath) : '') . "/${strDestination}"; - } - - $oStat = lstat($strDestinationFile); - - my $strSection = MANIFEST_SECTION_TARGET_PATH; - - if (S_ISREG($oStat->mode)) - { - $strSection = MANIFEST_SECTION_TARGET_FILE; - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_SIZE} = $oStat->size; - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_TIMESTAMP} = $oStat->mtime; - (${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_CHECKSUM}) = storageTest()->hashSize($strDestinationFile); - - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_FILE} = - basename(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_PATH}); - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_PATH} = - dirname(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_PATH}); - } - # Allow a link to a link to be created to test that backrest errors out correctly - elsif (S_ISLNK($oStat->mode)) - { - $strSection = MANIFEST_SECTION_TARGET_LINK; - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_DESTINATION} = $strDestination; - } - elsif (!S_ISDIR($oStat->mode)) - { - confess &log(ASSERT, "unrecognized file type for file $strDestinationFile"); - } - - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid); - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_MODE} = sprintf('%04o', S_IMODE($oStat->mode)); -} - -#################################################################################################################################### -# manifestLinkMap -# -# Remap links to new directories/files -#################################################################################################################################### -sub manifestLinkMap -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strDestination = shift; - - if ($$oManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TYPE} ne MANIFEST_VALUE_LINK) - { - confess "cannot map target ${strTarget} because it is not a link"; - } - - if (defined($$oManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID})) - { - confess "tablespace ${strTarget} cannot be remapped with this function"; - } - - if (defined($strDestination)) - { - confess "GENERAL LINK REMAP NOT IMPLEMENTED"; - } - else - { - delete($$oManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}); - delete($$oManifestRef{&MANIFEST_SECTION_TARGET_LINK}{$strTarget}); - } -} - -#################################################################################################################################### -# manifestLinkRemove -# -# Create a link and add it to the manifest. -#################################################################################################################################### -sub manifestLinkRemove -{ - my $self = shift; - my $oManifestRef = shift; - my $strPath = shift; - my $strFile = shift; - - # Delete the link - my $strDbFile = $self->dbFileRemove($oManifestRef, $strPath, $strFile); - - # Determine the manifest key - my $strManifestKey = $self->manifestKeyGet($oManifestRef, $strPath, $strFile); - - # Delete from manifest - delete(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}); - delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strManifestKey}); - delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}); - delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strManifestKey}); -} - -#################################################################################################################################### -# manifestPathCreate -# -# Create a path specifying mode and add it to the manifest. -#################################################################################################################################### -sub manifestPathCreate -{ - my $self = shift; - my $oManifestRef = shift; - my $strPath = shift; - my $strSubPath = shift; - my $strMode = shift; - - # Determine the manifest key - my $strManifestKey = $self->manifestKeyGet($oManifestRef, $strPath, $strSubPath); - - # Create the db path - my $strDbPath = $self->dbPathCreate($oManifestRef, $strPath, $strSubPath, defined($strMode) ? $strMode : '0700'); - - # Stat the file - my $oStat = lstat($strDbPath); - - # Check for errors in stat - if (!defined($oStat)) - { - confess 'unable to stat ${strSubPath}'; - } - - # Load file into manifest - my $strSection = MANIFEST_SECTION_TARGET_PATH; - - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid); - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_MODE} = sprintf('%04o', S_IMODE($oStat->mode)); -} - -#################################################################################################################################### -# manifestReference -# -# Update all files that do not have a reference with the supplied reference. -#################################################################################################################################### -sub manifestReference -{ - my $self = shift; - my $oManifestRef = shift; - my $strReference = shift; - my $bClear = shift; - - # Set prior backup - if (defined($strReference)) - { - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR} = $strReference; - } - else - { - delete(${$oManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR}); - } - - # Find all file sections - foreach my $strSectionFile (sort(keys(%$oManifestRef))) - { - # Skip non-file sections - if ($strSectionFile !~ /\:file$/) - { - next; - } - - foreach my $strFile (sort(keys(%{${$oManifestRef}{$strSectionFile}}))) - { - if (!defined($strReference)) - { - delete(${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE}); - } - elsif (defined($bClear) && $bClear) - { - if (defined(${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE}) && - ${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE} ne $strReference) - { - delete(${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE}); - } - } - elsif (!defined(${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE})) - { - ${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE} = $strReference; - } - } - } -} - -#################################################################################################################################### -# manifestTablespaceCreate -# -# Create a tablespace specifying mode and add it to the manifest. -#################################################################################################################################### -sub manifestTablespaceCreate -{ - my $self = shift; - my $oManifestRef = shift; - my $iOid = shift; - my $strMode = shift; - - # Load linked path into manifest - my $strLinkPath = $self->tablespacePath($iOid); - my $strTarget = MANIFEST_TARGET_PGTBLSPC . "/${iOid}"; - my $oStat = lstat($strLinkPath); - - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget}{&MANIFEST_SUBKEY_MODE} = - sprintf('%04o', S_IMODE($oStat->mode)); - - # Create the tablespace path if it does not exist - my $strTablespacePath = $strLinkPath; - my $strPathTarget = $strTarget; - my $iCatalog = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG}; - my $strTablespaceId = 'PG_' . ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} . "_${iCatalog}"; - - $strTablespacePath .= "/${strTablespaceId}"; - $strPathTarget .= "/${strTablespaceId}"; - - if (!-e $strTablespacePath) - { - storageTest()->pathCreate($strTablespacePath, {strMode => defined($strMode) ? $strMode : '0700'}); - } - - # Load tablespace path into manifest - $oStat = lstat($strTablespacePath); - - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGTBLSPC} = - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA}; - - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strPathTarget}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strPathTarget}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strPathTarget}{&MANIFEST_SUBKEY_MODE} = - sprintf('%04o', S_IMODE($oStat->mode)); - - # Create the link in DB_PATH_PGTBLSPC - my $strLink = $self->dbBasePath() . '/' . DB_PATH_PGTBLSPC . "/${iOid}"; - - symlink($strLinkPath, $strLink) - or confess "unable to link ${strLink} to ${strLinkPath}"; - - # Load link into the manifest - $oStat = lstat($strLink); - my $strLinkTarget = MANIFEST_TARGET_PGDATA . "/${strTarget}"; - - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strLinkTarget}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strLinkTarget}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strLinkTarget}{&MANIFEST_SUBKEY_DESTINATION} = $strLinkPath; - - # Load tablespace target into the manifest - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_PATH} = $strLinkPath; - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TYPE} = MANIFEST_VALUE_LINK; - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID} = $iOid; - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_NAME} = "ts${iOid}"; -} - -#################################################################################################################################### -# manifestTablespaceDrop -# -# Drop a tablespace add remove it from the manifest. -#################################################################################################################################### -sub manifestTablespaceDrop -{ - my $self = shift; - my $oManifestRef = shift; - my $iOid = shift; - my $iIndex = shift; - - # Remove tablespace path/file/link from manifest - my $strTarget = DB_PATH_PGTBLSPC . "/${iOid}"; - - # Remove manifest path, link, target - delete(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}); - delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{&MANIFEST_TARGET_PGDATA . "/${strTarget}"}); - delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget}); - - # Remove nested manifest files and paths - foreach my $strSection (&MANIFEST_SECTION_TARGET_PATH, &MANIFEST_SECTION_TARGET_FILE) - { - foreach my $strFile (keys(%{${$oManifestRef}{$strSection}})) - { - if (index($strFile, "${strTarget}/") == 0) - { - delete($$oManifestRef{$strSection}{$strFile}); - } - } - } - - # Drop the link in DB_PATH_PGTBLSPC - testFileRemove($self->dbBasePath($iIndex) . "/${strTarget}"); -} - -#################################################################################################################################### -# dbPathCreate -# -# Create a path specifying mode. -#################################################################################################################################### -sub dbPathCreate -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strSubPath = shift; - my $strMode = shift; - - # Create final file location - my $strFinalPath = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_PATH}; - - # Get tablespace path if this is a tablespace - if (index($strTarget, DB_PATH_PGTBLSPC . '/') == 0) - { - my $iCatalog = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG}; - - $strFinalPath .= '/PG_' . ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} . "_${iCatalog}"; - } - - $strFinalPath .= (defined($strSubPath) ? "/${strSubPath}" : ''); - - # Create the path - if (!(-e $strFinalPath)) - { - storageTest()->pathCreate($strFinalPath, {strMode => $strMode}); - } - - return $strFinalPath; -} - -#################################################################################################################################### -# dbPathMode -# -# Change the mode of a path. -#################################################################################################################################### -sub dbPathMode -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strPath = shift; - my $strMode = shift; - - # Get the db path - my $strDbPath = $self->manifestDbPathGet($oManifestRef, $strTarget, $strPath); - - testPathMode($strDbPath, $strMode); - - return $strDbPath; -} - -#################################################################################################################################### -# dbPathRemove -# -# Remove a path. -#################################################################################################################################### -sub dbPathRemove -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strPath = shift; - - # Get the db path - my $strDbPath = $self->manifestDbPathGet($oManifestRef, $strTarget, $strPath); - - # Create the path - testPathRemove($strDbPath); - - return $strDbPath; -} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostDbTest.pm b/test/lib/pgBackRestTest/Env/Host/HostDbTest.pm deleted file mode 100644 index 10a292d4ff..0000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostDbTest.pm +++ /dev/null @@ -1,528 +0,0 @@ -#################################################################################################################################### -# HostDbTest.pm - Database host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostDbTest; -use parent 'pgBackRestTest::Env::Host::HostDbCommonTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use DBI; -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(basename); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::Common::String; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Host::HostDbCommonTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# Db defaults -#################################################################################################################################### -use constant HOST_DB_DEFAULT => 'postgres'; -use constant HOST_DB_TIMEOUT => 30; - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oParam', required => false, trace => true}, - ); - - # Get db version - my $strDbVersion = testRunGet()->pgVersion(); - - my $self = $class->SUPER::new( - { - strImage => containerRepo() . ':' . testRunGet()->vm() . "-test", - bTls => $oParam->{bTls}, - strBackupDestination => $$oParam{strBackupDestination}, - bStandby => $$oParam{bStandby}, - bRepoLocal => $oParam->{bRepoLocal}, - bRepoEncrypt => $oParam->{bRepoEncrypt}, - }); - bless $self, $class; - - # Set parameters - $self->{strPgSocketPath} = $self->dbPath(); - $self->{iPgPort} = defined($$oParam{bStandby}) && $$oParam{bStandby} ? 6544 : 6543; - - $self->{strPgLogPath} = $self->testPath(); - $self->{strPgLogFile} = $self->pgLogPath() . '/postgresql.log'; - - # Get Db version - if (defined($strDbVersion)) - { - my $strOutLog = $self->executeSimple($self->pgBinPath() . '/postgres --version'); - - my @stryVersionToken = split(/ /, $strOutLog); - @stryVersionToken = split(/\./, $stryVersionToken[2]); - my $strDbVersionActual = - trim($stryVersionToken[0]) . - (defined($stryVersionToken[1]) && trim($stryVersionToken[0]) < 10 ? '.' . trim($stryVersionToken[1]) : ''); - - # Warn if this is a devel/alpha/beta version - my $strVersionRegExp = '(devel|((alpha|beta|rc)[0-9]+))$'; - - if ($strDbVersionActual =~ /$strVersionRegExp/) - { - my $strDevVersion = $strDbVersionActual; - $strDbVersionActual =~ s/$strVersionRegExp//; - $strDevVersion = substr($strDevVersion, length($strDbVersionActual)); - - if (!defined($$oParam{bStandby}) || !$$oParam{bStandby}) - { - &log(WARN, 'Testing against ' . trim($strOutLog) . " ${strDevVersion}"); - } - } - elsif (!defined($$oParam{bStandby}) || !$$oParam{bStandby}) - { - &log(INFO, 'Testing against ' . trim($strOutLog)); - } - - # Don't run unit tests for unsupported versions - my @stryVersionSupport = versionSupport(); - - if ($strDbVersionActual < $stryVersionSupport[0]) - { - confess &log(ERROR, "only PostgreSQL version $stryVersionSupport[0] and up are supported"); - } - - if ($strDbVersion ne $strDbVersionActual) - { - confess &log(ERROR, "actual database version ${strDbVersionActual} does not match expected version ${strDbVersion}"); - } - } - - # Create wal directory - storageTest()->pathCreate($self->dbPath() . '/pg_' . $self->walId(), {strMode => '0700'}); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# sqlConnect -#################################################################################################################################### -sub sqlConnect -{ - my $self = shift; - my $hParam = shift; - - # Set defaults - my $iTimeout = defined($$hParam{iTimeout}) ? $$hParam{iTimeout} : HOST_DB_TIMEOUT; - my $strDb = defined($$hParam{strDb}) ? $$hParam{strDb} : HOST_DB_DEFAULT; - - # If not connected - if (!defined($self->{db}{$strDb}{hDb})) - { - # Retry until connection is successful - my $oWait = waitInit($iTimeout); - - do - { - # Connect to the db (whether it is local or remote) - $self->{db}{$strDb}{hDb} = - DBI->connect( - "dbi:Pg:dbname=${strDb};port=" . $self->pgPort() . ';host=' . $self->pgSocketPath(), - $self->userGet(), undef, - {AutoCommit => 0, RaiseError => 0, PrintError => 0}); - - return $self->{db}{$strDb}{hDb} if $self->{db}{$strDb}{hDb}; - } - while (!defined($self->{db}{$strDb}{hDb}) && waitMore($oWait)); - - # Error if unable to connect - if (!defined($self->{db}{$strDb}{hDb})) - { - confess &log(ERROR, "unable to connect to PostgreSQL after ${iTimeout} second(s):\n" . $DBI::errstr, ERROR_DB_CONNECT); - } - } - - return $self->{db}{$strDb}{hDb}; -} - -#################################################################################################################################### -# sqlDisconnect -#################################################################################################################################### -sub sqlDisconnect -{ - my $self = shift; - my $hParam = shift; - - foreach my $strDb (keys(%{$self->{db}})) - { - if (defined($$hParam{$strDb}) && $$hParam{$strDb} ne $strDb) - { - next; - } - - if (defined($self->{db}{$strDb}{hDb})) - { - $self->{db}{$strDb}{hDb}->disconnect(); - undef($self->{db}{$strDb}{hDb}); - } - } -} - -#################################################################################################################################### -# sqlExecute -#################################################################################################################################### -sub sqlExecute -{ - my $self = shift; - my $strSql = shift; - my $hParam = shift; - - # Set defaults - my $bCheckPoint = defined($$hParam{bCheckPoint}) ? $$hParam{bCheckPoint} : false; - my $bCommit = defined($$hParam{bCommit}) ? $$hParam{bCommit} : true; - - # Get the db handle - my $hDb = $self->sqlConnect({strDb => $$hParam{strDb}}); - - # Set autocommit on/off - $hDb->{AutoCommit} = defined($$hParam{bAutoCommit}) ? ($$hParam{bAutoCommit} ? true : false) : false; - - # Log and execute the statement - &log(DETAIL, "SQL: ${strSql}"); - - my $hStatement = $hDb->prepare($strSql); - - $hStatement->execute() or - confess &log(ERROR, "Unable to execute: ${strSql}\n" . $DBI::errstr); - $hStatement->finish(); - - if ($bCommit && !$hDb->{AutoCommit}) - { - $self->sqlCommit(); - } - - # Perform a checkpoint if requested - if ($bCheckPoint) - { - $self->sqlExecute('checkpoint', {bCommit => false, bCheckPoint => false}); - } - - # Set autocommit off - $hDb->{AutoCommit} = 0; -} - -#################################################################################################################################### -# sqlSelect -#################################################################################################################################### -sub sqlSelect -{ - my $self = shift; - my $strSql = shift; - my $hParam = shift; - - # Get the db handle - my $hDb = $self->sqlConnect({strDb => $$hParam{strDb}}); - - # Log and execute the statement - &log(DEBUG, (defined($$hParam{strDb}) ? "DB: $$hParam{strDb}, " : "") . "SQL: ${strSql}"); - my $hStatement = $hDb->prepare($strSql); - - $hStatement = $hDb->prepare($strSql); - - $hStatement->execute() or - confess &log(ERROR, "Unable to execute: ${strSql}\n" . $DBI::errstr); - - my @oyRow = $hStatement->fetchrow_array(); - - $hStatement->finish(); - - return @oyRow; -} - -#################################################################################################################################### -# sqlSelectOne -#################################################################################################################################### -sub sqlSelectOne -{ - my $self = shift; - my $strSql = shift; - my $hParam = shift; - - return ($self->sqlSelect($strSql, $hParam))[0]; -} - -#################################################################################################################################### -# sqlSelectOneTest -#################################################################################################################################### -sub sqlSelectOneTest -{ - my $self = shift; - my $strSql = shift; - my $strExpectedValue = shift; - my $hParam = shift; - - # Set defaults - my $iTimeout = defined($$hParam{iTimeout}) ? $$hParam{iTimeout} : HOST_DB_TIMEOUT; - - my $lStartTime = time(); - my $strActualValue; - - do - { - $self->sqlConnect($hParam); - $strActualValue = $self->sqlSelectOne($strSql, $hParam); - - if (defined($strActualValue) && $strActualValue eq $strExpectedValue) - { - return; - } - - $self->sqlDisconnect(); - } - while (defined($iTimeout) && (time() - $lStartTime) <= $iTimeout); - - confess &log( - ERROR, "expected value '${strExpectedValue}' from '${strSql}' but actual was '" . - (defined($strActualValue) ? $strActualValue : '[undef]') . "'"); -} - -#################################################################################################################################### -# sqlCommit -#################################################################################################################################### -sub sqlCommit -{ - my $self = shift; - my $hParam = shift; - - my $bCheckPoint = defined($$hParam{bCheckPoint}) ? $$hParam{bCheckPoint} : false; - - $self->sqlExecute('commit', {bCommit => false, bCheckPoint => $bCheckPoint}); -} - -#################################################################################################################################### -# sqlWalRotate -#################################################################################################################################### -sub sqlWalRotate -{ - my $self = shift; - - $self->sqlExecute('select pg_switch_' . $self->walId() . '()', {bCommit => false, bCheckPoint => false}); -} - -#################################################################################################################################### -# clusterCreate -# -# Create the PostgreSQL cluster and start it. -#################################################################################################################################### -sub clusterCreate -{ - my $self = shift; - my $hParam = shift; - - # Set defaults - my $strWalPath = defined($$hParam{strWalPath}) ? $$hParam{strWalPath} : $self->dbPath() . '/pg_' . $self->walId(); - - $self->executeSimple( - $self->pgBinPath() . '/initdb -k' . - ($self->pgVersion() >= PG_VERSION_11 ? ' --wal-segsize=1' : '') . - ' --' . $self->walId() . "dir=${strWalPath}" . ' --pgdata=' . $self->dbBasePath() . ' --auth=trust'); - - if (!$self->standby()) - { - $self->executeSimple( - "echo 'host replication replicator db-standby trust' >> " . $self->dbBasePath() . '/pg_hba.conf'); - } - - $self->clusterStart( - {bHotStandby => $$hParam{bHotStandby}, bArchive => $$hParam{bArchive}, bArchiveAlways => $$hParam{bArchiveAlways}, - bArchiveInvalid => $$hParam{bArchiveInvalid}}); - - if (!$self->standby()) - { - $self->sqlExecute("create user replicator replication", {bCommit =>true}); - } -} - -#################################################################################################################################### -# clusterStart -# -# Start the PostgreSQL cluster with various test options. -#################################################################################################################################### -sub clusterStart -{ - my $self = shift; - my $hParam = shift; - - # Set defaults - my $bHotStandby = defined($$hParam{bHotStandby}) ? $$hParam{bHotStandby} : false; - my $bArchive = defined($$hParam{bArchive}) ? $$hParam{bArchive} : true; - my $bArchiveAlways = defined($$hParam{bArchiveAlways}) ? $$hParam{bArchiveAlways} : false; - my $bArchiveInvalid = defined($$hParam{bArchiveInvalid}) ? $$hParam{bArchiveInvalid} : false; - my $bArchiveEnabled = defined($$hParam{bArchiveEnabled}) ? $$hParam{bArchiveEnabled} : true; - - # Make sure postgres is not running - if (-e $self->dbBasePath() . '/' . DB_FILE_POSTMTRPID) - { - confess DB_FILE_POSTMTRPID . ' exists'; - } - - # Create the archive command - my $strArchive = - $self->backrestExe() . ' --stanza=' . ($bArchiveInvalid ? 'bogus' : $self->stanza()) . - ' --config=' . $self->backrestConfig() . ' archive-push %p'; - - # Start the cluster - my $strCommand = - $self->pgBinPath() . '/pg_ctl start -o "-c port=' . $self->pgPort() . - ($self->pgVersion() < PG_VERSION_95 ? ' -c checkpoint_segments=1' : ''); - - if ($bArchiveEnabled) - { - if ($self->pgVersion() >= PG_VERSION_95 && $bArchiveAlways) - { - $strCommand .= " -c archive_mode=always"; - } - else - { - $strCommand .= " -c archive_mode=on"; - } - } - else - { - $strCommand .= " -c archive_mode=off"; - } - - if ($bArchive) - { - $strCommand .= " -c archive_command='${strArchive}'"; - } - else - { - $strCommand .= " -c archive_command=true"; - } - - $strCommand .= ' -c wal_level=hot_standby -c hot_standby=' . ($bHotStandby ? 'on' : 'off'); - - # Force parallel mode on to make sure we are disabling it and there are no issues. This is important for testing that 9.6 - # works since pg_stop_backup() is marked parallel safe and will error if run in a worker. - if ($self->pgVersion() >= PG_VERSION_96) - { - if ($self->pgVersion() >= PG_VERSION_16) - { - $strCommand .= " -c debug_parallel_query='on'"; - } - else - { - $strCommand .= " -c force_parallel_mode='on'"; - } - - $strCommand .= " -c max_parallel_workers_per_gather=2"; - } - - $strCommand .= - ' -c max_wal_senders=3' . - ' -c listen_addresses=\'*\'' . - ' -c log_directory=\'' . $self->pgLogPath() . "'" . - ' -c log_filename=\'' . basename($self->pgLogFile()) . "'" . - ' -c log_rotation_age=0' . - ' -c log_rotation_size=0' . - ' -c log_error_verbosity=verbose' . - ' -c unix_socket_directories=\'' . $self->dbPath() . '\'"' . - ' -D ' . $self->dbBasePath() . ' -l ' . $self->pgLogFile() . ' -s'; - - $self->executeSimple($strCommand); - - # Connect user session - $self->sqlConnect(); -} - -#################################################################################################################################### -# clusterStop -# -# Stop the PostgreSQL cluster and optionally check for errors in the server log. -#################################################################################################################################### -sub clusterStop -{ - my $self = shift; - my $hParam = shift; - - # Set defaults - my $bIgnoreLogError = defined($$hParam{bIgnoreLogError}) ? $$hParam{bIgnoreLogError} : false; - my $bStop = defined($hParam->{bStop}) ? $$hParam{bStop} : true; - - # Disconnect user session - $self->sqlDisconnect(); - - # Grep for errors in postgresql.log - this is done first because we want to ignore any errors that happen during shutdown. - if (!$bIgnoreLogError && storageTest()->exists($self->pgLogFile())) - { - $self->executeSimple( - 'grep -v "FATAL\: 57P03\: the database system is (starting up|not yet accepting connections|" - "not accepting connections)" ' . $self->pgLogFile() . ' | grep "ERROR\|FATAL"', - {iExpectedExitStatus => 1}); - } - - # If pg process is running then stop the cluster - if ($bStop && -e $self->dbBasePath() . '/' . DB_FILE_POSTMTRPID) - { - $self->executeSimple($self->pgBinPath() . '/pg_ctl stop -D ' . $self->dbBasePath() . ' -w -s -m fast'); - } - - # Remove the log file - storageTest()->remove($self->pgLogFile(), {bIgnoreMissing => true}); -} - -#################################################################################################################################### -# clusterRestart -# -# Restart the PostgreSQL cluster. -#################################################################################################################################### -sub clusterRestart -{ - my $self = shift; - my $hParam = shift; - - $self->clusterStop($hParam); - $self->clusterStart($hParam); -} - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub walId {return shift->pgVersion() >= PG_VERSION_10 ? 'wal' : 'xlog'} -sub pgBinPath {return testRunGet()->pgBinPath()} -sub pgLogFile {return shift->{strPgLogFile}} -sub pgLogPath {return shift->{strPgLogPath}} -sub pgPort {return shift->{iPgPort}} -sub pgSocketPath {return shift->{strPgSocketPath}} -sub pgVersion {return testRunGet()->pgVersion()} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostGcsTest.pm b/test/lib/pgBackRestTest/Env/Host/HostGcsTest.pm deleted file mode 100644 index e9f89c1591..0000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostGcsTest.pm +++ /dev/null @@ -1,80 +0,0 @@ -#################################################################################################################################### -# GCS Test Host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostGcsTest; -use parent 'pgBackRestTest::Common::HostTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Cwd qw(abs_path); -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); -use Storable qw(dclone); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# GCS defaults -#################################################################################################################################### -use constant HOST_GCS_BUCKET => 'gcsbucket'; - push @EXPORT, qw(HOST_GCS_BUCKET); -use constant HOST_GCS_KEY => 'testkey'; - push @EXPORT, qw(HOST_GCS_KEY); -use constant HOST_GCS_KEY_TYPE => 'token'; - push @EXPORT, qw(HOST_GCS_KEY_TYPE); -use constant HOST_GCS_PORT => 4443; - push @EXPORT, qw(HOST_GCS_PORT); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - ); - - # Create the host - my $strProjectPath = dirname(dirname(abs_path($0))); - my $strFakeCertPath = "${strProjectPath}/doc/resource/fake-cert"; - - my $self = $class->SUPER::new( - HOST_GCS, 'test-' . testRunGet()->vmId() . '-' . HOST_GCS, 'fsouza/fake-gcs-server', 'root', undef, undef, undef, - false); - bless $self, $class; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostS3Test.pm b/test/lib/pgBackRestTest/Env/Host/HostS3Test.pm deleted file mode 100644 index 274619cdc1..0000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostS3Test.pm +++ /dev/null @@ -1,86 +0,0 @@ -#################################################################################################################################### -# S3 Test Host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostS3Test; -use parent 'pgBackRestTest::Common::HostTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Cwd qw(abs_path); -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); -use Storable qw(dclone); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# S3 defaults -#################################################################################################################################### -use constant HOST_S3_ACCESS_KEY => 'accessKey1'; - push @EXPORT, qw(HOST_S3_ACCESS_KEY); -use constant HOST_S3_ACCESS_SECRET_KEY => 'verySecretKey1'; - push @EXPORT, qw(HOST_S3_ACCESS_SECRET_KEY); -use constant HOST_S3_BUCKET => 'pgbackrest-dev'; - push @EXPORT, qw(HOST_S3_BUCKET); -use constant HOST_S3_ENDPOINT => 's3.amazonaws.com'; - push @EXPORT, qw(HOST_S3_ENDPOINT); -use constant HOST_S3_REGION => 'us-east-1'; - push @EXPORT, qw(HOST_S3_REGION); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - ); - - # Create the host - my $strProjectPath = dirname(dirname(abs_path($0))); - my $strFakeCertPath = "${strProjectPath}/doc/resource/fake-cert"; - - my $self = $class->SUPER::new( - HOST_S3, 'test-' . testRunGet()->vmId() . '-s3-server', 'minio/minio:RELEASE.2023-09-30T07-02-29Z', 'root', - ["${strFakeCertPath}/s3-server.crt:/root/.minio/certs/public.crt:ro", - "${strFakeCertPath}/s3-server.key:/root/.minio/certs/private.key:ro"], - '-e MINIO_REGION=' . HOST_S3_REGION . ' -e MINIO_DOMAIN=' . HOST_S3_ENDPOINT . ' -e MINIO_BROWSER=off' . - ' -e MINIO_ACCESS_KEY=' . HOST_S3_ACCESS_KEY . ' -e MINIO_SECRET_KEY=' . HOST_S3_ACCESS_SECRET_KEY, - 'server /data --address :443', false); - bless $self, $class; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostSftpTest.pm b/test/lib/pgBackRestTest/Env/Host/HostSftpTest.pm deleted file mode 100644 index 83efedc6cf..0000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostSftpTest.pm +++ /dev/null @@ -1,72 +0,0 @@ -#################################################################################################################################### -# SFTP Test Host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostSftpTest; -use parent 'pgBackRestTest::Common::HostTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Cwd qw(abs_path); -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); -use Storable qw(dclone); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# SFTP defaults -#################################################################################################################################### -use constant HOST_SFTP_ACCOUNT => TEST_USER; - push @EXPORT, qw(HOST_SFTP_ACCOUNT); -use constant HOST_SFTP_HOSTKEY_HASH_TYPE => 'sha1'; - push @EXPORT, qw(HOST_SFTP_HOSTKEY_HASH_TYPE); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - ); - - # Create the host - my $self = $class->SUPER::new( - HOST_SFTP, 'test-' . testRunGet()->vmId() . '-' . HOST_SFTP, containerRepo() . ':' . testRunGet()->vm() . '-test', 'root'); - bless $self, $class; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -1; diff --git a/test/lib/pgBackRestTest/Env/HostEnvTest.pm b/test/lib/pgBackRestTest/Env/HostEnvTest.pm deleted file mode 100644 index e7ca012d98..0000000000 --- a/test/lib/pgBackRestTest/Env/HostEnvTest.pm +++ /dev/null @@ -1,543 +0,0 @@ -#################################################################################################################################### -# FullCommonTest.pm - Common code for backup tests -#################################################################################################################################### -package pgBackRestTest::Env::HostEnvTest; -use parent 'pgBackRestTest::Common::RunTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Digest::SHA qw(sha1_hex); -use Exporter qw(import); - our @EXPORT = qw(); -use Storable qw(dclone); - -use pgBackRestDoc::Common::Log; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageBase; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Env::ArchiveInfo; -use pgBackRestTest::Env::Host::HostAzureTest; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Host::HostDbCommonTest; -use pgBackRestTest::Env::Host::HostDbTest; -use pgBackRestTest::Env::Host::HostDbSyntheticTest; -use pgBackRestTest::Env::Host::HostGcsTest; -use pgBackRestTest::Env::Host::HostS3Test; - -#################################################################################################################################### -# Constants -#################################################################################################################################### -use constant ENCRYPTION_KEY_ARCHIVE => 'archive'; - push @EXPORT, qw(ENCRYPTION_KEY_ARCHIVE); -use constant ENCRYPTION_KEY_MANIFEST => 'manifest'; - push @EXPORT, qw(ENCRYPTION_KEY_MANIFEST); -use constant ENCRYPTION_KEY_BACKUPSET => 'backupset'; - push @EXPORT, qw(ENCRYPTION_KEY_BACKUPSET); - -#################################################################################################################################### -# setup -#################################################################################################################################### -sub setup -{ - my $self = shift; - my $bSynthetic = shift; - my $oConfigParam = shift; - - # Start object server first since it takes the longest - #------------------------------------------------------------------------------------------------------------------------------- - my $oHostObject; - - if ($oConfigParam->{strStorage} eq S3) - { - $oHostObject = new pgBackRestTest::Env::Host::HostS3Test(); - } - elsif ($oConfigParam->{strStorage} eq AZURE) - { - $oHostObject = new pgBackRestTest::Env::Host::HostAzureTest(); - } - elsif ($oConfigParam->{strStorage} eq GCS) - { - $oHostObject = new pgBackRestTest::Env::Host::HostGcsTest(); - } - elsif ($oConfigParam->{strStorage} eq SFTP) - { - $oHostObject = new pgBackRestTest::Env::Host::HostSftpTest(); - } - - # Get host group - my $oHostGroup = hostGroupGet(); - - # Create the backup host - my $strBackupDestination; - my $bHostBackup = defined($$oConfigParam{bHostBackup}) ? $$oConfigParam{bHostBackup} : false; - my $oHostBackup = undef; - - my $bRepoEncrypt = defined($$oConfigParam{bRepoEncrypt}) ? $$oConfigParam{bRepoEncrypt} : false; - - if ($bHostBackup) - { - $strBackupDestination = defined($$oConfigParam{strBackupDestination}) ? $$oConfigParam{strBackupDestination} : HOST_BACKUP; - - $oHostBackup = new pgBackRestTest::Env::Host::HostBackupTest( - {strBackupDestination => $strBackupDestination, bSynthetic => $bSynthetic, - bRepoLocal => $oConfigParam->{strStorage} eq POSIX, bRepoEncrypt => $bRepoEncrypt, bTls => $oConfigParam->{bTls}}); - $oHostGroup->hostAdd($oHostBackup); - } - else - { - $strBackupDestination = - defined($$oConfigParam{strBackupDestination}) ? $$oConfigParam{strBackupDestination} : HOST_DB_PRIMARY; - } - - # Create the db-primary host - my $oHostDbPrimary = undef; - - if ($bSynthetic) - { - $oHostDbPrimary = new pgBackRestTest::Env::Host::HostDbSyntheticTest( - {strBackupDestination => $strBackupDestination, - bRepoLocal => $oConfigParam->{strStorage} eq POSIX, bRepoEncrypt => $bRepoEncrypt, bTls => $oConfigParam->{bTls}}); - } - else - { - $oHostDbPrimary = new pgBackRestTest::Env::Host::HostDbTest( - {strBackupDestination => $strBackupDestination, bRepoLocal => $oConfigParam->{strStorage} eq POSIX, - bRepoEncrypt => $bRepoEncrypt, bTls => $oConfigParam->{bTls}}); - } - - $oHostGroup->hostAdd($oHostDbPrimary); - - # Create the db-standby host - my $oHostDbStandby = undef; - - if (defined($$oConfigParam{bStandby}) && $$oConfigParam{bStandby}) - { - $oHostDbStandby = new pgBackRestTest::Env::Host::HostDbTest( - {strBackupDestination => $strBackupDestination, bStandby => true, bRepoLocal => $oConfigParam->{strStorage} eq POSIX, - bTls => $oConfigParam->{bTls}}); - - $oHostGroup->hostAdd($oHostDbStandby); - } - - # Finalize object server - #------------------------------------------------------------------------------------------------------------------------------- - if ($oConfigParam->{strStorage} eq S3) - { - $oHostGroup->hostAdd($oHostObject, {rstryHostName => ['pgbackrest-dev.s3.amazonaws.com', 's3.amazonaws.com']}); - } - elsif ($oConfigParam->{strStorage} eq AZURE || $oConfigParam->{strStorage} eq GCS || $oConfigParam->{strStorage} eq SFTP) - { - $oHostGroup->hostAdd($oHostObject); - } - - # Create db-primary config - $oHostDbPrimary->configCreate({ - bTls => $oConfigParam->{bTls}, - strBackupSource => $$oConfigParam{strBackupSource}, - strCompressType => $$oConfigParam{strCompressType}, - bHardlink => $bHostBackup ? undef : $$oConfigParam{bHardLink}, - bArchiveAsync => $$oConfigParam{bArchiveAsync}, - strStorage => $oConfigParam->{strStorage}, - iRepoTotal => $oConfigParam->{iRepoTotal}, - bBundle => $oConfigParam->{bBundle}, - bBlockIncr => $oConfigParam->{bBlockIncr}}); - - # Create backup config if backup host exists - if (defined($oHostBackup)) - { - $oHostBackup->configCreate({ - bTls => $oConfigParam->{bTls}, - strCompressType => $$oConfigParam{strCompressType}, - bHardlink => $$oConfigParam{bHardLink}, - strStorage => $oConfigParam->{strStorage}, - iRepoTotal => $oConfigParam->{iRepoTotal}, - bBundle => $oConfigParam->{bBundle}, - bBlockIncr => $oConfigParam->{bBlockIncr}}); - } - # If backup host is not defined set it to db-primary - else - { - $oHostBackup = $strBackupDestination eq HOST_DB_PRIMARY || $strBackupDestination eq HOST_SFTP ? $oHostDbPrimary : - $oHostDbStandby; - } - - storageRepoCommandSet( - $self->backrestExeHelper() . - ' --config=' . $oHostBackup->backrestConfig() . ' --stanza=' . $self->stanza() . ' --log-level-console=off' . - ' --log-level-stderr=error' . - ($oConfigParam->{strStorage} ne POSIX ? - ($oConfigParam->{strStorage} ne SFTP ? " --no-repo1-storage-verify-tls" : '') . - " --repo1-$oConfigParam->{strStorage}-" . - ($oConfigParam->{strStorage} eq GCS ? 'endpoint' : 'host') . "=" . $oHostObject->ipGet() : '') . - ($oConfigParam->{strStorage} eq GCS ? ':' . HOST_GCS_PORT : ''), - $oConfigParam->{strStorage} eq POSIX ? STORAGE_POSIX : STORAGE_OBJECT); - - # Create db-standby config - if (defined($oHostDbStandby)) - { - $oHostDbStandby->configCreate({ - bTls => $oConfigParam->{bTls}, - strBackupSource => $$oConfigParam{strBackupSource}, - strCompressType => $$oConfigParam{strCompressType}, - bHardlink => $bHostBackup ? undef : $$oConfigParam{bHardLink}, - bArchiveAsync => $$oConfigParam{bArchiveAsync}, - strStorage => $oConfigParam->{strStorage}, - iRepoTotal => $oConfigParam->{iRepoTotal}, - bBundle => $oConfigParam->{bBundle}, - bBlockIncr => $oConfigParam->{bBlockIncr}}); - } - - # Create object storage - if (defined($oHostObject)) - { - storageRepo()->create(); - } - - return $oHostDbPrimary, $oHostDbStandby, $oHostBackup; -} - -#################################################################################################################################### -# Generate database system id for the db version -#################################################################################################################################### -sub dbSysId -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPgVersion, - ) = - logDebugParam - ( - __PACKAGE__ . '->dbSysId', \@_, - {name => 'strPgVersion', trace => true}, - ); - - return (1000000000000000000 + ($strPgVersion * 10)); -} - -#################################################################################################################################### -# Get database catalog version for the db version -#################################################################################################################################### -sub dbCatalogVersion -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPgVersion, - ) = - logDebugParam - ( - __PACKAGE__ . '->sysId', \@_, - {name => 'strPgVersion', trace => true}, - ); - - my $hCatalogVersion = - { - &PG_VERSION_94 => 201409291, - &PG_VERSION_95 => 201510051, - &PG_VERSION_96 => 201608131, - &PG_VERSION_10 => 201707211, - &PG_VERSION_11 => 201806231, - &PG_VERSION_12 => 201909212, - &PG_VERSION_13 => 202007201, - &PG_VERSION_14 => 202105121, - &PG_VERSION_15 => 202209061, - &PG_VERSION_16 => 202307071, - }; - - if (!defined($hCatalogVersion->{$strPgVersion})) - { - confess &log(ASSERT, "no catalog version defined for pg version ${strPgVersion}"); - } - - return $hCatalogVersion->{$strPgVersion}; -} - -#################################################################################################################################### -# Generate control file content -#################################################################################################################################### -sub controlGenerateContent -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPgVersion, - ) = - logDebugParam - ( - __PACKAGE__ . '->controlGenerateContent', \@_, - {name => 'strPgVersion', trace => true}, - ); - - my $hControlContent = - { - 32 => - { - &PG_VERSION_94 => - "5e0064a7b3b6e00dae0300000b43010c00000000000000000000000001000000000000000000000000000000000000000000000001000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000002000000000000000000000000000010000000000000000000000000000000000000000" . - "000000006b0756c8", - &PG_VERSION_95 => - "5f0064a7b3b6e00dae030000a3cc020c00000000000000000000000001000000000000000000000000000000000000000000000001000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" . - "000000000000000000000000000000003bfe413a", - &PG_VERSION_96 => - "600064a7b3b6e00dc0030000c34b040c00000000000000000000000001000000000000000000000000000000000000000000000001000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" . - "000000000000000000000000000000005d135da6", - &PG_VERSION_10 => - "640064a7b3b6e00dea030000cbce050c00000000000000000000000001000000000000000000000000000000000000000000000001000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" . - "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f8556c34", - }, - 64 => - { - &PG_VERSION_94 => - "5e0064a7b3b6e00dae0300000b43010c00000000000000000000000000000000010000000000000000000000000000000000000000000000" . - "0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" . - "00000000000000000000000000000000ee6cf996", - &PG_VERSION_95 => - "5f0064a7b3b6e00dae030000a3cc020c00000000000000000000000000000000010000000000000000000000000000000000000000000000" . - "0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000" . - "0000000000000001000000000000000000000000000000000000000000000000381ec2de", - &PG_VERSION_96 => - "600064a7b3b6e00dc0030000c34b040c00000000000000000000000000000000010000000000000000000000000000000000000000000000" . - "0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000" . - "00000000000000010000000000000000000000000000000000000000000000002d96a4c0", - &PG_VERSION_10 => - "640064a7b3b6e00dea030000cbce050c00000000000000000000000000000000010000000000000000000000000000000000000000000000" . - "0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000" . - "0000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "00000000000000008d543cdf", - }, - }; - - my $strControlContent = $hControlContent->{$self->archBits()}{$strPgVersion}; - - if (!defined($strControlContent)) - { - confess &log(ASSERT, "no control content defined for pg version ${strPgVersion}"); - } - - my $tControlContent = ''; - - for (my $iIdx = 0; $iIdx < length($strControlContent) / 2; $iIdx++) - { - my $iChar = hex(substr($strControlContent, $iIdx * 2, 2)); - $tControlContent .= pack('C', $iChar); - } - - - # Pad bytes - for (my $iIdx = length($tControlContent); $iIdx < 8192; $iIdx++) - { - $tControlContent .= pack('C', 0); - } - - return \$tControlContent; -} - -#################################################################################################################################### -# Generate control file and write to disk -#################################################################################################################################### -sub controlGenerate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbPath, - $strPgVersion, - ) = - logDebugParam - ( - __PACKAGE__ . '->controlGenerate', \@_, - {name => 'strDbPath', trace => true}, - {name => 'strPgVersion', trace => true}, - ); - - storageTest()->put("${strDbPath}/global/pg_control", $self->controlGenerateContent($strPgVersion)); -} - -#################################################################################################################################### -# walSegment -# -# Generate name of WAL segment from component parts. -#################################################################################################################################### -sub walSegment -{ - my $self = shift; - my $iTimeline = shift; - my $iMajor = shift; - my $iMinor = shift; - - return uc(sprintf('%08x%08x%08x', $iTimeline, $iMajor, $iMinor)); -} - -#################################################################################################################################### -# Generate WAL file content -#################################################################################################################################### -sub walGenerateContent -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPgVersion, - $iSourceNo, - ) = - logDebugParam - ( - __PACKAGE__ . '->walGenerateContent', \@_, - {name => 'strPgVersion', trace => true}, - {name => 'iSourceNo', optional => true, default => 1, trace => true}, - ); - - # Get WAL magic for the PG version - my $hWalMagic = - { - &PG_VERSION_94 => hex('0xD07E'), - &PG_VERSION_95 => hex('0xD087'), - &PG_VERSION_96 => hex('0xD093'), - &PG_VERSION_10 => hex('0xD097'), - &PG_VERSION_11 => hex('0xD098'), - &PG_VERSION_12 => hex('0xD101'), - &PG_VERSION_13 => hex('0xD106'), - }; - - my $tWalContent = pack('S', $hWalMagic->{$strPgVersion}); - - # Indicate that the long header is present - $tWalContent .= pack('S', 2); - - # Add junk (H for header) for the bytes that won't be read by the tests - my $iOffset = 12 + (testRunGet()->archBits() / 8); - $tWalContent .= ('H' x $iOffset); - - # Add the system identifier - $tWalContent .= pack('Q', $self->dbSysId($strPgVersion)); - - # Add segment size - $tWalContent .= pack('L', PG_WAL_SEGMENT_SIZE); - - # Add the source number to produce WAL segments with different checksums - $tWalContent .= pack('S', $iSourceNo); - - # Pad out to the required size (B for body) - $tWalContent .= ('B' x (PG_WAL_SEGMENT_SIZE - length($tWalContent))); - - return \$tWalContent; -} - -#################################################################################################################################### -# Generate WAL file content checksum -#################################################################################################################################### -sub walGenerateContentChecksum -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPgVersion, - $hParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->walGenerateContent', \@_, - {name => 'strPgVersion', trace => true}, - {name => 'hParam', required => false, trace => true}, - ); - - return sha1_hex(${$self->walGenerateContent($strPgVersion, $hParam)}); -} - -#################################################################################################################################### -# walGenerate -# -# Generate a WAL segment and ready file for testing. -#################################################################################################################################### -sub walGenerate -{ - my $self = shift; - my $strWalPath = shift; - my $strPgVersion = shift; - my $iSourceNo = shift; - my $strWalSegment = shift; - my $bPartial = shift; - my $bChecksum = shift; - my $bReady = shift; - - my $rtWalContent = $self->walGenerateContent($strPgVersion, {iSourceNo => $iSourceNo}); - my $strWalFile = - "${strWalPath}/${strWalSegment}" . ($bChecksum ? '-' . sha1_hex($rtWalContent) : '') . - (defined($bPartial) && $bPartial ? '.partial' : ''); - - # Put the WAL segment and the ready file - storageTest()->put($strWalFile, $rtWalContent); - - if (!defined($bReady) || $bReady) - { - storageTest()->put("${strWalPath}/archive_status/${strWalSegment}.ready"); - } - - return $strWalFile; -} - -#################################################################################################################################### -# walRemove -# -# Remove WAL file and ready file. -#################################################################################################################################### -sub walRemove -{ - my $self = shift; - my $strWalPath = shift; - my $strWalFile = shift; - - storageTest()->remove("$self->{strWalPath}/${strWalFile}"); - storageTest()->remove("$self->{strWalPath}/archive_status/${strWalFile}.ready"); -} - -1; diff --git a/test/lib/pgBackRestTest/Env/InfoCommon.pm b/test/lib/pgBackRestTest/Env/InfoCommon.pm deleted file mode 100644 index 22990a81ec..0000000000 --- a/test/lib/pgBackRestTest/Env/InfoCommon.pm +++ /dev/null @@ -1,32 +0,0 @@ -#################################################################################################################################### -# INFO MODULE -# Constants, variables and functions common to the info files -#################################################################################################################################### -package pgBackRestTest::Env::InfoCommon; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Exporter qw(import); - our @EXPORT = qw(); - -#################################################################################################################################### -# DB section constants -#################################################################################################################################### -use constant INFO_BACKUP_SECTION_DB => 'db'; - push @EXPORT, qw(INFO_BACKUP_SECTION_DB); -use constant INFO_BACKUP_SECTION_DB_HISTORY => INFO_BACKUP_SECTION_DB . ':history'; - push @EXPORT, qw(INFO_BACKUP_SECTION_DB_HISTORY); - -#################################################################################################################################### -# History section constants -#################################################################################################################################### -use constant INFO_HISTORY_ID => 'id'; - push @EXPORT, qw(INFO_HISTORY_ID); -use constant INFO_DB_VERSION => 'version'; - push @EXPORT, qw(INFO_DB_VERSION); -use constant INFO_SYSTEM_ID => 'system-id'; - push @EXPORT, qw(INFO_SYSTEM_ID); - -1; diff --git a/test/lib/pgBackRestTest/Env/Manifest.pm b/test/lib/pgBackRestTest/Env/Manifest.pm deleted file mode 100644 index 564e6db657..0000000000 --- a/test/lib/pgBackRestTest/Env/Manifest.pm +++ /dev/null @@ -1,1446 +0,0 @@ -#################################################################################################################################### -# MANIFEST MODULE -#################################################################################################################################### -package pgBackRestTest::Env::Manifest; -use parent 'pgBackRestDoc::Common::Ini'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname basename); -use Fcntl qw(:mode); -use Time::Local qw(timelocal); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; - -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; - -#################################################################################################################################### -# File/path constants -#################################################################################################################################### -use constant PATH_BACKUP_HISTORY => 'backup.history'; - push @EXPORT, qw(PATH_BACKUP_HISTORY); -use constant FILE_MANIFEST => 'backup.manifest'; - push @EXPORT, qw(FILE_MANIFEST); -use constant FILE_MANIFEST_COPY => FILE_MANIFEST . INI_COPY_EXT; - push @EXPORT, qw(FILE_MANIFEST_COPY); - -#################################################################################################################################### -# Default match factor -#################################################################################################################################### -use constant MANIFEST_DEFAULT_MATCH_FACTOR => 0.1; - push @EXPORT, qw(MANIFEST_DEFAULT_MATCH_FACTOR); - -#################################################################################################################################### -# MANIFEST Constants -#################################################################################################################################### -use constant MANIFEST_TARGET_PGDATA => 'pg_data'; - push @EXPORT, qw(MANIFEST_TARGET_PGDATA); -use constant MANIFEST_TARGET_PGTBLSPC => 'pg_tblspc'; - push @EXPORT, qw(MANIFEST_TARGET_PGTBLSPC); - -use constant MANIFEST_VALUE_PATH => 'path'; - push @EXPORT, qw(MANIFEST_VALUE_PATH); -use constant MANIFEST_VALUE_LINK => 'link'; - push @EXPORT, qw(MANIFEST_VALUE_LINK); - -# Manifest sections -use constant MANIFEST_SECTION_BACKUP => 'backup'; - push @EXPORT, qw(MANIFEST_SECTION_BACKUP); -use constant MANIFEST_SECTION_BACKUP_DB => 'backup:db'; - push @EXPORT, qw(MANIFEST_SECTION_BACKUP_DB); -use constant MANIFEST_SECTION_BACKUP_INFO => 'backup:info'; - push @EXPORT, qw(MANIFEST_SECTION_BACKUP_INFO); -use constant MANIFEST_SECTION_BACKUP_OPTION => 'backup:option'; - push @EXPORT, qw(MANIFEST_SECTION_BACKUP_OPTION); -use constant MANIFEST_SECTION_BACKUP_TARGET => 'backup:target'; - push @EXPORT, qw(MANIFEST_SECTION_BACKUP_TARGET); -use constant MANIFEST_SECTION_DB => 'db'; - push @EXPORT, qw(MANIFEST_SECTION_DB); -use constant MANIFEST_SECTION_TARGET_PATH => 'target:path'; - push @EXPORT, qw(MANIFEST_SECTION_TARGET_PATH); -use constant MANIFEST_SECTION_TARGET_FILE => 'target:file'; - push @EXPORT, qw(MANIFEST_SECTION_TARGET_FILE); -use constant MANIFEST_SECTION_TARGET_LINK => 'target:link'; - push @EXPORT, qw(MANIFEST_SECTION_TARGET_LINK); - -# Backup metadata required for restores -use constant MANIFEST_KEY_ARCHIVE_START => 'backup-archive-start'; - push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_START); -use constant MANIFEST_KEY_ARCHIVE_STOP => 'backup-archive-stop'; - push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_STOP); -use constant MANIFEST_KEY_LABEL => 'backup-label'; - push @EXPORT, qw(MANIFEST_KEY_LABEL); -use constant MANIFEST_KEY_LSN_START => 'backup-lsn-start'; - push @EXPORT, qw(MANIFEST_KEY_LSN_START); -use constant MANIFEST_KEY_LSN_STOP => 'backup-lsn-stop'; - push @EXPORT, qw(MANIFEST_KEY_LSN_STOP); -use constant MANIFEST_KEY_PRIOR => 'backup-prior'; - push @EXPORT, qw(MANIFEST_KEY_PRIOR); -use constant MANIFEST_KEY_TIMESTAMP_COPY_START => 'backup-timestamp-copy-start'; - push @EXPORT, qw(MANIFEST_KEY_TIMESTAMP_COPY_START); -use constant MANIFEST_KEY_TIMESTAMP_START => 'backup-timestamp-start'; - push @EXPORT, qw(MANIFEST_KEY_TIMESTAMP_START); -use constant MANIFEST_KEY_TIMESTAMP_STOP => 'backup-timestamp-stop'; - push @EXPORT, qw(MANIFEST_KEY_TIMESTAMP_STOP); -use constant MANIFEST_KEY_TYPE => 'backup-type'; - push @EXPORT, qw(MANIFEST_KEY_TYPE); - -# Options that were set when the backup was made -use constant MANIFEST_KEY_BACKUP_STANDBY => 'option-backup-standby'; - push @EXPORT, qw(MANIFEST_KEY_BACKUP_STANDBY); -use constant MANIFEST_KEY_HARDLINK => 'option-hardlink'; - push @EXPORT, qw(MANIFEST_KEY_HARDLINK); -use constant MANIFEST_KEY_ARCHIVE_CHECK => 'option-archive-check'; - push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_CHECK); -use constant MANIFEST_KEY_ARCHIVE_COPY => 'option-archive-copy'; - push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_COPY); -use constant MANIFEST_KEY_BUFFER_SIZE => 'option-buffer-size'; - push @EXPORT, qw(MANIFEST_KEY_BUFFER_SIZE); -use constant MANIFEST_KEY_CHECKSUM_PAGE => 'option-checksum-page'; - push @EXPORT, qw(MANIFEST_KEY_CHECKSUM_PAGE); -use constant MANIFEST_KEY_COMPRESS => 'option-compress'; - push @EXPORT, qw(MANIFEST_KEY_COMPRESS); -use constant MANIFEST_KEY_COMPRESS_TYPE => 'option-compress-type'; - push @EXPORT, qw(MANIFEST_KEY_COMPRESS_TYPE); -use constant MANIFEST_KEY_COMPRESS_LEVEL => 'option-compress-level'; - push @EXPORT, qw(MANIFEST_KEY_COMPRESS_LEVEL); -use constant MANIFEST_KEY_COMPRESS_LEVEL_NETWORK => 'option-compress-level-network'; - push @EXPORT, qw(MANIFEST_KEY_COMPRESS_LEVEL_NETWORK); -use constant MANIFEST_KEY_ONLINE => 'option-online'; - push @EXPORT, qw(MANIFEST_KEY_ONLINE); -use constant MANIFEST_KEY_DELTA => 'option-delta'; - push @EXPORT, qw(MANIFEST_KEY_DELTA); -use constant MANIFEST_KEY_PROCESS_MAX => 'option-process-max'; - push @EXPORT, qw(MANIFEST_KEY_PROCESS_MAX); - -# Information about the database that was backed up -use constant MANIFEST_KEY_DB_ID => 'db-id'; - push @EXPORT, qw(MANIFEST_KEY_DB_ID); -use constant MANIFEST_KEY_SYSTEM_ID => 'db-system-id'; - push @EXPORT, qw(MANIFEST_KEY_SYSTEM_ID); -use constant MANIFEST_KEY_CATALOG => 'db-catalog-version'; - push @EXPORT, qw(MANIFEST_KEY_CATALOG); -use constant MANIFEST_KEY_CONTROL => 'db-control-version'; - push @EXPORT, qw(MANIFEST_KEY_CONTROL); -use constant MANIFEST_KEY_DB_LAST_SYSTEM_ID => 'db-last-system-id'; - push @EXPORT, qw(MANIFEST_KEY_DB_LAST_SYSTEM_ID); -use constant MANIFEST_KEY_DB_VERSION => 'db-version'; - push @EXPORT, qw(MANIFEST_KEY_DB_VERSION); - -# Subkeys used for path/file/link info -use constant MANIFEST_SUBKEY_CHECKSUM => 'checksum'; - push @EXPORT, qw(MANIFEST_SUBKEY_CHECKSUM); -use constant MANIFEST_SUBKEY_CHECKSUM_PAGE => 'checksum-page'; - push @EXPORT, qw(MANIFEST_SUBKEY_CHECKSUM_PAGE); -use constant MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR => 'checksum-page-error'; - push @EXPORT, qw(MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR); -use constant MANIFEST_SUBKEY_DESTINATION => 'destination'; - push @EXPORT, qw(MANIFEST_SUBKEY_DESTINATION); -use constant MANIFEST_SUBKEY_FILE => 'file'; - push @EXPORT, qw(MANIFEST_SUBKEY_FILE); -use constant MANIFEST_SUBKEY_FUTURE => 'future'; - push @EXPORT, qw(MANIFEST_SUBKEY_FUTURE); -use constant MANIFEST_SUBKEY_GROUP => 'group'; - push @EXPORT, qw(MANIFEST_SUBKEY_GROUP); -use constant MANIFEST_SUBKEY_MODE => 'mode'; - push @EXPORT, qw(MANIFEST_SUBKEY_MODE); -use constant MANIFEST_SUBKEY_TIMESTAMP => 'timestamp'; - push @EXPORT, qw(MANIFEST_SUBKEY_TIMESTAMP); -use constant MANIFEST_SUBKEY_TYPE => 'type'; - push @EXPORT, qw(MANIFEST_SUBKEY_TYPE); -use constant MANIFEST_SUBKEY_PATH => 'path'; - push @EXPORT, qw(MANIFEST_SUBKEY_PATH); -use constant MANIFEST_SUBKEY_REFERENCE => 'reference'; - push @EXPORT, qw(MANIFEST_SUBKEY_REFERENCE); -use constant MANIFEST_SUBKEY_REPO_SIZE => 'repo-size'; - push @EXPORT, qw(MANIFEST_SUBKEY_REPO_SIZE); -use constant MANIFEST_SUBKEY_SIZE => 'size'; - push @EXPORT, qw(MANIFEST_SUBKEY_SIZE); -use constant MANIFEST_SUBKEY_TABLESPACE_ID => 'tablespace-id'; - push @EXPORT, qw(MANIFEST_SUBKEY_TABLESPACE_ID); -use constant MANIFEST_SUBKEY_TABLESPACE_NAME => 'tablespace-name'; - push @EXPORT, qw(MANIFEST_SUBKEY_TABLESPACE_NAME); -use constant MANIFEST_SUBKEY_USER => 'user'; - push @EXPORT, qw(MANIFEST_SUBKEY_USER); - -#################################################################################################################################### -# Database locations for important files/paths -#################################################################################################################################### -use constant DB_PATH_ARCHIVESTATUS => 'archive_status'; - push @EXPORT, qw(DB_PATH_ARCHIVESTATUS); -use constant DB_PATH_BASE => 'base'; - push @EXPORT, qw(DB_PATH_BASE); -use constant DB_PATH_GLOBAL => 'global'; - push @EXPORT, qw(DB_PATH_GLOBAL); -use constant DB_PATH_PGDYNSHMEM => 'pg_dynshmem'; - push @EXPORT, qw(DB_PATH_PGDYNSHMEM); -use constant DB_PATH_PGMULTIXACT => 'pg_multixact'; - push @EXPORT, qw(DB_PATH_PGMULTIXACT); -use constant DB_PATH_PGNOTIFY => 'pg_notify'; - push @EXPORT, qw(DB_PATH_PGNOTIFY); -use constant DB_PATH_PGREPLSLOT => 'pg_replslot'; - push @EXPORT, qw(DB_PATH_PGREPLSLOT); -use constant DB_PATH_PGSERIAL => 'pg_serial'; - push @EXPORT, qw(DB_PATH_PGSERIAL); -use constant DB_PATH_PGSNAPSHOTS => 'pg_snapshots'; - push @EXPORT, qw(DB_PATH_PGSNAPSHOTS); -use constant DB_PATH_PGSTATTMP => 'pg_stat_tmp'; - push @EXPORT, qw(DB_PATH_PGSTATTMP); -use constant DB_PATH_PGSUBTRANS => 'pg_subtrans'; - push @EXPORT, qw(DB_PATH_PGSUBTRANS); -use constant DB_PATH_PGTBLSPC => 'pg_tblspc'; - push @EXPORT, qw(DB_PATH_PGTBLSPC); - -use constant DB_FILE_BACKUPLABEL => 'backup_label'; - push @EXPORT, qw(DB_FILE_BACKUPLABEL); -use constant DB_FILE_BACKUPLABELOLD => DB_FILE_BACKUPLABEL . '.old'; - push @EXPORT, qw(DB_FILE_BACKUPLABELOLD); -use constant DB_FILE_PGCONTROL => DB_PATH_GLOBAL . '/pg_control'; - push @EXPORT, qw(DB_FILE_PGCONTROL); -use constant DB_FILE_PGFILENODEMAP => 'pg_filenode.map'; - push @EXPORT, qw(DB_FILE_PGFILENODEMAP); -use constant DB_FILE_PGINTERNALINIT => 'pg_internal.init'; - push @EXPORT, qw(DB_FILE_PGINTERNALINIT); -use constant DB_FILE_PGVERSION => 'PG_VERSION'; - push @EXPORT, qw(DB_FILE_PGVERSION); -use constant DB_FILE_POSTGRESQLAUTOCONFTMP => 'postgresql.auto.conf.tmp'; - push @EXPORT, qw(DB_FILE_POSTGRESQLAUTOCONFTMP); -use constant DB_FILE_POSTMTROPTS => 'postmas'.'ter.opts'; - push @EXPORT, qw(DB_FILE_POSTMTROPTS); -use constant DB_FILE_POSTMTRPID => 'postmas'.'ter.pid'; - push @EXPORT, qw(DB_FILE_POSTMTRPID); -use constant DB_FILE_RECOVERYCONF => 'recovery.conf'; - push @EXPORT, qw(DB_FILE_RECOVERYCONF); -use constant DB_FILE_RECOVERYSIGNAL => 'recovery.signal'; - push @EXPORT, qw(DB_FILE_RECOVERYSIGNAL); -use constant DB_FILE_RECOVERYDONE => 'recovery.done'; - push @EXPORT, qw(DB_FILE_RECOVERYDONE); -use constant DB_FILE_STANDBYSIGNAL => 'standby.signal'; - push @EXPORT, qw(DB_FILE_STANDBYSIGNAL); -use constant DB_FILE_TABLESPACEMAP => 'tablespace_map'; - push @EXPORT, qw(DB_FILE_TABLESPACEMAP); - -use constant DB_FILE_PREFIX_TMP => 'pgsql_tmp'; - push @EXPORT, qw(DB_FILE_PREFIX_TMP); - -#################################################################################################################################### -# Manifest locations for important files/paths -#################################################################################################################################### -use constant MANIFEST_PATH_BASE => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_BASE; - push @EXPORT, qw(MANIFEST_PATH_BASE); -use constant MANIFEST_PATH_GLOBAL => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_GLOBAL; - push @EXPORT, qw(MANIFEST_PATH_GLOBAL); -use constant MANIFEST_PATH_PGDYNSHMEM => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGDYNSHMEM; - push @EXPORT, qw(MANIFEST_PATH_PGDYNSHMEM); -use constant MANIFEST_PATH_PGMULTIXACT => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGMULTIXACT; - push @EXPORT, qw(MANIFEST_PATH_PGMULTIXACT); -use constant MANIFEST_PATH_PGNOTIFY => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGNOTIFY; - push @EXPORT, qw(MANIFEST_PATH_PGNOTIFY); -use constant MANIFEST_PATH_PGREPLSLOT => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGREPLSLOT; - push @EXPORT, qw(MANIFEST_PATH_PGREPLSLOT); -use constant MANIFEST_PATH_PGSERIAL => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSERIAL; - push @EXPORT, qw(MANIFEST_PATH_PGSERIAL); -use constant MANIFEST_PATH_PGSNAPSHOTS => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSNAPSHOTS; - push @EXPORT, qw(MANIFEST_PATH_PGSNAPSHOTS); -use constant MANIFEST_PATH_PGSTATTMP => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSTATTMP; - push @EXPORT, qw(MANIFEST_PATH_PGSTATTMP); -use constant MANIFEST_PATH_PGSUBTRANS => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSUBTRANS; - push @EXPORT, qw(MANIFEST_PATH_PGSUBTRANS); -use constant MANIFEST_PATH_PGTBLSPC => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC; - push @EXPORT, qw(MANIFEST_PATH_PGTBLSPC); - -use constant MANIFEST_FILE_BACKUPLABEL => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_BACKUPLABEL; - push @EXPORT, qw(MANIFEST_FILE_BACKUPLABEL); -use constant MANIFEST_FILE_BACKUPLABELOLD => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_BACKUPLABELOLD; - push @EXPORT, qw(MANIFEST_FILE_BACKUPLABELOLD); -use constant MANIFEST_FILE_PGCONTROL => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_PGCONTROL; - push @EXPORT, qw(MANIFEST_FILE_PGCONTROL); -use constant MANIFEST_FILE_POSTGRESQLAUTOCONFTMP => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_POSTGRESQLAUTOCONFTMP; - push @EXPORT, qw(MANIFEST_FILE_PGCONTROL); -use constant MANIFEST_FILE_POSTMTROPTS => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_POSTMTROPTS; - push @EXPORT, qw(MANIFEST_FILE_POSTMTROPTS); -use constant MANIFEST_FILE_POSTMTRPID => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_POSTMTRPID; - push @EXPORT, qw(MANIFEST_FILE_POSTMTRPID); -use constant MANIFEST_FILE_RECOVERYCONF => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_RECOVERYCONF; - push @EXPORT, qw(MANIFEST_FILE_RECOVERYCONF); -use constant MANIFEST_FILE_RECOVERYSIGNAL => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_RECOVERYSIGNAL; - push @EXPORT, qw(MANIFEST_FILE_RECOVERYSIGNAL); -use constant MANIFEST_FILE_RECOVERYDONE => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_RECOVERYDONE; - push @EXPORT, qw(MANIFEST_FILE_RECOVERYDONE); -use constant MANIFEST_FILE_STANDBYSIGNAL => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_STANDBYSIGNAL; - push @EXPORT, qw(MANIFEST_FILE_STANDBYSIGNAL); -use constant MANIFEST_FILE_TABLESPACEMAP => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_TABLESPACEMAP; - push @EXPORT, qw(MANIFEST_FILE_TABLESPACEMAP); - -#################################################################################################################################### -# Minimum ID for a user object in postgres -#################################################################################################################################### -use constant DB_USER_OBJECT_MINIMUM_ID => 16384; - push @EXPORT, qw(DB_USER_OBJECT_MINIMUM_ID); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFileName, - $bLoad, - $oStorage, - $strDbVersion, - $iDbCatalogVersion, - $strCipherPass, # Passphrase to open the manifest if encrypted - $strCipherPassSub, # Passphrase to encrypt the backup files - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'strFileName', trace => true}, - {name => 'bLoad', optional => true, default => true, trace => true}, - {name => 'oStorage', optional => true, default => storageRepo(), trace => true}, - {name => 'strDbVersion', optional => true, trace => true}, - {name => 'iDbCatalogVersion', optional => true, trace => true}, - {name => 'strCipherPass', optional => true, redact => true}, - {name => 'strCipherPassSub', optional => true, redact => true}, - ); - - # Init object and store variables - my $self = $class->SUPER::new( - $oStorage, $strFileName, {bLoad => $bLoad, strCipherPass => $strCipherPass, strCipherPassSub => $strCipherPassSub}); - - # If manifest not loaded from a file then the db version and catalog version must be set - if (!$bLoad) - { - if (!(defined($strDbVersion) && defined($iDbCatalogVersion))) - { - confess &log(ASSERT, 'strDbVersion and iDbCatalogVersion must be provided with bLoad = false'); - } - - # Force the version to a string since newer versions of JSON::PP lose track of the fact that it is one - $self->set(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_VERSION, undef, $strDbVersion . ''); - $self->numericSet(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_CATALOG, undef, $iDbCatalogVersion); - } - - # Mark the manifest as built if it was loaded from a file - $self->{bBuilt} = $bLoad; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# save -# -# Save the manifest. -#################################################################################################################################### -sub save -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '->save'); - - # Call inherited save - $self->SUPER::save(); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# get -# -# Get a value. -#################################################################################################################################### -sub get -{ - my $self = shift; - my $strSection = shift; - my $strKey = shift; - my $strSubKey = shift; - my $bRequired = shift; - my $oDefault = shift; - - my $oValue = $self->SUPER::get($strSection, $strKey, $strSubKey, false); - - if (!defined($oValue) && defined($strKey) && defined($strSubKey) && - ($strSection eq MANIFEST_SECTION_TARGET_FILE || $strSection eq MANIFEST_SECTION_TARGET_PATH || - $strSection eq MANIFEST_SECTION_TARGET_LINK) && - ($strSubKey eq MANIFEST_SUBKEY_USER || $strSubKey eq MANIFEST_SUBKEY_GROUP || $strSubKey eq MANIFEST_SUBKEY_MODE) && - $self->test($strSection, $strKey)) - { - $oValue = $self->SUPER::get("${strSection}:default", $strSubKey, undef, $bRequired, $oDefault); - } - else - { - $oValue = $self->SUPER::get($strSection, $strKey, $strSubKey, $bRequired, $oDefault); - } - - return $oValue; -} - -#################################################################################################################################### -# boolGet -# -# Get a numeric value. -#################################################################################################################################### -sub boolGet -{ - my $self = shift; - my $strSection = shift; - my $strValue = shift; - my $strSubValue = shift; - my $bRequired = shift; - my $bDefault = shift; - - return $self->get($strSection, $strValue, $strSubValue, $bRequired, - defined($bDefault) ? ($bDefault ? INI_TRUE : INI_FALSE) : undef) ? true : false; -} - -#################################################################################################################################### -# numericGet -# -# Get a numeric value. -#################################################################################################################################### -sub numericGet -{ - my $self = shift; - my $strSection = shift; - my $strValue = shift; - my $strSubValue = shift; - my $bRequired = shift; - my $nDefault = shift; - - return $self->get($strSection, $strValue, $strSubValue, $bRequired, - defined($nDefault) ? $nDefault + 0 : undef) + 0; -} - -#################################################################################################################################### -# tablespacePathGet -# -# Get the unique path assigned by Postgres for the tablespace. -#################################################################################################################################### -sub tablespacePathGet -{ - my $self = shift; - - return('PG_' . $self->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_VERSION) . - '_' . $self->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_CATALOG)); -} - -#################################################################################################################################### -# dbPathGet -# -# Convert a repo path to where the file actually belongs in the db. -#################################################################################################################################### -sub dbPathGet -{ - my $self = shift; - my $strDbPath = shift; - my $strFile = shift; - - my $strDbFile = defined($strDbPath) ? "${strDbPath}/" : ''; - - if (index($strFile, MANIFEST_TARGET_PGDATA . '/') == 0) - { - $strDbFile .= substr($strFile, length(MANIFEST_TARGET_PGDATA) + 1); - } - else - { - $strDbFile .= $strFile; - } - - return $strDbFile; -} - -#################################################################################################################################### -# repoPathGet -# -# Convert a database path to where to file is located in the repo. -#################################################################################################################################### -sub repoPathGet -{ - my $self = shift; - my $strTarget = shift; - my $strFile = shift; - - my $strRepoFile = $strTarget; - - if ($self->isTargetTablespace($strTarget)) - { - $strRepoFile .= '/' . $self->tablespacePathGet(); - } - - if (defined($strFile)) - { - $strRepoFile .= "/${strFile}"; - } - - return $strRepoFile; -} - -#################################################################################################################################### -# isTargetValid -# -# Determine if a target is valid. -#################################################################################################################################### -sub isTargetValid -{ - my $self = shift; - my $strTarget = shift; - my $bError = shift; - - if (!defined($strTarget)) - { - confess &log(ASSERT, 'target is not defined'); - } - - if (!$self->test(MANIFEST_SECTION_BACKUP_TARGET, $strTarget)) - { - if (defined($bError) && $bError) - { - confess &log(ASSERT, "${strTarget} is not a valid target"); - } - - return false; - } - - return true; -} - -#################################################################################################################################### -# isTargetLink -# -# Determine if a target is a link. -#################################################################################################################################### -sub isTargetLink -{ - my $self = shift; - my $strTarget = shift; - - $self->isTargetValid($strTarget, true); - - return $self->test(MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_TYPE, MANIFEST_VALUE_LINK); -} - -#################################################################################################################################### -# isTargetFile -# -# Determine if a target is a file link. -#################################################################################################################################### -sub isTargetFile -{ - my $self = shift; - my $strTarget = shift; - - $self->isTargetValid($strTarget, true); - - return $self->test(MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_FILE); -} - -#################################################################################################################################### -# isTargetTablespace -# -# Determine if a target is a tablespace. -#################################################################################################################################### -sub isTargetTablespace -{ - my $self = shift; - my $strTarget = shift; - - $self->isTargetValid($strTarget, true); - - return $self->test(MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_TABLESPACE_ID); -} - -#################################################################################################################################### -# checkDelta -# -# Determine if the delta option should be enabled. Only called if delta has not yet been enabled. -#################################################################################################################################### -sub checkDelta -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strLastBackupSource, - $bOnlineSame, - $strTimelineCurrent, - $strTimelineLast, - ) = - logDebugParam - ( - __PACKAGE__ . '->checkDelta', \@_, - {name => 'strLastBackupSource'}, - {name => 'bOnlineSame'}, - {name => 'strTimelineCurrent', required => false}, - {name => 'strTimelineLast', required => false}, - ); - - my $bDelta = false; - - # Determine if a timeline switch has occurred - if (defined($strTimelineLast) && defined($strTimelineCurrent)) - { - # If there is a prior backup, check if a timeline switch has occurred since then - if ($strTimelineLast ne $strTimelineCurrent) - { - &log(WARN, "a timeline switch has occurred since the ${strLastBackupSource} backup, enabling delta checksum"); - $bDelta = true; - } - } - - # If delta was not set above and there is a change in the online option, then set delta option - if (!$bDelta && !$bOnlineSame) - { - &log(WARN, "the online option has changed since the ${strLastBackupSource} backup, enabling delta checksum"); - $bDelta = true; - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bDelta', value => $bDelta, trace => true}, - ); -} - -#################################################################################################################################### -# checkDeltaFile -# -# Determine if the delta option should be enabled. Only called if delta has not yet been enabled. -#################################################################################################################################### -sub checkDeltaFile -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $stryFileList, - $oPriorManifest, - $lTimeBegin, - ) = - logDebugParam - ( - __PACKAGE__ . '->checkDeltaFile', \@_, - {name => 'stryFileList'}, - {name => 'oPriorManifest', required => false}, - {name => 'lTimeBegin', required => false}, - ); - - my $bDelta = false; - - # Loop though all files - foreach my $strName (@{$stryFileList}) - { - # If $lTimeBegin is defined, then this is not an aborted manifest so check if modification time is in the future (in this - # backup OR the last backup) then enable delta and exit - if (defined($lTimeBegin) && - ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) > $lTimeBegin || - (defined($oPriorManifest) && - $oPriorManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_FUTURE, 'y')))) - { - &log(WARN, "file $strName has timestamp in the future, enabling delta checksum"); - $bDelta = true; - last; - } - - # If the time on the file is earlier than the last manifest time or the size is different but the timestamp is the - # same, then enable delta and exit - if (defined($oPriorManifest) && $oPriorManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName) && - ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) < - $oPriorManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) || - ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) != - $oPriorManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) && - $self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) == - $oPriorManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP)))) - { - &log(WARN, "file $strName timestamp in the past or size changed but timestamp did not, enabling delta checksum"); - $bDelta = true; - last; - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bDelta', value => $bDelta, trace => true}, - ); -} - -#################################################################################################################################### -# build -# -# Build the manifest object. -#################################################################################################################################### -sub build -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oStorageDbPrimary, - $strPath, - $oLastManifest, - $bOnline, - $bDelta, - $hTablespaceMap, - $hDatabaseMap, - $rhExclude, - $strTimelineCurrent, - $strTimelineLast, - $strLevel, - $bTablespace, - $strParentPath, - $strFilter, - $iLevel, - ) = - logDebugParam - ( - __PACKAGE__ . '->build', \@_, - {name => 'oStorageDbPrimary'}, - {name => 'strPath'}, - {name => 'oLastManifest', required => false}, - {name => 'bOnline'}, - {name => 'bDelta'}, - {name => 'hTablespaceMap', required => false}, - {name => 'hDatabaseMap', required => false}, - {name => 'rhExclude', required => false}, - {name => 'strTimelineCurrent', required => false}, - {name => 'strTimelineLast', required => false}, - {name => 'strLevel', required => false}, - {name => 'bTablespace', required => false}, - {name => 'strParentPath', required => false}, - {name => 'strFilter', required => false}, - {name => 'iLevel', required => false, default => 0}, - ); - - # Limit recursion to something reasonable (if more then we are very likely in a link loop) - if ($iLevel >= 16) - { - confess &log( - ERROR, - "recursion in manifest build exceeds depth of ${iLevel}: ${strLevel}\n" . - 'HINT: is there a link loop in $PGDATA?', - ERROR_FORMAT); - } - - if (!defined($strLevel)) - { - # Don't allow the manifest to be built more than once - if ($self->{bBuilt}) - { - confess &log(ASSERT, "manifest has already been built"); - } - - $self->{bBuilt} = true; - - # Set initial level - $strLevel = MANIFEST_TARGET_PGDATA; - - # If not online then build the tablespace map from pg_tblspc path - if (!$bOnline && !defined($hTablespaceMap)) - { - my $hTablespaceManifest = $oStorageDbPrimary->manifest($strPath . '/' . DB_PATH_PGTBLSPC); - $hTablespaceMap = {}; - - foreach my $strOid (sort(CORE::keys(%{$hTablespaceManifest}))) - { - if ($strOid eq '.' or $strOid eq '..') - { - next; - } - - logDebugMisc($strOperation, "found tablespace ${strOid} in offline mode"); - - $hTablespaceMap->{$strOid} = "ts${strOid}"; - } - } - - # If there is a last manifest, then check to see if delta checksum should be enabled - if (defined($oLastManifest) && !$bDelta) - { - $bDelta = $self->checkDelta( - 'last', $oLastManifest->boolTest(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ONLINE, undef, $bOnline), - $strTimelineCurrent, $strTimelineLast); - } - } - - $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_PATH, $strPath); - $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_TYPE, - $strLevel eq MANIFEST_TARGET_PGDATA ? MANIFEST_VALUE_PATH : MANIFEST_VALUE_LINK); - - if ($bTablespace) - { - my $iTablespaceId = (split('\/', $strLevel))[1]; - - if (!defined($hTablespaceMap->{$iTablespaceId})) - { - confess &log(ASSERT, "tablespace with oid ${iTablespaceId} not found in tablespace map\n" . - "HINT: was a tablespace created or dropped during the backup?"); - } - - $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_TABLESPACE_ID, $iTablespaceId); - $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_TABLESPACE_NAME, - $hTablespaceMap->{$iTablespaceId}); - } - - if (index($strPath, '/') != 0) - { - if (!defined($strParentPath)) - { - confess &log(ASSERT, "cannot get manifest for '${strPath}' when no parent path is specified"); - } - - $strPath = $oStorageDbPrimary->pathAbsolute($strParentPath, $strPath); - } - - # Get the manifest for this level - my $hManifest = $oStorageDbPrimary->manifest($strPath, {strFilter => $strFilter}); - my $strManifestType = MANIFEST_VALUE_LINK; - - # Loop though all paths/files/links in the manifest - foreach my $strName (sort(CORE::keys(%{$hManifest}))) - { - my $strFile = $strLevel; - - if ($strName ne '.') - { - if ($strManifestType eq MANIFEST_VALUE_LINK && $hManifest->{$strName}{type} eq 'l') - { - confess &log(ERROR, 'link \'' . - $self->dbPathGet( - $self->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH), $strLevel) . - '\' -> \'' . $self->get(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_PATH) . - '\' cannot reference another link', ERROR_LINK_DESTINATION); - } - - if ($strManifestType eq MANIFEST_VALUE_LINK) - { - $strFile = dirname($strFile); - $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_PATH, - dirname($self->get(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_PATH))); - $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_FILE, $strName); - } - - $strFile .= "/${strName}"; - } - else - { - $strManifestType = MANIFEST_VALUE_PATH; - } - - # Skip wal directory when doing an online backup. WAL will be restored from the archive or stored in the wal directory at - # the end of the backup if the archive-copy option is set. - next if ($bOnline && $strFile =~ (qw{^} . MANIFEST_TARGET_PGDATA . qw{/} . $self->walPath() . '\/') && - $strFile !~ ('^' . MANIFEST_TARGET_PGDATA . qw{/} . $self->walPath() . qw{/} . DB_PATH_ARCHIVESTATUS . '$')); - - # Skip all directories and files that start with pgsql_tmp. The files are removed when the server is restarted and the - # directories are recreated. - next if $strName =~ ('(^|\/)' . DB_FILE_PREFIX_TMP); - - # Skip pg_dynshmem/* since these files cannot be reused on recovery - next if $strFile =~ ('^' . MANIFEST_PATH_PGDYNSHMEM . '\/') && $self->dbVersion() >= PG_VERSION_94; - - # Skip pg_notify/* since these files cannot be reused on recovery - next if $strFile =~ ('^' . MANIFEST_PATH_PGNOTIFY . '\/'); - - # Skip pg_replslot/* since these files are generally not useful after a restore - next if $strFile =~ ('^' . MANIFEST_PATH_PGREPLSLOT . '\/') && $self->dbVersion() >= PG_VERSION_94; - - # Skip pg_serial/* since these files are reset - next if $strFile =~ ('^' . MANIFEST_PATH_PGSERIAL . '\/'); - - # Skip pg_snapshots/* since these files cannot be reused on recovery - next if $strFile =~ ('^' . MANIFEST_PATH_PGSNAPSHOTS . '\/'); - - # Skip temporary statistics in pg_stat_tmp even when stats_temp_directory is set because PGSS_TEXT_FILE is always created - # there. - next if $strFile =~ ('^' . MANIFEST_PATH_PGSTATTMP . '\/'); - - # Skip pg_subtrans/* since these files are reset - next if $strFile =~ ('^' . MANIFEST_PATH_PGSUBTRANS . '\/'); - - # Skip pg_internal.init since it is recreated on startup - next if $strFile =~ (DB_FILE_PGINTERNALINIT . '$'); - - # Skip recovery files - if ($self->dbVersion() >= PG_VERSION_12) - { - next if ($strFile eq MANIFEST_FILE_RECOVERYSIGNAL || $strFile eq MANIFEST_FILE_STANDBYSIGNAL); - } - else - { - next if ($strFile eq MANIFEST_FILE_RECOVERYDONE || $strFile eq MANIFEST_FILE_RECOVERYCONF); - } - - # Skip ignored files - if ($strFile eq MANIFEST_FILE_POSTGRESQLAUTOCONFTMP || # postgresql.auto.conf.tmp - temp file for safe writes - $strFile eq MANIFEST_FILE_BACKUPLABELOLD || # backup_label.old - old backup labels are not useful - $strFile eq MANIFEST_FILE_POSTMTROPTS || # not useful for backup - $strFile eq MANIFEST_FILE_POSTMTRPID) # to avoid confusing postgres after restore - { - next; - } - - # Check for files to exclude - if ($hManifest->{$strName}{type} eq 'f') - { - # Get the directory name from the manifest; it will be used later to search for existence in the keys - my $strDir = dirname($strName); - - # If it is a database data directory (base or tablespace) then check for files to skip - if ($strDir =~ '^base\/[0-9]+$' || - $strDir =~ ('^' . $self->tablespacePathGet() . '\/[0-9]+$')) - { - # Get just the filename - my $strBaseName = basename($strName); - - # Skip temp tables (lower t followed by numbers underscore numbers and a dot (segment) or underscore (fork) and/or - # segment, e.g. t1234_123, t1234_123.1, t1234_123_vm, t1234_123_fsm.1 - if ($strBaseName =~ '^t[0-9]+\_[0-9]+(|\_(fsm|vm)){0,1}(\.[0-9]+){0,1}$') - { - next; - } - - # Check for unlogged tables to skip - # Exclude all forks for unlogged tables except the init fork (numbers underscore init and optional dot segment) - if ($strBaseName =~ '^[0-9]+(|\_(fsm|vm)){0,1}(\.[0-9]+){0,1}$') - { - # Get the filenode (OID) - my ($strFileNode) = $strBaseName =~ '^(\d+)'; - - # Add _init to the OID to see if this is an unlogged object - $strFileNode = $strDir. "/" . $strFileNode . "_init"; - - # If key exists in manifest then skip - if (exists($hManifest->{$strFileNode}) && $hManifest->{$strFileNode}{type} eq 'f') - { - next; - } - } - } - } - - # Exclude files requested by the user - if (defined($rhExclude)) - { - # Exclusions are based on the name of the file relative to PGDATA - my $strPgFile = $self->dbPathGet(undef, $strFile); - my $bExclude = false; - - # Iterate through exclusions - foreach my $strExclude (sort(keys(%{$rhExclude}))) - { - # If the exclusion ends in / then we must do a prefix match - if ($strExclude =~ /\/$/) - { - if (index($strPgFile, $strExclude) == 0) - { - $bExclude = true; - } - } - # Else an exact match or a prefix match with / appended is required - elsif ($strPgFile eq $strExclude || index($strPgFile, "${strExclude}/") == 0) - { - $bExclude = true; - } - - # Log everything that gets excluded at a high level so it will hopefully be seen if wrong - if ($bExclude) - { - &log(INFO, "exclude ${strPgFile} from backup using '${strExclude}' exclusion"); - last; - } - } - - # Skip the file if it was excluded - next if $bExclude; - } - - my $cType = $hManifest->{$strName}{type}; - my $strSection = MANIFEST_SECTION_TARGET_PATH; - - if ($cType eq 'f') - { - $strSection = MANIFEST_SECTION_TARGET_FILE; - } - elsif ($cType eq 'l') - { - $strSection = MANIFEST_SECTION_TARGET_LINK; - } - elsif ($cType ne 'd') - { - &log(WARN, "exclude special file '" . $self->dbPathGet(undef, $strFile) . "' from backup"); - next; - } - - # Make sure that DB_PATH_PGTBLSPC contains only absolute links that do not point inside PGDATA - my $bTablespace = false; - - if (index($strName, DB_PATH_PGTBLSPC . '/') == 0 && $strLevel eq MANIFEST_TARGET_PGDATA) - { - $bTablespace = true; - $strFile = MANIFEST_TARGET_PGDATA . '/' . $strName; - - # Check for files in DB_PATH_PGTBLSPC that are not links - if ($hManifest->{$strName}{type} ne 'l') - { - confess &log(ERROR, "${strName} is not a symlink - " . DB_PATH_PGTBLSPC . ' should contain only symlinks', - ERROR_LINK_EXPECTED); - } - - # Check for tablespaces in PGDATA - if (index($hManifest->{$strName}{link_destination}, "${strPath}/") == 0 || - (index($hManifest->{$strName}{link_destination}, '/') != 0 && - index($oStorageDbPrimary->pathAbsolute($strPath . '/' . DB_PATH_PGTBLSPC, - $hManifest->{$strName}{link_destination}) . '/', "${strPath}/") == 0)) - { - confess &log(ERROR, 'tablespace symlink ' . $hManifest->{$strName}{link_destination} . - ' destination must not be in $PGDATA', ERROR_LINK_DESTINATION); - } - } - - # User and group required for all types - if (defined($hManifest->{$strName}{user})) - { - $self->set($strSection, $strFile, MANIFEST_SUBKEY_USER, $hManifest->{$strName}{user}); - } - else - { - $self->boolSet($strSection, $strFile, MANIFEST_SUBKEY_USER, false); - } - - if (defined($hManifest->{$strName}{group})) - { - $self->set($strSection, $strFile, MANIFEST_SUBKEY_GROUP, $hManifest->{$strName}{group}); - } - else - { - $self->boolSet($strSection, $strFile, MANIFEST_SUBKEY_GROUP, false); - } - - # Mode for required file and path type only - if ($cType eq 'f' || $cType eq 'd') - { - $self->set($strSection, $strFile, MANIFEST_SUBKEY_MODE, $hManifest->{$strName}{mode}); - } - - # Modification time and size required for file type only - if ($cType eq 'f') - { - $self->set($strSection, $strFile, MANIFEST_SUBKEY_TIMESTAMP, - $hManifest->{$strName}{modification_time} + 0); - $self->set($strSection, $strFile, MANIFEST_SUBKEY_SIZE, $hManifest->{$strName}{size} + 0); - } - - # Link destination required for link type only - if ($cType eq 'l') - { - my $strLinkDestination = $hManifest->{$strName}{link_destination}; - $self->set($strSection, $strFile, MANIFEST_SUBKEY_DESTINATION, $strLinkDestination); - - # If this is a tablespace then set the filter to use for the next level - my $strFilter; - - if ($bTablespace) - { - $strFilter = $self->tablespacePathGet(); - - $self->set(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGTBLSPC, undef, - $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA)); - - # PGDATA prefix was only needed for the link so strip it off before recursing - $strFile = substr($strFile, length(MANIFEST_TARGET_PGDATA) + 1); - } - - $bDelta = $self->build( - $oStorageDbPrimary, $strLinkDestination, undef, $bOnline, $bDelta, $hTablespaceMap, $hDatabaseMap, $rhExclude, - undef, undef, $strFile, $bTablespace, dirname("${strPath}/${strName}"), $strFilter, $iLevel + 1); - } - } - - # If this is the base level then do post-processing - if ($strLevel eq MANIFEST_TARGET_PGDATA) - { - my $bTimeInFuture = false; - - # Wait for the remainder of the second when doing online backups. This is done because most filesystems only have a one - # second resolution and Postgres will still be modifying files during the second that the manifest is built and this could - # lead to an invalid diff/incr backup later when using timestamps to determine which files have changed. Offline backups do - # not wait because it makes testing much faster and Postgres should not be running (if it is the backup will not be - # consistent anyway and the one-second resolution problem is the least of our worries). - my $lTimeBegin = waitRemainder($bOnline); - - if (defined($oLastManifest)) - { - $self->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR, undef, - $oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL)); - } - - # Store database map information when provided during an online backup. - foreach my $strDbName (sort(keys(%{$hDatabaseMap}))) - { - $self->numericSet(MANIFEST_SECTION_DB, $strDbName, MANIFEST_KEY_DB_ID, - $hDatabaseMap->{$strDbName}{&MANIFEST_KEY_DB_ID}); - $self->numericSet(MANIFEST_SECTION_DB, $strDbName, MANIFEST_KEY_DB_LAST_SYSTEM_ID, - $hDatabaseMap->{$strDbName}{&MANIFEST_KEY_DB_LAST_SYSTEM_ID}); - } - - # Determine if delta checksum should be enabled - if (!$bDelta) - { - my @stryFileList = $self->keys(MANIFEST_SECTION_TARGET_FILE); - - if (@stryFileList) - { - $bDelta = $self->checkDeltaFile(\@stryFileList, $oLastManifest, $lTimeBegin); - } - } - - # Loop though all files - foreach my $strName ($self->keys(MANIFEST_SECTION_TARGET_FILE)) - { - # If modification time is in the future (in this backup OR the last backup) set warning flag and do not - # allow a reference - if ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) > $lTimeBegin || - (defined($oLastManifest) && - $oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_FUTURE, 'y'))) - { - $bTimeInFuture = true; - - # Only mark as future if still in the future in the current backup - if ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) > $lTimeBegin) - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_FUTURE, 'y'); - } - } - # Else check if the size and timestamp match OR if the size matches and the delta option is set, then keep the file. - # In the latter case, if there had been a timestamp change then rather than removing and recopying the file, the file - # will be tested in backupFile to see if the db/repo checksum still matches: if so, it is not necessary to recopy, - # else it will need to be copied to the new backup. For zero sized files, the reference will be set and copying - # will be skipped later. - elsif (defined($oLastManifest) && $oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName) && - $self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) == - $oLastManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) && - ($bDelta || ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) == 0 || - $self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) == - $oLastManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP)))) - { - # Copy reference from previous backup if possible - if ($oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE)) - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE, - $oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE)); - } - # Otherwise the reference is to the previous backup - else - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE, - $oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL)); - } - - # Copy the checksum from previous manifest (if it exists - zero sized files don't have checksums) - if ($oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM)) - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM, - $oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM)); - } - - # Copy repo size from the previous manifest (if it exists) - if ($oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE)) - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE, - $oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE)); - } - - # Copy checksum page from the previous manifest (if it exists) - my $bChecksumPage = $oLastManifest->get( - MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE, false); - - if (defined($bChecksumPage)) - { - $self->boolSet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE, $bChecksumPage); - - if (!$bChecksumPage && - $oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR)) - { - $self->set( - MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR, - $oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR)); - } - } - } - } - - # Warn if any files in the current backup are in the future - if ($bTimeInFuture) - { - &log(WARN, "some files have timestamps in the future - they will be copied to prevent possible race conditions"); - } - - # Record the time when copying will start - $self->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START, undef, $lTimeBegin + ($bOnline ? 1 : 0)); - - # Build default sections - $self->buildDefault(); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bDelta', value => $bDelta, trace => true}, - ); -} - -#################################################################################################################################### -# fileAdd -# -# Add files to the manifest that were generated after the initial manifest build, e.g. backup_label, tablespace_map, and copied WAL -# files. Since the files were not in the original cluster the user, group, and mode must be defaulted. -#################################################################################################################################### -sub fileAdd -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strManifestFile, - $lModificationTime, - $lSize, - $strChecksum, - $bPrimary, - ) = - logDebugParam - ( - __PACKAGE__ . '->fileAdd', \@_, - {name => 'strManifestFile'}, - {name => 'lModificationTime'}, - {name => 'lSize'}, - {name => 'lChecksum'}, - {name => 'bPrimary'}, - ); - - # Set manifest values - if (!$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_USER) || - !$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_USER, undef, - $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_USER))) - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_USER, - $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_USER)); - } - - if (!$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_GROUP) || - !$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_GROUP, undef, - $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_GROUP))) - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_GROUP, - $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_GROUP)); - } - - if (!$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_MODE) || - !$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_MODE, undef, '0600')) - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_MODE, '0600'); - } - - $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_TIMESTAMP, $lModificationTime); - $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_SIZE, $lSize); - $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_CHECKSUM, $strChecksum); -} - -#################################################################################################################################### -# buildDefault -# -# Builds the default section. -#################################################################################################################################### -sub buildDefault -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '->buildDefault'); - - # Defaults for subkeys that tend to repeat - my $strDefaultUser = $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_USER); - my $strDefaultGroup = $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_GROUP); - my $strDefaultPathMode = $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_MODE); - my $strDefaultFileMode = sprintf('%04o', oct($strDefaultPathMode) & (S_IRUSR | S_IWUSR | S_IRGRP)); - - # Remove subkeys that match the defaults - foreach my $strSection (&MANIFEST_SECTION_TARGET_FILE, &MANIFEST_SECTION_TARGET_PATH, &MANIFEST_SECTION_TARGET_LINK) - { - next if !$self->test($strSection); - - foreach my $strFile ($self->keys($strSection)) - { - if ($self->test($strSection, $strFile, MANIFEST_SUBKEY_USER, $strDefaultUser)) - { - $self->remove($strSection, $strFile, MANIFEST_SUBKEY_USER); - } - - if ($self->test($strSection, $strFile, MANIFEST_SUBKEY_GROUP, $strDefaultGroup)) - { - $self->remove($strSection, $strFile, MANIFEST_SUBKEY_GROUP); - } - - if ($self->test( - $strSection, $strFile, MANIFEST_SUBKEY_MODE, - $strSection eq MANIFEST_SECTION_TARGET_PATH ? $strDefaultPathMode : $strDefaultFileMode)) - { - $self->remove($strSection, $strFile, MANIFEST_SUBKEY_MODE); - } - } - } - - # Write defaults - $self->set(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_USER, undef, $strDefaultUser); - $self->set(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_GROUP, undef, $strDefaultGroup); - $self->set(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_MODE, undef, $strDefaultFileMode); - - if ($self->test(MANIFEST_SECTION_TARGET_LINK)) - { - $self->set(MANIFEST_SECTION_TARGET_LINK . ':default', MANIFEST_SUBKEY_USER, undef, $strDefaultUser); - $self->set(MANIFEST_SECTION_TARGET_LINK . ':default', MANIFEST_SUBKEY_GROUP, undef, $strDefaultGroup); - } - - $self->set(MANIFEST_SECTION_TARGET_PATH . ':default', MANIFEST_SUBKEY_USER, undef, $strDefaultUser); - $self->set(MANIFEST_SECTION_TARGET_PATH . ':default', MANIFEST_SUBKEY_GROUP, undef, $strDefaultGroup); - $self->set(MANIFEST_SECTION_TARGET_PATH . ':default', MANIFEST_SUBKEY_MODE, undef, $strDefaultPathMode); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# validate -# -# Checks for any missing values or inconsistencies in the manifest. -#################################################################################################################################### -sub validate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . 'validate'); - - # Make sure that all files have size and checksum (when size > 0). Since these values are removed before the backup file copy - # starts this ensures that all files had results stored in the manifest during the file copy. - foreach my $strFile ($self->keys(MANIFEST_SECTION_TARGET_FILE)) - { - # Ensure size is set - if (!$self->test(MANIFEST_SECTION_TARGET_FILE, $strFile, MANIFEST_SUBKEY_SIZE)) - { - confess &log(ASSERT, "manifest subvalue 'size' not set for file '${strFile}'"); - } - - # If size > 0 then checksum must also be set - if ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strFile, MANIFEST_SUBKEY_SIZE) > 0 && - !$self->test(MANIFEST_SECTION_TARGET_FILE, $strFile, MANIFEST_SUBKEY_CHECKSUM)) - { - confess &log(ASSERT, "manifest subvalue 'checksum' not set for file '${strFile}'"); - } - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# dbVersion - version of PostgreSQL that the manifest is being built for -#################################################################################################################################### -sub dbVersion -{ - my $self = shift; - - return $self->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_VERSION); -} - -#################################################################################################################################### -# xactPath - return the transaction directory based on the PostgreSQL version -#################################################################################################################################### -sub xactPath -{ - my $self = shift; - - return $self->dbVersion() >= PG_VERSION_10 ? 'pg_xact' : 'pg_clog'; -} - -#################################################################################################################################### -# walPath - return the wal directory based on the PostgreSQL version -#################################################################################################################################### -sub walPath -{ - my $self = shift; - - return $self->dbVersion() >= PG_VERSION_10 ? 'pg_wal' : 'pg_xlog'; -} - -#################################################################################################################################### -# isPrimaryFile -# -# Is this file required to be copied from the primary? -#################################################################################################################################### -sub isPrimaryFile -{ - my $self = shift; - my $strFile = shift; - - return - $strFile !~ ('^(' . MANIFEST_TARGET_PGDATA . '\/' . '(' . DB_PATH_BASE . '|' . DB_PATH_GLOBAL . '|' . - $self->xactPath() . '|' . DB_PATH_PGMULTIXACT . ')|' . DB_PATH_PGTBLSPC . ')\/'); -} - -#################################################################################################################################### -# isChecksumPage -# -# Can this file have page checksums? -#################################################################################################################################### -sub isChecksumPage -{ - my $strFile = shift; - - if (($strFile =~ ('^' . MANIFEST_TARGET_PGDATA . '\/' . DB_PATH_BASE . '\/[0-9]+\/|^' . MANIFEST_TARGET_PGTBLSPC . - '\/[0-9]+\/[^\/]+\/[0-9]+\/') && - $strFile !~ ('(' . DB_FILE_PGFILENODEMAP . '|' . DB_FILE_PGINTERNALINIT . '|' . DB_FILE_PGVERSION . ')$')) || - ($strFile =~ ('^' . MANIFEST_TARGET_PGDATA . '\/' . DB_PATH_GLOBAL . '\/') && - $strFile !~ ('(' . DB_FILE_PGFILENODEMAP . '|' . DB_FILE_PGINTERNALINIT . '|' . DB_FILE_PGVERSION . '|' . - DB_FILE_PGCONTROL . ')$'))) - { - return true; - } - - return false; -} - -push @EXPORT, qw(isChecksumPage); - -1; diff --git a/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm b/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm deleted file mode 100644 index 911c42a444..0000000000 --- a/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm +++ /dev/null @@ -1,747 +0,0 @@ -#################################################################################################################################### -# Test All Commands On PostgreSQL Clusters -#################################################################################################################################### -package pgBackRestTest::Module::Real::RealAllTest; -use parent 'pgBackRestTest::Env::HostEnvTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use File::Basename qw(dirname); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::FileTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::VmTest; -use pgBackRestTest::Common::Storage; -use pgBackRestTest::Common::StoragePosix; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::ArchiveInfo; -use pgBackRestTest::Env::BackupInfo; -use pgBackRestTest::Env::InfoCommon; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Host::HostDbTest; -use pgBackRestTest::Env::Host::HostDbTest; -use pgBackRestTest::Env::HostEnvTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# Backup advisory lock -#################################################################################################################################### -use constant DB_BACKUP_ADVISORY_LOCK => '12340078987004321'; - -#################################################################################################################################### -# run -#################################################################################################################################### -sub run -{ - my $self = shift; - - foreach my $rhRun - ( - {pg => '9.4', dst => 'db-standby', tls => 0, stg => POSIX, enc => 1, cmp => LZ4, rt => 1, bnd => 1, bi => 0}, - {pg => '9.5', dst => 'backup', tls => 1, stg => GCS, enc => 0, cmp => BZ2, rt => 1, bnd => 0, bi => 1}, - {pg => '9.6', dst => 'backup', tls => 0, stg => POSIX, enc => 0, cmp => NONE, rt => 2, bnd => 1, bi => 1}, - {pg => '10', dst => 'sftp-srvr', tls => 0, stg => SFTP, enc => 1, cmp => GZ, rt => 1, bnd => 1, bi => 0}, - {pg => '11', dst => 'backup', tls => 1, stg => AZURE, enc => 0, cmp => ZST, rt => 2, bnd => 0, bi => 0}, - {pg => '12', dst => 'backup', tls => 0, stg => S3, enc => 1, cmp => LZ4, rt => 1, bnd => 0, bi => 1}, - {pg => '13', dst => 'db-standby', tls => 1, stg => GCS, enc => 0, cmp => ZST, rt => 1, bnd => 1, bi => 1}, - {pg => '14', dst => 'sftp-srvr', tls => 0, stg => SFTP, enc => 0, cmp => LZ4, rt => 1, bnd => 1, bi => 0}, - {pg => '15', dst => 'db-standby', tls => 0, stg => AZURE, enc => 0, cmp => NONE, rt => 2, bnd => 1, bi => 1}, - {pg => '16', dst => 'backup', tls => 0, stg => S3, enc => 1, cmp => NONE, rt => 1, bnd => 0, bi => 0}, - ) - { - # Only run tests for this pg version - next if ($rhRun->{pg} ne $self->pgVersion()); - - # Get run parameters - my $bHostBackup = $rhRun->{dst} eq HOST_BACKUP ? true : false; - my $bTls = $rhRun->{tls}; - my $strBackupDestination = $rhRun->{dst}; - my $strStorage = $rhRun->{stg}; - my $bRepoEncrypt = $rhRun->{enc}; - my $strCompressType = $rhRun->{cmp}; - my $iRepoTotal = $rhRun->{rt}; - my $bBundle = $rhRun->{bnd}; - my $bBlockIncr = $rhRun->{bi}; - - # Some tests are not version specific so only run them on a single version of PostgreSQL - my $bNonVersionSpecific = $self->pgVersion() eq PG_VERSION_96; - - # Increment the run, log, and decide whether this unit test should be run - next if !$self->begin( - "bkp ${bHostBackup}, tls ${bTls}, dst ${strBackupDestination}, cmp ${strCompressType}, storage ${strStorage}" . - ", enc ${bRepoEncrypt}, bi ${bBlockIncr}"); - - # Create hosts, file object, and config - my ($oHostDbPrimary, $oHostDbStandby, $oHostBackup) = $self->setup( - false, - {bHostBackup => $bHostBackup, bStandby => true, bTls => $bTls, strBackupDestination => $strBackupDestination, - strCompressType => $strCompressType, bArchiveAsync => false, strStorage => $strStorage, - bRepoEncrypt => $bRepoEncrypt, iRepoTotal => $iRepoTotal, bBundle => $bBundle, bBlockIncr => $bBlockIncr}); - - # Some commands will fail because of the bogus host created when a standby is present. These options reset the bogus host - # so it won't interfere with commands that won't tolerate a connection failure. - my $strBogusReset = $oHostBackup->bogusHost() ? - ' --reset-pg2-host --reset-pg2-host-type --reset-pg2-host-cmd --reset-pg2-host-config --reset-pg2-host-user' . - ' --reset-pg2-path' : - ''; - - # If S3 set process max to 2. This seems like the best place for parallel testing since it will help speed S3 processing - # without slowing down the other tests too much. - if ($strStorage eq S3) - { - $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}}); - $oHostDbPrimary->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}}); - } - - $oHostDbPrimary->clusterCreate(); - - # Create the stanza - $oHostDbPrimary->stanzaCreate('main create stanza info files'); - - # Get passphrase to access the Manifest file from backup.info - returns undefined if repo not encrypted - my $strCipherPass = - (new pgBackRestTest::Env::BackupInfo($oHostBackup->repoBackupPath()))->cipherPassSub(); - - # Create a manifest with the pg version to get version-specific paths - my $oManifest = new pgBackRestTest::Env::Manifest(BOGUS, {bLoad => false, strDbVersion => $self->pgVersion(), - iDbCatalogVersion => $self->dbCatalogVersion($self->pgVersion()), - strCipherPass => $strCipherPass, strCipherPassSub => $bRepoEncrypt ? ENCRYPTION_KEY_BACKUPSET : undef}); - - # Static backup parameters - my $fTestDelay = 1; - - # Restore test string - my $strDefaultMessage = 'default'; - my $strFullMessage = 'full'; - my $strStandbyMessage = 'standby'; - my $strIncrMessage = 'incr'; - my $strTimeMessage = 'time'; - my $strXidMessage = 'xid'; - my $strNameMessage = 'name'; - my $strTimelineMessage = 'timeline'; - - # Create two new databases - $oHostDbPrimary->sqlExecute('create database test1', {bAutoCommit => true}); - $oHostDbPrimary->sqlExecute('create database test2', {bAutoCommit => true}); - - # ??? Removed temporarily until manifest build can be brought back into the check command - # Create a directory in pg_data location that is only readable by root to ensure manifest->build is called by check - # -------------------------------------------------------------------------------------------------------------------------- - # my $strDir = $oHostDbPrimary->dbBasePath() . '/rootreaddir'; - # executeTest('sudo mkdir ' . $strDir); - # executeTest("sudo chown root:root ${strDir}"); - # executeTest("sudo chmod 400 ${strDir}"); - # - # $strComment = 'confirm primary manifest->build executed'; - # $oHostDbPrimary->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_PATH_OPEN}); - # executeTest("sudo rmdir ${strDir}"); - - # -------------------------------------------------------------------------------------------------------------------------- - my $strComment = 'verify check command runs successfully'; - - $oHostDbPrimary->check($strComment, {iTimeout => 10, bStanza => false}); - - # Also run check on the backup host when present - if ($bHostBackup) - { - $oHostBackup->check($strComment, {iTimeout => 10, strOptionalParam => $strBogusReset}); - } - - # Restart the cluster ignoring any errors in the postgresql log - $oHostDbPrimary->clusterRestart({bIgnoreLogError => true}); - - # Full backup - #--------------------------------------------------------------------------------------------------------------------------- - # Create the table where test messages will be stored - $oHostDbPrimary->sqlExecute("create table test (message text not null)"); - $oHostDbPrimary->sqlWalRotate(); - $oHostDbPrimary->sqlExecute("insert into test values ('$strDefaultMessage')"); - - # Acquire the backup advisory lock so it looks like a backup is running - if (!$oHostDbPrimary->sqlSelectOne('select pg_try_advisory_lock(' . DB_BACKUP_ADVISORY_LOCK . ')')) - { - confess 'unable to acquire advisory lock for testing'; - } - - $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_FULL, 'fail on backup lock exists', {iExpectedExitStatus => ERROR_LOCK_ACQUIRE}); - - # Release the backup advisory lock so the next backup will succeed - if (!$oHostDbPrimary->sqlSelectOne('select pg_advisory_unlock(' . DB_BACKUP_ADVISORY_LOCK . ')')) - { - confess 'unable to release advisory lock'; - } - - $oHostDbPrimary->sqlExecute("update test set message = '$strFullMessage'"); - - # Required to set hint bits to be sent to the standby to make the heap match on both sides - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strFullMessage); - - # Backup to repo1 - my $strFullBackup = $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_FULL, 'repo1', - {strOptionalParam => ' --buffer-size=16384'}); - - # Backup to repo2 if it exists - if ($iRepoTotal == 2) - { - $oHostBackup->backup(CFGOPTVAL_BACKUP_TYPE_FULL, 'repo2', {iRepo => 2}); - } - - # Make a new backup with expire-auto disabled then run the expire command and compare backup numbers to ensure that expire - # was really disabled. This test is not version specific so is run on only one version. - #--------------------------------------------------------------------------------------------------------------------------- - if ($bNonVersionSpecific) - { - my $oBackupInfo = new pgBackRestTest::Env::BackupInfo($oHostBackup->repoBackupPath()); - push(my @backupLst1, $oBackupInfo->list()); - - $strFullBackup = $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_FULL, 'with disabled expire-auto', - {strOptionalParam => ' --repo1-retention-full='.scalar(@backupLst1). ' --no-expire-auto'}); - - $oBackupInfo = new pgBackRestTest::Env::BackupInfo($oHostBackup->repoBackupPath()); - push(my @backupLst2, $oBackupInfo->list()); - - &log(INFO, " run the expire command"); - $oHostBackup->expire({iRetentionFull => scalar(@backupLst1)}); - $oBackupInfo = new pgBackRestTest::Env::BackupInfo($oHostBackup->repoBackupPath()); - push(my @backupLst3, $oBackupInfo->list()); - - unless (scalar(@backupLst2) == scalar(@backupLst1) + 1 && scalar(@backupLst1) == scalar(@backupLst3)) - { - confess "expire-auto option didn't work as expected"; - } - } - - # Enabled async archiving - $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'archive-async' => 'y'}}); - - # Kick out a bunch of archive logs to exercise async archiving. Only do this when compressed and remote to slow it down - # enough to make it evident that the async process is working. - if ($strCompressType ne NONE && $strBackupDestination eq HOST_BACKUP) - { - &log(INFO, ' multiple wal switches to exercise async archiving'); - $oHostDbPrimary->sqlExecute("create table wal_activity (id int)"); - $oHostDbPrimary->sqlWalRotate(); - $oHostDbPrimary->sqlExecute("insert into wal_activity values (1)"); - $oHostDbPrimary->sqlWalRotate(); - $oHostDbPrimary->sqlExecute("insert into wal_activity values (2)"); - $oHostDbPrimary->sqlWalRotate(); - $oHostDbPrimary->sqlExecute("insert into wal_activity values (3)"); - $oHostDbPrimary->sqlWalRotate(); - $oHostDbPrimary->sqlExecute("insert into wal_activity values (4)"); - $oHostDbPrimary->sqlWalRotate(); - } - - # Setup replica - #--------------------------------------------------------------------------------------------------------------------------- - my %oRemapHash; - $oRemapHash{&MANIFEST_TARGET_PGDATA} = $oHostDbStandby->dbBasePath(); - - $oHostDbStandby->linkRemap($oManifest->walPath(), $oHostDbStandby->dbPath() . '/' . $oManifest->walPath()); - - $oHostDbStandby->restore( - 'restore backup on replica', 'latest', - {rhRemapHash => \%oRemapHash, strType => CFGOPTVAL_RESTORE_TYPE_STANDBY, - strOptionalParam => - ' --recovery-option="primary_conninfo=host=' . HOST_DB_PRIMARY . - ' port=' . $oHostDbPrimary->pgPort() . ' user=replicator"'}); - - $oHostDbStandby->clusterStart({bHotStandby => true}); - - # Make sure streaming replication is on - $oHostDbPrimary->sqlSelectOneTest( - "select client_addr || '-' || state from pg_stat_replication", $oHostDbStandby->ipGet() . '/32-streaming'); - - # Check that the cluster was restored properly - $oHostDbStandby->sqlSelectOneTest('select message from test', $strFullMessage); - - # Update message for standby - $oHostDbPrimary->sqlExecute("update test set message = '$strStandbyMessage'"); - - if (!$bTls) - { - # If there is only a primary and a replica and the replica is the backup destination, then if pg2-host and - # pg256-host are BOGUS, confirm failure to reach the primary - if (!$bHostBackup && $strBackupDestination eq HOST_DB_STANDBY) - { - my $strStandbyBackup = $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_FULL, 'backup from standby, failure to reach primary', - {bStandby => true, iExpectedExitStatus => ERROR_DB_CONNECT, strOptionalParam => '--pg256-host=' . BOGUS}); - } - else - { - my $strStandbyBackup = $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_FULL, 'backup from standby, failure to access at least one standby', - {bStandby => true, iExpectedExitStatus => ERROR_DB_CONNECT, strOptionalParam => '--pg256-host=' . BOGUS}); - } - } - - my $strStandbyBackup = $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_FULL, 'backup from standby', - {bStandby => true, iExpectedExitStatus => undef, strOptionalParam => '--repo1-retention-full=1'}); - - $strFullBackup = $strStandbyBackup; - - # ??? Removed temporarily until manifest build can be brought back into the check command - # # Create a directory in pg_data location that is only readable by root to ensure manifest->build is called by check - # my $strDir = $oHostDbStandby->dbBasePath() . '/rootreaddir'; - # executeTest('sudo mkdir ' . $strDir); - # executeTest("sudo chown root:root ${strDir}"); - # executeTest("sudo chmod 400 ${strDir}"); - # - # my $strComment = 'confirm standby manifest->build executed'; - # - # # If there is an invalid host, the final error returned from check will be the inability to resolve the name which is - # # an open error instead of a read error - # if (!$oHostDbStandby->bogusHost()) - # { - # $oHostDbStandby->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_PATH_OPEN}); - # } - # else - # { - # $oHostDbStandby->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_FILE_READ}); - # } - # - # # Remove the directory in pg_data location that is only readable by root - # executeTest("sudo rmdir ${strDir}"); - - # Confirm the check command runs without error on a standby (when a bogus host is not configured) - $oHostDbStandby->check('verify check command on standby', {strOptionalParam => $strBogusReset}); - - # Shutdown the standby before creating tablespaces (this will error since paths are different) - $oHostDbStandby->clusterStop({bIgnoreLogError => true}); - - my $strAdhocBackup; - - # Execute stop and make sure the backup fails - #--------------------------------------------------------------------------------------------------------------------------- - # Restart the cluster to check for any errors before continuing since the stop tests will definitely create errors and the - # logs will need to be deleted to avoid causing issues further down the line. This test is not version specific so is run on - # only one version. - if ($bNonVersionSpecific) - { - confess "test must be performed on posix storage" if $strStorage ne POSIX; - - $oHostDbPrimary->clusterRestart(); - - # Add backup for adhoc expire - $strAdhocBackup = $oHostBackup->backup(CFGOPTVAL_BACKUP_TYPE_DIFF, 'backup for adhoc expire'); - - $oHostDbPrimary->stop(); - - $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_INCR, 'attempt backup when stopped', - {iExpectedExitStatus => $oHostBackup == $oHostDbPrimary ? ERROR_STOP : ERROR_DB_CONNECT}); - - $oHostDbPrimary->start(); - } - - # Setup the time targets - #--------------------------------------------------------------------------------------------------------------------------- - # If the tests are running quickly then the time target might end up the same as the end time of the prior full backup. That - # means restore auto-select will not pick it as a candidate and restore the last backup instead causing the restore compare - # to fail. So, sleep one second. - sleep(1); - - $oHostDbPrimary->sqlExecute("update test set message = '$strTimeMessage'"); - $oHostDbPrimary->sqlWalRotate(); - my $strTimeTarget = $oHostDbPrimary->sqlSelectOne("select current_timestamp"); - &log(INFO, " time target is ${strTimeTarget}"); - - # Incr backup - fail on archive_mode=always when version >= 9.5 - #--------------------------------------------------------------------------------------------------------------------------- - if ($oHostDbPrimary->pgVersion() >= PG_VERSION_95) - { - # Set archive_mode=always - $oHostDbPrimary->clusterRestart({bArchiveAlways => true}); - - $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_INCR, 'fail on archive_mode=always', {iExpectedExitStatus => ERROR_FEATURE_NOT_SUPPORTED}); - - # Reset the cluster to a normal state so the next test will work - $oHostDbPrimary->clusterRestart(); - } - - # Incr backup - #--------------------------------------------------------------------------------------------------------------------------- - # Create a tablespace directory - storageTest()->pathCreate($oHostDbPrimary->tablespacePath(1), {strMode => '0700', bCreateParent => true}); - - # Also create it on the standby so replay won't fail - if (defined($oHostDbStandby)) - { - storageTest()->pathCreate($oHostDbStandby->tablespacePath(1), {strMode => '0700', bCreateParent => true}); - } - - $oHostDbPrimary->sqlExecute( - "create tablespace ts1 location '" . $oHostDbPrimary->tablespacePath(1) . "'", {bAutoCommit => true}); - $oHostDbPrimary->sqlExecute("alter table test set tablespace ts1"); - - # Create a table in the tablespace that will not be modified again to be sure it does get full page writes in the WAL later - $oHostDbPrimary->sqlExecute("create table test_exists (id int) tablespace ts1", {bCommit => true, bCheckPoint => true}); - - # Create a table in the tablespace - $oHostDbPrimary->sqlExecute("create table test_remove (id int)"); - $oHostDbPrimary->sqlWalRotate(); - $oHostDbPrimary->sqlExecute("update test set message = '$strDefaultMessage'"); - $oHostDbPrimary->sqlWalRotate(); - - # Create a database in the tablespace and a table to check - $oHostDbPrimary->sqlExecute("create database test3 with tablespace ts1", {bAutoCommit => true}); - $oHostDbPrimary->sqlExecute( - 'create table test3_exists (id int);' . - 'insert into test3_exists values (1);', - {strDb => 'test3', bAutoCommit => true}); - - # Create a table in test1 to check - test1 will not be restored - $oHostDbPrimary->sqlExecute( - 'create table test1_zeroed (id int);' . - 'insert into test1_zeroed values (1);', - {strDb => 'test1', bAutoCommit => true}); - - # Start a backup so the next backup has to restart it. This test is not required for PostgreSQL >= 9.6 since backups are run - # in non-exclusive mode. - if ($oHostDbPrimary->pgVersion() < PG_VERSION_96) - { - $oHostDbPrimary->sqlSelectOne("select pg_start_backup('test backup that will cause an error', true)"); - - # Verify that an error is returned if the backup is already running - $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_INCR, 'fail on backup already running', {iExpectedExitStatus => ERROR_DB_QUERY}); - - # Restart the cluster ignoring any errors in the postgresql log - $oHostDbPrimary->clusterRestart({bIgnoreLogError => true}); - - # Start a new backup to make the next test restarts it - $oHostDbPrimary->sqlSelectOne("select pg_start_backup('test backup that will be restarted', true)"); - } - - if (defined($strAdhocBackup)) - { - # Adhoc expire the latest backup - no other tests should be affected - $oHostBackup->expire({strOptionalParam => '--set=' . $strAdhocBackup}); - } - - # Drop a table - $oHostDbPrimary->sqlExecute('drop table test_remove'); - $oHostDbPrimary->sqlWalRotate(); - $oHostDbPrimary->sqlExecute("update test set message = '$strIncrMessage'", {bCommit => true}); - - # Exercise --delta checksum option - my $strIncrBackup = $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_INCR, 'delta', - {strOptionalParam => '--stop-auto --buffer-size=32768 --delta', iRepo => $iRepoTotal}); - - # Ensure the check command runs properly with a tablespace - $oHostBackup->check( 'check command with tablespace', {iTimeout => 10, strOptionalParam => $strBogusReset}); - - # Setup the xid target - #--------------------------------------------------------------------------------------------------------------------------- - my $strXidTarget = undef; - - $oHostDbPrimary->sqlExecute("update test set message = '$strXidMessage'", {bCommit => false}); - $oHostDbPrimary->sqlWalRotate(); - $strXidTarget = $oHostDbPrimary->sqlSelectOne("select txid_current()"); - $oHostDbPrimary->sqlCommit(); - &log(INFO, " xid target is ${strXidTarget}"); - - # Setup the name target - #--------------------------------------------------------------------------------------------------------------------------- - my $strNameTarget = 'backrest'; - - $oHostDbPrimary->sqlExecute("update test set message = '$strNameMessage'", {bCommit => true}); - $oHostDbPrimary->sqlWalRotate(); - - $oHostDbPrimary->sqlExecute("select pg_create_restore_point('${strNameTarget}')"); - - &log(INFO, " name target is ${strNameTarget}"); - - # Create a table and data in database test2 - #--------------------------------------------------------------------------------------------------------------------------- - # Initialize variables for SHA1 and path of the pg_filenode.map for the database that will not be restored - my $strDb1TablePath; - my $strDb1TableSha1; - - $oHostDbPrimary->sqlExecute( - 'create table test (id int);' . - 'insert into test values (1);' . - 'create table test_ts1 (id int) tablespace ts1;' . - 'insert into test_ts1 values (2);', - {strDb => 'test2', bAutoCommit => true}); - - $oHostDbPrimary->sqlWalRotate(); - - # Get the SHA1 and path of the table for the database that will not be restored - $strDb1TablePath = $oHostDbPrimary->dbBasePath(). "/base/" . - $oHostDbPrimary->sqlSelectOne("select oid from pg_database where datname='test1'") . "/" . - $oHostDbPrimary->sqlSelectOne("select relfilenode from pg_class where relname='test1_zeroed'", {strDb => 'test1'}); - $strDb1TableSha1 = storageTest()->hashSize($strDb1TablePath); - - # Restore (type = default) - #--------------------------------------------------------------------------------------------------------------------------- - # Expect failure because pg (appears to be) running - $oHostDbPrimary->restore('pg running', 'latest', {iExpectedExitStatus => ERROR_PG_RUNNING}); - - $oHostDbPrimary->clusterStop(); - - # Expect failure because db path is not empty - $oHostDbPrimary->restore('path not empty', 'latest', {iExpectedExitStatus => ERROR_PATH_NOT_EMPTY}); - - # Drop and recreate db path - testPathRemove($oHostDbPrimary->dbBasePath()); - storageTest()->pathCreate($oHostDbPrimary->dbBasePath(), {strMode => '0700'}); - testPathRemove($oHostDbPrimary->dbPath() . qw{/} . $oManifest->walPath()); - storageTest()->pathCreate($oHostDbPrimary->dbPath() . qw{/} . $oManifest->walPath(), {strMode => '0700'}); - testPathRemove($oHostDbPrimary->tablespacePath(1)); - storageTest()->pathCreate($oHostDbPrimary->tablespacePath(1), {strMode => '0700'}); - - # Now the restore should work - $oHostDbPrimary->restore( - undef, 'latest', - {strOptionalParam => ' --db-include=test2 --db-include=test3 --buffer-size=16384', iRepo => $iRepoTotal}); - - # Test that the first database has not been restored since --db-include did not include test1 - my ($strSHA1, $lSize) = storageTest()->hashSize($strDb1TablePath); - - # Create a zeroed sparse file in the test directory that is the same size as the filenode.map. We need to use the posix - # driver directly to do this because handles cannot be passed back from the C code. - my $oStorageTrunc = new pgBackRestTest::Common::Storage($self->testPath(), new pgBackRestTest::Common::StoragePosix()); - - my $strTestTable = $self->testPath() . "/testtable"; - my $oDestinationFileIo = $oStorageTrunc->openWrite($strTestTable); - $oDestinationFileIo->open(); - - # Truncate to the original size which will create a sparse file. - if (!truncate($oDestinationFileIo->handle(), $lSize)) - { - confess "unable to truncate '$strTestTable' with handle " . $oDestinationFileIo->handle(); - } - $oDestinationFileIo->close(); - - # Confirm the test filenode.map and the database test1 filenode.map are zeroed - my ($strSHA1Test, $lSizeTest) = storageTest()->hashSize($strTestTable); - $self->testResult(sub {($strSHA1Test eq $strSHA1) && ($lSizeTest == $lSize) && ($strSHA1 ne $strDb1TableSha1)}, - true, 'database test1 not restored'); - - $oHostDbPrimary->clusterStart(); - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strNameMessage); - - # Once the cluster is back online, make sure the database & table in the tablespace exists properly - $oHostDbPrimary->sqlSelectOneTest('select id from test_ts1', 2, {strDb => 'test2'}); - $oHostDbPrimary->sqlDisconnect({strDb => 'test2'}); - - $oHostDbPrimary->sqlSelectOneTest('select id from test3_exists', 1, {strDb => 'test3'}); - $oHostDbPrimary->sqlDisconnect({strDb => 'test3'}); - - # The tablespace path should exist and have files in it - my $strTablespacePath = $oHostDbPrimary->tablespacePath(1); - - # Backup info will have the catalog number - my $oBackupInfo = new pgBackRestDoc::Common::Ini( - storageRepo(), $oHostBackup->repoBackupPath(FILE_BACKUP_INFO), - {bLoad => false, strContent => ${storageRepo()->get($oHostBackup->repoBackupPath(FILE_BACKUP_INFO))}}); - - # Construct the special path - $strTablespacePath .= - '/PG_' . $oHostDbPrimary->pgVersion() . qw{_} . $oBackupInfo->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CATALOG); - - # Check that path exists - if (!storageTest()->pathExists($strTablespacePath)) - { - confess &log(ASSERT, "unable to find tablespace path '${strTablespacePath}'"); - } - - # Make sure there are some files in the tablespace path - if (grep(!/^PG\_VERSION$/i, storageTest()->list($strTablespacePath)) == 0) - { - confess &log(ASSERT, "no files found in tablespace path '${strTablespacePath}'"); - } - - # This table should exist to prove that the tablespace was restored. It has not been updated since it was created so it - # should not be created by any full page writes. Once it is verified to exist it can be dropped. - $oHostDbPrimary->sqlSelectOneTest("select count(*) from test_exists", 0); - $oHostDbPrimary->sqlExecute('drop table test_exists'); - - # Now it should be OK to drop database test2 and test3 - $oHostDbPrimary->sqlExecute('drop database test2', {bAutoCommit => true}); - - # The test table lives in ts1 so it needs to be moved or dropped - $oHostDbPrimary->sqlExecute('alter table test set tablespace pg_default'); - - # And drop the tablespace - $oHostDbPrimary->sqlExecute('drop database test3', {bAutoCommit => true}); - $oHostDbPrimary->sqlExecute("drop tablespace ts1", {bAutoCommit => true}); - - # Restore (restore type = immediate, inclusive) - #--------------------------------------------------------------------------------------------------------------------------- - if ($oHostDbPrimary->pgVersion() >= PG_VERSION_94) - { - &log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_IMMEDIATE); - - $oHostDbPrimary->clusterStop(); - - $oHostDbPrimary->restore( - undef, $strFullBackup, {bForce => true, strType => CFGOPTVAL_RESTORE_TYPE_IMMEDIATE, strTargetAction => 'promote'}); - - $oHostDbPrimary->clusterStart(); - $oHostDbPrimary->sqlSelectOneTest('select message from test', ($strStandbyMessage)); - } - - # Restore (restore type = xid, inclusive) - #--------------------------------------------------------------------------------------------------------------------------- - my $strRecoveryFile = undef; - - &log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_XID); - - $oHostDbPrimary->clusterStop(); - - executeTest('rm -rf ' . $oHostDbPrimary->dbBasePath() . "/*"); - executeTest('rm -rf ' . $oHostDbPrimary->dbPath() . qw{/} . $oManifest->walPath() . '/*'); - - $oHostDbPrimary->restore( - undef, $strIncrBackup, - {bForce => true, strType => CFGOPTVAL_RESTORE_TYPE_XID, strTarget => $strXidTarget, strTargetAction => 'promote', - strTargetTimeline => $oHostDbPrimary->pgVersion() >= PG_VERSION_12 ? 'current' : undef, - strOptionalParam => '--tablespace-map-all=../../tablespace', bTablespace => false, - iRepo => $iRepoTotal}); - - # Save recovery file to test so we can use it in the next test - $strRecoveryFile = $oHostDbPrimary->pgVersion() >= PG_VERSION_12 ? 'postgresql.auto.conf' : DB_FILE_RECOVERYCONF; - - storageTest()->copy( - $oHostDbPrimary->dbBasePath() . qw{/} . $strRecoveryFile, $self->testPath() . qw{/} . $strRecoveryFile); - - $oHostDbPrimary->clusterStart(); - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strXidMessage); - - $oHostDbPrimary->sqlExecute("update test set message = '$strTimelineMessage'"); - - # Restore (restore type = preserve, inclusive) - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_PRESERVE); - - $oHostDbPrimary->clusterStop(); - - executeTest('rm -rf ' . $oHostDbPrimary->dbBasePath() . "/*"); - executeTest('rm -rf ' . $oHostDbPrimary->dbPath() . qw{/} . $oManifest->walPath() . '/*'); - executeTest('rm -rf ' . $oHostDbPrimary->tablespacePath(1) . "/*"); - - # Restore recovery file that was saved in last test - storageTest()->move($self->testPath . "/${strRecoveryFile}", $oHostDbPrimary->dbBasePath() . "/${strRecoveryFile}"); - - # Also touch recovery.signal when required - if ($oHostDbPrimary->pgVersion() >= PG_VERSION_12) - { - storageTest()->put($oHostDbPrimary->dbBasePath() . "/" . DB_FILE_RECOVERYSIGNAL); - } - - $oHostDbPrimary->restore(undef, 'latest', {strType => CFGOPTVAL_RESTORE_TYPE_PRESERVE}); - - $oHostDbPrimary->clusterStart(); - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strXidMessage); - - $oHostDbPrimary->sqlExecute("update test set message = '$strTimelineMessage'"); - - # Restore (restore type = time, inclusive, automatically select backup) - there is no exclusive time test because I can't - # find a way to find the exact commit time of a transaction. - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_TIME); - - $oHostDbPrimary->clusterStop(); - - $oHostDbPrimary->restore( - undef, 'latest', - {bDelta => true, strType => CFGOPTVAL_RESTORE_TYPE_TIME, strTarget => $strTimeTarget, strTargetAction => 'promote', - strTargetTimeline => $oHostDbPrimary->pgVersion() >= PG_VERSION_12 ? 'current' : undef, - strBackupExpected => $strFullBackup}); - - $oHostDbPrimary->clusterStart(); - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strTimeMessage); - - # Restore (restore type = xid, exclusive) - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_XID); - - $oHostDbPrimary->clusterStop(); - - $oHostDbPrimary->restore( - undef, $strIncrBackup, - {bDelta => true, strType => CFGOPTVAL_RESTORE_TYPE_XID, strTarget => $strXidTarget, bTargetExclusive => true, - strTargetAction => 'promote', - strTargetTimeline => $oHostDbPrimary->pgVersion() >= PG_VERSION_12 ? 'current' : undef, - iRepo => $iRepoTotal}); - - $oHostDbPrimary->clusterStart(); - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strIncrMessage); - - # Restore (restore type = name) - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_NAME); - - $oHostDbPrimary->clusterStop(); - - $oHostDbPrimary->restore( - undef, 'latest', - {bDelta => true, bForce => true, strType => CFGOPTVAL_RESTORE_TYPE_NAME, strTarget => $strNameTarget, - strTargetAction => 'promote', - strTargetTimeline => $oHostDbPrimary->pgVersion() >= PG_VERSION_12 ? 'current' : undef}); - - $oHostDbPrimary->clusterStart(); - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strNameMessage); - - # Restore (restore type = default, timeline = created by type = xid, inclusive recovery) - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_DEFAULT); - - $oHostDbPrimary->clusterStop(); - - # The timeline to use for this test is subject to change based on tests being added or removed above. The best thing would - # be to automatically grab the timeline after the restore, but since this test has been stable for a long time it does not - # seem worth the effort to automate. - $oHostDbPrimary->restore( - undef, $strIncrBackup, - {bDelta => true, strType => CFGOPTVAL_RESTORE_TYPE_STANDBY, strTargetTimeline => 4, iRepo => $iRepoTotal}); - - $oHostDbPrimary->clusterStart({bHotStandby => true}); - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strTimelineMessage, {iTimeout => 120}); - - # Stop clusters to catch any errors in the postgres log - #--------------------------------------------------------------------------------------------------------------------------- - $oHostDbPrimary->clusterStop(); - - # Stanza-delete --force without access to pgbackrest on database host. This test is not version specific so is run on only - # one version. - #--------------------------------------------------------------------------------------------------------------------------- - if ($bNonVersionSpecific) - { - # Make sure this test has a backup host to work with - confess "test must run with backup dst = " . HOST_BACKUP if !$bHostBackup; - - $oHostDbPrimary->stop(); - $oHostBackup->stop({strStanza => $self->stanza}); - $oHostBackup->stanzaDelete( - "delete stanza with --force when pgbackrest on pg host not accessible", {strOptionalParam => ' --force'}); - $oHostDbPrimary->start(); - $oHostBackup->start(); - } - } -} - -1; diff --git a/test/src/build/config/config.yaml b/test/src/build/config/config.yaml index a12fc79cd5..8006823c4f 100644 --- a/test/src/build/config/config.yaml +++ b/test/src/build/config/config.yaml @@ -70,6 +70,12 @@ option: command: test: {} + pg-version: + type: string + default: invalid + command: + test: {} + profile: type: boolean default: false diff --git a/test/src/build/help/help.xml b/test/src/build/help/help.xml index 1901798953..b9b93b01a7 100644 --- a/test/src/build/help/help.xml +++ b/test/src/build/help/help.xml @@ -120,6 +120,16 @@ n + +