diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2d27fabdba..cc9de1b705 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,13 +32,13 @@ jobs: - param: test --vm=d10 --param=no-performance --param=c-only # All integration tests for 32-bit - - param: test --vm=d10 --param=module=mock --param=module=real + - param: test --vm=d10 --param=module=mock --param=module=integration # Debian/Ubuntu documentation - param: doc --vm=u20 # All integration tests - - param: test --vm=u22 --param=build-package --param=module=mock --param=module=real + - param: test --vm=u22 --param=build-package --param=module=mock --param=module=integration # All unit tests with coverage, backtrace and alternate timezone - param: test --vm=u22 --param=c-only --param=no-valgrind --param=tz=America/New_York @@ -53,7 +53,7 @@ jobs: - param: doc --vm=rh8 # All integration tests - - param: test --vm=rh7 --param=module=mock --param=module=real + - param: test --vm=rh7 --param=module=mock --param=module=integration steps: - name: Checkout Code diff --git a/doc/lib/pgBackRestDoc/Common/DocExecute.pm b/doc/lib/pgBackRestDoc/Common/DocExecute.pm index 91f360b783..695166d348 100644 --- a/doc/lib/pgBackRestDoc/Common/DocExecute.pm +++ b/doc/lib/pgBackRestDoc/Common/DocExecute.pm @@ -16,11 +16,11 @@ use File::Basename qw(dirname); use Storable qw(dclone); use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostTest; -use pgBackRestTest::Common::HostGroupTest; use pgBackRestDoc::Common::DocManifest; use pgBackRestDoc::Common::Exception; +use pgBackRestDoc::Common::Host; +use pgBackRestDoc::Common::HostGroup; use pgBackRestDoc::Common::Ini; use pgBackRestDoc::Common::Log; use pgBackRestDoc::Common::String; @@ -1050,7 +1050,7 @@ sub sectionChildProcess $strOption =~ s/\{\[host\-repo\-path\]\}/${strHostRepoPath}/g; } - my $oHost = new pgBackRestTest::Common::HostTest( + my $oHost = new pgBackRestDoc::Common::Host( $$hCacheKey{name}, "doc-$$hCacheKey{name}", $strImage, $strHostUser, defined($strMount) ? [$strMount] : undef, $strOption, $$hCacheKey{param}, $$hCacheKey{'update-hosts'}); diff --git a/test/lib/pgBackRestTest/Common/HostTest.pm b/doc/lib/pgBackRestDoc/Common/Host.pm similarity index 99% rename from test/lib/pgBackRestTest/Common/HostTest.pm rename to doc/lib/pgBackRestDoc/Common/Host.pm index d7d76a4e04..20606976b2 100644 --- a/test/lib/pgBackRestTest/Common/HostTest.pm +++ b/doc/lib/pgBackRestDoc/Common/Host.pm @@ -1,7 +1,7 @@ #################################################################################################################################### -# HostTest.pm - Encapsulate a docker host for testing +# HostTest.pm - Encapsulate a docker host #################################################################################################################################### -package pgBackRestTest::Common::HostTest; +package pgBackRestDoc::Common::Host; #################################################################################################################################### # Perl includes diff --git a/test/lib/pgBackRestTest/Common/HostGroupTest.pm b/doc/lib/pgBackRestDoc/Common/HostGroup.pm similarity index 96% rename from test/lib/pgBackRestTest/Common/HostGroupTest.pm rename to doc/lib/pgBackRestDoc/Common/HostGroup.pm index 345a54e04b..e49bc44639 100644 --- a/test/lib/pgBackRestTest/Common/HostGroupTest.pm +++ b/doc/lib/pgBackRestDoc/Common/HostGroup.pm @@ -1,7 +1,7 @@ #################################################################################################################################### -# HostGroupTest.pm - Encapsulate a group of docker containers for testing +# HostGroupTest.pm - Encapsulate a group of docker containers #################################################################################################################################### -package pgBackRestTest::Common::HostGroupTest; +package pgBackRestDoc::Common::HostGroup; #################################################################################################################################### # Perl includes @@ -177,7 +177,7 @@ sub hostGroupGet { if (!defined($oHostGroup)) { - $oHostGroup = new pgBackRestTest::Common::HostGroupTest(); + $oHostGroup = new pgBackRestDoc::Common::HostGroup(); } return $oHostGroup; diff --git a/src/build/common/exec.c b/src/build/common/exec.c index ebe27d1954..eb1e52d294 100644 --- a/src/build/common/exec.c +++ b/src/build/common/exec.c @@ -9,10 +9,11 @@ Execute Process Extensions /**********************************************************************************************************************************/ static String * -execProcess(Exec *const this) +execProcess(Exec *const this, const ExecOneParam param) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(EXEC, this); + FUNCTION_LOG_PARAM(INT, param.resultExpect); FUNCTION_LOG_END(); String *const result = strNew(); @@ -35,7 +36,7 @@ execProcess(Exec *const this) // If the process exited normally but without a success status if (WIFEXITED(processStatus)) { - if (WEXITSTATUS(processStatus) != 0) + if (WEXITSTATUS(processStatus) != param.resultExpect) execCheckStatusError(this, processStatus, strTrim(result)); } // Else if the process did not exit normally then it must have been a signal @@ -53,7 +54,8 @@ execOne(const String *const command, const ExecOneParam param) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STRING, command); - (void)param; + FUNCTION_LOG_PARAM(STRING, param.shell); + FUNCTION_LOG_PARAM(INT, param.resultExpect); FUNCTION_LOG_END(); String *result = NULL; @@ -61,23 +63,23 @@ execOne(const String *const command, const ExecOneParam param) MEM_CONTEXT_TEMP_BEGIN() { const StringList *const shellList = strLstNewSplitZ(param.shell != NULL ? param.shell : STRDEF("sh -c"), " "); - StringList *const param = strLstNew(); + StringList *const paramList = strLstNew(); ASSERT(strLstSize(shellList) != 0); for (unsigned int shellIdx = 1; shellIdx < strLstSize(shellList); shellIdx++) - strLstAdd(param, strLstGet(shellList, shellIdx)); + strLstAdd(paramList, strLstGet(shellList, shellIdx)); - strLstAddFmt(param, "%s 2>&1", strZ(command)); - strLstAddZ(param, "2>&1"); + strLstAddFmt(paramList, "%s 2>&1", strZ(command)); + strLstAddZ(paramList, "2>&1"); - Exec *const exec = execNew(strLstGet(shellList, 0), param, command, ioTimeoutMs()); + Exec *const exec = execNew(strLstGet(shellList, 0), paramList, command, ioTimeoutMs()); execOpen(exec); MEM_CONTEXT_PRIOR_BEGIN() { - result = execProcess(exec); + result = execProcess(exec, param); } MEM_CONTEXT_PRIOR_END(); } diff --git a/src/build/common/exec.h b/src/build/common/exec.h index cf53a43770..94171dcd0a 100644 --- a/src/build/common/exec.h +++ b/src/build/common/exec.h @@ -17,6 +17,7 @@ typedef struct ExecOneParam { VAR_PARAM_HEADER; const String *shell; // Shell command to use for exec (default is sh -c) + int resultExpect; // Expected result, if not 0 } ExecOneParam; #define execOneP(command, ...) \ diff --git a/test/certificate/pgbackrest-test-server.cnf b/test/certificate/pgbackrest-test-server.cnf index 0349097535..7443b43b55 100644 --- a/test/certificate/pgbackrest-test-server.cnf +++ b/test/certificate/pgbackrest-test-server.cnf @@ -37,6 +37,6 @@ DNS.4 = 127.0.0.1 IP.1 = 127.0.0.1 # Used in integration tests -DNS.5 = db-primary -DNS.6 = db-standby -DNS.7 = backup +DNS.5 = pg1 +DNS.6 = pg2 +DNS.7 = repo diff --git a/test/certificate/pgbackrest-test-server.crt b/test/certificate/pgbackrest-test-server.crt index 95040f2beb..01c9042654 100644 --- a/test/certificate/pgbackrest-test-server.crt +++ b/test/certificate/pgbackrest-test-server.crt @@ -1,8 +1,8 @@ -----BEGIN CERTIFICATE----- -MIIGAjCCA+qgAwIBAgIUW0gPWoZD5DqjIWIP3PliYA0IAOQwDQYJKoZIhvcNAQEL +MIIF8jCCA9qgAwIBAgIUJCya0E5vFzyH2AgiM3HSAHmpZ1QwDQYJKoZIhvcNAQEL BQAwXDELMAkGA1UEBhMCVVMxDDAKBgNVBAgMA0FsbDEMMAoGA1UEBwwDQWxsMRMw EQYDVQQKDApwZ0JhY2tSZXN0MRwwGgYDVQQDDBN0ZXN0LnBnYmFja3Jlc3Qub3Jn -MCAXDTIxMDgyNjEyMjkwM1oYDzIyOTUwNjEwMTIyOTAzWjB6MQswCQYDVQQGEwJV +MCAXDTI0MDMwNDAxMzgzMFoYDzIyOTcxMjE3MDEzODMwWjB6MQswCQYDVQQGEwJV UzEMMAoGA1UECAwDQWxsMQwwCgYDVQQHDANBbGwxEzARBgNVBAoMCnBnQmFja1Jl c3QxHDAaBgNVBAsME1VuaXQgVGVzdGluZyBEb21haW4xHDAaBgNVBAMME3Rlc3Qu cGdiYWNrcmVzdC5vcmcwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDD @@ -16,20 +16,19 @@ dCdyXuSftGFx0JxvmDhl9qFGarv1BKgwO83j7sy3IREte1K21JaIHNBVWP+NwU0N 4Z4OMqnpnnnGiyi0xnfJVqOXghu5BLWl9MuOntZ0nnzLmFD7w795uNRgjE6jmRmF FlX+PGqhHsZr0wZsBDsE9xO4i2l8aqJZx1hT5l3LIXC+lei/qo2gJi3nyePuz4UB t53sTNEdrZndFUaRyq/aJfkR13J0eaoqKn5BRRHhw8tRef6S84e0kQ6ABYNRGHQN -V+GswPl1fV37114FTBnz2Bi/GSQSs8vWjw49HHKK5wIDAQABo4GbMIGYMAkGA1Ud -EwQCMAAwCwYDVR0PBAQDAgXgMH4GA1UdEQR3MHWCE3Rlc3QucGdiYWNrcmVzdC5v +V+GswPl1fV37114FTBnz2Bi/GSQSs8vWjw49HHKK5wIDAQABo4GLMIGIMAkGA1Ud +EwQCMAAwCwYDVR0PBAQDAgXgMG4GA1UdEQRnMGWCE3Rlc3QucGdiYWNrcmVzdC5v cmeCFSoudGVzdC5wZ2JhY2tyZXN0Lm9yZ4IWKi50ZXN0Mi5wZ2JhY2tyZXN0Lm9y -Z4IJMTI3LjAuMC4xhwR/AAABggpkYi1wcmltYXJ5ggpkYi1zdGFuZGJ5ggZiYWNr -dXAwDQYJKoZIhvcNAQELBQADggIBAMbSq1/hjvQZJ2PFE/VVz9OcA7vlewq632eE -P5BalSJJgLVEsv1AxPx8VT08xfFQHQtEcCg/PFqT3RQ5yb1kHfa6glJkjYIdKQbn -lv9OVc/iutQwKPwk32QQjSgQFb/m0tXv9SlQ+gNTdkK4UKffXPj5rpgwaSiVwuLF -d+3TUpJihS48LLRC27kcL5Ur69/fu0ZD7xZSoCr/n8MUq4f9LwOhBqq+h64wM9cV -V79iPWmEJXoNAJrPYmK+XNhcro071c4m+HR4CCNikjxz/GUUf/NGHWT3pL0Ildku -X3dHmsNRVT/wLqi2v2oa6zr9FfVzjDAdCfnvTLOJ6H6dmofzQUFJBSWfhqGNDR8U -oblwirM2sjaOUjnkBS6Cb26yHSClStI+GZvS0KZfSVd2Qbe4YmtQMTNl/hdZGK3z -ZoqV++idVR+A0NQP8xR4VWqQdq0BR5eQOXDA4wtqvivqlIXpbJvqh1kBHPU9cAF+ -g/t3Wa7EomwLazRaV9djLUpon6wGwScKJGzv+vyQSgXN1tQG9tLV4NCFUKDueUUZ -U/j1t64KF9hp5NU2A16zLp6V5GPIJhufXOYa66AFjV8c880eLd5YlkfzgyYwReOx -7vHkiLylbx2tc6aYUqdwjpMwnkxTsn52BBVxDvXToBIRdq/ea/LnZ/yhpnaac/Um -bJOTMee+ +Z4IJMTI3LjAuMC4xhwR/AAABggNwZzGCA3BnMoIEcmVwbzANBgkqhkiG9w0BAQsF +AAOCAgEAX0MEXH6ANllRhdQz6neQ7SVG48Aj1lEAGeOhfpoNKzuyBcRjVw7+NyNN +IwlPKSSBDpnxaWQ5rCLtBtXod6yPMGKTRlFHwFFzfOps6nlRQjPsA848d6daLBvj +unpUQx4NFGPZJSs6z5z4BlT/+5mJVHC9qsrZBtkndYpRWo37xbVhRqP0+FSTbzrx +Gj2td3PoqQBgA/AmSKIpwagGzw7cSor1r4uEjkVxxyOMRbjuuASHMHUM7MtQV3YR +rz9UspvGfoKBdUkzMoqKRwxZWuh+uAoM+1GWXBjqlN6uAdQxpV2wZ75iRJp3Y4Bk +/CkXTLZ83lARGLqS/E5EFfg7Z9Bre2f5fHzV8C/h6WGpvCt/GlZqTx8fix/mMPT2 +CFq+FcSmvF5JsIMxUnpvTw1hcTDNRPnOkFKnO1bjf9+jGCwzDUoGReYpb5veFSxh +IFkQ3oyw3/6v11aPstXSADTvRTFMyRklqu5NIHkMVQCPwQCAE3346KpEsT5HO2T7 +X57sP05alMESjUv1sR260yYC3GUr0dbmf8gthVDhH1SsP3Drn/A3l70xGASnJA46 +LCZwAJZ0KI6G/ok17lTZe9Hjwn2DkmVf1CKD7gXjmroYQI9O+etUtD/g+B2AISTG +SP60NakteOBWqmcSirV5npKh/SZR8Il5oaLS+3HerhgwvNDJ078= -----END CERTIFICATE----- diff --git a/test/ci.pl b/test/ci.pl index 5553243001..824402c776 100755 --- a/test/ci.pl +++ b/test/ci.pl @@ -183,7 +183,7 @@ sub processEnd # Build list of packages that need to be installed my $strPackage = "make gcc ccache python3-pip git rsync zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config" . - " uncrustify libssh2-1-dev"; + " uncrustify libssh2-1-dev valgrind"; # Add lcov when testing coverage if (vmCoverageC($strVm)) @@ -194,7 +194,7 @@ sub processEnd # Extra packages required when testing without containers if ($strVm eq VM_NONE) { - $strPackage .= " valgrind liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev"; + $strPackage .= " liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev"; } # Else packages needed for integration tests on containers else diff --git a/test/define.yaml b/test/define.yaml index 324648ebba..9334c6316a 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -608,7 +608,9 @@ unit: # ---------------------------------------------------------------------------------------------------------------------------- - name: sftp total: 19 - harness: libSsh2 + harness: + name: libSsh2 + integration: false harness: name: fd shim: @@ -635,7 +637,9 @@ unit: # ---------------------------------------------------------------------------------------------------------------------------- - name: client total: 1 - harness: pq + harness: + name: pq + integration: false coverage: - postgres/client @@ -881,6 +885,7 @@ unit: total: 12 harness: name: backup + integration: false shim: command/backup/backup: function: @@ -1004,13 +1009,14 @@ unit: integration: # ******************************************************************************************************************************** - - name: real + - name: integration db: true test: # ---------------------------------------------------------------------------------------------------------------------------- - name: all total: 1 + harness: host # ********************************************************************************************************************************** # Performance tests diff --git a/test/lib/pgBackRestTest/Common/CoverageTest.pm b/test/lib/pgBackRestTest/Common/CoverageTest.pm index 13c860c7ff..4a681aab27 100644 --- a/test/lib/pgBackRestTest/Common/CoverageTest.pm +++ b/test/lib/pgBackRestTest/Common/CoverageTest.pm @@ -26,7 +26,31 @@ use pgBackRestTest::Common::ContainerTest; use pgBackRestTest::Common::DefineTest; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::ListTest; -use pgBackRestTest::Common::RunTest; + +#################################################################################################################################### +# testRunName +# +# Create module/test names by upper-casing the first letter and then inserting capitals after each -. +#################################################################################################################################### +sub testRunName +{ + my $strName = shift; + my $bInitCapFirst = shift; + + $bInitCapFirst = defined($bInitCapFirst) ? $bInitCapFirst : true; + my $bFirst = true; + + my @stryName = split('\-', $strName); + $strName = undef; + + foreach my $strPart (@stryName) + { + $strName .= ($bFirst && $bInitCapFirst) || !$bFirst ? ucfirst($strPart) : $strPart; + $bFirst = false; + } + + return $strName; +} #################################################################################################################################### # Generate an lcov configuration file diff --git a/test/lib/pgBackRestTest/Common/FileTest.pm b/test/lib/pgBackRestTest/Common/FileTest.pm deleted file mode 100644 index 6fddeddd22..0000000000 --- a/test/lib/pgBackRestTest/Common/FileTest.pm +++ /dev/null @@ -1,277 +0,0 @@ -#################################################################################################################################### -# CommonTest.pm - Common globals used for testing -#################################################################################################################################### -package pgBackRestTest::Common::FileTest; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Cwd qw(abs_path cwd); -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); -use File::Copy qw(move); -use File::Path qw(remove_tree); -use IO::Select; -use IPC::Open3; -use POSIX ':sys_wait_h'; -use Symbol 'gensym'; - -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::Common::String; - -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageBase; -use pgBackRestTest::Common::VmTest; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Host::HostDbCommonTest; -use pgBackRestTest::Env::Host::HostDbTest; -use pgBackRestTest::Env::Host::HostS3Test; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# testLinkCreate -# -# Create a symlink -#################################################################################################################################### -sub testLinkCreate -{ - my $strLink = shift; - my $strDestination = shift; - - # Create the file - symlink($strDestination, $strLink) - or confess "unable to link ${strLink} to ${strDestination}"; -} - -push(@EXPORT, qw(testLinkCreate)); - -#################################################################################################################################### -# testPathMode -# -# Set mode of an existing path. -#################################################################################################################################### -sub testPathMode -{ - my $strPath = shift; - my $strMode = shift; - - # Set the mode - chmod(oct($strMode), $strPath) - or confess 'unable to set mode ${strMode} for ${strPath}'; -} - -push(@EXPORT, qw(testPathMode)); - -#################################################################################################################################### -# testPathRemove -# -# Remove a path and all subpaths. -#################################################################################################################################### -sub testPathRemove -{ - my $strPath = shift; - my $bSuppressError = shift; - - executeTest('rm -rf ' . $strPath, {bSuppressError => $bSuppressError}); -} - -push(@EXPORT, qw(testPathRemove)); - -#################################################################################################################################### -# testFileCreate -# -# Create a file specifying content, mode, and time. -#################################################################################################################################### -sub testFileCreate -{ - my $strFile = shift; - my $strContent = shift; - my $lTime = shift; - my $strMode = shift; - - # Open the file and save strContent to it - my $hFile = shift; - - open($hFile, '>', $strFile) - or confess "unable to open ${strFile} for writing"; - - if (defined($strContent) && $strContent ne '') - { - syswrite($hFile, $strContent) - or confess "unable to write to ${strFile}: $!"; - } - - close($hFile); - - # Set the time - if (defined($lTime)) - { - utime($lTime, $lTime, $strFile) - or confess 'unable to set time ${lTime} for ${strPath}'; - } - - # Set the mode - chmod(oct(defined($strMode) ? $strMode : '0600'), $strFile) - or confess 'unable to set mode ${strMode} for ${strFile}'; -} - -push(@EXPORT, qw(testFileCreate)); - -#################################################################################################################################### -# testFileRemove -# -# Remove a file. -#################################################################################################################################### -sub testFileRemove -{ - my $strFile = shift; - - unlink($strFile) - or confess "unable to remove ${strFile}: $!"; -} - -push(@EXPORT, qw(testFileRemove)); - -#################################################################################################################################### -# forceStorageMode - force mode on a file or path -#################################################################################################################################### -sub forceStorageMode -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oStorage, - $strPath, - $strMode, - $bRecurse - ) = - logDebugParam - ( - __PACKAGE__ . '::forceStorageMode', \@_, - {name => 'oStorage'}, - {name => 'strPath'}, - {name => 'strMode'}, - {name => 'bRecurse', optional => true, default => false}, - ); - - # Mode commands are ignored on object storage - if ($oStorage->type() ne STORAGE_OBJECT) - { - executeTest('chmod ' . ($bRecurse ? '-R ' : '') . "${strMode} ${strPath}"); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -push(@EXPORT, qw(forceStorageMode)); - -#################################################################################################################################### -# forceStorageMove - force move a directory or file -#################################################################################################################################### -sub forceStorageMove -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oStorage, - $strSourcePath, - $strDestinationPath, - $bRecurse, - ) = - logDebugParam - ( - __PACKAGE__ . '->forceStorageMove', \@_, - {name => 'oStorage'}, - {name => 'strSourcePath'}, - {name => 'strDestinationPath'}, - {name => 'bRecurse', optional => true, default => true}, - ); - - # If object storage then use storage commands to remove - if ($oStorage->type() eq STORAGE_OBJECT) - { - if ($bRecurse) - { - my $rhManifest = $oStorage->manifest($strSourcePath); - - foreach my $strName (sort(keys(%{$rhManifest}))) - { - if ($rhManifest->{$strName}{type} eq 'f') - { - $oStorage->put( - "${strDestinationPath}/${strName}", ${$oStorage->get("${strSourcePath}/${strName}", {bRaw => true})}, - {bRaw => true}); - } - } - - $oStorage->pathRemove($strSourcePath, {bRecurse => true}); - } - else - { - $oStorage->put($strDestinationPath, ${$oStorage->get($strSourcePath, {bRaw => true})}, {bRaw => true}); - $oStorage->remove($strSourcePath); - } - } - # Else remove using filesystem commands - else - { - executeTest("mv ${strSourcePath} ${strDestinationPath}"); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -push(@EXPORT, qw(forceStorageMove)); - -#################################################################################################################################### -# forceStorageRemove - force remove a file or path from storage -#################################################################################################################################### -sub forceStorageRemove -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oStorage, - $strPath, - $bRecurse - ) = - logDebugParam - ( - __PACKAGE__ . '->forceStorageRemove', \@_, - {name => 'oStorage'}, - {name => 'strPath'}, - {name => 'bRecurse', optional => true, default => false}, - ); - - # If object storage then use storage commands to remove - if ($oStorage->type() eq STORAGE_OBJECT) - { - $oStorage->pathRemove($strPath, {bRecurse => true}); - } - else - { - executeTest('rm -f' . ($bRecurse ? 'r ' : ' ') . $strPath); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -push(@EXPORT, qw(forceStorageRemove)); - -1; diff --git a/test/lib/pgBackRestTest/Common/JobTest.pm b/test/lib/pgBackRestTest/Common/JobTest.pm index 0a02b5de53..55fd291d37 100644 --- a/test/lib/pgBackRestTest/Common/JobTest.pm +++ b/test/lib/pgBackRestTest/Common/JobTest.pm @@ -30,7 +30,6 @@ use pgBackRestTest::Common::DbVersion; use pgBackRestTest::Common::DefineTest; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::ListTest; -use pgBackRestTest::Common::RunTest; use pgBackRestTest::Common::VmTest; #################################################################################################################################### @@ -115,7 +114,9 @@ sub new $self->{iTry} = 0; # Setup the path where unit test will be built - $self->{strUnitPath} = "$self->{strTestPath}/unit-$self->{iVmIdx}/$self->{oTest}->{&TEST_VM}"; + $self->{strUnitPath} = + "$self->{strTestPath}/unit-$self->{iVmIdx}/" . + ($self->{oTest}->{&TEST_TYPE} eq TESTDEF_INTEGRATION ? 'none' : $self->{oTest}->{&TEST_VM}); $self->{strDataPath} = "$self->{strTestPath}/data-$self->{iVmIdx}"; $self->{strRepoPath} = "$self->{strTestPath}/repo"; @@ -246,58 +247,32 @@ sub run if (!$self->{bDryRun} || $self->{bVmOut}) { - my $strCommand = undef; # Command to run test - - # If testing with C harness - if ($self->{oTest}->{&TEST_C}) - { - # Create command - # ------------------------------------------------------------------------------------------------------------------ - # Build filename for valgrind suppressions - my $strValgrindSuppress = $self->{strRepoPath} . '/test/src/valgrind.suppress.' . $self->{oTest}->{&TEST_VM}; - - $strCommand = - ($self->{oTest}->{&TEST_VM} ne VM_NONE ? "docker exec -i -u ${\TEST_USER} ${strImage} bash -l -c '" : '') . - " \\\n" . - $self->{strTestPath} . '/build/' . $self->{oTest}->{&TEST_VM} . '/test/src/test-pgbackrest' . - ' --repo-path=' . $self->{strTestPath} . '/repo' . ' --test-path=' . $self->{strTestPath} . - " --log-level=$self->{strLogLevel}" . ' --vm=' . $self->{oTest}->{&TEST_VM} . - ' --vm-id=' . $self->{iVmIdx} . ($self->{bProfile} ? ' --profile' : '') . - ($self->{bLogTimestamp} ? '' : ' --no-log-timestamp') . - ($self->{strTimeZone} ? " --tz='$self->{strTimeZone}'" : '') . - ($self->{bBackTraceUnit} ? '' : ' --no-back-trace') . ($bCoverage ? '' : ' --no-coverage') . ' test ' . - $self->{oTest}->{&TEST_MODULE} . '/' . $self->{oTest}->{&TEST_NAME} . " && \\\n" . - # Allow stderr to be copied to stderr and stdout - "exec 3>&1 && \\\n" . - # Test with valgrind when requested - ($self->{bValgrindUnit} && $self->{oTest}->{&TEST_TYPE} ne TESTDEF_PERFORMANCE ? - 'valgrind -q --gen-suppressions=all' . - ($self->{oStorageTest}->exists($strValgrindSuppress) ? " --suppressions=${strValgrindSuppress}" : '') . - " --exit-on-first-error=yes --leak-check=full --leak-resolution=high --error-exitcode=25" . ' ' : '') . - "$self->{strUnitPath}/build/test-unit 2>&1 1>&3 | tee /dev/stderr" . - ($self->{oTest}->{&TEST_VM} ne VM_NONE ? "'" : ''); - } - else - { - $strCommand = - ($self->{oTest}->{&TEST_CONTAINER} ? 'docker exec -i -u ' . TEST_USER . " ${strImage} " : '') . - abs_path($0) . - " --test-path=${strVmTestPath}" . - " --vm=$self->{oTest}->{&TEST_VM}" . - " --vm-id=$self->{iVmIdx}" . - " --module=" . $self->{oTest}->{&TEST_MODULE} . - ' --test=' . $self->{oTest}->{&TEST_NAME} . - $strCommandRunParam . - (defined($self->{oTest}->{&TEST_DB}) ? ' --pg-version=' . $self->{oTest}->{&TEST_DB} : '') . - ($self->{strLogLevel} ne lc(INFO) ? " --log-level=$self->{strLogLevel}" : '') . - ($self->{strLogLevelTestFile} ne lc(TRACE) ? " --log-level-test-file=$self->{strLogLevelTestFile}" : '') . + my $bValgrind = $self->{bValgrindUnit} && $self->{oTest}->{&TEST_TYPE} ne TESTDEF_PERFORMANCE; + my $strValgrindSuppress = + $self->{strRepoPath} . '/test/src/valgrind.suppress.' . + ($self->{oTest}->{&TEST_TYPE} eq TESTDEF_INTEGRATION ? VM_NONE : $self->{oTest}->{&TEST_VM}); + my $strVm = $self->{oTest}->{&TEST_TYPE} eq TESTDEF_INTEGRATION ? VM_NONE : $self->{oTest}->{&TEST_VM}; + + my $strCommand = + ($strVm ne VM_NONE ? "docker exec -i -u ${\TEST_USER} ${strImage} bash -l -c '\\\n" : '') . + $self->{strTestPath} . "/build/${strVm}/test/src/test-pgbackrest" . + ' --repo-path=' . $self->{strTestPath} . '/repo' . ' --test-path=' . $self->{strTestPath} . + " --log-level=$self->{strLogLevel}" . ' --vm=' . $self->{oTest}->{&TEST_VM} . + ' --vm-id=' . $self->{iVmIdx} . ($self->{bProfile} ? ' --profile' : '') . ($self->{bLogTimestamp} ? '' : ' --no-log-timestamp') . - ' --psql-bin=' . $self->{oTest}->{&TEST_PGSQL_BIN} . ($self->{strTimeZone} ? " --tz='$self->{strTimeZone}'" : '') . - ($self->{bDryRun} ? ' --dry-run' : '') . - ($self->{bDryRun} ? ' --vm-out' : '') . - ($self->{bNoCleanup} ? " --no-cleanup" : ''); - } + (defined($self->{oTest}->{&TEST_DB}) ? ' --pg-version=' . $self->{oTest}->{&TEST_DB} : '') . + ($self->{bBackTraceUnit} ? '' : ' --no-back-trace') . ($bCoverage ? '' : ' --no-coverage') . ' test ' . + $self->{oTest}->{&TEST_MODULE} . '/' . $self->{oTest}->{&TEST_NAME} . " && \\\n" . + # Allow stderr to be copied to stderr and stdout + "exec 3>&1 && \\\n" . + # Test with valgrind when requested + ($bValgrind ? + 'valgrind -q --gen-suppressions=all' . + ($self->{oStorageTest}->exists($strValgrindSuppress) ? " --suppressions=${strValgrindSuppress}" : '') . + " --exit-on-first-error=yes --leak-check=full --leak-resolution=high --error-exitcode=25" . ' ' : '') . + "$self->{strUnitPath}/build/test-unit 2>&1 1>&3 | tee /dev/stderr" . + ($strVm ne VM_NONE ? "'" : ''); my $oExec = new pgBackRestTest::Common::ExecuteTest( $strCommand, {bSuppressError => true, bShowOutputAsync => $self->{bShowOutputAsync}}); diff --git a/test/lib/pgBackRestTest/Common/RunTest.pm b/test/lib/pgBackRestTest/Common/RunTest.pm deleted file mode 100644 index 2c1ce7e4e9..0000000000 --- a/test/lib/pgBackRestTest/Common/RunTest.pm +++ /dev/null @@ -1,426 +0,0 @@ -#################################################################################################################################### -# RunTest.pm - All tests are inherited from this object -#################################################################################################################################### -package pgBackRestTest::Common::RunTest; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::Common::String; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::BuildTest; -use pgBackRestTest::Common::DefineTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::Storage; -use pgBackRestTest::Common::StoragePosix; -use pgBackRestTest::Common::VmTest; -use pgBackRestTest::Common::Wait; - -#################################################################################################################################### -# Constant to use when bogus data is required -#################################################################################################################################### -use constant BOGUS => 'bogus'; - push @EXPORT, qw(BOGUS); - -#################################################################################################################################### -# The current test run that is executing. Only a single run should ever occur in a process to prevent various cleanup issues from -# affecting the next run. Of course multiple subtests can be executed in a single run. -#################################################################################################################################### -my $oTestRun; -my $oStorage; - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Create the class hash - my $self = {}; - bless $self, $class; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '->new'); - - # Initialize run counter - $self->{iRun} = 0; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# initModule -# -# Empty init sub in case the ancestor class does not declare one. -#################################################################################################################################### -sub initModule {} - -#################################################################################################################################### -# initTest -# -# Empty init sub in case the ancestor class does not declare one. -#################################################################################################################################### -sub initTest {} - -#################################################################################################################################### -# cleanTest -# -# Delete all files in test directory. -#################################################################################################################################### -sub cleanTest -{ - my $self = shift; - - executeTest('rm -rf ' . $self->testPath() . '/*'); -} - -#################################################################################################################################### -# cleanModule -# -# Empty final sub in case the ancestor class does not declare one. -#################################################################################################################################### -sub cleanModule {} - -#################################################################################################################################### -# process -#################################################################################################################################### -sub process -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - ( - my $strOperation, - $self->{strVm}, - $self->{iVmId}, - $self->{strBasePath}, - $self->{strTestPath}, - $self->{strBackRestExe}, - $self->{strBackRestExeHelper}, - $self->{strPgBinPath}, - $self->{strPgVersion}, - $self->{strModule}, - $self->{strModuleTest}, - $self->{iyModuleTestRun}, - $self->{bOutput}, - $self->{bDryRun}, - $self->{bCleanup}, - $self->{strLogLevelTestFile}, - $self->{strPgUser}, - $self->{strGroup}, - ) = - logDebugParam - ( - __PACKAGE__ . '->process', \@_, - {name => 'strVm'}, - {name => 'iVmId'}, - {name => 'strBasePath'}, - {name => 'strTestPath'}, - {name => 'strBackRestExe'}, - {name => 'strBackRestExeHelper'}, - {name => 'strPgBinPath', required => false}, - {name => 'strPgVersion', required => false}, - {name => 'strModule'}, - {name => 'strModuleTest'}, - {name => 'iModuleTestRun', required => false}, - {name => 'bOutput'}, - {name => 'bDryRun'}, - {name => 'bCleanup'}, - {name => 'strLogLevelTestFile'}, - {name => 'strPgUser'}, - {name => 'strGroup'}, - ); - - # Init will only be run on first test, clean/init on subsequent tests - $self->{bFirstTest} = true; - - # Initialize test storage - $oStorage = new pgBackRestTest::Common::Storage( - $self->testPath(), new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false})); - - # Init, run, and clean the test(s) - $self->initModule(); - $self->run(); - $self->cleanModule(); - - # Make sure the correct number of tests ran - my $hModuleTest = testDefModuleTest($self->{strModule}, $self->{strModuleTest}); - - if ($hModuleTest->{&TESTDEF_TOTAL} != $self->runCurrent()) - { - confess &log(ASSERT, "expected $hModuleTest->{&TESTDEF_TOTAL} tests to run but $self->{iRun} ran"); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# begin -#################################################################################################################################### -sub begin -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDescription, - ) = - logDebugParam - ( - __PACKAGE__ . '->begin', \@_, - {name => 'strDescription'}, - ); - - # Increment the run counter; - $self->{iRun}++; - - # Return if this test should not be run - if (@{$self->{iyModuleTestRun}} != 0 && !grep(/^$self->{iRun}$/i, @{$self->{iyModuleTestRun}})) - { - return false; - } - - # Output information about test to run - &log(INFO, 'run ' . sprintf('%03d', $self->runCurrent()) . ' - ' . $strDescription); - - if ($self->isDryRun()) - { - return false; - } - - if (!$self->{bFirstTest}) - { - $self->cleanTest(); - } - - $self->initTest(); - $self->{bFirstTest} = false; - - return true; -} - -#################################################################################################################################### -# testResult -#################################################################################################################################### -sub testResult -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $fnSub, - $strExpected, - $strDescription, - $iWaitSeconds, - ) = - logDebugParam - ( - __PACKAGE__ . '::testResult', \@_, - {name => 'fnSub', trace => true}, - {name => 'strExpected', required => false, trace => true}, - {name => 'strDescription', trace => true}, - {name => 'iWaitSeconds', optional => true, default => 0, trace => true}, - ); - - &log(INFO, ' ' . $strDescription); - my $strActual; - my $bWarnValid = true; - - my $oWait = waitInit($iWaitSeconds); - my $bDone = false; - - # Clear the cache for this test - logFileCacheClear(); - - my @stryResult; - - do - { - eval - { - @stryResult = ref($fnSub) eq 'CODE' ? $fnSub->() : $fnSub; - - if (@stryResult <= 1) - { - $strActual = ${logDebugBuild($stryResult[0])}; - } - else - { - $strActual = ${logDebugBuild(\@stryResult)}; - } - - return true; - } - or do - { - if (!isException(\$EVAL_ERROR)) - { - confess "unexpected standard Perl exception" . (defined($EVAL_ERROR) ? ": ${EVAL_ERROR}" : ''); - } - - confess &logException($EVAL_ERROR); - }; - - if ($strActual ne (defined($strExpected) ? $strExpected : "[undef]")) - { - if (!waitMore($oWait)) - { - confess - "expected:\n" . (defined($strExpected) ? "\"${strExpected}\"" : '[undef]') . - "\nbut actual was:\n" . (defined($strActual) ? "\"${strActual}\"" : '[undef]'); - } - } - else - { - $bDone = true; - } - } while (!$bDone); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'result', value => \@stryResult, trace => true} - ); -} - -#################################################################################################################################### -# testRunName -# -# Create module/test names by upper-casing the first letter and then inserting capitals after each -. -#################################################################################################################################### -sub testRunName -{ - my $strName = shift; - my $bInitCapFirst = shift; - - $bInitCapFirst = defined($bInitCapFirst) ? $bInitCapFirst : true; - my $bFirst = true; - - my @stryName = split('\-', $strName); - $strName = undef; - - foreach my $strPart (@stryName) - { - $strName .= ($bFirst && $bInitCapFirst) || !$bFirst ? ucfirst($strPart) : $strPart; - $bFirst = false; - } - - return $strName; -} - -push @EXPORT, qw(testRunName); - -#################################################################################################################################### -# testRun -#################################################################################################################################### -sub testRun -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strModule, - $strModuleTest, - ) = - logDebugParam - ( - __PACKAGE__ . '::testRun', \@_, - {name => 'strModule', trace => true}, - {name => 'strModuleTest', trace => true}, - ); - - # Error if the test run is already defined - only one run per process is allowed - if (defined($oTestRun)) - { - confess &log(ASSERT, 'a test run has already been created in this process'); - } - - my $strModuleName = - 'pgBackRestTest::Module::' . testRunName($strModule) . '::' . testRunName($strModule) . testRunName($strModuleTest) . - 'Test'; - - $oTestRun = eval("require ${strModuleName}; ${strModuleName}->import(); return new ${strModuleName}();") - or do {confess $EVAL_ERROR}; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'oRun', value => $oTestRun, trace => true} - ); -} - -push @EXPORT, qw(testRun); - -#################################################################################################################################### -# testRunGet -#################################################################################################################################### -sub testRunGet -{ - return $oTestRun; -} - -push @EXPORT, qw(testRunGet); - -#################################################################################################################################### -# storageTest - get the storage for the current test -#################################################################################################################################### -sub storageTest -{ - return $oStorage; -} - -push(@EXPORT, qw(storageTest)); - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub archBits {return vmArchBits(shift->{strVm})} -sub backrestExe {return shift->{strBackRestExe}} -sub backrestExeHelper {return shift->{strBackRestExeHelper}} -sub basePath {return shift->{strBasePath}} -sub dataPath {return shift->basePath() . '/test/data'} -sub doCleanup {return shift->{bCleanup}} -sub logLevelTestFile {return shift->{strLogLevelTestFile}} -sub group {return shift->{strGroup}} -sub isDryRun {return shift->{bDryRun}} -sub module {return shift->{strModule}} -sub moduleTest {return shift->{strModuleTest}} -sub pgBinPath {return shift->{strPgBinPath}} -sub pgUser {return shift->{strPgUser}} -sub pgVersion {return shift->{strPgVersion}} -sub runCurrent {return shift->{iRun}} -sub stanza {return 'db'} -sub testPath {return shift->{strTestPath}} -sub vm {return shift->{strVm}} -sub vmId {return shift->{iVmId}} - -1; diff --git a/test/lib/pgBackRestTest/Common/StorageRepo.pm b/test/lib/pgBackRestTest/Common/StorageRepo.pm deleted file mode 100644 index 7469342d2d..0000000000 --- a/test/lib/pgBackRestTest/Common/StorageRepo.pm +++ /dev/null @@ -1,623 +0,0 @@ -#################################################################################################################################### -# C Storage Interface -#################################################################################################################################### -package pgBackRestTest::Common::StorageRepo; -use parent 'pgBackRestTest::Common::StorageBase'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Digest::SHA qw(sha1_hex); -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); -use Fcntl qw(:mode); -use File::stat qw{lstat}; -use JSON::PP; - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::Io::Handle; -use pgBackRestTest::Common::Io::Process; -use pgBackRestTest::Common::StorageBase; - -#################################################################################################################################### -# Temp file extension -#################################################################################################################################### -use constant STORAGE_TEMP_EXT => PROJECT_EXE . '.tmp'; - push @EXPORT, qw(STORAGE_TEMP_EXT); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Create the class hash - my $self = {}; - bless $self, $class; - - # Assign function parameters, defaults, and log debug info - ( - my $strOperation, - $self->{strCommand}, - $self->{strType}, - $self->{lBufferMax}, - $self->{iTimeoutIo}, - $self->{iRepo}, - $self->{strDefaultPathMode}, - $self->{strDefaultFileMode}, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'strCommand'}, - {name => 'strType'}, - {name => 'lBufferMax'}, - {name => 'iTimeoutIo'}, - {name => 'iRepo'}, - {name => 'strDefaultPathMode', optional => true, default => '0750'}, - {name => 'strDefaultFileMode', optional => true, default => '0640'}, - ); - - # Create JSON object - $self->{oJSON} = JSON::PP->new()->allow_nonref(); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# Escape characteres that have special meaning on the command line -#################################################################################################################################### -sub escape -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strValue, - ) = - logDebugParam - ( - __PACKAGE__ . '->escape', \@_, - {name => 'strValue', trace => true}, - ); - - $strValue =~ s/\\/\\\\/g; - $strValue =~ s/\\\\/\\\>/g; - $strValue =~ s/\!/\\\!/g; - $strValue =~ s/\*/\\\*/g; - $strValue =~ s/\(/\\\(/g; - $strValue =~ s/\)/\\\)/g; - $strValue =~ s/\&/\\\&/g; - $strValue =~ s/\'/\\\'/g; - $strValue =~ s/\;/\\\;/g; - $strValue =~ s/\?/\\\?/g; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strValue', value => $strValue}, - ); -} - -#################################################################################################################################### -# Execute command and return the output -#################################################################################################################################### -sub exec -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strCommand, - ) = - logDebugParam - ( - __PACKAGE__ . '->exec', \@_, - {name => 'strCommand'}, - ); - - $strCommand = "$self->{strCommand} ${strCommand}"; - my $oBuffer = new pgBackRestTest::Common::Io::Buffered( - new pgBackRestTest::Common::Io::Handle($strCommand), $self->{iTimeoutIo}, $self->{lBufferMax}); - my $oProcess = new pgBackRestTest::Common::Io::Process($oBuffer, $strCommand); - - my $tResult; - - while (!$oBuffer->eof()) - { - $oBuffer->read(\$tResult, $self->{lBufferMax}, false); - } - - $oProcess->close(); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'tResult', value => $tResult}, - {name => 'iExitStatus', value => $oProcess->exitStatus()}, - ); -} - -#################################################################################################################################### -# Create storage -#################################################################################################################################### -sub create -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '->create'); - - $self->exec("--repo=$self->{iRepo} repo-create"); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# Check if file exists (not a path) -#################################################################################################################################### -sub exists -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFileExp, - ) = - logDebugParam - ( - __PACKAGE__ . '->exists', \@_, - {name => 'strFileExp'}, - ); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bExists', value => $self->info($strFileExp, {bIgnoreMissing => true})->{type} eq 'f'} - ); -} - -#################################################################################################################################### -# Read a buffer from storage all at once -#################################################################################################################################### -sub get -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $xFile, - $strCipherPass, - $bRaw, - ) = - logDebugParam - ( - __PACKAGE__ . '->get', \@_, - {name => 'xFile', required => false}, - {name => 'strCipherPass', optional => true, redact => true}, - {name => 'bRaw', optional => true, default => false}, - ); - - # If openRead() was called first set values from that call - my $strFile = $xFile; - my $bIgnoreMissing = false; - - if (ref($xFile)) - { - $strFile = $xFile->{strFile}; - $bIgnoreMissing = $xFile->{bIgnoreMissing}; - $strCipherPass = $xFile->{strCipherPass}; - } - - # Check invalid params - if ($bRaw && defined($strCipherPass)) - { - confess &log(ERROR, 'bRaw and strCipherPass cannot both be set'); - } - - # Get file - my ($tResult, $iExitStatus) = $self->exec( - (defined($strCipherPass) ? ' --cipher-pass=' . $self->escape($strCipherPass) : '') . ($bRaw ? ' --raw' : '') . - ($bIgnoreMissing ? ' --ignore-missing' : '') . " --repo=$self->{iRepo} repo-get " . $self->escape($strFile)); - - # Error if missing an not ignored - if ($iExitStatus == 1 && !$bIgnoreMissing) - { - confess &log(ERROR, "unable to open '${strFile}'", ERROR_FILE_OPEN); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'rtContent', value => $iExitStatus == 0 ? \$tResult : undef, trace => true}, - ); -} - -#################################################################################################################################### -# Get information for path/file -#################################################################################################################################### -sub info -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPathFileExp, - $bIgnoreMissing, - ) = - logDebugParam - ( - __PACKAGE__ . '->info', \@_, - {name => 'strPathFileExp'}, - {name => 'bIgnoreMissing', optional => true, default => false}, - ); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'rhInfo', value => $self->manifest($strPathFileExp, {bRecurse => false})->{'.'}, trace => true} - ); -} - -#################################################################################################################################### -# List all files/paths in path -#################################################################################################################################### -sub list -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPathExp, - $strExpression, - $strSortOrder, - $bIgnoreMissing, - ) = - logDebugParam - ( - __PACKAGE__ . '->list', \@_, - {name => 'strPathExp', required => false}, - {name => 'strExpression', optional => true}, - {name => 'strSortOrder', optional => true, default => 'forward'}, - {name => 'bIgnoreMissing', optional => true, default => false}, - ); - - # Get file list - my $rstryFileList = []; - my $rhManifest = $self->manifest($strPathExp, {bRecurse => false}); - - foreach my $strKey ($strSortOrder eq 'reverse' ? sort {$b cmp $a} keys(%{$rhManifest}) : sort keys(%{$rhManifest})) - { - next if $strKey eq '.'; - next if defined($strExpression) && $strKey !~ $strExpression; - - push(@{$rstryFileList}, $strKey); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'stryFileList', value => $rstryFileList} - ); -} - -#################################################################################################################################### -# Build path/file/link manifest starting with base path and including all subpaths -#################################################################################################################################### -sub manifest -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPathExp, - $bRecurse, - ) = - logDebugParam - ( - __PACKAGE__ . '->manifest', \@_, - {name => 'strPathExp'}, - {name => 'bRecurse', optional => true, default => true}, - ); - - my $rhManifest = $self->{oJSON}->decode( - $self->exec( - "--output=json" . ($bRecurse ? ' --recurse' : '') . " --repo=$self->{iRepo} repo-ls " . $self->escape($strPathExp))); - - # Transform the manifest to the old format - foreach my $strKey (keys(%{$rhManifest})) - { - if ($rhManifest->{$strKey}{type} eq 'file') - { - $rhManifest->{$strKey}{type} = 'f'; - - if (defined($rhManifest->{$strKey}{time})) - { - $rhManifest->{$strKey}{modified_time} = $rhManifest->{$strKey}{time}; - delete($rhManifest->{$strKey}{time}); - } - } - elsif ($rhManifest->{$strKey}{type} eq 'path') - { - $rhManifest->{$strKey}{type} = 'd'; - } - elsif ($rhManifest->{$strKey}{type} eq 'link') - { - $rhManifest->{$strKey}{type} = 'l'; - } - elsif ($rhManifest->{$strKey}{type} eq 'special') - { - $rhManifest->{$strKey}{type} = 's'; - } - else - { - confess "invalid file type '$rhManifest->{type}'"; - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'rhManifest', value => $rhManifest, trace => true} - ); -} - -#################################################################################################################################### -# Open file for reading -#################################################################################################################################### -sub openRead -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFile, - $bIgnoreMissing, - $strCipherPass, - ) = - logDebugParam - ( - __PACKAGE__ . '->openRead', \@_, - {name => 'strFile'}, - {name => 'bIgnoreMissing', optional => true, default => false}, - {name => 'strCipherPass', optional => true, redact => true}, - ); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'rhFileIo', value => {strFile => $strFile, bIgnoreMissing => $bIgnoreMissing, strCipherPass => $strCipherPass}, - trace => true}, - ); -} - -#################################################################################################################################### -# Remove path and all files below it -#################################################################################################################################### -sub pathRemove -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPath, - $bRecurse, - ) = - logDebugParam - ( - __PACKAGE__ . '->pathRemove', \@_, - {name => 'strPath'}, - {name => 'bRecurse', optional => true, default => false}, - ); - - $self->exec("--repo=$self->{iRepo} repo-rm " . ($bRecurse ? '--recurse ' : '') . $self->escape($strPath)); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# put - writes a buffer out to storage all at once -#################################################################################################################################### -sub put -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFile, - $tContent, - $strCipherPass, - $bRaw, - ) = - logDebugParam - ( - __PACKAGE__ . '->put', \@_, - {name => 'strFile'}, - {name => 'tContent', required => false}, - {name => 'strCipherPass', optional => true, redact => true}, - {name => 'bRaw', optional => true, default => false}, - ); - - # Check invalid params - if ($bRaw && defined($strCipherPass)) - { - confess &log(ERROR, 'bRaw and strCipherPass cannot both be set'); - } - - # Put file - my $strCommand = - "$self->{strCommand}" . (defined($strCipherPass) ? ' --cipher-pass=' . $self->escape($strCipherPass) : '') . - ($bRaw ? ' --raw' : '') . " --repo=$self->{iRepo} repo-put " . $self->escape($strFile); - - my $oBuffer = new pgBackRestTest::Common::Io::Buffered( - new pgBackRestTest::Common::Io::Handle($strCommand), $self->{iTimeoutIo}, $self->{lBufferMax}); - my $oProcess = new pgBackRestTest::Common::Io::Process($oBuffer, $strCommand); - - if (defined($tContent)) - { - $oBuffer->write(\$tContent); - } - - close($oBuffer->handleWrite()); - - my $tResult; - - while (!$oBuffer->eof()) - { - $oBuffer->read(\$tResult, $self->{lBufferMax}, false); - } - - close($oBuffer->handleRead()); - $oProcess->close(); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# Remove file -#################################################################################################################################### -sub remove -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFile, - ) = - logDebugParam - ( - __PACKAGE__ . '->remove', \@_, - {name => 'xFileExp'}, - ); - - $self->exec("--repo=$self->{iRepo} repo-rm " . $self->escape($strFile)); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# Cache storage so it can be retrieved quickly -#################################################################################################################################### -my $oRepoStorage; - -#################################################################################################################################### -# storageRepoCommandSet -#################################################################################################################################### -my $strStorageRepoCommand; -my $strStorageRepoType; - -sub storageRepoCommandSet -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strCommand, - $strStorageType, - ) = - logDebugParam - ( - __PACKAGE__ . '::storageRepoCommandSet', \@_, - {name => 'strCommand'}, - {name => 'strStorageType'}, - ); - - $strStorageRepoCommand = $strCommand; - $strStorageRepoType = $strStorageType; - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -push @EXPORT, qw(storageRepoCommandSet); - -#################################################################################################################################### -# storageRepo - get repository storage -#################################################################################################################################### -sub storageRepo -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strStanza, - $iRepo, - ) = - logDebugParam - ( - __PACKAGE__ . '::storageRepo', \@_, - {name => 'strStanza', optional => true, trace => true}, - {name => 'iRepo', optional => true, default => 1, trace => true}, - ); - - # Create storage if not defined - if (!defined($oRepoStorage->{$iRepo})) - { - $oRepoStorage->{$iRepo} = new pgBackRestTest::Common::StorageRepo( - $strStorageRepoCommand, $strStorageRepoType, 64 * 1024, 60, $iRepo); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'oStorageRepo', value => $oRepoStorage->{$iRepo}, trace => true}, - ); -} - -push @EXPORT, qw(storageRepo); - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub capability {shift->type() eq STORAGE_POSIX} -sub type {shift->{strType}} - -1; diff --git a/test/lib/pgBackRestTest/Env/ArchiveInfo.pm b/test/lib/pgBackRestTest/Env/ArchiveInfo.pm deleted file mode 100644 index 42257b9ea7..0000000000 --- a/test/lib/pgBackRestTest/Env/ArchiveInfo.pm +++ /dev/null @@ -1,478 +0,0 @@ -#################################################################################################################################### -# ARCHIVE INFO MODULE -# -# The archive.info file is created when archiving begins. It is located under the stanza directory. The file contains information -# regarding the stanza database version, database WAL segment system id and other information to ensure that archiving is being -# performed on the proper database. -#################################################################################################################################### -package pgBackRestTest::Env::ArchiveInfo; -use parent 'pgBackRestDoc::Common::Ini'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname basename); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; - -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::StorageBase; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Env::InfoCommon; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# File/path constants -#################################################################################################################################### -use constant ARCHIVE_INFO_FILE => 'archive.info'; - push @EXPORT, qw(ARCHIVE_INFO_FILE); - -#################################################################################################################################### -# RegEx constants -#################################################################################################################################### -use constant REGEX_ARCHIVE_DIR_DB_VERSION => '^[0-9]+(\.[0-9]+)*-[0-9]+$'; - push @EXPORT, qw(REGEX_ARCHIVE_DIR_DB_VERSION); -use constant REGEX_ARCHIVE_DIR_WAL => '^[0-F]{16}$'; - push @EXPORT, qw(REGEX_ARCHIVE_DIR_WAL); - -#################################################################################################################################### -# WAL segment size -#################################################################################################################################### -use constant PG_WAL_SEGMENT_SIZE => 16777216; - push @EXPORT, qw(PG_WAL_SEGMENT_SIZE); - -#################################################################################################################################### -# Archive info constants -#################################################################################################################################### -use constant INFO_ARCHIVE_SECTION_DB => INFO_BACKUP_SECTION_DB; - push @EXPORT, qw(INFO_ARCHIVE_SECTION_DB); -use constant INFO_ARCHIVE_SECTION_DB_HISTORY => INFO_BACKUP_SECTION_DB_HISTORY; - push @EXPORT, qw(INFO_ARCHIVE_SECTION_DB_HISTORY); - -use constant INFO_ARCHIVE_KEY_DB_VERSION => MANIFEST_KEY_DB_VERSION; - push @EXPORT, qw(INFO_ARCHIVE_KEY_DB_VERSION); -use constant INFO_ARCHIVE_KEY_DB_ID => MANIFEST_KEY_DB_ID; - push @EXPORT, qw(INFO_ARCHIVE_KEY_DB_ID); -use constant INFO_ARCHIVE_KEY_DB_SYSTEM_ID => MANIFEST_KEY_SYSTEM_ID; - push @EXPORT, qw(INFO_ARCHIVE_KEY_DB_SYSTEM_ID); - -#################################################################################################################################### -# Global variables -#################################################################################################################################### -my $strArchiveInfoMissingMsg = - ARCHIVE_INFO_FILE . " does not exist but is required to push/get WAL segments\n" . - "HINT: is archive_command configured in postgresql.conf?\n" . - "HINT: has a stanza-create been performed?\n" . - "HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme."; - -#################################################################################################################################### -# CONSTRUCTOR -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strArchiveClusterPath, # Archive cluster path - $bRequired, # Is archive info required? - $bLoad, # Should the file attempt to be loaded? - $bIgnoreMissing, # Don't error on missing files - $strCipherPassSub, # Passphrase to encrypt the subsequent archive files if repo is encrypted - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'strArchiveClusterPath'}, - {name => 'bRequired', default => true}, - {name => 'bLoad', optional => true, default => true}, - {name => 'bIgnoreMissing', optional => true, default => false}, - {name => 'strCipherPassSub', optional => true}, - ); - - # Build the archive info path/file name - my $strArchiveInfoFile = "${strArchiveClusterPath}/" . ARCHIVE_INFO_FILE; - my $self = {}; - my $iResult = 0; - my $strResultMessage; - - # Init object and store variables - eval - { - $self = $class->SUPER::new( - storageRepo(), $strArchiveInfoFile, - {bLoad => $bLoad, bIgnoreMissing => $bIgnoreMissing, strCipherPassSub => $strCipherPassSub}); - return true; - } - or do - { - # Capture error information - $iResult = exceptionCode($EVAL_ERROR); - $strResultMessage = exceptionMessage($EVAL_ERROR); - }; - - if ($iResult != 0) - { - # If the file does not exist but is required to exist, then error - # The archive info is only allowed not to exist when running a stanza-create on a new install - if ($iResult == ERROR_FILE_MISSING) - { - if ($bRequired) - { - confess &log(ERROR, $strArchiveInfoMissingMsg, ERROR_FILE_MISSING); - } - } - elsif ($iResult == ERROR_CRYPTO && $strResultMessage =~ "^unable to flush") - { - confess &log(ERROR, "unable to parse '$strArchiveInfoFile'\nHINT: is or was the repo encrypted?", $iResult); - } - else - { - confess $EVAL_ERROR; - } - } - - $self->{strArchiveClusterPath} = $strArchiveClusterPath; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# check -# -# Check archive info file and make sure it is compatible with the current version of the database for the stanza. If the file does -# not exist an error will occur. -#################################################################################################################################### -sub check -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $ullDbSysId, - $bRequired, - ) = - logDebugParam - ( - __PACKAGE__ . '->check', \@_, - {name => 'strDbVersion'}, - {name => 'ullDbSysId'}, - {name => 'bRequired', default => true}, - ); - - # ??? remove bRequired after stanza-upgrade - if ($bRequired) - { - # Confirm the info file exists with the DB section - $self->confirmExists(); - } - - my $strError = undef; - - if (!$self->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef, $strDbVersion)) - { - $strError = "WAL segment version ${strDbVersion} does not match archive version " . - $self->get(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION); - } - - if (!$self->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_SYSTEM_ID, undef, $ullDbSysId)) - { - $strError = (defined($strError) ? ($strError . "\n") : "") . - "WAL segment system-id ${ullDbSysId} does not match archive system-id " . - $self->get(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_SYSTEM_ID); - } - - if (defined($strError)) - { - confess &log(ERROR, "${strError}\nHINT: are you archiving to the correct stanza?", ERROR_ARCHIVE_MISMATCH); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strArchiveId', value => $self->archiveId()} - ); -} - -#################################################################################################################################### -# archiveId -# -# Get the archive id which is a combination of the DB version and the db-id setting (e.g. 9.4-1) -#################################################################################################################################### -sub archiveId -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $ullDbSysId, - ) = logDebugParam - ( - __PACKAGE__ . '->archiveId', \@_, - {name => 'strDbVersion', optional => true}, - {name => 'ullDbSysId', optional => true}, - ); - - my $strArchiveId = undef; - - # If neither optional version and system-id are passed then set the archive id to the current one - if (!defined($strDbVersion) && !defined($ullDbSysId)) - { - $strArchiveId = $self->get(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION) . "-" . - $self->get(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_ID); - } - # If both the optional version and system-id are passed - elsif (defined($strDbVersion) && defined($ullDbSysId)) - { - # Get the newest archiveId for the version/system-id passed - $strArchiveId = ($self->archiveIdList($strDbVersion, $ullDbSysId))[0]; - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strArchiveId', value => $strArchiveId} - ); -} - -#################################################################################################################################### -# archiveIdList -# -# Get a sorted list of the archive ids for the db-version and db-system-id passed. -#################################################################################################################################### -sub archiveIdList -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $ullDbSysId, - ) = logDebugParam - ( - __PACKAGE__ . '->archiveIdList', \@_, - {name => 'strDbVersion'}, - {name => 'ullDbSysId'}, - ); - - my @stryArchiveId; - - # Get the version and system-id for all known databases - my $hDbList = $self->dbHistoryList(); - - foreach my $iDbHistoryId (sort {$a <=> $b} keys %$hDbList) - { - # If the version and system-id match then construct the archive id so that the constructed array has the newest match first - if (($hDbList->{$iDbHistoryId}{&INFO_DB_VERSION} eq $strDbVersion) && - ($hDbList->{$iDbHistoryId}{&INFO_SYSTEM_ID} eq $ullDbSysId)) - { - unshift(@stryArchiveId, $strDbVersion . "-" . $iDbHistoryId); - } - } - - # If the archive id has still not been found, then error - if (@stryArchiveId == 0) - { - confess &log( - ERROR, "unable to retrieve the archive id for database version '$strDbVersion' and system-id '$ullDbSysId'", - ERROR_ARCHIVE_MISMATCH); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'stryArchiveId', value => \@stryArchiveId} - ); -} - -#################################################################################################################################### -# create -# -# Creates the archive.info file. WARNING - this function should only be called from stanza-create or tests. -#################################################################################################################################### -sub create -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $ullDbSysId, - $bSave, - ) = - logDebugParam - ( - __PACKAGE__ . '->create', \@_, - {name => 'strDbVersion'}, - {name => 'ullDbSysId'}, - {name => 'bSave', default => true}, - ); - - # Fill db section and db history section - $self->dbSectionSet($strDbVersion, $ullDbSysId, $self->dbHistoryIdGet(false)); - - if ($bSave) - { - $self->save(); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# dbHistoryIdGet -# -# Get the db history ID -#################################################################################################################################### -sub dbHistoryIdGet -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $bFileRequired, - ) = - logDebugParam - ( - __PACKAGE__ . '->dbHistoryIdGet', \@_, - {name => 'bFileRequired', default => true}, - ); - - # Confirm the info file exists if it is required - if ($bFileRequired) - { - $self->confirmExists(); - } - - # If the DB section does not exist, initialize the history to one, else return the latest ID - my $iDbHistoryId = (!$self->test(INFO_ARCHIVE_SECTION_DB)) - ? 1 : $self->numericGet(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_ID); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'iDbHistoryId', value => $iDbHistoryId} - ); -} - -#################################################################################################################################### -# dbHistoryList -# -# Get the data from the db history section. -#################################################################################################################################### -sub dbHistoryList -{ - my $self = shift; - my - ( - $strOperation, - ) = logDebugParam - ( - __PACKAGE__ . '->dbHistoryList', - ); - - my %hDbHash; - - foreach my $iHistoryId ($self->keys(INFO_ARCHIVE_SECTION_DB_HISTORY)) - { - $hDbHash{$iHistoryId}{&INFO_DB_VERSION} = - $self->get(INFO_ARCHIVE_SECTION_DB_HISTORY, $iHistoryId, INFO_ARCHIVE_KEY_DB_VERSION); - $hDbHash{$iHistoryId}{&INFO_SYSTEM_ID} = - $self->get(INFO_ARCHIVE_SECTION_DB_HISTORY, $iHistoryId, INFO_ARCHIVE_KEY_DB_ID); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'hDbHash', value => \%hDbHash} - ); -} - -#################################################################################################################################### -# dbSectionSet -# -# Set the db and db:history sections. -#################################################################################################################################### -sub dbSectionSet -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $ullDbSysId, - $iDbHistoryId, - ) = - logDebugParam - ( - __PACKAGE__ . '->dbSectionSet', \@_, - {name => 'strDbVersion', trace => true}, - {name => 'ullDbSysId', trace => true}, - {name => 'iDbHistoryId', trace => true} - ); - - # Fill db section - $self->numericSet(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_SYSTEM_ID, undef, $ullDbSysId); - # Force the version to a string since newer versions of JSON::PP lose track of the fact that it is one - $self->set(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef, $strDbVersion . ''); - $self->numericSet(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_ID, undef, $iDbHistoryId); - - # Fill db history - $self->numericSet(INFO_ARCHIVE_SECTION_DB_HISTORY, $iDbHistoryId, INFO_ARCHIVE_KEY_DB_ID, $ullDbSysId); - # Force the version to a string since newer versions of JSON::PP lose track of the fact that it is one - $self->set(INFO_ARCHIVE_SECTION_DB_HISTORY, $iDbHistoryId, INFO_ARCHIVE_KEY_DB_VERSION, $strDbVersion . ''); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# confirmExists -# -# Ensure that the archive.info file and the db section exist. -#################################################################################################################################### -sub confirmExists -{ - my $self = shift; - - # Confirm the file exists and the DB section is filled out - if (!$self->test(INFO_ARCHIVE_SECTION_DB) || !$self->{bExists}) - { - confess &log(ERROR, $strArchiveInfoMissingMsg, ERROR_FILE_MISSING); - } -} - -1; diff --git a/test/lib/pgBackRestTest/Env/BackupInfo.pm b/test/lib/pgBackRestTest/Env/BackupInfo.pm deleted file mode 100644 index 3b8b4061ac..0000000000 --- a/test/lib/pgBackRestTest/Env/BackupInfo.pm +++ /dev/null @@ -1,898 +0,0 @@ -#################################################################################################################################### -# BACKUP INFO MODULE -#################################################################################################################################### -package pgBackRestTest::Env::BackupInfo; -use parent 'pgBackRestDoc::Common::Ini'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); -use English '-no_match_vars'; - -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname basename); -use File::stat; - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; - -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Env::ArchiveInfo; -use pgBackRestTest::Env::InfoCommon; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# Backup type constants -#################################################################################################################################### -use constant CFGOPTVAL_BACKUP_TYPE_FULL => 'full'; - push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_FULL); -use constant CFGOPTVAL_BACKUP_TYPE_DIFF => 'diff'; - push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_DIFF); -use constant CFGOPTVAL_BACKUP_TYPE_INCR => 'incr'; - push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_INCR); - -#################################################################################################################################### -# File/path constants -#################################################################################################################################### -use constant FILE_BACKUP_INFO => 'backup.info'; - push @EXPORT, qw(FILE_BACKUP_INFO); - -#################################################################################################################################### -# Backup info constants -#################################################################################################################################### -use constant INFO_BACKUP_SECTION_BACKUP => MANIFEST_SECTION_BACKUP; - push @EXPORT, qw(INFO_BACKUP_SECTION_BACKUP); -use constant INFO_BACKUP_SECTION_BACKUP_CURRENT => INFO_BACKUP_SECTION_BACKUP . ':current'; - push @EXPORT, qw(INFO_BACKUP_SECTION_BACKUP_CURRENT); - -use constant INFO_BACKUP_KEY_ARCHIVE_CHECK => MANIFEST_KEY_ARCHIVE_CHECK; - push @EXPORT, qw(INFO_BACKUP_KEY_ARCHIVE_CHECK); -use constant INFO_BACKUP_KEY_ARCHIVE_COPY => MANIFEST_KEY_ARCHIVE_COPY; - push @EXPORT, qw(INFO_BACKUP_KEY_ARCHIVE_COPY); -use constant INFO_BACKUP_KEY_ARCHIVE_START => MANIFEST_KEY_ARCHIVE_START; - push @EXPORT, qw(INFO_BACKUP_KEY_ARCHIVE_START); -use constant INFO_BACKUP_KEY_ARCHIVE_STOP => MANIFEST_KEY_ARCHIVE_STOP; - push @EXPORT, qw(INFO_BACKUP_KEY_ARCHIVE_STOP); -use constant INFO_BACKUP_KEY_BACKUP_STANDBY => MANIFEST_KEY_BACKUP_STANDBY; - push @EXPORT, qw(INFO_BACKUP_KEY_BACKUP_STANDBY); -use constant INFO_BACKUP_KEY_BACKUP_REPO_SIZE => 'backup-info-repo-size'; - push @EXPORT, qw(INFO_BACKUP_KEY_BACKUP_REPO_SIZE); -use constant INFO_BACKUP_KEY_BACKUP_REPO_SIZE_DELTA => 'backup-info-repo-size-delta'; - push @EXPORT, qw(INFO_BACKUP_KEY_BACKUP_REPO_SIZE_DELTA); -use constant INFO_BACKUP_KEY_BACKUP_SIZE => 'backup-info-size'; - push @EXPORT, qw(INFO_BACKUP_KEY_BACKUP_SIZE); -use constant INFO_BACKUP_KEY_BACKUP_SIZE_DELTA => 'backup-info-size-delta'; - push @EXPORT, qw(INFO_BACKUP_KEY_BACKUP_SIZE_DELTA); -use constant INFO_BACKUP_KEY_CATALOG => MANIFEST_KEY_CATALOG; - push @EXPORT, qw(INFO_BACKUP_KEY_CATALOG); -use constant INFO_BACKUP_KEY_CONTROL => MANIFEST_KEY_CONTROL; - push @EXPORT, qw(INFO_BACKUP_KEY_CONTROL); -use constant INFO_BACKUP_KEY_COMPRESS => MANIFEST_KEY_COMPRESS; - push @EXPORT, qw(INFO_BACKUP_KEY_COMPRESS); -use constant INFO_BACKUP_KEY_CHECKSUM_PAGE => MANIFEST_KEY_CHECKSUM_PAGE; - push @EXPORT, qw(INFO_BACKUP_KEY_CHECKSUM_PAGE); -use constant INFO_BACKUP_KEY_DB_VERSION => MANIFEST_KEY_DB_VERSION; - push @EXPORT, qw(INFO_BACKUP_KEY_DB_VERSION); -use constant INFO_BACKUP_KEY_FORMAT => INI_KEY_FORMAT; - push @EXPORT, qw(INFO_BACKUP_KEY_FORMAT); -use constant INFO_BACKUP_KEY_HARDLINK => MANIFEST_KEY_HARDLINK; - push @EXPORT, qw(INFO_BACKUP_KEY_HARDLINK); -use constant INFO_BACKUP_KEY_HISTORY_ID => MANIFEST_KEY_DB_ID; - push @EXPORT, qw(INFO_BACKUP_KEY_HISTORY_ID); -use constant INFO_BACKUP_KEY_LABEL => MANIFEST_KEY_LABEL; - push @EXPORT, qw(INFO_BACKUP_KEY_LABEL); -use constant INFO_BACKUP_KEY_PRIOR => MANIFEST_KEY_PRIOR; - push @EXPORT, qw(INFO_BACKUP_KEY_PRIOR); -use constant INFO_BACKUP_KEY_REFERENCE => 'backup-reference'; - push @EXPORT, qw(INFO_BACKUP_KEY_REFERENCE); -use constant INFO_BACKUP_KEY_ONLINE => MANIFEST_KEY_ONLINE; - push @EXPORT, qw(INFO_BACKUP_KEY_ONLINE); -use constant INFO_BACKUP_KEY_SYSTEM_ID => MANIFEST_KEY_SYSTEM_ID; - push @EXPORT, qw(INFO_BACKUP_KEY_SYSTEM_ID); -use constant INFO_BACKUP_KEY_TIMESTAMP_START => MANIFEST_KEY_TIMESTAMP_START; - push @EXPORT, qw(INFO_BACKUP_KEY_TIMESTAMP_START); -use constant INFO_BACKUP_KEY_TIMESTAMP_STOP => MANIFEST_KEY_TIMESTAMP_STOP; - push @EXPORT, qw(INFO_BACKUP_KEY_TIMESTAMP_STOP); -use constant INFO_BACKUP_KEY_TYPE => MANIFEST_KEY_TYPE; - push @EXPORT, qw(INFO_BACKUP_KEY_TYPE); -use constant INFO_BACKUP_KEY_VERSION => INI_KEY_VERSION; - push @EXPORT, qw(INFO_BACKUP_KEY_VERSION); - -#################################################################################################################################### -# Global variables -#################################################################################################################################### -my $strBackupInfoMissingMsg = - FILE_BACKUP_INFO . " does not exist and is required to perform a backup.\n" . - "HINT: has a stanza-create been performed?"; - -#################################################################################################################################### -# CONSTRUCTOR -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strBackupClusterPath, - $bRequired, - $oStorage, - $bLoad, # Should the file attemp to be loaded? - $bIgnoreMissing, # Don't error on missing files - $strCipherPassSub, # Generated passphrase to encrypt manifest files if the repo is encrypted - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'strBackupClusterPath'}, - {name => 'bRequired', default => true}, - {name => 'oStorage', optional => true, default => storageRepo()}, - {name => 'bLoad', optional => true, default => true}, - {name => 'bIgnoreMissing', optional => true, default => false}, - {name => 'strCipherPassSub', optional => true}, - ); - - # Build the backup info path/file name - my $strBackupInfoFile = "${strBackupClusterPath}/" . FILE_BACKUP_INFO; - my $self = {}; - my $iResult = 0; - my $strResultMessage; - - # Init object and store variables - eval - { - $self = $class->SUPER::new( - $oStorage, $strBackupInfoFile, - {bLoad => $bLoad, bIgnoreMissing => $bIgnoreMissing, strCipherPassSub => $strCipherPassSub}); - return true; - } - or do - { - # Capture error information - $iResult = exceptionCode($EVAL_ERROR); - $strResultMessage = exceptionMessage($EVAL_ERROR); - }; - - if ($iResult != 0) - { - # If the backup info file does not exist and is required, then throw an error - # The backup info is only allowed not to exist when running a stanza-create on a new install - if ($iResult == ERROR_FILE_MISSING) - { - if ($bRequired) - { - confess &log(ERROR, "${strBackupClusterPath}/$strBackupInfoMissingMsg", ERROR_FILE_MISSING); - } - } - elsif ($iResult == ERROR_CRYPTO && $strResultMessage =~ "^unable to flush") - { - confess &log(ERROR, "unable to parse '$strBackupInfoFile'\nHINT: is or was the repo encrypted?", $iResult); - } - else - { - confess $EVAL_ERROR; - } - } - - $self->{strBackupClusterPath} = $strBackupClusterPath; - $self->{oStorage} = $oStorage; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# check -# -# Check db info and make sure it matches what is already in the repository. Return the db-id if everything matches. -#################################################################################################################################### -sub check -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $iControlVersion, - $iCatalogVersion, - $ullDbSysId, - $bRequired, - ) = - logDebugParam - ( - __PACKAGE__ . '->check', \@_, - {name => 'strDbVersion', trace => true}, - {name => 'iControlVersion', trace => true}, - {name => 'iCatalogVersion', trace => true}, - {name => 'ullDbSysId', trace => true}, - {name => 'bRequired', default => true}, - ); - - # Confirm the info file exists with the DB section - if ($bRequired) - { - $self->confirmExists(); - } - - if (!$self->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_SYSTEM_ID, undef, $ullDbSysId) || - !$self->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION, undef, $strDbVersion)) - { - confess &log(ERROR, "database version = ${strDbVersion}, system-id ${ullDbSysId} does not match backup version = " . - $self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION) . ", system-id = " . - $self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_SYSTEM_ID) . "\n" . - "HINT: is this the correct stanza?", ERROR_BACKUP_MISMATCH); - } - - if (!$self->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CATALOG, undef, $iCatalogVersion) || - !$self->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CONTROL, undef, $iControlVersion)) - { - confess &log(ERROR, "database control-version = ${iControlVersion}, catalog-version ${iCatalogVersion}" . - " does not match backup control-version = " . - $self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CONTROL) . ", catalog-version = " . - $self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CATALOG) . "\n" . - "HINT: this may be a symptom of database or repository corruption!", ERROR_BACKUP_MISMATCH); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'iDbHistoryId', value => $self->numericGet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_HISTORY_ID)} - ); -} - -#################################################################################################################################### -# add -# -# Add a backup to the info file. -#################################################################################################################################### -sub add -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oBackupManifest, - $bSave, - $bRequired, - ) = - logDebugParam - ( - __PACKAGE__ . '->add', \@_, - {name => 'oBackupManifest', trace => true}, - {name => 'bSave', default => true, trace => true}, - {name => 'bRequired', default => true, trace => true}, - ); - - # Confirm the info file exists with the DB section - if ($bRequired) - { - $self->confirmExists(); - } - - # Get the backup label - my $strBackupLabel = $oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL); - - # Calculate backup sizes and references - my $lBackupSize = 0; - my $lBackupSizeDelta = 0; - my $lBackupRepoSize = 0; - my $lBackupRepoSizeDelta = 0; - my $oReferenceHash = undef; - - foreach my $strFileKey ($oBackupManifest->keys(MANIFEST_SECTION_TARGET_FILE)) - { - my $lFileSize = - $oBackupManifest->get(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_SIZE); - my $lRepoSize = - $oBackupManifest->get(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE, false, $lFileSize); - my $strFileReference = - $oBackupManifest->get(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REFERENCE, false); - - # Temporary until compressed size is back in - $lBackupSize += $lFileSize; - $lBackupRepoSize += $lRepoSize; - - if (defined($strFileReference)) - { - $$oReferenceHash{$strFileReference} = true; - } - else - { - $lBackupSizeDelta += $lFileSize; - $lBackupRepoSizeDelta += $lRepoSize; - } - } - - # Set backup size info - $self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_BACKUP_SIZE, $lBackupSize); - $self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_BACKUP_SIZE_DELTA, $lBackupSizeDelta); - $self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_BACKUP_REPO_SIZE, $lBackupRepoSize); - $self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_BACKUP_REPO_SIZE_DELTA, - $lBackupRepoSizeDelta); - - $self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_ARCHIVE_CHECK, - $oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_CHECK)); - $self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_ARCHIVE_COPY, - $oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_COPY)); - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_ARCHIVE_START, - $oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_START, undef, false)); - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_ARCHIVE_STOP, - $oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_STOP, undef, false)); - $self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_BACKUP_STANDBY, - $oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_BACKUP_STANDBY)); - $self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_CHECKSUM_PAGE, - $oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_CHECKSUM_PAGE)); - $self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_COMPRESS, - $oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS)); - $self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_FORMAT, - $oBackupManifest->numericGet(INI_SECTION_BACKREST, INI_KEY_FORMAT)); - $self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_HARDLINK, - $oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_HARDLINK)); - $self->boolSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_ONLINE, - $oBackupManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ONLINE)); - $self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_TIMESTAMP_START, - $oBackupManifest->numericGet(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_START)); - $self->numericSet(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_TIMESTAMP_STOP, - $oBackupManifest->numericGet(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP)); - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_TYPE, - $oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE)); - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_VERSION, - $oBackupManifest->get(INI_SECTION_BACKREST, INI_KEY_VERSION)); - - if ($bRequired) - { - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_HISTORY_ID, - $self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_HISTORY_ID)); - } - # If we are reconstructing, then the history id must be taken from the manifest - else - { - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_HISTORY_ID, - $oBackupManifest->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_ID)); - } - - if (!$oBackupManifest->test(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE, undef, CFGOPTVAL_BACKUP_TYPE_FULL)) - { - my @stryReference = sort(keys(%$oReferenceHash)); - - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_PRIOR, - $oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR)); - $self->set(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel, INFO_BACKUP_KEY_REFERENCE, - \@stryReference); - } - - if ($bSave) - { - $self->save(); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# current -# -# Test if a backup is current. -#################################################################################################################################### -sub current -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strBackup - ) = - logDebugParam - ( - __PACKAGE__ . '->current', \@_, - {name => 'strBackup'} - ); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bTest', value => $self->test(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackup)} - ); -} - -#################################################################################################################################### -# list -# -# Get backup keys. -#################################################################################################################################### -sub list -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFilter, - $strOrder - ) = - logDebugParam - ( - __PACKAGE__ . '->list', \@_, - {name => 'strFilter', required => false}, - {name => 'strOrder', default => 'forward'} - ); - - # List of backups - my @stryBackup; - - # Iterate through the backups and filter - for my $strBackup ($self->keys(INFO_BACKUP_SECTION_BACKUP_CURRENT)) - { - if (!defined($strFilter) || $strBackup =~ $strFilter) - { - if ($strOrder eq 'reverse') - { - unshift(@stryBackup, $strBackup) - } - else - { - push(@stryBackup, $strBackup) - } - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'stryBackup', value => \@stryBackup} - ); -} - -#################################################################################################################################### -# backupArchiveDbHistoryId -# -# Gets the backup.info db-id for the archiveId passed. -#################################################################################################################################### -sub backupArchiveDbHistoryId -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strArchiveId, - $strPathBackupArchive, - ) = - logDebugParam - ( - __PACKAGE__ . '->backupArchiveDbHistoryId', \@_, - {name => 'strArchiveId'}, - {name => 'strPathBackupArchive'}, - ); - - # List of backups associated with the db-id provided - my @stryArchiveBackup; - - # Build the db list from the history in the backup info and archive info file - my $oArchiveInfo = new pgBackRestTest::Env::ArchiveInfo($strPathBackupArchive, true); - my $hDbListArchive = $oArchiveInfo->dbHistoryList(); - my $hDbListBackup = $self->dbHistoryList(); - my $iDbHistoryId = undef; - - # Get the db-version and db-id (history id) from the archiveId - my ($strDbVersionArchive, $iDbIdArchive) = split("-", $strArchiveId); - - # Get the DB system ID to map back to the backup info if it exists in the archive info file - if (exists($hDbListArchive->{$iDbIdArchive})) - { - my $ullDbSysIdArchive = $$hDbListArchive{$iDbIdArchive}{&INFO_SYSTEM_ID}; - - # Get the db-id from backup info history that corresponds to the archive db-version and db-system-id - # Sort from newest (highest db-id) to oldest - foreach my $iDbIdBackup (sort {$b <=> $a} keys %{$hDbListBackup}) - { - if ($$hDbListBackup{$iDbIdBackup}{&INFO_SYSTEM_ID} == $ullDbSysIdArchive && - $$hDbListBackup{$iDbIdBackup}{&INFO_DB_VERSION} eq $strDbVersionArchive) - { - $iDbHistoryId = $iDbIdBackup; - last; - } - } - } - - # If the database is not found in the backup.info history list - if (!defined($iDbHistoryId)) - { - # Check to see that the current DB sections match for the archive and backup info files - if (!($oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef, - ($self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION)))) || - !($oArchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_SYSTEM_ID, undef, - ($self->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_SYSTEM_ID))))) - { - # This should never happen unless the backup.info file is corrupt - confess &log(ASSERT, "the archive and backup database sections do not match", ERROR_FILE_INVALID); - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'iDbHistoryId', value => $iDbHistoryId} - ); -} - -#################################################################################################################################### -# listByArchiveId -# -# Filters a list of backups by the archiveId passed. -#################################################################################################################################### -sub listByArchiveId -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strArchiveId, - $strPathBackupArchive, - $stryBackup, - $strOrder, - ) = - logDebugParam - ( - __PACKAGE__ . '->listByArchiveId', \@_, - {name => 'strArchiveId'}, - {name => 'strPathBackupArchive'}, - {name => 'stryBackup'}, - {name => 'strOrder', default => 'forward'} - ); - - # List of backups associated with the db-id provided - my @stryArchiveBackup; - - my $iDbHistoryId = $self->backupArchiveDbHistoryId($strArchiveId, $strPathBackupArchive); - - # If history found, then build list of backups associated with the archive id passed, else return empty array - if (defined($iDbHistoryId)) - { - # Iterate through the backups and filter - foreach my $strBackup (@$stryBackup) - { - # From the backup.info current backup section, get the db-id for the backup and if it is the same, add to the list - if ($self->test(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackup, INFO_BACKUP_KEY_HISTORY_ID, $iDbHistoryId)) - { - if ($strOrder eq 'reverse') - { - unshift(@stryArchiveBackup, $strBackup) - } - else - { - push(@stryArchiveBackup, $strBackup) - } - } - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'stryArchiveBackup', value => \@stryArchiveBackup} - ); -} - -#################################################################################################################################### -# last -# -# Find the last backup depending on the type. -#################################################################################################################################### -sub last -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strType - ) = - logDebugParam - ( - __PACKAGE__ . '->last', \@_, - {name => 'strType'} - ); - - my $strFilter = backupRegExpGet(true, $strType ne CFGOPTVAL_BACKUP_TYPE_FULL, $strType eq CFGOPTVAL_BACKUP_TYPE_INCR); - my $strBackup = ($self->list($strFilter, 'reverse'))[0]; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strBackup', value => $strBackup} - ); -} - -#################################################################################################################################### -# delete -# -# Delete a backup from the info file. -#################################################################################################################################### -sub delete -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strBackupLabel - ) = - logDebugParam - ( - __PACKAGE__ . '->delete', \@_, - {name => 'strBackupLabel'} - ); - - $self->remove(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackupLabel); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# create -# -# Create the info file. WARNING - this file should only be called from stanza-create or test modules. -#################################################################################################################################### -sub create -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $ullDbSysId, - $iControlVersion, - $iCatalogVersion, - $bSave, - ) = - logDebugParam - ( - __PACKAGE__ . '->create', \@_, - {name => 'strDbVersion'}, - {name => 'ullDbSysId'}, - {name => 'iControlVersion'}, - {name => 'iCatalogVersion'}, - {name => 'bSave', default => true}, - ); - - # Fill db section and db history section - $self->dbSectionSet($strDbVersion, $iControlVersion, $iCatalogVersion, $ullDbSysId, $self->dbHistoryIdGet(false)); - - if ($bSave) - { - $self->save(); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# dbHistoryIdGet -# -# Get the db history ID -#################################################################################################################################### -sub dbHistoryIdGet -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $bFileRequired, - ) = logDebugParam - ( - __PACKAGE__ . '->dbHistoryIdGet', \@_, - {name => 'bFileRequired', default => true}, - ); - - # Confirm the info file exists if it is required - if ($bFileRequired) - { - $self->confirmExists(); - } - - # If the DB section does not exist, initialize the history to one, else return the latest ID - my $iDbHistoryId = (!$self->test(INFO_BACKUP_SECTION_DB)) - ? 1 : $self->numericGet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_HISTORY_ID); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'iDbHistoryId', value => $iDbHistoryId} - ); -} - -#################################################################################################################################### -# dbHistoryList -# -# Get the data from the db history section. -#################################################################################################################################### -sub dbHistoryList -{ - my $self = shift; - my - ( - $strOperation, - ) = logDebugParam - ( - __PACKAGE__ . '->dbHistoryList', - ); - - my %hDbHash; - - foreach my $iHistoryId ($self->keys(INFO_BACKUP_SECTION_DB_HISTORY)) - { - $hDbHash{$iHistoryId}{&INFO_DB_VERSION} = - $self->get(INFO_BACKUP_SECTION_DB_HISTORY, $iHistoryId, INFO_BACKUP_KEY_DB_VERSION); - $hDbHash{$iHistoryId}{&INFO_SYSTEM_ID} = - $self->get(INFO_BACKUP_SECTION_DB_HISTORY, $iHistoryId, INFO_BACKUP_KEY_SYSTEM_ID); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'hDbHash', value => \%hDbHash} - ); -} - -#################################################################################################################################### -# dbSectionSet -# -# Set the db and db:history sections. -#################################################################################################################################### -sub dbSectionSet -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbVersion, - $iControlVersion, - $iCatalogVersion, - $ullDbSysId, - $iDbHistoryId, - ) = - logDebugParam - ( - __PACKAGE__ . '->dbSectionSet', \@_, - {name => 'strDbVersion', trace => true}, - {name => 'iControlVersion', trace => true}, - {name => 'iCatalogVersion', trace => true}, - {name => 'ullDbSysId', trace => true}, - {name => 'iDbHistoryId', trace => true}, - ); - - # Fill db section - $self->numericSet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CATALOG, undef, $iCatalogVersion); - $self->numericSet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CONTROL, undef, $iControlVersion); - $self->numericSet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_SYSTEM_ID, undef, $ullDbSysId); - # Force the version to a string since newer versions of JSON::PP lose track of the fact that it is one - $self->set(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION, undef, $strDbVersion . ''); - $self->numericSet(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_HISTORY_ID, undef, $iDbHistoryId); - - # Fill db history - $self->numericSet(INFO_BACKUP_SECTION_DB_HISTORY, $iDbHistoryId, INFO_BACKUP_KEY_CATALOG, $iCatalogVersion); - $self->numericSet(INFO_BACKUP_SECTION_DB_HISTORY, $iDbHistoryId, INFO_BACKUP_KEY_CONTROL, $iControlVersion); - $self->numericSet(INFO_BACKUP_SECTION_DB_HISTORY, $iDbHistoryId, INFO_BACKUP_KEY_SYSTEM_ID, $ullDbSysId); - $self->set(INFO_BACKUP_SECTION_DB_HISTORY, $iDbHistoryId, INFO_BACKUP_KEY_DB_VERSION, $strDbVersion . ''); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# confirmDb -# -# Ensure that the backup is associated with the database passed. -# NOTE: The backup must exist in the backup:current section. -#################################################################################################################################### -sub confirmDb -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strBackup, - $strDbVersion, - $ullDbSysId, - ) = - logDebugParam - ( - __PACKAGE__ . '->confirmDb', \@_, - {name => 'strBackup', trace => true}, - {name => 'strDbVersion', trace => true}, - {name => 'ullDbSysId', trace => true}, - ); - - my $bConfirmDb = undef; - - # Get the db-id associated with the backup - my $iDbHistoryId = $self->get(INFO_BACKUP_SECTION_BACKUP_CURRENT, $strBackup, INFO_BACKUP_KEY_HISTORY_ID); - - # Get the version and system-id for all known databases - my $hDbList = $self->dbHistoryList(); - - # If the db-id for the backup exists in the list - if (exists $hDbList->{$iDbHistoryId}) - { - # If the version and system-id match then database is confirmed for the backup - if (($hDbList->{$iDbHistoryId}{&INFO_DB_VERSION} eq $strDbVersion) && - ($hDbList->{$iDbHistoryId}{&INFO_SYSTEM_ID} eq $ullDbSysId)) - { - $bConfirmDb = true; - } - else - { - $bConfirmDb = false; - } - } - # If not, the backup.info file must be corrupt - else - { - confess &log(ERROR, "backup info file is missing database history information for an existing backup", ERROR_FILE_INVALID); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bConfirmDb', value => $bConfirmDb} - ); -} - -#################################################################################################################################### -# confirmExists -# -# Ensure that the backup.info file and the db section exist. -#################################################################################################################################### -sub confirmExists -{ - my $self = shift; - - # Confirm the file exists and the DB section is filled out - if (!$self->test(INFO_BACKUP_SECTION_DB) || !$self->{bExists}) - { - confess &log(ERROR, $self->{strBackupClusterPath} . "/" . $strBackupInfoMissingMsg, ERROR_FILE_MISSING); - } -} - -1; diff --git a/test/lib/pgBackRestTest/Env/ExpireEnvTest.pm b/test/lib/pgBackRestTest/Env/ExpireEnvTest.pm deleted file mode 100644 index 541bc8a2dc..0000000000 --- a/test/lib/pgBackRestTest/Env/ExpireEnvTest.pm +++ /dev/null @@ -1,639 +0,0 @@ -#################################################################################################################################### -# ExpireCommonTest.pm - Common code for expire tests -#################################################################################################################################### -package pgBackRestTest::Env::ExpireEnvTest; -use parent 'pgBackRestTest::Env::HostEnvTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Fcntl qw(O_RDONLY); -use File::Basename qw(basename); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::FileTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Env::ArchiveInfo; -use pgBackRestTest::Env::BackupInfo; -use pgBackRestTest::Env::HostEnvTest; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Create the class hash - my $self = {}; - bless $self, $class; - - # Assign function parameters, defaults, and log debug info - ( - my $strOperation, - $self->{oHostBackup}, - $self->{strBackRestExe}, - $self->{oStorageRepo}, - $self->{strPgPath}, - $self->{oRunTest}, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oHostBackup', required => false, trace => true}, - {name => 'strBackRestExe', trace => true}, - {name => 'oStorageRepo', trace => true}, - {name => 'strPgPath', trace => true}, - {name => 'oRunTest', required => false, trace => true}, - ); - - $self->{strVm} = $self->{oRunTest}->vm(); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# get into from pg_control -#################################################################################################################################### -my $oPgControlVersionHash = -{ - # iControlVersion => {iCatalogVersion => strDbVersion} - 942 => - { - 201409291 => PG_VERSION_94, - 201510051 => PG_VERSION_95, - }, - 960 => - { - 201608131 => PG_VERSION_96, - }, - 1002 => - { - 201707211 => PG_VERSION_10, - }, - 1100 => - { - 201809051 => PG_VERSION_11, - }, - 1201 => - { - 201909212 => PG_VERSION_12, - }, - 1300 => - { - 202007201 => PG_VERSION_13, - }, -}; - -sub info -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbPath - ) = - logDebugParam - ( - __PACKAGE__ . '->info', \@_, - {name => 'strDbPath', default => $self->{strPgPath}} - ); - - # Open the control file and read system id and versions - #----------------------------------------------------------------------------------------------------------------------- - my $strControlFile = "${strDbPath}/" . DB_FILE_PGCONTROL; - my $hFile; - my $tBlock; - - sysopen($hFile, $strControlFile, O_RDONLY) - or confess &log(ERROR, "unable to open ${strControlFile}", ERROR_FILE_OPEN); - - # Read system identifier - sysread($hFile, $tBlock, 8) == 8 - or confess &log(ERROR, "unable to read database system identifier"); - - $self->{info}{$strDbPath}{ullDbSysId} = unpack('Q', $tBlock); - - # Read control version - sysread($hFile, $tBlock, 4) == 4 - or confess &log(ERROR, "unable to read control version"); - - $self->{info}{$strDbPath}{iDbControlVersion} = unpack('L', $tBlock); - - # Read catalog version - sysread($hFile, $tBlock, 4) == 4 - or confess &log(ERROR, "unable to read catalog version"); - - $self->{info}{$strDbPath}{iDbCatalogVersion} = unpack('L', $tBlock); - - # Close the control file - close($hFile); - - # Get PostgreSQL version - $self->{info}{$strDbPath}{strDbVersion} = - $oPgControlVersionHash->{$self->{info}{$strDbPath}{iDbControlVersion}} - {$self->{info}{$strDbPath}{iDbCatalogVersion}}; - - if (!defined($self->{info}{$strDbPath}{strDbVersion})) - { - confess &log( - ERROR, - 'unexpected control version = ' . $self->{info}{$strDbPath}{iDbControlVersion} . - ' and catalog version = ' . $self->{info}{$strDbPath}{iDbCatalogVersion} . "\n" . - 'HINT: is this version of PostgreSQL supported?'); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strDbVersion', value => $self->{info}{$strDbPath}{strDbVersion}}, - {name => 'iDbControlVersion', value => $self->{info}{$strDbPath}{iDbControlVersion}}, - {name => 'iDbCatalogVersion', value => $self->{info}{$strDbPath}{iDbCatalogVersion}}, - {name => 'ullDbSysId', value => $self->{info}{$strDbPath}{ullDbSysId}} - ); -} - -#################################################################################################################################### -# stanzaSet - set the local stanza object -#################################################################################################################################### -sub stanzaSet -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strStanza, - $strDbVersion, - $bStanzaUpgrade, - ) = - logDebugParam - ( - __PACKAGE__ . '->stanzaSet', \@_, - {name => 'strStanza'}, - {name => 'strDbVersion'}, - {name => 'bStanzaUpgrade'}, - ); - - # Assign variables - my $oStanza = {}; - my $oArchiveInfo = {}; - my $oBackupInfo = {}; - my $iArchiveDbId = 1; - my $iBackupDbId = 1; - - # If we're not upgrading, then create the info files - if (!$bStanzaUpgrade) - { - $oArchiveInfo = - new pgBackRestTest::Env::ArchiveInfo($self->{oHostBackup}->repoArchivePath(), false, - {bIgnoreMissing => true, strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_ARCHIVE : undef}); - $oBackupInfo = - new pgBackRestTest::Env::BackupInfo($self->{oHostBackup}->repoBackupPath(), false, - {bIgnoreMissing => true, strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_MANIFEST : undef}); - } - # Else get the info data from disk - else - { - $oArchiveInfo = - new pgBackRestTest::Env::ArchiveInfo($self->{oHostBackup}->repoArchivePath(), - {strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_ARCHIVE : undef}); - $oBackupInfo = - new pgBackRestTest::Env::BackupInfo($self->{oHostBackup}->repoBackupPath(), - {strCipherPassSub => $self->{oHostBackup}->repoEncrypt() ? ENCRYPTION_KEY_MANIFEST : undef}); - } - - # Get the database info for the stanza - (my $strVersion, $$oStanza{iControlVersion}, $$oStanza{iCatalogVersion}, $$oStanza{ullDbSysId}) = $self->info(); - $$oStanza{strDbVersion} = $strDbVersion; - - if ($bStanzaUpgrade) - { - $iArchiveDbId = $oArchiveInfo->dbHistoryIdGet() + 1; - $iBackupDbId = $oBackupInfo->dbHistoryIdGet() + 1; - } - - $oArchiveInfo->dbSectionSet($$oStanza{strDbVersion}, $$oStanza{ullDbSysId}, $iArchiveDbId); - $oArchiveInfo->save(); - - $oBackupInfo->dbSectionSet($$oStanza{strDbVersion}, $$oStanza{iControlVersion}, $$oStanza{iCatalogVersion}, - $$oStanza{ullDbSysId}, $iBackupDbId); - $oBackupInfo->save(); - - # Get the archive and directory paths for the stanza - $$oStanza{strArchiveClusterPath} = $self->{oHostBackup}->repoArchivePath($oArchiveInfo->archiveId()); - $$oStanza{strBackupClusterPath} = $self->{oHostBackup}->repoBackupPath(); - - $self->{oStanzaHash}{$strStanza} = $oStanza; - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# stanzaCreate -#################################################################################################################################### -sub stanzaCreate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strStanza, - $strDbVersion, - ) = - logDebugParam - ( - __PACKAGE__ . '->stanzaCreate', \@_, - {name => 'strStanza'}, - {name => 'strDbVersion'}, - ); - - my $strDbVersionTemp = $strDbVersion; - $strDbVersionTemp =~ s/\.//; - - # Create the test path for pg_control - storageTest()->pathCreate(($self->{strPgPath} . '/' . DB_PATH_GLOBAL), {bIgnoreExists => true}); - - # Generate pg_control for stanza-create - $self->controlGenerate($self->{strPgPath}, $strDbVersion); - executeTest('chmod 600 ' . $self->{strPgPath} . '/' . DB_FILE_PGCONTROL); - - # Create the stanza and set the local stanza object - $self->stanzaSet($strStanza, $strDbVersion, false); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# stanzaUpgrade -#################################################################################################################################### -sub stanzaUpgrade -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strStanza, - $strDbVersion, - ) = - logDebugParam - ( - __PACKAGE__ . '->stanzaUpgrade', \@_, - {name => 'strStanza'}, - {name => 'strDbVersion'}, - ); - - my $strDbVersionTemp = $strDbVersion; - $strDbVersionTemp =~ s/\.//; - - # Remove pg_control - storageTest()->remove($self->{strPgPath} . '/' . DB_FILE_PGCONTROL); - - # Copy pg_control for stanza-upgrade - $self->controlGenerate($self->{strPgPath}, $strDbVersion); - executeTest('chmod 600 ' . $self->{strPgPath} . '/' . DB_FILE_PGCONTROL); - - $self->stanzaSet($strStanza, $strDbVersion, true); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} -#################################################################################################################################### -# backupCreate -#################################################################################################################################### -sub backupCreate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strStanza, - $strType, - $lTimestamp, - $iArchiveBackupTotal, - $iArchiveBetweenTotal - ) = - logDebugParam - ( - __PACKAGE__ . '->backupCreate', \@_, - {name => 'strStanza'}, - {name => 'strType'}, - {name => 'lTimestamp'}, - {name => 'iArchiveBackupTotal', default => 3}, - {name => 'iArchiveBetweenTotal', default => 3} - ); - - my $oStanza = $self->{oStanzaHash}{$strStanza}; - - my ($strArchiveStart, $strArchiveStop); - - if ($iArchiveBackupTotal != -1) - { - ($strArchiveStart, $strArchiveStop) = $self->archiveCreate($strStanza, $iArchiveBackupTotal); - } - - # Create the manifest - my $oLastManifest = $strType ne CFGOPTVAL_BACKUP_TYPE_FULL ? $$oStanza{oManifest} : undef; - - my $strBackupLabel = - backupLabelFormat($strType, - defined($oLastManifest) ? $oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL) : undef, - $lTimestamp); - - my $strBackupClusterSetPath = "$$oStanza{strBackupClusterPath}/${strBackupLabel}"; - - &log(INFO, "create backup ${strBackupLabel}"); - - # Get passphrase (returns undefined if repo not encrypted) to access the manifest - my $strCipherPassManifest = - (new pgBackRestTest::Env::BackupInfo($self->{oHostBackup}->repoBackupPath()))->cipherPassSub(); - my $strCipherPassBackupSet; - - # If repo is encrypted then get passphrase for accessing the backup files from the last manifest if it exists provide one - if (defined($strCipherPassManifest)) - { - $strCipherPassBackupSet = (defined($oLastManifest)) ? $oLastManifest->cipherPassSub() : - ENCRYPTION_KEY_BACKUPSET; - } - - my $strManifestFile = "$$oStanza{strBackupClusterPath}/${strBackupLabel}/" . FILE_MANIFEST; - - my $oManifest = new pgBackRestTest::Env::Manifest($strManifestFile, {bLoad => false, strDbVersion => PG_VERSION_94, - iDbCatalogVersion => $self->dbCatalogVersion(PG_VERSION_94), - strCipherPass => $strCipherPassManifest, strCipherPassSub => $strCipherPassBackupSet}); - - # Store information about the backup into the backup section - $oManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL, undef, $strBackupLabel); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_CHECK, undef, true); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_COPY, undef, false); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_BACKUP_STANDBY, undef, false); - $oManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_START, undef, $strArchiveStart); - $oManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_STOP, undef, $strArchiveStop); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, 'backup-bundle', undef, true); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_CHECKSUM_PAGE, undef, true); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS, undef, true); - $oManifest->numericSet(INI_SECTION_BACKREST, INI_KEY_FORMAT, undef, REPOSITORY_FORMAT); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_HARDLINK, undef, false); - $oManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ONLINE, undef, true); - $oManifest->numericSet(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_START, undef, $lTimestamp); - $oManifest->numericSet(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP, undef, $lTimestamp); - $oManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE, undef, $strType); - $oManifest->set(INI_SECTION_BACKREST, INI_KEY_VERSION, undef, PROJECT_VERSION); - - if ($strType ne CFGOPTVAL_BACKUP_TYPE_FULL) - { - if (!defined($oLastManifest)) - { - confess &log(ERROR, "oLastManifest must be defined when strType = ${strType}"); - } - - # Set backup-prior - if ($strType eq CFGOPTVAL_BACKUP_TYPE_INCR) - { - # If this is an incremental backup, then it is always based on the prior backup so use the label from the last backup - $oManifest->set( - MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR, undef, - $oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL)); - } - else - { - # If it is a differential then backup-prior must be set to the newest full backup so get the full backup label from - # the prior label - $oManifest->set( - MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR, undef, - substr($oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL), 0, 16)); - } - } - - $oManifest->save(); - $$oStanza{oManifest} = $oManifest; - - # Add the backup to info - my $oBackupInfo = new pgBackRestTest::Env::BackupInfo($$oStanza{strBackupClusterPath}, false); - - $oBackupInfo->check($$oStanza{strDbVersion}, $$oStanza{iControlVersion}, $$oStanza{iCatalogVersion}, $$oStanza{ullDbSysId}); - $oBackupInfo->add($oManifest); - - # Create the backup description string - if (defined($$oStanza{strBackupDescription})) - { - $$oStanza{strBackupDescription} .= "\n"; - } - - $$oStanza{strBackupDescription} .= - "* ${strType} backup: label = ${strBackupLabel}" . - (defined($oLastManifest) ? ', prior = ' . $oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL) : '') . - (defined($strArchiveStart) ? ", start = ${strArchiveStart}, stop = ${strArchiveStop}" : ', not online'); - - if ($iArchiveBetweenTotal != -1) - { - $self->archiveCreate($strStanza, $iArchiveBetweenTotal); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# archiveNext -#################################################################################################################################### -sub archiveNext -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strArchive, - $bSkipFF - ) = - logDebugParam - ( - __PACKAGE__ . '->archiveNext', \@_, - {name => 'strArchive', trace => true}, - {name => 'bSkipFF', trace => true} - ); - - # Break archive log into components - my $lTimeline = hex(substr($strArchive, 0, 8)); - my $lMajor = hex(substr($strArchive, 8, 8)); - my $lMinor = hex(substr($strArchive, 16, 8)); - - # Increment the minor component (and major when needed) - $lMinor += 1; - - if ($bSkipFF && $lMinor == 255 || !$bSkipFF && $lMinor == 256) - { - $lMajor += 1; - $lMinor = 0; - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strArchiveNext', value => uc(sprintf("%08x%08x%08x", $lTimeline, $lMajor, $lMinor)), trace => true} - ); -} - -#################################################################################################################################### -# archiveCreate -#################################################################################################################################### -sub archiveCreate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strStanza, - $iArchiveTotal - ) = - logDebugParam - ( - __PACKAGE__ . '->archiveCreate', \@_, - {name => 'strStanza'}, - {name => 'iArchiveTotal'} - ); - - my $oStanza = $self->{oStanzaHash}{$strStanza}; - my $iArchiveIdx = 0; - - my $strArchive = defined($$oStanza{strArchiveLast}) ? $self->archiveNext($$oStanza{strArchiveLast}, false) : - '000000010000000000000000'; - - # Get passphrase (returns undefined if repo not encrypted) to access the archive files - my $strCipherPass = - (new pgBackRestTest::Env::ArchiveInfo($self->{oHostBackup}->repoArchivePath()))->cipherPassSub(); - - push(my @stryArchive, $strArchive); - - do - { - my $strPath = "$$oStanza{strArchiveClusterPath}/" . substr($strArchive, 0, 16); - my $strFile = "${strPath}/${strArchive}-0000000000000000000000000000000000000000" . ($iArchiveIdx % 2 == 0 ? '.gz' : ''); - - storageRepo()->put($strFile, 'ARCHIVE', {strCipherPass => $strCipherPass}); - - $iArchiveIdx++; - - if ($iArchiveIdx < $iArchiveTotal) - { - $strArchive = $self->archiveNext($strArchive, false); - } - } - while ($iArchiveIdx < $iArchiveTotal); - - push(@stryArchive, $strArchive); - $$oStanza{strArchiveLast} = $strArchive; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'stryArchive', value => \@stryArchive} - ); -} - -#################################################################################################################################### -# process -#################################################################################################################################### -sub process -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strStanza, - $iExpireFull, - $iExpireDiff, - $strExpireArchiveType, - $iExpireArchive, - $strDescription - ) = - logDebugParam - ( - __PACKAGE__ . '->process', \@_, - {name => 'strStanza'}, - {name => 'iExpireFull', required => false}, - {name => 'iExpireDiff', required => false}, - {name => 'strExpireArchiveType'}, - {name => 'iExpireArchive', required => false}, - {name => 'strDescription'} - ); - - my $oStanza = $self->{oStanzaHash}{$strStanza}; - - undef($$oStanza{strBackupDescription}); - - my $strCommand = - $self->{strBackRestExe} . ' --config="' . $self->{oHostBackup}->backrestConfig() . '"' . ' --stanza=' . $strStanza . - ' --log-level-console=' . lc(DETAIL); - - if (defined($iExpireFull)) - { - $strCommand .= ' --repo1-retention-full=' . $iExpireFull; - } - - if (defined($iExpireDiff)) - { - $strCommand .= ' --repo1-retention-diff=' . $iExpireDiff; - } - - if (defined($strExpireArchiveType)) - { - if (defined($iExpireArchive)) - { - $strCommand .= ' --repo1-retention-archive-type=' . $strExpireArchiveType . - ' --repo1-retention-archive=' . $iExpireArchive; - } - else - { - $strCommand .= ' --repo1-retention-archive-type=' . $strExpireArchiveType; - } - } - - $strCommand .= ' expire'; - - $self->{oHostBackup}->executeSimple($strCommand, {strComment => $strDescription}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostAzureTest.pm b/test/lib/pgBackRestTest/Env/Host/HostAzureTest.pm deleted file mode 100644 index d882c2275c..0000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostAzureTest.pm +++ /dev/null @@ -1,82 +0,0 @@ -#################################################################################################################################### -# Azure Test Host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostAzureTest; -use parent 'pgBackRestTest::Common::HostTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Cwd qw(abs_path); -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); -use Storable qw(dclone); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# Azure defaults -#################################################################################################################################### -use constant HOST_AZURE_ACCOUNT => 'azaccount'; - push @EXPORT, qw(HOST_AZURE_ACCOUNT); -use constant HOST_AZURE_KEY => 'YXpLZXk='; - push @EXPORT, qw(HOST_AZURE_KEY); -use constant HOST_AZURE_CONTAINER => 'azcontainer'; - push @EXPORT, qw(HOST_AZURE_CONTAINER); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - ); - - # Create the host - my $strProjectPath = dirname(dirname(abs_path($0))); - my $strFakeCertPath = "${strProjectPath}/doc/resource/fake-cert"; - - my $self = $class->SUPER::new( - HOST_AZURE, 'test-' . testRunGet()->vmId() . '-' . HOST_AZURE, 'mcr.microsoft.com/azure-storage/azurite', 'root', - ["${strFakeCertPath}/s3-server.crt:/root/public.crt:ro", "${strFakeCertPath}/s3-server.key:/root/private.key:ro"], - '-e AZURITE_ACCOUNTS="' . HOST_AZURE_ACCOUNT . ':' . HOST_AZURE_KEY . '"', - 'azurite-blob --blobPort 443 --blobHost 0.0.0.0 --cert=/root/public.crt --key=/root/private.key -d debug.log" - " --disableProductStyleUrl', - false); - bless $self, $class; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm b/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm deleted file mode 100644 index 6e0e0e2afa..0000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm +++ /dev/null @@ -1,2321 +0,0 @@ -#################################################################################################################################### -# HostBackupTest.pm - Backup host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostBackupTest; -use parent 'pgBackRestTest::Env::Host::HostBaseTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Exporter qw(import); - our @EXPORT = qw(); -use Fcntl ':mode'; -use File::Basename qw(dirname); -use File::stat qw{lstat}; -use Storable qw(dclone); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::Common::String; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::StorageBase; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Env::ArchiveInfo; -use pgBackRestTest::Env::BackupInfo; -use pgBackRestTest::Env::Host::HostAzureTest; -use pgBackRestTest::Env::Host::HostGcsTest; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Host::HostS3Test; -use pgBackRestTest::Env::Host::HostSftpTest; -use pgBackRestTest::Env::Manifest; -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::VmTest; - -#################################################################################################################################### -# Error constants -#################################################################################################################################### -use constant ERROR_REPO_INVALID => 103; -push @EXPORT, qw(ERROR_REPO_INVALID); - -#################################################################################################################################### -# Latest backup link constant -#################################################################################################################################### -use constant LINK_LATEST => 'latest'; - push @EXPORT, qw(LINK_LATEST); - -#################################################################################################################################### -# Host defaults -#################################################################################################################################### -use constant HOST_PATH_LOCK => 'lock'; - push @EXPORT, qw(HOST_PATH_LOCK); -use constant HOST_PATH_LOG => 'log'; - push @EXPORT, qw(HOST_PATH_LOG); -use constant HOST_PATH_REPO => 'repo'; - -use constant HOST_PROTOCOL_TIMEOUT => 10; - push @EXPORT, qw(HOST_PROTOCOL_TIMEOUT); - -#################################################################################################################################### -# Configuration constants -#################################################################################################################################### -use constant CFGDEF_SECTION_GLOBAL => 'global'; - push @EXPORT, qw(CFGDEF_SECTION_GLOBAL); -use constant CFGDEF_SECTION_STANZA => 'stanza'; - push @EXPORT, qw(CFGDEF_SECTION_STANZA); - -use constant CFGOPTVAL_BACKUP_TYPE_FULL => 'full'; - push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_FULL); -use constant CFGOPTVAL_BACKUP_TYPE_DIFF => 'diff'; - push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_DIFF); -use constant CFGOPTVAL_BACKUP_TYPE_INCR => 'incr'; - push @EXPORT, qw(CFGOPTVAL_BACKUP_TYPE_INCR); - -use constant CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC => 'aes-256-cbc'; - push @EXPORT, qw(CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC); - -use constant AZURE => 'azure'; - push @EXPORT, qw(AZURE); -use constant CIFS => 'cifs'; - push @EXPORT, qw(CIFS); -use constant GCS => 'gcs'; - push @EXPORT, qw(GCS); -use constant POSIX => STORAGE_POSIX; - push @EXPORT, qw(POSIX); -use constant S3 => 's3'; - push @EXPORT, qw(S3); -use constant SFTP => 'sftp'; - push @EXPORT, qw(SFTP); - -use constant CFGOPTVAL_RESTORE_TYPE_DEFAULT => 'default'; - push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_DEFAULT); -use constant CFGOPTVAL_RESTORE_TYPE_IMMEDIATE => 'immediate'; - push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_IMMEDIATE); -use constant CFGOPTVAL_RESTORE_TYPE_NAME => 'name'; - push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_NAME); -use constant CFGOPTVAL_RESTORE_TYPE_PRESERVE => 'preserve'; - push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_PRESERVE); -use constant CFGOPTVAL_RESTORE_TYPE_STANDBY => 'standby'; - push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_STANDBY); -use constant CFGOPTVAL_RESTORE_TYPE_TIME => 'time'; - push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_TIME); -use constant CFGOPTVAL_RESTORE_TYPE_XID => 'xid'; - push @EXPORT, qw(CFGOPTVAL_RESTORE_TYPE_XID); - -use constant NONE => 'none'; - push @EXPORT, qw(NONE); -use constant BZ2 => 'bz2'; - push @EXPORT, qw(BZ2); -use constant GZ => 'gz'; - push @EXPORT, qw(GZ); -use constant LZ4 => 'lz4'; - push @EXPORT, qw(LZ4); -use constant ZST => 'zst'; - push @EXPORT, qw(ZST); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oParam', required => false, trace => true}, - ); - - # If params are not passed - my $oHostGroup = hostGroupGet(); - - my ($strName, $strImage, $strUser); - - if (!defined($$oParam{strName}) || $$oParam{strName} eq HOST_BACKUP) - { - $strName = HOST_BACKUP; - $strImage = containerRepo() . ':' . testRunGet()->vm() . '-test'; - } - else - { - $strName = $$oParam{strName}; - $strImage = $$oParam{strImage}; - } - - $strUser = testRunGet()->pgUser(); - - # Create the host - my $self = $class->SUPER::new($strName, {strImage => $strImage, strUser => $strUser, bTls => $oParam->{bTls}}); - bless $self, $class; - - # If repo is on local filesystem then set the repo-path locally - if ($oParam->{bRepoLocal} || $oParam->{strBackupDestination} eq HOST_SFTP) - { - $self->{strRepoPath} = $self->testRunGet()->testPath() . "/$$oParam{strBackupDestination}/" . HOST_PATH_REPO; - } - # Else on KV store and repo will be in root - else - { - $self->{strRepoPath} = '/'; - } - - # If there is a repo2 it will always be posix on the repo host - $self->{strRepo2Path} = $self->testRunGet()->testPath() . "/$$oParam{strBackupDestination}/" . HOST_PATH_REPO . "2"; - - # Set log/lock paths - $self->{strLogPath} = $self->testPath() . '/' . HOST_PATH_LOG; - storageTest()->pathCreate($self->{strLogPath}, {strMode => '0770'}); - $self->{strLockPath} = $self->testPath() . '/' . HOST_PATH_LOCK; - - # Set conf file - $self->{strBackRestConfig} = $self->testPath() . '/' . PROJECT_CONF; - - # Set synthetic - $self->{bSynthetic} = defined($$oParam{bSynthetic}) && $$oParam{bSynthetic} ? true : false; - - # Set the backup destination - $self->{strBackupDestination} = $$oParam{strBackupDestination}; - - # Default hardlink to false - $self->{bHardLink} = false; - - # By default there is no bogus host - $self->{bBogusHost} = false; - - # Create a placeholder hash for file munging - $self->{hInfoFile} = {}; - - # Set whether repo should be encrypted or not - $self->{bRepoEncrypt} = defined($$oParam{bRepoEncrypt}) ? $$oParam{bRepoEncrypt} : false; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# timestampFileFormat -#################################################################################################################################### -sub timestampFileFormat -{ - my $strFormat = shift; - my $lTime = shift; - - return timestampFormat(defined($strFormat) ? $strFormat : '%4d%02d%02d-%02d%02d%02d', $lTime); -} - -push @EXPORT, qw(timestampFileFormat); - -#################################################################################################################################### -# backupLabelFormat -# -# Format the label for a backup. -#################################################################################################################################### -sub backupLabelFormat -{ - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strType, - $strBackupLabelLast, - $lTimestampStart - ) = - logDebugParam - ( - __PACKAGE__ . '::backupLabelFormat', \@_, - {name => 'strType', trace => true}, - {name => 'strBackupLabelLast', required => false, trace => true}, - {name => 'lTimestampTart', trace => true} - ); - - # Full backup label - my $strBackupLabel; - - if ($strType eq CFGOPTVAL_BACKUP_TYPE_FULL) - { - # Last backup label must not be defined - if (defined($strBackupLabelLast)) - { - confess &log(ASSERT, "strBackupLabelLast must not be defined when strType = '${strType}'"); - } - - # Format the timestamp and add the full indicator - $strBackupLabel = timestampFileFormat(undef, $lTimestampStart) . 'F'; - } - # Else diff or incr label - else - { - # Last backup label must be defined - if (!defined($strBackupLabelLast)) - { - confess &log(ASSERT, "strBackupLabelLast must be defined when strType = '${strType}'"); - } - - # Get the full backup portion of the last backup label - $strBackupLabel = substr($strBackupLabelLast, 0, 16); - - # Format the timestamp - $strBackupLabel .= '_' . timestampFileFormat(undef, $lTimestampStart); - - # Add the diff indicator - if ($strType eq CFGOPTVAL_BACKUP_TYPE_DIFF) - { - $strBackupLabel .= 'D'; - } - # Else incr indicator - else - { - $strBackupLabel .= 'I'; - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strBackupLabel', value => $strBackupLabel, trace => true} - ); -} - -push @EXPORT, qw(backupLabelFormat); - -#################################################################################################################################### -# backupBegin -#################################################################################################################################### -sub backupBegin -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strType, - $strComment, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->backupBegin', \@_, - {name => 'strType', trace => true}, - {name => 'strComment', trace => true}, - {name => 'oParam', required => false, trace => true}, - ); - - # Set defaults - my $oExpectedManifest = defined($$oParam{oExpectedManifest}) ? $$oParam{oExpectedManifest} : undef; - - $strComment = - "${strType} backup" . (defined($strComment) ? " - ${strComment}" : '') . - ' (' . $self->nameGet() . ' host)'; - - &log(INFO, " $strComment"); - - # Execute the backup command - my $oExecuteBackup = $self->execute( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - (defined($oExpectedManifest) ? " --no-online" : '') . - (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . - (defined($$oParam{bStandby}) && $$oParam{bStandby} ? " --backup-standby" : '') . - (defined($oParam->{strRepoType}) ? " --repo1-type=$oParam->{strRepoType}" : '') . - (defined($oParam->{iRepo}) ? ' --repo=' . $oParam->{iRepo} : '') . - ($strType ne 'incr' ? " --type=${strType}" : '') . - ' --stanza=' . (defined($oParam->{strStanza}) ? $oParam->{strStanza} : $self->stanza()) . ' backup', - {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, bLogOutput => $self->synthetic()}); - - $oExecuteBackup->begin(); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'oExecuteBackup', value => $oExecuteBackup, trace => true}, - ); -} - -#################################################################################################################################### -# backupEnd -#################################################################################################################################### -sub backupEnd -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strType, - $oExecuteBackup, - $oParam, - $bManifestCompare, - ) = - logDebugParam - ( - __PACKAGE__ . '->backupEnd', \@_, - {name => 'strType', trace => true}, - {name => 'oExecuteBackup', trace => true}, - {name => 'oParam', required => false, trace => true}, - {name => 'bManifestCompare', required => false, default => true, trace => true}, - ); - - # Set defaults - my $oExpectedManifest = defined($$oParam{oExpectedManifest}) ? dclone($$oParam{oExpectedManifest}) : undef; - - my $iExitStatus = $oExecuteBackup->end(); - - return if ($oExecuteBackup->{iExpectedExitStatus} != 0); - - # If an alternate stanza was specified - if (defined($oParam->{strStanza})) - { - confess &log(ASSERT, - 'if an alternate stanza is specified it must generate an error - the remaining code will not be aware of the stanza'); - } - - my $strBackup = $self->backupLast($oParam->{iRepo}); - - # Only compare backups that are in repo1. There is not a lot of value in comparing backups in other repos and it would require a - # lot of changes to the test harness. - if (!defined($oParam->{iRepo}) || $oParam->{iRepo} == 1) - { - # If a real backup then load the expected manifest from the actual manifest. An expected manifest can't be generated - # perfectly because a running database is always in flux. Even so, it allows us to test many things. - if (!$self->synthetic()) - { - $oExpectedManifest = iniParse( - ${storageRepo()->get( - storageRepo()->openRead( - 'backup/' . $self->stanza() . "/${strBackup}/" . FILE_MANIFEST, - {strCipherPass => $self->cipherPassManifest()}))}); - } - - # Make sure tablespace links are correct - if ($self->hasLink()) - { - if (($strType eq CFGOPTVAL_BACKUP_TYPE_FULL || $self->hardLink()) && - !$oExpectedManifest->{&MANIFEST_SECTION_BACKUP}{'backup-bundle'}) - { - my $hTablespaceManifest = storageTest()->manifest( - $self->repoBackupPath("${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC)); - - # Remove . and .. - delete($hTablespaceManifest->{'.'}); - delete($hTablespaceManifest->{'..'}); - - # Iterate file links - for my $strFile (sort(keys(%{$hTablespaceManifest}))) - { - # Make sure the link is in the expected manifest - my $hManifestTarget = - $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGTBLSPC . "/${strFile}"}; - - if (!defined($hManifestTarget) || $hManifestTarget->{&MANIFEST_SUBKEY_TYPE} ne MANIFEST_VALUE_LINK || - $hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID} ne $strFile) - { - confess &log(ERROR, "'${strFile}' is not in expected manifest as a link with the correct tablespace id"); - } - - # Make sure the link really is a link - if ($hTablespaceManifest->{$strFile}{type} ne 'l') - { - confess &log(ERROR, "'${strFile}' in tablespace directory is not a link"); - } - - # Make sure the link destination is correct - my $strLinkDestination = '../../' . MANIFEST_TARGET_PGTBLSPC . "/${strFile}"; - - if ($hTablespaceManifest->{$strFile}{link_destination} ne $strLinkDestination) - { - confess &log(ERROR, - "'${strFile}' link should reference '${strLinkDestination}' but actually references " . - "'$hTablespaceManifest->{$strFile}{link_destination}'"); - } - } - - # Iterate manifest targets - for my $strTarget (sort(keys(%{$oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}}))) - { - my $hManifestTarget = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}; - my $strTablespaceId = $hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID}; - - # Make sure the target exists as a link on disk - if ($hManifestTarget->{&MANIFEST_SUBKEY_TYPE} eq MANIFEST_VALUE_LINK && defined($strTablespaceId) && - !defined($hTablespaceManifest->{$strTablespaceId})) - { - confess &log(ERROR, - "target '${strTarget}' does not have a link at '" . DB_PATH_PGTBLSPC. "/${strTablespaceId}'"); - } - } - } - # Else there should not be a tablespace directory at all. This is only valid for storage that supports links. - elsif (storageRepo()->capability(STORAGE_CAPABILITY_LINK) && - storageTest()->pathExists( - $self->repoBackupPath("${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC))) - { - confess &log(ERROR, 'backup must be full or hard-linked to have ' . DB_PATH_PGTBLSPC . ' directory'); - } - } - - # Check that latest link exists unless repo links are disabled - my $strLatestLink = $self->repoBackupPath(LINK_LATEST); - my $bLatestLinkExists = storageRepo()->exists($strLatestLink); - - if ((!defined($oParam->{strRepoType}) || $oParam->{strRepoType} eq POSIX) && $self->hasLink()) - { - my $strLatestLinkDestination = readlink($strLatestLink); - - if ($strLatestLinkDestination ne $strBackup) - { - confess &log(ERROR, "'" . LINK_LATEST . "' link should be '${strBackup}' but is '${strLatestLinkDestination}"); - } - } - elsif ($bLatestLinkExists) - { - confess &log(ERROR, "'" . LINK_LATEST . "' link should not exist"); - } - - # Only do compare for synthetic backups since for real backups the expected manifest *is* the actual manifest. - if ($self->synthetic()) - { - # Compare only if expected to do so - if ($bManifestCompare) - { - # Set backup type in the expected manifest - ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TYPE} = $strType; - - $self->backupCompare($strBackup, $oExpectedManifest); - } - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strBackup', value => $strBackup, trace => true}, - ); -} - -#################################################################################################################################### -# backup -#################################################################################################################################### -sub backup -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strType, - $strComment, - $oParam, - $bManifestCompare, - ) = - logDebugParam - ( - __PACKAGE__ . '->backup', \@_, - {name => 'strType'}, - {name => 'strComment'}, - {name => 'oParam', required => false}, - {name => 'bManifestCompare', required => false, default => true}, - ); - - my $oExecuteBackup = $self->backupBegin($strType, $strComment, $oParam); - my $strBackup = $self->backupEnd($strType, $oExecuteBackup, $oParam, $bManifestCompare); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'strBackup', value => $strBackup}, - ); -} - -#################################################################################################################################### -# backupCompare -#################################################################################################################################### -sub backupCompare -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strBackup, - $oExpectedManifest, - ) = - logDebugParam - ( - __PACKAGE__ . '->backupCompare', \@_, - {name => 'strBackup', trace => true}, - {name => 'oExpectedManifest', trace => true}, - ); - - ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL} = $strBackup; - - my $oActualManifest = new pgBackRestTest::Env::Manifest( - $self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), {strCipherPass => $self->cipherPassManifest()}); - - ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{'backup-reference'} = - $oActualManifest->get(MANIFEST_SECTION_BACKUP, 'backup-reference'); - ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START} = - $oActualManifest->get(MANIFEST_SECTION_BACKUP, &MANIFEST_KEY_TIMESTAMP_START); - ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_STOP} = - $oActualManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP); - ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_COPY_START} = - $oActualManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START); - ${$oExpectedManifest}{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM} = - $oActualManifest->get(INI_SECTION_BACKREST, INI_KEY_CHECKSUM); - ${$oExpectedManifest}{&INI_SECTION_BACKREST}{&INI_KEY_FORMAT} = REPOSITORY_FORMAT + 0; - - if (defined($oExpectedManifest->{&INI_SECTION_CIPHER}) && - defined($oExpectedManifest->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS}) && - $oActualManifest->test(INI_SECTION_CIPHER, INI_KEY_CIPHER_PASS)) - { - $oExpectedManifest->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS} = - $oActualManifest->get(INI_SECTION_CIPHER, INI_KEY_CIPHER_PASS); - } - - # Update the expected manifest with whether the --delta option was used or not to perform the backup. - $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA} = - $oActualManifest->boolGet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_DELTA) ? INI_TRUE : INI_FALSE; - - my $strSectionPath = $oActualManifest->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH); - - foreach my $strFileKey ($oActualManifest->keys(MANIFEST_SECTION_TARGET_FILE)) - { - # Remove repo checksum - $oActualManifest->remove(&MANIFEST_SECTION_TARGET_FILE, $strFileKey, 'rck'); - - # Determine repo size if compression or encryption is enabled - my $strCompressType = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE}; - - if ($strCompressType ne NONE || - (defined($oExpectedManifest->{&INI_SECTION_CIPHER}) && - defined($oExpectedManifest->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS}))) - { - - my $lRepoSize = - $oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REFERENCE) ? - $oActualManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE, false) : - (storageRepo()->info( - $self->repoBackupPath("${strBackup}/${strFileKey}") . - ($strCompressType eq NONE ? '' : ".${strCompressType}")))->{size}; - - if (defined($lRepoSize) && - $lRepoSize != $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_SIZE}) - { - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_REPO_SIZE} = $lRepoSize; - } - } - - # If the backup does not have page checksums then no need to compare - if (!$oExpectedManifest->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_CHECKSUM_PAGE}) - { - delete($oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_CHECKSUM_PAGE}); - delete($oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR}); - } - # Else make sure things that should have checks do have checks - elsif ($oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_CHECKSUM_PAGE) != - isChecksumPage($strFileKey)) - { - confess - "check-page actual for ${strFileKey} is " . - ($oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, - MANIFEST_SUBKEY_CHECKSUM_PAGE) ? 'set' : '[undef]') . - ' but isChecksumPage() says it should be ' . - (isChecksumPage($strFileKey) ? 'set' : 'undef') . '.'; - } - } - - $self->manifestDefault($oExpectedManifest); - - my $strTestPath = $self->testPath(); - - storageTest()->put("${strTestPath}/actual.manifest", iniRender($oActualManifest->{oContent})); - storageTest()->put("${strTestPath}/expected.manifest", iniRender($oExpectedManifest)); - - executeTest("diff ${strTestPath}/expected.manifest ${strTestPath}/actual.manifest"); - - storageTest()->remove("${strTestPath}/expected.manifest"); - storageTest()->remove("${strTestPath}/actual.manifest"); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# manifestDefault -#################################################################################################################################### -sub manifestDefault -{ - my $self = shift; - my $oExpectedManifest = shift; - - # Defaults for subkeys that tend to repeat - my $strDefaultUser = $oExpectedManifest->{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_USER}; - my $strDefaultGroup = $oExpectedManifest->{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_GROUP}; - my $strDefaultPathMode = $oExpectedManifest->{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_MODE}; - my $strDefaultFileMode = sprintf('%04o', oct($strDefaultPathMode) & (S_IRUSR | S_IWUSR | S_IRGRP)); - - # Remove subkeys that match the defaults - foreach my $strSection (&MANIFEST_SECTION_TARGET_FILE, &MANIFEST_SECTION_TARGET_PATH, &MANIFEST_SECTION_TARGET_LINK) - { - next if !defined($oExpectedManifest->{$strSection}); - - foreach my $strFile (keys(%{$oExpectedManifest->{$strSection}})) - { - if (defined($oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_USER}) && - $oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_USER} eq $strDefaultUser) - { - delete($oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_USER}); - } - - if (defined($oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_GROUP}) && - $oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_GROUP} eq $strDefaultGroup) - { - delete($oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_GROUP}); - } - - if (defined($oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_MODE}) && - $oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_MODE} eq - ($strSection eq MANIFEST_SECTION_TARGET_PATH ? $strDefaultPathMode : $strDefaultFileMode)) - { - delete($oExpectedManifest->{$strSection}{$strFile}{&MANIFEST_SUBKEY_MODE}); - } - } - } - - # Write defaults - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE . ':default'}{&MANIFEST_SUBKEY_USER} = $strDefaultUser; - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE . ':default'}{&MANIFEST_SUBKEY_GROUP} = $strDefaultGroup; - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE . ':default'}{&MANIFEST_SUBKEY_MODE} = $strDefaultFileMode; - - if (defined($oExpectedManifest->{&MANIFEST_SECTION_TARGET_LINK})) - { - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_LINK . ':default'}{&MANIFEST_SUBKEY_USER} = $strDefaultUser; - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_LINK . ':default'}{&MANIFEST_SUBKEY_GROUP} = $strDefaultGroup; - } - - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_PATH . ':default'}{&MANIFEST_SUBKEY_USER} = $strDefaultUser; - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_PATH . ':default'}{&MANIFEST_SUBKEY_GROUP} = $strDefaultGroup; - $oExpectedManifest->{&MANIFEST_SECTION_TARGET_PATH . ':default'}{&MANIFEST_SUBKEY_MODE} = $strDefaultPathMode; -} - -#################################################################################################################################### -# backupLast -#################################################################################################################################### -sub backupLast -{ - my $self = shift; - my $iRepo = shift; - - my @stryBackup = storageRepo({iRepo => $iRepo})->list( - $self->repoBackupPath(undef, $iRepo), - {strExpression => '[0-9]{8}-[0-9]{6}F(_[0-9]{8}-[0-9]{6}(D|I)){0,1}', strSortOrder => 'reverse'}); - - if (!defined($stryBackup[0])) - { - confess 'no backup was found: ' . join(@stryBackup, ', '); - } - - return $stryBackup[0]; -} - -#################################################################################################################################### -# check -#################################################################################################################################### -sub check -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strComment, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->check', \@_, - {name => 'strComment'}, - {name => 'oParam', required => false}, - ); - - $strComment = - 'check ' . $self->stanza() . ' - ' . $strComment . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " $strComment"); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - (defined($$oParam{iTimeout}) ? " --archive-timeout=$$oParam{iTimeout}" : '') . - (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . - (!defined($oParam->{bStanza}) || $oParam->{bStanza} ? ' --stanza=' . $self->stanza() : '') . ' check', - {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, bLogOutput => $self->synthetic()}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# expire -#################################################################################################################################### -sub expire -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->check', \@_, - {name => 'oParam', required => false}, - ); - - my $strComment = - 'expire' . - (defined($$oParam{iRetentionFull}) ? " full=$$oParam{iRetentionFull}" : '') . - (defined($$oParam{iRetentionDiff}) ? " diff=$$oParam{iRetentionDiff}" : '') . - (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " ${strComment}"); - - # Determine whether or not to expect an error - my $oHostGroup = hostGroupGet(); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - (defined($$oParam{iRetentionFull}) ? " --repo1-retention-full=$$oParam{iRetentionFull}" : '') . - (defined($$oParam{iRetentionDiff}) ? " --repo1-retention-diff=$$oParam{iRetentionDiff}" : '') . - (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . - ' --repo=' . (defined($oParam->{iRepo}) ? $oParam->{iRepo} : '1') . - ' --stanza=' . $self->stanza() . ' expire', - {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, bLogOutput => $self->synthetic()}); -} - -#################################################################################################################################### -# info -#################################################################################################################################### -sub info -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strComment, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->info', \@_, - {name => 'strComment'}, - {name => 'oParam', required => false}, - ); - - $strComment = - 'info' . (defined($$oParam{strStanza}) ? " $$oParam{strStanza} stanza" : ' all stanzas') . ' - ' . $strComment . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " $strComment"); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - ' --log-level-console=warn' . - (defined($$oParam{strStanza}) ? " --stanza=$$oParam{strStanza}" : '') . - (defined($$oParam{strOutput}) ? " --output=$$oParam{strOutput}" : '') . ' info', - {strComment => $strComment, bLogOutput => $self->synthetic()}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# stanzaCreate -#################################################################################################################################### -sub stanzaCreate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strComment, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->stanzaCreate', \@_, - {name => 'strComment'}, - {name => 'oParam', required => false}, - ); - - $strComment = - 'stanza-create ' . $self->stanza() . ' - ' . $strComment . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " $strComment"); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - ' --stanza=' . $self->stanza() . - (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . - ' stanza-create', - {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, bLogOutput => $self->synthetic()}); - - if (storageRepo()->exists('backup/' . $self->stanza() . qw{/} . FILE_BACKUP_INFO)) - { - # Get the passphrase for accessing the manifest file - $self->{strCipherPassManifest} = (new pgBackRestTest::Env::BackupInfo($self->repoBackupPath()))->cipherPassSub(); - } - - if (storageRepo()->exists('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE)) - { - - # Get the passphrase for accessing the archived files - $self->{strCipherPassArchive} = - (new pgBackRestTest::Env::ArchiveInfo($self->repoArchivePath()))->cipherPassSub(); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# stanzaUpgrade -#################################################################################################################################### -sub stanzaUpgrade -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strComment, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->stanzaUpgrade', \@_, - {name => 'strComment'}, - {name => 'oParam', required => false}, - ); - - $strComment = - 'stanza-upgrade ' . $self->stanza() . ' - ' . $strComment . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " $strComment"); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - ' --stanza=' . $self->stanza() . - (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . - ' stanza-upgrade', - {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, bLogOutput => $self->synthetic()}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# stanzaDelete -#################################################################################################################################### -sub stanzaDelete -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strComment, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->stanzaDelete', \@_, - {name => 'strComment'}, - {name => 'oParam', required => false}, - ); - - $strComment = - 'stanza-delete ' . $self->stanza() . ' - ' . $strComment . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " $strComment"); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - ' --repo=' . (defined($oParam->{iRepo}) ? $oParam->{iRepo} : '1') . - ' --stanza=' . $self->stanza() . - (defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') . - ' stanza-delete', - {strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, bLogOutput => $self->synthetic()}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# start -#################################################################################################################################### -sub start -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->start', \@_, - {name => 'oParam', required => false}, - ); - - my $strComment = - 'start' . (defined($$oParam{strStanza}) ? " $$oParam{strStanza} stanza" : ' all stanzas') . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " $strComment"); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - (defined($$oParam{strStanza}) ? " --stanza=$$oParam{strStanza}" : '') . ' start', - {strComment => $strComment, bLogOutput => $self->synthetic()}); -} - -#################################################################################################################################### -# stop -#################################################################################################################################### -sub stop -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->stop', \@_, - {name => 'oParam', required => false}, - ); - - my $strComment = - 'stop' . (defined($$oParam{strStanza}) ? " $$oParam{strStanza} stanza" : ' all stanzas') . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " $strComment"); - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - (defined($$oParam{strStanza}) ? " --stanza=$$oParam{strStanza}" : '') . - (defined($$oParam{bForce}) && $$oParam{bForce} ? ' --force' : '') . ' stop', - {strComment => $strComment, bLogOutput => $self->synthetic()}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# configCreate -#################################################################################################################################### -sub configCreate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->stop', \@_, - {name => 'oParam', required => false}, - ); - - my %oParamHash; - my $strStanza = $self->stanza(); - my $oHostGroup = hostGroupGet(); - my $oHostBackup = $oHostGroup->hostGet($self->backupDestination()); - my $oHostDbPrimary = $oHostGroup->hostGet(HOST_DB_PRIMARY); - my $oHostDbStandby = $oHostGroup->hostGet(HOST_DB_STANDBY, true); - - my $bArchiveAsync = defined($$oParam{bArchiveAsync}) ? $$oParam{bArchiveAsync} : false; - - my $iRepoTotal = defined($oParam->{iRepoTotal}) ? $oParam->{iRepoTotal} : 1; - - if ($iRepoTotal < 1 || $iRepoTotal > 2) - { - confess "invalid repo total ${iRepoTotal}"; - } - - # General options - # ------------------------------------------------------------------------------------------------------------------------------ - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'beta'} = 'y'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'job-retry'} = 0; - - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-level-console'} = lc(DETAIL); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-level-file'} = testRunGet()->logLevelTestFile(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-level-stderr'} = lc(OFF); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-subprocess'} = - testRunGet()->logLevelTestFile() eq lc(OFF) ? 'n' : 'y'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-timestamp'} = 'n'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'buffer-size'} = '64k'; - - if ($oParam->{bBundle}) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-bundle'} = 'y'; - # Set bundle size/limit smaller for testing and because FakeGCS does not do multi-part upload - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-bundle-size'} = '1MiB'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-bundle-limit'} = '64KiB'; - } - - if ($oParam->{bBlockIncr}) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-block'} = 'y'; - } - - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-path'} = $self->logPath(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'lock-path'} = $self->lockPath(); - - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'protocol-timeout'} = 60; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'db-timeout'} = 45; - - # Set to make sure that changing the default works and to speed compression for testing - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'compress-level'} = 3; - - # Only set network compress level if there is more than one host - if ($oHostBackup != $oHostDbPrimary) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'compress-level-network'} = 1; - } - - if (defined($oParam->{strCompressType}) && $oParam->{strCompressType} ne 'gz') - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'compress-type'} = $oParam->{strCompressType}; - } - - if ($self->isHostBackup()) - { - if ($self->repoEncrypt()) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-cipher-type'} = CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-cipher-pass'} = 'x'; - } - - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-path'} = $self->repoPath(); - - # S3 settings - if ($oParam->{strStorage} eq S3) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-type'} = S3; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-key'} = HOST_S3_ACCESS_KEY; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-key-secret'} = HOST_S3_ACCESS_SECRET_KEY; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-bucket'} = HOST_S3_BUCKET; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-endpoint'} = HOST_S3_ENDPOINT; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-region'} = HOST_S3_REGION; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-verify-ssl'} = 'n'; - } - elsif ($oParam->{strStorage} eq AZURE) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-type'} = AZURE; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-azure-account'} = HOST_AZURE_ACCOUNT; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-azure-key'} = HOST_AZURE_KEY; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-azure-container'} = HOST_AZURE_CONTAINER; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-azure-host'} = HOST_AZURE; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-azure-verify-tls'} = 'n'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-azure-uri-style'} = 'path'; - } - elsif ($oParam->{strStorage} eq GCS) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-type'} = GCS; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-gcs-bucket'} = HOST_GCS_BUCKET; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-gcs-key-type'} = HOST_GCS_KEY_TYPE; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-gcs-key'} = HOST_GCS_KEY; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-gcs-endpoint'} = HOST_GCS . ':' . HOST_GCS_PORT; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-storage-verify-tls'} = 'n'; - } - - if ($iRepoTotal == 2) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-path'} = $self->repo2Path(); - } - - if (defined($$oParam{bHardlink}) && $$oParam{bHardlink}) - { - $self->{bHardLink} = true; - $oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'repo1-s3-hardlink'} = 'y'; - } - - $oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'archive-copy'} = 'y'; - $oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'start-fast'} = 'y'; - } - - # Host specific options - # ------------------------------------------------------------------------------------------------------------------------------ - - # If this is the backup host - if ($self->isHostBackup()) - { - my $oHostDb1 = $oHostDbPrimary; - my $oHostDb2 = $oHostDbStandby; - - if ($self->nameTest(HOST_DB_STANDBY)) - { - $oHostDb1 = $oHostDbStandby; - $oHostDb2 = $oHostDbPrimary; - } - - if ($self->nameTest(HOST_BACKUP)) - { - $oParamHash{$strStanza}{'pg1-host'} = $oHostDb1->nameGet(); - $oParamHash{$strStanza}{'pg1-host-user'} = $oHostDb1->userGet(); - $oParamHash{$strStanza}{'pg1-host-cmd'} = $oHostDb1->backrestExe(); - $oParamHash{$strStanza}{'pg1-host-config'} = $oHostDb1->backrestConfig(); - - if ($oParam->{bTls}) - { - $oParamHash{$strStanza}{'pg1-host-type'} = 'tls'; - $oParamHash{$strStanza}{'pg1-host-cert-file'} = testRunGet()->basePath() . HOST_CLIENT_CERT; - $oParamHash{$strStanza}{'pg1-host-key-file'} = testRunGet()->basePath() . HOST_CLIENT_KEY; - } - - # Port can't be configured for a synthetic host - if (!$self->synthetic()) - { - $oParamHash{$strStanza}{'pg1-port'} = $oHostDb1->pgPort(); - } - } - - $oParamHash{$strStanza}{'pg1-path'} = $oHostDb1->dbBasePath(); - - if (defined($oHostDb2)) - { - # Add an invalid replica to simulate more than one replica. A warning should be thrown when a stanza is created and a - # valid replica should be chosen. Only do this for SSH since TLS takes longer to timeout. - if (!$oParam->{bTls}) - { - $oParamHash{$strStanza}{"pg2-host"} = BOGUS; - $oParamHash{$strStanza}{"pg2-host-user"} = $oHostDb2->userGet(); - $oParamHash{$strStanza}{"pg2-host-cmd"} = $oHostDb2->backrestExe(); - $oParamHash{$strStanza}{"pg2-host-config"} = $oHostDb2->backrestConfig(); - $oParamHash{$strStanza}{"pg2-path"} = $oHostDb2->dbBasePath(); - } - - # Set a flag so we know there's a bogus host - $self->{bBogusHost} = true; - - # Set a valid replica to a higher index to ensure skipping indexes does not make a difference - $oParamHash{$strStanza}{"pg256-host"} = $oHostDb2->nameGet(); - $oParamHash{$strStanza}{"pg256-host-user"} = $oHostDb2->userGet(); - $oParamHash{$strStanza}{"pg256-host-cmd"} = $oHostDb2->backrestExe(); - $oParamHash{$strStanza}{"pg256-host-config"} = $oHostDb2->backrestConfig(); - $oParamHash{$strStanza}{"pg256-path"} = $oHostDb2->dbBasePath(); - - if ($oParam->{bTls}) - { - $oParamHash{$strStanza}{'pg256-host-type'} = 'tls'; - $oParamHash{$strStanza}{'pg256-host-cert-file'} = testRunGet()->basePath() . HOST_CLIENT_CERT; - $oParamHash{$strStanza}{'pg256-host-key-file'} = testRunGet()->basePath() . HOST_CLIENT_KEY; - } - - # Only test explicit ports on the backup server. This is so locally configured ports are also tested. - if (!$self->synthetic() && $self->nameTest(HOST_BACKUP)) - { - $oParamHash{$strStanza}{"pg256-port"} = $oHostDb2->pgPort(); - } - } - } - elsif ($oParam->{strStorage} eq SFTP) - { - my $oHostDb1 = $oHostDbPrimary; - my $oHostDb2 = $oHostDbStandby; - - if ($self->nameTest(HOST_DB_STANDBY)) - { - $oHostDb1 = $oHostDbStandby; - $oHostDb2 = $oHostDbPrimary; - } - - # Set a flag so we know there's a bogus host - $self->{bBogusHost} = true; - - # Set a valid replica to a higher index to ensure skipping indexes does not make a difference - $oParamHash{$strStanza}{"pg256-host"} = $oHostDb2->nameGet(); - $oParamHash{$strStanza}{"pg256-host-user"} = $oHostDb2->userGet(); - $oParamHash{$strStanza}{"pg256-host-cmd"} = $oHostDb2->backrestExe(); - $oParamHash{$strStanza}{"pg256-host-config"} = $oHostDb2->backrestConfig(); - $oParamHash{$strStanza}{"pg256-path"} = $oHostDb2->dbBasePath(); - $oParamHash{$strStanza}{"pg256-port"} = $oHostDb2->pgPort(); - } - - # If this is a database host - if ($self->isHostDb()) - { - $oParamHash{$strStanza}{'pg1-path'} = $self->dbBasePath(); - - if (!$self->synthetic()) - { - $oParamHash{$strStanza}{'pg1-socket-path'} = $self->pgSocketPath(); - $oParamHash{$strStanza}{'pg1-port'} = $self->pgPort(); - } - - if ($bArchiveAsync) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL . ':archive-push'}{'archive-async'} = 'y'; - } - - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'spool-path'} = $self->spoolPath(); - - # If the backup host is remote - if (!$self->isHostBackup()) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-user'} = $oHostBackup->userGet(); - - if ($oHostBackup->nameGet() eq HOST_SFTP) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-type'} = "sftp"; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-sftp-host'} = HOST_SFTP; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-sftp-host-key-hash-type'} = "sha1"; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-sftp-host-user'} = TEST_USER; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-sftp-private-key-file'} = testRunGet()->basePath() . SSH_PRIVATE_KEY; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-sftp-public-key-file'} = testRunGet()->basePath() . SSH_PUBLIC_KEY; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-sftp-host-key-check-type'} = "none"; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-path'} = $self->repoPath(); - - # At what count do we hit diminishing returns - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'process-max'} = 8; - - $oParamHash{&CFGDEF_SECTION_GLOBAL . ':backup'}{'start-fast'} = 'y'; - } - else - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host'} = $oHostBackup->nameGet(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-cmd'} = $oHostBackup->backrestExe(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-config'} = $oHostBackup->backrestConfig(); - } - - if ($oParam->{bTls}) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-type'} = 'tls'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-cert-file'} = testRunGet()->basePath() . HOST_CLIENT_CERT; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo1-host-key-file'} = testRunGet()->basePath() . HOST_CLIENT_KEY; - } - - if ($iRepoTotal == 2) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host'} = $oHostBackup->nameGet(); - $oParam->{bTls} ? $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-type'} = 'tls' : undef; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-user'} = $oHostBackup->userGet(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-cmd'} = $oHostBackup->backrestExe(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-config'} = $oHostBackup->backrestConfig(); - - if ($oParam->{bTls}) - { - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-type'} = 'tls'; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-cert-file'} = testRunGet()->basePath() . HOST_CLIENT_CERT; - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'repo2-host-key-file'} = testRunGet()->basePath() . HOST_CLIENT_KEY; - } - } - - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'log-path'} = $self->logPath(); - $oParamHash{&CFGDEF_SECTION_GLOBAL}{'lock-path'} = $self->lockPath(); - } - } - - # Write out the configuration file - storageTest()->put($self->backrestConfig(), iniRender(\%oParamHash, true)); -} - -#################################################################################################################################### -# configUpdate - update configuration with new options -#################################################################################################################################### -sub configUpdate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $hParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->configUpdate', \@_, - {name => 'hParam'}, - ); - - # Load db config file - my $oConfig = iniParse(${storageTest()->get($self->backrestConfig())}, {bRelaxed => true}); - - # Load params - foreach my $strSection (keys(%{$hParam})) - { - foreach my $strKey (keys(%{$hParam->{$strSection}})) - { - if (defined($hParam->{$strSection}{$strKey})) - { - $oConfig->{$strSection}{$strKey} = $hParam->{$strSection}{$strKey}; - } - else - { - delete($oConfig->{$strSection}{$strKey}); - } - } - } - - storageTest()->put($self->backrestConfig(), iniRender($oConfig, true)); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# manifestMunge -# -# Allows for munging of the manifest while making it appear to be valid. This is used to create various error conditions that should -# be caught by the unit tests. -#################################################################################################################################### -sub manifestMunge -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strBackup, - $hParam, - $bCache, - ) = - logDebugParam - ( - __PACKAGE__ . '->manifestMunge', \@_, - {name => 'strBackup'}, - {name => '$hParam'}, - {name => 'bCache', default => true}, - ); - - $self->infoMunge($self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), $hParam, $bCache, true); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# manifestRestore -#################################################################################################################################### -sub manifestRestore -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strBackup, - $bSave, - ) = - logDebugParam - ( - __PACKAGE__ . '->manifestRestore', \@_, - {name => 'strBackup'}, - {name => 'bSave', default => true}, - ); - - $self->infoRestore($self->repoBackupPath("${strBackup}/" . FILE_MANIFEST), $bSave); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# infoMunge -# -# With the file name specified (e.g. /repo/archive/db/archive.info) copy the current values from the file into the global hash and -# update the file with the new values passed. Later, using infoRestore, the global variable will be used to restore the file to its -# original state. -#################################################################################################################################### -sub infoMunge -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFileName, - $hParam, - $bCache, - $bManifest, - ) = - logDebugParam - ( - __PACKAGE__ . '->infoMunge', \@_, - {name => 'strFileName'}, - {name => 'hParam'}, - {name => 'bCache', default => true}, - {name => 'bManifest', default => false}, - ); - - # If the original file content does not exist then load it - if (!defined($self->{hInfoFile}{$strFileName})) - { - $self->{hInfoFile}{$strFileName} = new pgBackRestDoc::Common::Ini( - storageRepo(), $strFileName, - {strCipherPass => !$bManifest ? undef : $self->cipherPassManifest()}); - } - - # Make a copy of the original file contents - my $oMungeIni = new pgBackRestDoc::Common::Ini( - storageRepo(), $strFileName, - {bLoad => false, strContent => iniRender($self->{hInfoFile}{$strFileName}->{oContent}), - strCipherPass => !$bManifest ? undef : $self->cipherPassManifest()}); - - # Load params - foreach my $strSection (keys(%{$hParam})) - { - foreach my $strKey (keys(%{$hParam->{$strSection}})) - { - if (ref($hParam->{$strSection}{$strKey}) eq 'HASH') - { - foreach my $strSubKey (keys(%{$hParam->{$strSection}{$strKey}})) - { - # Munge the copy with the new parameter values - if (defined($hParam->{$strSection}{$strKey}{$strSubKey})) - { - $oMungeIni->set($strSection, $strKey, $strSubKey, $hParam->{$strSection}{$strKey}{$strSubKey}); - } - else - { - $oMungeIni->remove($strSection, $strKey, $strSubKey); - } - } - } - else - { - # Munge the copy with the new parameter values - if (defined($hParam->{$strSection}{$strKey})) - { - $oMungeIni->set($strSection, $strKey, undef, $hParam->{$strSection}{$strKey}); - } - else - { - $oMungeIni->remove($strSection, $strKey); - } - } - } - } - - # Save the munged data to the file - $oMungeIni->save(); - - # Clear the cache is requested - if (!$bCache) - { - delete($self->{hInfoFile}{$strFileName}); - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# infoRestore -# -# With the file name specified (e.g. /repo/archive/db/archive.info) use the original file contents in the global hash to restore the -# file to its original state after modifying the values with infoMunge. -#################################################################################################################################### -sub infoRestore -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFileName, - $bSave, - ) = - logDebugParam - ( - __PACKAGE__ . '->infoRestore', \@_, - {name => 'strFileName'}, - {name => 'bSave', default => true}, - ); - - # If the original file content exists in the global hash, then save it to the file - if (defined($self->{hInfoFile}{$strFileName})) - { - if ($bSave) - { - # Save the munged data to the file - $self->{hInfoFile}{$strFileName}->{bModified} = true; - $self->{hInfoFile}{$strFileName}->save(); - } - } - else - { - confess &log(ASSERT, "There is no original data cached for $strFileName. infoMunge must be called first."); - } - - # Remove the element from the hash - delete($self->{hInfoFile}{$strFileName}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# configRecovery -#################################################################################################################################### -sub configRecovery -{ - my $self = shift; - my $oHostBackup = shift; - my $oRecoveryHashRef = shift; - - # Get stanza - my $strStanza = $self->stanza(); - - # Load db config file - my $oConfig = iniParse(${storageTest->get($self->backrestConfig())}, {bRelaxed => true}); - - # Rewrite recovery options - my @stryRecoveryOption; - - foreach my $strOption (sort(keys(%$oRecoveryHashRef))) - { - push (@stryRecoveryOption, "${strOption}=${$oRecoveryHashRef}{$strOption}"); - } - - if (@stryRecoveryOption) - { - $oConfig->{$strStanza}{'recovery-option'} = \@stryRecoveryOption; - } - - # Save db config file - storageTest()->put($self->backrestConfig(), iniRender($oConfig, true)); -} - -#################################################################################################################################### -# configRemap -#################################################################################################################################### -sub configRemap -{ - my $self = shift; - my $oRemapHashRef = shift; - my $oManifestRef = shift; - - # Get stanza name - my $strStanza = $self->stanza(); - - # Load db config file - my $oConfig = iniParse(${storageTest()->get($self->backrestConfig())}, {bRelaxed => true}); - - # Load backup config file - my $oRemoteConfig; - my $oHostBackup = - !$self->standby() && !$self->nameTest($self->backupDestination()) ? - hostGroupGet()->hostGet($self->backupDestination()) : undef; - - if (defined($oHostBackup)) - { - $oRemoteConfig = iniParse(${storageTest()->get($oHostBackup->backrestConfig())}, {bRelaxed => true}); - } - - # Rewrite recovery section - delete($oConfig->{"${strStanza}:restore"}{'tablespace-map'}); - my @stryTablespaceMap; - - foreach my $strRemap (sort(keys(%$oRemapHashRef))) - { - my $strRemapPath = ${$oRemapHashRef}{$strRemap}; - - if ($strRemap eq MANIFEST_TARGET_PGDATA) - { - $oConfig->{$strStanza}{'pg1-path'} = $strRemapPath; - - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_PATH} = $strRemapPath; - - if (defined($oHostBackup)) - { - $oRemoteConfig->{$strStanza}{'pg1-path'} = $strRemapPath; - } - } - else - { - my $strTablespaceOid = (split('\/', $strRemap))[1]; - push (@stryTablespaceMap, "${strTablespaceOid}=${strRemapPath}"); - - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strRemap}{&MANIFEST_SUBKEY_PATH} = $strRemapPath; - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{MANIFEST_TARGET_PGDATA . "/${strRemap}"}{destination} = $strRemapPath; - } - } - - if (@stryTablespaceMap) - { - $oConfig->{"${strStanza}:restore"}{'tablespace-map'} = \@stryTablespaceMap; - } - - # Save db config file - storageTest()->put($self->backrestConfig(), iniRender($oConfig, true)); - - # Save backup config file (but not if this is the standby which is not the source of backups) - if (defined($oHostBackup)) - { - storageTest()->put($oHostBackup->backrestConfig(), iniRender($oRemoteConfig, true)); - } -} - -#################################################################################################################################### -# restore -#################################################################################################################################### -sub restore -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strComment, - $strBackup, - $rhExpectedManifest, - $rhRemapHash, - $bDelta, - $bForce, - $strType, - $strTarget, - $bTargetExclusive, - $strTargetAction, - $strTargetTimeline, - $rhRecoveryHash, - $iExpectedExitStatus, - $strOptionalParam, - $bTablespace, - $strUser, - $strBackupExpected, - $iRepo, - ) = - logDebugParam - ( - __PACKAGE__ . '->restore', \@_, - {name => 'strComment', required => false}, - {name => 'strBackup'}, - {name => 'rhExpectedManifest', optional => true}, - {name => 'rhRemapHash', optional => true}, - {name => 'bDelta', optional => true, default => false}, - {name => 'bForce', optional => true, default => false}, - {name => 'strType', optional => true}, - {name => 'strTarget', optional => true}, - {name => 'bTargetExclusive', optional => true, default => false}, - {name => 'strTargetAction', optional => true}, - {name => 'strTargetTimeline', optional => true}, - {name => 'rhRecoveryHash', optional => true}, - {name => 'iExpectedExitStatus', optional => true}, - {name => 'strOptionalParam', optional => true}, - {name => 'bTablespace', optional => true}, - {name => 'strUser', optional => true}, - {name => 'strBackupExpected', optional => true}, - {name => 'iRepo', optional => true}, - ); - - # Build link map options - my $strLinkMap; - - foreach my $strTarget (sort(keys(%{$self->{hLinkRemap}}))) - { - $strLinkMap .= " --link-map=\"${strTarget}=${$self->{hLinkRemap}}{$strTarget}\""; - } - - $strComment = 'restore' . - ($bDelta ? ' delta' : '') . - ($bForce ? ', force' : '') . - ($strBackup ne 'latest' ? ", backup '${strBackup}'" : '') . - # This does not output 'default' for synthetic tests to make expect logs match up (may change later) - ($strType ? ", type '${strType}'" : (defined($rhExpectedManifest) ? '' : ", type 'default'")) . - ($strTarget ? ", target '${strTarget}'" : '') . - ($strTargetTimeline ? ", timeline '${strTargetTimeline}'" : '') . - ($bTargetExclusive ? ', exclusive' : '') . - (defined($strTargetAction) && $strTargetAction ne 'pause' ? ", target-action=${strTargetAction}" : '') . - (defined($rhRemapHash) ? ', remap' : '') . - (defined($iExpectedExitStatus) ? ", expect exit ${iExpectedExitStatus}" : '') . - (defined($strComment) ? " - ${strComment}" : '') . - ' (' . $self->nameGet() . ' host)'; - &log(INFO, " ${strComment}"); - - # Get the backup host - my $oHostGroup = hostGroupGet(); - my $oHostBackup = defined($oHostGroup->hostGet(HOST_BACKUP, true)) ? $oHostGroup->hostGet(HOST_BACKUP) : $self; - - # If the repo was not passed, then use repo1 as the repo for getting the expected manifest/backup - my $iRepoDefault = !defined($iRepo) ? 1 : $iRepo; - - # Load the expected manifest if it was not defined - my $oExpectedManifest = undef; - - # If an expected backup is defined, then the strBackup should be the default to allow the restore process to select the backup - # - which should be the backup passed as strBackupExpected. If it is not defined, then set it based on the strBackup passed. - if (!defined($strBackupExpected)) - { - $strBackupExpected = $strBackup eq 'latest' ? $oHostBackup->backupLast($iRepoDefault) : $strBackup; - } - - if (!defined($rhExpectedManifest)) - { - # Load the manifest from the backup expected to be chosen/processed by restore - my $oExpectedManifest = new pgBackRestTest::Env::Manifest( - $self->repoBackupPath($strBackupExpected . qw{/} . FILE_MANIFEST, $iRepoDefault), - {strCipherPass => $oHostBackup->cipherPassManifest(), oStorage => storageRepo({iRepo => $iRepoDefault})}); - - $rhExpectedManifest = $oExpectedManifest->{oContent}; - - # Remap links in the expected manifest - foreach my $strTarget (sort(keys(%{$self->{hLinkRemap}}))) - { - my $strDestination = ${$self->{hLinkRemap}}{$strTarget}; - my $strTarget = 'pg_data/' . $strTarget; - my $strTargetPath = $strDestination; - - # If this link is to a file then the specified path must be split into file and path parts - if ($oExpectedManifest->isTargetFile($strTarget)) - { - $strTargetPath = dirname($strTargetPath); - - # Error when the path is not deep enough to be valid - if (!defined($strTargetPath)) - { - confess &log(ERROR, "${strDestination} is not long enough to be target for ${strTarget}"); - } - - # Set the file part - $oExpectedManifest->set( - MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_FILE, - substr($strDestination, length($strTargetPath) + 1)); - - # Set the link target - $oExpectedManifest->set( - MANIFEST_SECTION_TARGET_LINK, $strTarget, MANIFEST_SUBKEY_DESTINATION, $strDestination); - } - else - { - # Set the link target - $oExpectedManifest->set(MANIFEST_SECTION_TARGET_LINK, $strTarget, MANIFEST_SUBKEY_DESTINATION, $strTargetPath); - } - - # Set the target path - $oExpectedManifest->set(MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_PATH, $strTargetPath); - } - } - - # Get the backup host - if (defined($rhRemapHash)) - { - $self->configRemap($rhRemapHash, $rhExpectedManifest); - } - - if (defined($rhRecoveryHash)) - { - $self->configRecovery($oHostBackup, $rhRecoveryHash); - } - - # Create the restore command - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - ($bDelta ? ' --delta' : '') . - ($bForce ? ' --force' : '') . - ($strBackup ne 'latest' ? " --set=${strBackup}" : '') . - (defined($strOptionalParam) ? " ${strOptionalParam} " : '') . - (defined($strType) && $strType ne CFGOPTVAL_RESTORE_TYPE_DEFAULT ? " --type=${strType}" : '') . - (defined($strTarget) ? " --target=\"${strTarget}\"" : '') . - (defined($strTargetTimeline) ? " --target-timeline=\"${strTargetTimeline}\"" : '') . - ($bTargetExclusive ? ' --target-exclusive' : '') . - (defined($strLinkMap) ? $strLinkMap : '') . - ($self->synthetic() ? '' : ' --link-all') . - (defined($strTargetAction) && $strTargetAction ne 'pause' ? " --target-action=${strTargetAction}" : '') . - (defined($iRepo) ? " --repo=${iRepo}" : '') . - " --stanza=" . $self->stanza() . ' restore', - {strComment => $strComment, iExpectedExitStatus => $iExpectedExitStatus, bLogOutput => $self->synthetic()}, - $strUser); - - if (!defined($iExpectedExitStatus)) - { - # Only compare restores in repo1. There is not a lot of value in comparing restores in other repos and it would require a - # lot of changes to the Perl test harness. - if ($iRepoDefault == 1) - { - $self->restoreCompare($strBackupExpected, dclone($rhExpectedManifest), $bTablespace); - } - } -} - -#################################################################################################################################### -# restoreCompare -#################################################################################################################################### -sub restoreCompare -{ - my $self = shift; - my $strBackup = shift; - my $oExpectedManifestRef = shift; - my $bTablespace = shift; - - my $strTestPath = $self->testPath(); - - # Get the backup host - my $oHostGroup = hostGroupGet(); - my $oHostBackup = defined($oHostGroup->hostGet(HOST_BACKUP, true)) ? $oHostGroup->hostGet(HOST_BACKUP) : $self; - - # Load the last manifest if it exists - my $oLastManifest = undef; - - if (defined(${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR})) - { - my $oExpectedManifest = - new pgBackRestTest::Env::Manifest( - $self->repoBackupPath( - ($strBackup eq 'latest' ? $oHostBackup->backupLast() : $strBackup) . '/' . FILE_MANIFEST), - {strCipherPass => $oHostBackup->cipherPassManifest()}); - - # Get the --delta option from the backup manifest so the actual manifest can be built the same way for comparison - $$oExpectedManifestRef{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA} = - $oExpectedManifest->get(MANIFEST_SECTION_BACKUP_OPTION, &MANIFEST_KEY_DELTA); - - $oLastManifest = - new pgBackRestTest::Env::Manifest( - $self->repoBackupPath( - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR} . qw{/} . FILE_MANIFEST), - {strCipherPass => $oHostBackup->cipherPassManifest()}); - } - - # Generate the tablespace map for real backups - my $oTablespaceMap = undef; - - if (!$self->synthetic()) - { - # Tablespace_map file is not restored in versions >= 9.5 because it interferes with internal remapping features. - if (${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} >= PG_VERSION_95) - { - delete(${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{MANIFEST_TARGET_PGDATA . '/tablespace_map'}); - } - - foreach my $strTarget (keys(%{${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}})) - { - if (defined(${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID})) - { - my $iTablespaceId = - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID}; - - $$oTablespaceMap{$iTablespaceId} = - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_NAME}; - } - } - } - - # Generate the actual manifest - my $strDbClusterPath = - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_PATH}; - - if (defined($bTablespace) && !$bTablespace) - { - foreach my $strTarget (keys(%{${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}})) - { - if ($$oExpectedManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TYPE} eq - MANIFEST_VALUE_LINK && - defined($$oExpectedManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID})) - { - my $strRemapPath; - my $iTablespaceName = - $$oExpectedManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_NAME}; - - $strRemapPath = "../../tablespace/${iTablespaceName}"; - - $$oExpectedManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_PATH} = $strRemapPath; - $$oExpectedManifestRef{&MANIFEST_SECTION_TARGET_LINK}{MANIFEST_TARGET_PGDATA . "/${strTarget}"} - {&MANIFEST_SUBKEY_DESTINATION} = $strRemapPath; - } - } - } - - my $oActualManifest = new pgBackRestTest::Env::Manifest( - "${strTestPath}/" . FILE_MANIFEST, - {bLoad => false, strDbVersion => $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION}, - iDbCatalogVersion => $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG}, - oStorage => storageTest()}); - - # Build the actual manifest using the delta setting that was actually used by the latest backup if one exists - $oActualManifest->build(storageTest(), $strDbClusterPath, $oLastManifest, false, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA}, $oTablespaceMap); - $oActualManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_DELTA, undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS_TYPE, undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_TYPE}); - - my $strSectionPath = $oActualManifest->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH); - - foreach my $strName ($oActualManifest->keys(MANIFEST_SECTION_TARGET_FILE)) - { - # Remove repo checksum - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{'rck'}); - - # When bundling zero-length files will not have a reference - if ($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-bundle'} && - $oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_SIZE} == 0) - { - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE); - } - - # If synthetic match checksum errors since they can't be verified here - if ($self->synthetic) - { - my $bChecksumPage = $oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM_PAGE}; - - if (defined($bChecksumPage)) - { - $oActualManifest->boolSet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE, $bChecksumPage); - - if (!$bChecksumPage && - defined($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR})) - { - $oActualManifest->set( - MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR, - $oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR}); - } - } - } - # Else if page checksums are enabled make sure the correct files are being checksummed - else - { - if ($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_CHECKSUM_PAGE}) - { - if (defined($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM_PAGE}) != - isChecksumPage($strName)) - { - confess - "check-page actual for ${strName} is " . - ($oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, - MANIFEST_SUBKEY_CHECKSUM_PAGE) ? 'set' : '[undef]') . - ' but isChecksumPage() says it should be ' . - (isChecksumPage($strName) ? 'set' : '[undef]') . '.'; - } - - # Because the page checksum flag is copied to incr and diff from the previous backup but further processing is not - # done, they can't be expected to match so delete them. - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM_PAGE}); - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE); - } - } - - if (!$self->synthetic()) - { - $oActualManifest->set( - MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strName}{size}); - } - - # Remove repo-size, bn*, bi* from the manifest - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_REPO_SIZE}); - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, "bni"); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{"bni"}); - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, "bno"); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{"bno"}); - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, "bi"); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{"bi"}); - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, "bic"); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{"bic"}); - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, "bim"); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strName}{"bim"}); - - if ($oActualManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) != 0) - { - my $oStat = lstat($oActualManifest->dbPathGet($strSectionPath, $strName)); - - # When performing a selective restore, the files for the database(s) that are not restored are still copied but as empty - # sparse files (blocks == 0). If the file is not a sparse file or is a link, then get the actual checksum for comparison - if ($oStat->blocks > 0 || S_ISLNK($oStat->mode)) - { - my ($strHash) = storageTest()->hashSize($oActualManifest->dbPathGet($strSectionPath, $strName)); - - $oActualManifest->set( - MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM, $strHash); - - # If the delta option was set, it is possible that the checksum on the file changed from the last manifest. If so, - # then the file was expected to be copied by the backup and therefore the reference would have been removed. - if ($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA}) - { - # If the actual checksum and last manifest checksum don't match, remove the reference - if (defined($oLastManifest) && - $oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM) && - $strHash ne $oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM)) - { - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE); - } - } - } - else - { - # If there is a sparse file, remove the checksum and reference since they may or may not match. In this case, it is - # not important to check them since it is known that the file was intentionally not restored. - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM); - delete(${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_CHECKSUM}); - - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE); - delete(${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strName}{&MANIFEST_SUBKEY_REFERENCE}); - } - } - } - - # If PostgreSQL >= 12 don't compare postgresql.auto.conf since it will have recovery settings written into it - if (${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} >= PG_VERSION_12) - { - delete($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{'pg_data/postgresql.auto.conf'}); - $oActualManifest->remove(MANIFEST_SECTION_TARGET_FILE, 'pg_data/postgresql.auto.conf'); - } - - # If the link section is empty then delete it and the default section - if (keys(%{${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_LINK}}) == 0) - { - delete($$oExpectedManifestRef{&MANIFEST_SECTION_TARGET_LINK}); - delete($$oExpectedManifestRef{&MANIFEST_SECTION_TARGET_LINK . ':default'}); - } - - # Set actual to expected for settings that always change from backup to backup - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_CHECK, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ARCHIVE_CHECK}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ARCHIVE_COPY, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ARCHIVE_COPY}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_BACKUP_STANDBY, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BACKUP_STANDBY}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_BACKUP_STANDBY, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BACKUP_STANDBY}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_BUFFER_SIZE, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_BUFFER_SIZE}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS_LEVEL, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_LEVEL}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS_LEVEL_NETWORK, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_COMPRESS_LEVEL_NETWORK}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_HARDLINK, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_HARDLINK}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ONLINE, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ONLINE}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_PROCESS_MAX, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_PROCESS_MAX}); - $oActualManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_DELTA, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_DELTA}); - - $oActualManifest->set(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_VERSION, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION}); - $oActualManifest->numericSet(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_CONTROL, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CONTROL}); - $oActualManifest->numericSet(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_CATALOG, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG}); - $oActualManifest->numericSet(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_SYSTEM_ID, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_SYSTEM_ID}); - $oActualManifest->numericSet(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_ID, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_ID}); - - $oActualManifest->set(INI_SECTION_BACKREST, INI_KEY_VERSION, undef, - ${$oExpectedManifestRef}{&INI_SECTION_BACKREST}{&INI_KEY_VERSION}); - - # Copy passphrase if one exists - if (defined($oExpectedManifestRef->{&INI_SECTION_CIPHER}) && - defined($oExpectedManifestRef->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS})) - { - $oActualManifest->set(INI_SECTION_CIPHER, INI_KEY_CIPHER_PASS, undef, - $oExpectedManifestRef->{&INI_SECTION_CIPHER}{&INI_KEY_CIPHER_PASS}); - } - - # This option won't be set in the actual manifest - delete($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_CHECKSUM_PAGE}); - - if ($self->synthetic()) - { - $oActualManifest->remove(MANIFEST_SECTION_BACKUP); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}); - } - else - { - $oActualManifest->set( - INI_SECTION_BACKREST, INI_KEY_CHECKSUM, undef, $oExpectedManifestRef->{&INI_SECTION_BACKREST}{&INI_KEY_CHECKSUM}); - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL, undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL}); - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, 'backup-reference', undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-reference'}); - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START, undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_COPY_START}); - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_START, undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START}); - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_STOP, undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_STOP}); - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TYPE, undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TYPE}); - - if (defined($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-bundle'})) - { - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, 'backup-bundle', undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-bundle'}); - } - - if (defined($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-bundle-raw'})) - { - $oActualManifest->set( - MANIFEST_SECTION_BACKUP, 'backup-bundle-raw', undef, - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-bundle-raw'}); - } - - # Delete block incr headers since old Perl manifest code will not generate them - delete($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-block-incr'}); - delete($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP}{'backup-block-incr-size'}); - - $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LSN_START, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LSN_START}); - $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LSN_STOP, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LSN_STOP}); - - if (defined(${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_ARCHIVE_START})) - { - $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_START, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_ARCHIVE_START}); - } - - if (${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_ARCHIVE_STOP}) - { - $oActualManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_ARCHIVE_STOP, undef, - ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_ARCHIVE_STOP}); - } - } - - # Check that archive status exists in the manifest for an online backup - my $strArchiveStatusPath = MANIFEST_TARGET_PGDATA . qw{/} . $oActualManifest->walPath() . qw{/} . DB_PATH_ARCHIVESTATUS; - - if ($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ONLINE} && - !defined($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_PATH}{$strArchiveStatusPath})) - { - confess &log(ERROR, "${strArchiveStatusPath} expected for online backup", ERROR_ASSERT); - } - - # Delete the list of DBs - delete($$oExpectedManifestRef{&MANIFEST_SECTION_DB}); - - # Only update defaults if the expect manifest is synthetic. If loaded from a file the defaults will already be correct. - if ($self->synthetic()) - { - $self->manifestDefault($oExpectedManifestRef); - } - - # Newer Perls will change this variable to a number whenever a numeric comparison is performed. It is expected to be a string so - # make sure it is one before saving. - $oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} .= ''; - - storageTest()->put("${strTestPath}/actual.manifest", iniRender($oActualManifest->{oContent})); - storageTest()->put("${strTestPath}/expected.manifest", iniRender($oExpectedManifestRef)); - - executeTest("diff ${strTestPath}/expected.manifest ${strTestPath}/actual.manifest"); - - storageTest()->remove("${strTestPath}/expected.manifest"); - storageTest()->remove("${strTestPath}/actual.manifest"); -} - -#################################################################################################################################### -# Get repo backup/archive path -#################################################################################################################################### -sub repoSubPath -{ - my $self = shift; - my $strSubPath = shift; - my $strPath = shift; - my $iRepo = shift; - - my $strRepoPath = $self->repoPath(); - - if (defined($iRepo) && $iRepo == 2) - { - $strRepoPath = $self->repo2Path(); - } - - return - ($strRepoPath eq '/' ? '' : $strRepoPath) . "/${strSubPath}/" . $self->stanza() . - (defined($strPath) ? "/${strPath}" : ''); -} - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub backrestConfig {return shift->{strBackRestConfig}} -sub backupDestination {return shift->{strBackupDestination}} -sub backrestExe {return testRunGet()->backrestExe()} -sub bogusHost {return shift->{bBogusHost}} -sub hardLink {return shift->{bHardLink}} -sub hasLink {storageRepo()->capability(STORAGE_CAPABILITY_LINK)} -sub isFS {storageRepo()->type() ne STORAGE_OBJECT} -sub isHostBackup {my $self = shift; return $self->backupDestination() eq $self->nameGet()} -sub isHostDbPrimary {return shift->nameGet() eq HOST_DB_PRIMARY} -sub isHostDbStandby {return shift->nameGet() eq HOST_DB_STANDBY} -sub isHostDb {my $self = shift; return $self->isHostDbPrimary() || $self->isHostDbStandby()} -sub lockPath {return shift->{strLockPath}} -sub logPath {return shift->{strLogPath}} -sub repoArchivePath {return shift->repoSubPath('archive', shift)} -sub repoBackupPath {return shift->repoSubPath('backup', shift, shift)} -sub repoPath {return shift->{strRepoPath}} -sub repo2Path {return shift->{strRepo2Path}} -sub repoEncrypt {return shift->{bRepoEncrypt}} -sub stanza {return testRunGet()->stanza()} -sub synthetic {return shift->{bSynthetic}} -sub cipherPassManifest {return shift->{strCipherPassManifest}} -sub cipherPassArchive {return shift->{strCipherPassArchive}} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostBaseTest.pm b/test/lib/pgBackRestTest/Env/Host/HostBaseTest.pm deleted file mode 100644 index 229055bbe8..0000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostBaseTest.pm +++ /dev/null @@ -1,139 +0,0 @@ -#################################################################################################################################### -# HostBackupTest.pm - Backup host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostBaseTest; -use parent 'pgBackRestTest::Common::HostTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Cwd qw(abs_path); -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); - -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::JobTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::VmTest; - -#################################################################################################################################### -# Host constants -#################################################################################################################################### -use constant HOST_BASE => 'base'; - push @EXPORT, qw(HOST_BASE); -use constant HOST_DB_PRIMARY => 'db-primary'; - push @EXPORT, qw(HOST_DB_PRIMARY); -use constant HOST_DB_STANDBY => 'db-standby'; - push @EXPORT, qw(HOST_DB_STANDBY); -use constant HOST_BACKUP => 'backup'; - push @EXPORT, qw(HOST_BACKUP); -use constant HOST_GCS => 'gcs'; - push @EXPORT, qw(HOST_GCS); -use constant HOST_AZURE => 'azure'; - push @EXPORT, qw(HOST_AZURE); -use constant HOST_S3 => 's3-server'; - push @EXPORT, qw(HOST_S3); -use constant HOST_SFTP => 'sftp-srvr'; - push @EXPORT, qw(HOST_SFTP); - -#################################################################################################################################### -# CA/cert/key constants -#################################################################################################################################### -use constant HOST_CERT_PATH => '/test/certificate/'; - -use constant HOST_CLIENT_CERT => HOST_CERT_PATH . 'pgbackrest-test-client.crt'; - push @EXPORT, qw(HOST_CLIENT_CERT); -use constant HOST_CLIENT_KEY => HOST_CERT_PATH . 'pgbackrest-test-client.key'; - push @EXPORT, qw(HOST_CLIENT_KEY); - -use constant HOST_SERVER_CA => HOST_CERT_PATH . 'pgbackrest-test-ca.crt'; - push @EXPORT, qw(HOST_SERVER_CA); -use constant HOST_SERVER_CERT => HOST_CERT_PATH . 'pgbackrest-test-server.crt'; -use constant HOST_SERVER_KEY => HOST_CERT_PATH . 'pgbackrest-test-server.key'; - -#################################################################################################################################### -# SFTP key constants -#################################################################################################################################### -use constant SSH_KEY_PATH => '/test/certificate/ssh/'; - -use constant SSH_PRIVATE_KEY => SSH_KEY_PATH . 'id_rsa'; - push @EXPORT, qw(SSH_PRIVATE_KEY); -use constant SSH_PUBLIC_KEY => SSH_KEY_PATH . 'id_rsa.pub'; - push @EXPORT, qw(SSH_PUBLIC_KEY); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strName, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'strName', default => HOST_BASE, trace => true}, - {name => 'oParam', required => false, trace => true}, - ); - - my $strTestPath = testRunGet()->testPath() . ($strName eq HOST_BASE ? '' : "/${strName}"); - storageTest()->pathCreate($strTestPath, {strMode => '0770'}); - - # Make sure keys have the correct permissions - if (chmod(0600, testRunGet()->basePath() . HOST_SERVER_KEY, testRunGet()->basePath() . HOST_CLIENT_KEY) != 2) - { - confess "unable to set mode on keys"; - } - - # Create the host - my $strProjectPath = dirname(dirname(abs_path($0))); - my $strBinPath = dirname(dirname($strTestPath)) . '/bin/' . testRunGet()->vm() . '/' . PROJECT_EXE; - my $strContainer = 'test-' . testRunGet()->vmId() . "-$strName"; - - my $self = $class->SUPER::new( - $strName, $strContainer, $$oParam{strImage}, $$oParam{strUser}, - ["${strProjectPath}:${strProjectPath}", "${strTestPath}:${strTestPath}", "${strBinPath}:${strBinPath}:ro"], undef, - $oParam->{bTls} ? - 'server --log-level-console=debug --tls-server-ca-file=' . testRunGet()->basePath() . HOST_SERVER_CA . - ' --tls-server-cert-file=' . testRunGet()->basePath() . HOST_SERVER_CERT . ' --tls-server-key-file=' . - testRunGet()->basePath() . HOST_SERVER_KEY . ' --tls-server-auth=pgbackrest-client=* --tls-server-address=0.0.0.0' : - undef, - undef, $oParam->{bTls} ? testRunGet()->backrestExe() : undef); - bless $self, $class; - - # Set test path - $self->{strTestPath} = $strTestPath; - - # Set permissions on the test path - $self->executeSimple('chown -R ' . $self->userGet() . ':'. TEST_GROUP . ' ' . $self->testPath(), undef, 'root'); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub testPath {return shift->{strTestPath}} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm b/test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm deleted file mode 100644 index 9418f35ce5..0000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm +++ /dev/null @@ -1,217 +0,0 @@ -#################################################################################################################################### -# HostDbTest.pm - Database host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostDbCommonTest; -use parent 'pgBackRestTest::Env::Host::HostBackupTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); -use Storable qw(dclone); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::Common::String; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# Test WAL size -#################################################################################################################################### -use constant PG_WAL_SIZE_TEST => 16777216; - -#################################################################################################################################### -# Host defaults -#################################################################################################################################### -use constant HOST_PATH_SPOOL => 'spool'; -use constant HOST_PATH_DB => 'db'; -use constant HOST_PATH_DB_BASE => 'base'; - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oParam', required => false, trace => true}, - ); - - # Get host group - my $oHostGroup = hostGroupGet(); - - # Is standby? - my $bStandby = defined($$oParam{bStandby}) && $$oParam{bStandby} ? true : false; - - my $self = $class->SUPER::new( - { - strName => $bStandby ? HOST_DB_STANDBY : HOST_DB_PRIMARY, - strImage => $$oParam{strImage}, - bTls => $oParam->{bTls}, - strBackupDestination => $$oParam{strBackupDestination}, - bSynthetic => $$oParam{bSynthetic}, - bRepoLocal => $oParam->{bRepoLocal}, - bRepoEncrypt => $oParam->{bRepoEncrypt}, - }); - bless $self, $class; - - # Set parameters - $self->{bStandby} = $bStandby; - - $self->{strDbPath} = $self->testPath() . '/' . HOST_PATH_DB; - $self->{strDbBasePath} = $self->dbPath() . '/' . HOST_PATH_DB_BASE; - $self->{strTablespacePath} = $self->dbPath() . '/tablespace'; - - storageTest()->pathCreate($self->dbBasePath(), {strMode => '0700', bCreateParent => true}); - - $self->{strSpoolPath} = $self->testPath() . '/' . HOST_PATH_SPOOL; - storageTest()->pathCreate($self->spoolPath()); - - # Initialize linkRemap Hashes - $self->{hLinkRemap} = {}; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# archivePush -#################################################################################################################################### -sub archivePush -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strWalPath, - $strArchiveTestFile, - $iArchiveNo, - $iExpectedError, - $bAsync, - $strOptionalParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->archivePush', \@_, - {name => 'strWalPath'}, - {name => 'strArchiveTestFile', required => false}, - {name => 'iArchiveNo', required => false}, - {name => 'iExpectedError', required => false}, - {name => 'bAsync', default => true}, - {name => 'strOptionalParam', required => false}, - ); - - my $strSourceFile; - - if (defined($strArchiveTestFile)) - { - $strSourceFile = "${strWalPath}/" . uc(sprintf('0000000100000001%08x', $iArchiveNo)); - - storageTest()->copy($strArchiveTestFile, storageTest()->openWrite($strSourceFile, {bPathCreate => true})); - - storageTest()->pathCreate("${strWalPath}/archive_status/", {bIgnoreExists => true, bCreateParent => true}); - storageTest()->put("${strWalPath}/archive_status/" . uc(sprintf('0000000100000001%08x', $iArchiveNo)) . '.ready'); - } - - $self->executeSimple( - $self->backrestExe() . - ' --config=' . $self->backrestConfig() . - ' --log-level-console=warn --archive-push-queue-max=' . int(2 * PG_WAL_SIZE_TEST) . - ' --stanza=' . $self->stanza() . - ($bAsync ? '' : ' --no-archive-async') . - " archive-push" . (defined($strSourceFile) ? " ${strSourceFile}" : '') . - (defined($strOptionalParam) ? " ${strOptionalParam}" : ''), - {iExpectedExitStatus => $iExpectedError, bLogOutput => $self->synthetic()}); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# linkRemap -#################################################################################################################################### -sub linkRemap -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strTarget, - $strDestination - ) = - logDebugParam - ( - __PACKAGE__ . '->linkRemap', \@_, - {name => 'strTarget'}, - {name => 'strDestination'}, - ); - - ${$self->{hLinkRemap}}{$strTarget} = $strDestination; - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub dbPath {return shift->{strDbPath};} - -sub dbBasePath -{ - my $self = shift; - my $iIndex = shift; - - return $self->{strDbBasePath} . (defined($iIndex) ? "-${iIndex}" : ''); -} - -sub spoolPath {return shift->{strSpoolPath}} -sub standby {return shift->{bStandby}} - -sub tablespacePath -{ - my $self = shift; - my $iTablespace = shift; - my $iIndex = shift; - - return - $self->{strTablespacePath} . - (defined($iTablespace) ? "/ts${iTablespace}" . - (defined($iIndex) ? "-${iIndex}" : '') : ''); -} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm b/test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm deleted file mode 100644 index ed6af2de32..0000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm +++ /dev/null @@ -1,717 +0,0 @@ -#################################################################################################################################### -# HostDbTest.pm - Database host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostDbSyntheticTest; -use parent 'pgBackRestTest::Env::Host::HostDbCommonTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Exporter qw(import); - our @EXPORT = qw(); -use Fcntl ':mode'; -use File::Basename qw(basename dirname); -use File::stat; - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::Common::String; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::FileTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Host::HostDbCommonTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oParam', required => false, trace => true}, - ); - - my $self = $class->SUPER::new( - { - strImage => containerRepo() . ':' . testRunGet()->vm() . "-test", - bTls => $oParam->{bTls}, - strBackupDestination => $$oParam{strBackupDestination}, - bSynthetic => true, - bStandby => $$oParam{bStandby}, - bRepoLocal => $oParam->{bRepoLocal}, - bRepoEncrypt => $oParam->{bRepoEncrypt}, - }); - bless $self, $class; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# dbFileCreate -# -# Create a file specifying content, mode, and time. -#################################################################################################################################### -sub dbFileCreate -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strFile = shift; - my $strContent = shift; - my $lTime = shift; - my $strMode = shift; - - # Check that strTarget is a valid - my $strPath = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_PATH}; - - if (!defined($strPath)) - { - confess &log(ERROR, "${strTarget} not a valid target: \n" . Dumper(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET})); - } - - # Get tablespace path if this is a tablespace - my $strPgPath; - - if (index($strTarget, DB_PATH_PGTBLSPC . '/') == 0) - { - my $iCatalog = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG}; - - $strPgPath = 'PG_' . ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} . "_${iCatalog}"; - } - - # Create actual file location - my $strPathFile = $strPath . - (defined($strPgPath) ? "/${strPgPath}" : '') . "/${strFile}"; - - if (index($strPathFile, '/') != 0) - { - $strPathFile = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_PATH} . '/' . - (defined(dirname($strPathFile)) ? dirname($strPathFile) : '') . "/${strPathFile}"; - } - - # Create the file - testFileCreate($strPathFile, $strContent, $lTime, $strMode); - - # Return path to created file - return $strPathFile; -} - -#################################################################################################################################### -# dbFileRemove -#################################################################################################################################### -sub dbFileRemove -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strFile = shift; - my $bIgnoreMissing = shift; - - # Get actual path location - my $strDbFile = $self->manifestDbPathGet($oManifestRef, $strTarget, $strFile); - - # Remove the file - if (!(defined($bIgnoreMissing) && $bIgnoreMissing && !(-e $strDbFile))) - { - testFileRemove($strDbFile); - } - - return $strDbFile; -} - -#################################################################################################################################### -# dbLinkCreate -# -# Create a file specifying content, mode, and time. -#################################################################################################################################### -sub dbLinkCreate -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strFile = shift; - my $strDestination = shift; - - # Create actual file location - my $strDbFile = $self->manifestDbPathGet($oManifestRef, $strTarget, $strFile); - - # Create the file - testLinkCreate($strDbFile, $strDestination); - - # Return path to created file - return $strDbFile; -} - -#################################################################################################################################### -# manifestDbPathGet -# -# Get the db path based on the target and file passed. -#################################################################################################################################### -sub manifestDbPathGet -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strFile = shift; - - # Determine the manifest key - my $strDbPath = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_PATH}; - - # If target is a tablespace - if (defined(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID})) - { - my $iCatalog = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG}; - - $strDbPath .= '/PG_' . ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} . "_${iCatalog}"; - } - - $strDbPath .= defined($strFile) ? "/${strFile}" : ''; - - return $strDbPath; -} - -#################################################################################################################################### -# manifestFileCreate -# -# Create a file specifying content, mode, and time and add it to the manifest. -#################################################################################################################################### -sub manifestFileCreate -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strFile = shift; - my $strContent = shift; - my $strChecksum = shift; - my $lTime = shift; - my $strMode = shift; - my $bPrimary = shift; - my $strChecksumPageError = shift; - - # Determine the manifest key - my $strManifestKey = $self->manifestKeyGet($oManifestRef, $strTarget, $strFile); - - # Create the file - my $strPathFile = $self->dbFileCreate($oManifestRef, $strTarget, $strFile, $strContent, $lTime, $strMode); - - # Stat the file - my $oStat = lstat($strPathFile); - - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_MODE} = - sprintf('%04o', S_IMODE($oStat->mode)); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_TIMESTAMP} = $oStat->mtime; - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_SIZE} = $oStat->size; - delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_REFERENCE}); - - my $bChecksumPage = defined($strChecksumPageError) ? false : (isChecksumPage($strManifestKey) ? true : undef); - - if (defined($bChecksumPage)) - { - $oManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_CHECKSUM_PAGE} = - $bChecksumPage ? JSON::PP::true : JSON::PP::false; - - if (!$bChecksumPage && $strChecksumPageError ne '0') - { - my @iyChecksumPageError = eval($strChecksumPageError); - - $oManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR} = - \@iyChecksumPageError; - } - else - { - delete($oManifestRef->{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR}); - } - } - - if (defined($strChecksum)) - { - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{checksum} = $strChecksum; - } -} - -#################################################################################################################################### -# manifestFileRemove -# -# Remove a file from disk and (optionally) the manifest. -#################################################################################################################################### -sub manifestFileRemove -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strFile = shift; - - # Determine the manifest key - my $strManifestKey = $self->manifestKeyGet($oManifestRef, $strTarget, $strFile); - - # Remove the file - $self->dbFileRemove($oManifestRef, $strTarget, $strFile, true); - - # Remove from manifest - delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}); -} - -#################################################################################################################################### -# manifestKeyGet -# -# Get the manifest key based on the target and file/path/link passed. -#################################################################################################################################### -sub manifestKeyGet -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strFile = shift; - - # Determine the manifest key - my $strManifestKey = $strTarget; - - # If target is a tablespace - if (defined(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID})) - { - my $iCatalog = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG}; - - $strManifestKey .= '/PG_' . ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} . "_${iCatalog}"; - } - - $strManifestKey .= (defined($strFile) ? "/$strFile" : ''); - - return $strManifestKey; -} - -#################################################################################################################################### -# manifestLinkCreate -# -# Create a link and add it to the manifest. -#################################################################################################################################### -sub manifestLinkCreate -{ - my $self = shift; - my $oManifestRef = shift; - my $strPath = shift; - my $strFile = shift; - my $strDestination = shift; - my $bPrimary = shift; - - # Determine the manifest key - my $strManifestKey = $self->manifestKeyGet($oManifestRef, $strPath, $strFile); - - # Load target - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_PATH} = $strDestination; - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_TYPE} = MANIFEST_VALUE_LINK; - - # Create the link - my $strDbFile = $self->dbLinkCreate($oManifestRef, $strPath, $strFile, $strDestination); - - # Stat the link - my $oStat = lstat($strDbFile); - - # Check for errors in stat - if (!defined($oStat)) - { - confess 'unable to stat ${strDbFile}'; - } - - # Load file into manifest - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strManifestKey}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strManifestKey}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strManifestKey}{&MANIFEST_SUBKEY_DESTINATION} = $strDestination; - - # Stat what the link is pointing to - my $strDestinationFile = $strDestination; - - if (index($strDestinationFile, '/') != 0) - { - $strDestinationFile = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGDATA}{&MANIFEST_SUBKEY_PATH} . '/' . - (defined(dirname($strPath)) ? dirname($strPath) : '') . "/${strDestination}"; - } - - $oStat = lstat($strDestinationFile); - - my $strSection = MANIFEST_SECTION_TARGET_PATH; - - if (S_ISREG($oStat->mode)) - { - $strSection = MANIFEST_SECTION_TARGET_FILE; - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_SIZE} = $oStat->size; - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_TIMESTAMP} = $oStat->mtime; - (${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_CHECKSUM}) = storageTest()->hashSize($strDestinationFile); - - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_FILE} = - basename(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_PATH}); - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_PATH} = - dirname(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}{&MANIFEST_SUBKEY_PATH}); - } - # Allow a link to a link to be created to test that backrest errors out correctly - elsif (S_ISLNK($oStat->mode)) - { - $strSection = MANIFEST_SECTION_TARGET_LINK; - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_DESTINATION} = $strDestination; - } - elsif (!S_ISDIR($oStat->mode)) - { - confess &log(ASSERT, "unrecognized file type for file $strDestinationFile"); - } - - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid); - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_MODE} = sprintf('%04o', S_IMODE($oStat->mode)); -} - -#################################################################################################################################### -# manifestLinkMap -# -# Remap links to new directories/files -#################################################################################################################################### -sub manifestLinkMap -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strDestination = shift; - - if ($$oManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TYPE} ne MANIFEST_VALUE_LINK) - { - confess "cannot map target ${strTarget} because it is not a link"; - } - - if (defined($$oManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID})) - { - confess "tablespace ${strTarget} cannot be remapped with this function"; - } - - if (defined($strDestination)) - { - confess "GENERAL LINK REMAP NOT IMPLEMENTED"; - } - else - { - delete($$oManifestRef{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}); - delete($$oManifestRef{&MANIFEST_SECTION_TARGET_LINK}{$strTarget}); - } -} - -#################################################################################################################################### -# manifestLinkRemove -# -# Create a link and add it to the manifest. -#################################################################################################################################### -sub manifestLinkRemove -{ - my $self = shift; - my $oManifestRef = shift; - my $strPath = shift; - my $strFile = shift; - - # Delete the link - my $strDbFile = $self->dbFileRemove($oManifestRef, $strPath, $strFile); - - # Determine the manifest key - my $strManifestKey = $self->manifestKeyGet($oManifestRef, $strPath, $strFile); - - # Delete from manifest - delete(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strManifestKey}); - delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strManifestKey}); - delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}); - delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strManifestKey}); -} - -#################################################################################################################################### -# manifestPathCreate -# -# Create a path specifying mode and add it to the manifest. -#################################################################################################################################### -sub manifestPathCreate -{ - my $self = shift; - my $oManifestRef = shift; - my $strPath = shift; - my $strSubPath = shift; - my $strMode = shift; - - # Determine the manifest key - my $strManifestKey = $self->manifestKeyGet($oManifestRef, $strPath, $strSubPath); - - # Create the db path - my $strDbPath = $self->dbPathCreate($oManifestRef, $strPath, $strSubPath, defined($strMode) ? $strMode : '0700'); - - # Stat the file - my $oStat = lstat($strDbPath); - - # Check for errors in stat - if (!defined($oStat)) - { - confess 'unable to stat ${strSubPath}'; - } - - # Load file into manifest - my $strSection = MANIFEST_SECTION_TARGET_PATH; - - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid); - ${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_MODE} = sprintf('%04o', S_IMODE($oStat->mode)); -} - -#################################################################################################################################### -# manifestReference -# -# Update all files that do not have a reference with the supplied reference. -#################################################################################################################################### -sub manifestReference -{ - my $self = shift; - my $oManifestRef = shift; - my $strReference = shift; - my $bClear = shift; - - # Set prior backup - if (defined($strReference)) - { - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR} = $strReference; - } - else - { - delete(${$oManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR}); - } - - # Find all file sections - foreach my $strSectionFile (sort(keys(%$oManifestRef))) - { - # Skip non-file sections - if ($strSectionFile !~ /\:file$/) - { - next; - } - - foreach my $strFile (sort(keys(%{${$oManifestRef}{$strSectionFile}}))) - { - if (!defined($strReference)) - { - delete(${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE}); - } - elsif (defined($bClear) && $bClear) - { - if (defined(${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE}) && - ${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE} ne $strReference) - { - delete(${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE}); - } - } - elsif (!defined(${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE})) - { - ${$oManifestRef}{$strSectionFile}{$strFile}{&MANIFEST_SUBKEY_REFERENCE} = $strReference; - } - } - } -} - -#################################################################################################################################### -# manifestTablespaceCreate -# -# Create a tablespace specifying mode and add it to the manifest. -#################################################################################################################################### -sub manifestTablespaceCreate -{ - my $self = shift; - my $oManifestRef = shift; - my $iOid = shift; - my $strMode = shift; - - # Load linked path into manifest - my $strLinkPath = $self->tablespacePath($iOid); - my $strTarget = MANIFEST_TARGET_PGTBLSPC . "/${iOid}"; - my $oStat = lstat($strLinkPath); - - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget}{&MANIFEST_SUBKEY_MODE} = - sprintf('%04o', S_IMODE($oStat->mode)); - - # Create the tablespace path if it does not exist - my $strTablespacePath = $strLinkPath; - my $strPathTarget = $strTarget; - my $iCatalog = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG}; - my $strTablespaceId = 'PG_' . ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} . "_${iCatalog}"; - - $strTablespacePath .= "/${strTablespaceId}"; - $strPathTarget .= "/${strTablespaceId}"; - - if (!-e $strTablespacePath) - { - storageTest()->pathCreate($strTablespacePath, {strMode => defined($strMode) ? $strMode : '0700'}); - } - - # Load tablespace path into manifest - $oStat = lstat($strTablespacePath); - - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGTBLSPC} = - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA}; - - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strPathTarget}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strPathTarget}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strPathTarget}{&MANIFEST_SUBKEY_MODE} = - sprintf('%04o', S_IMODE($oStat->mode)); - - # Create the link in DB_PATH_PGTBLSPC - my $strLink = $self->dbBasePath() . '/' . DB_PATH_PGTBLSPC . "/${iOid}"; - - symlink($strLinkPath, $strLink) - or confess "unable to link ${strLink} to ${strLinkPath}"; - - # Load link into the manifest - $oStat = lstat($strLink); - my $strLinkTarget = MANIFEST_TARGET_PGDATA . "/${strTarget}"; - - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strLinkTarget}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strLinkTarget}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid); - ${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strLinkTarget}{&MANIFEST_SUBKEY_DESTINATION} = $strLinkPath; - - # Load tablespace target into the manifest - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_PATH} = $strLinkPath; - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TYPE} = MANIFEST_VALUE_LINK; - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_ID} = $iOid; - ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_TABLESPACE_NAME} = "ts${iOid}"; -} - -#################################################################################################################################### -# manifestTablespaceDrop -# -# Drop a tablespace add remove it from the manifest. -#################################################################################################################################### -sub manifestTablespaceDrop -{ - my $self = shift; - my $oManifestRef = shift; - my $iOid = shift; - my $iIndex = shift; - - # Remove tablespace path/file/link from manifest - my $strTarget = DB_PATH_PGTBLSPC . "/${iOid}"; - - # Remove manifest path, link, target - delete(${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}); - delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{&MANIFEST_TARGET_PGDATA . "/${strTarget}"}); - delete(${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget}); - - # Remove nested manifest files and paths - foreach my $strSection (&MANIFEST_SECTION_TARGET_PATH, &MANIFEST_SECTION_TARGET_FILE) - { - foreach my $strFile (keys(%{${$oManifestRef}{$strSection}})) - { - if (index($strFile, "${strTarget}/") == 0) - { - delete($$oManifestRef{$strSection}{$strFile}); - } - } - } - - # Drop the link in DB_PATH_PGTBLSPC - testFileRemove($self->dbBasePath($iIndex) . "/${strTarget}"); -} - -#################################################################################################################################### -# dbPathCreate -# -# Create a path specifying mode. -#################################################################################################################################### -sub dbPathCreate -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strSubPath = shift; - my $strMode = shift; - - # Create final file location - my $strFinalPath = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget}{&MANIFEST_SUBKEY_PATH}; - - # Get tablespace path if this is a tablespace - if (index($strTarget, DB_PATH_PGTBLSPC . '/') == 0) - { - my $iCatalog = ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG}; - - $strFinalPath .= '/PG_' . ${$oManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} . "_${iCatalog}"; - } - - $strFinalPath .= (defined($strSubPath) ? "/${strSubPath}" : ''); - - # Create the path - if (!(-e $strFinalPath)) - { - storageTest()->pathCreate($strFinalPath, {strMode => $strMode}); - } - - return $strFinalPath; -} - -#################################################################################################################################### -# dbPathMode -# -# Change the mode of a path. -#################################################################################################################################### -sub dbPathMode -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strPath = shift; - my $strMode = shift; - - # Get the db path - my $strDbPath = $self->manifestDbPathGet($oManifestRef, $strTarget, $strPath); - - testPathMode($strDbPath, $strMode); - - return $strDbPath; -} - -#################################################################################################################################### -# dbPathRemove -# -# Remove a path. -#################################################################################################################################### -sub dbPathRemove -{ - my $self = shift; - my $oManifestRef = shift; - my $strTarget = shift; - my $strPath = shift; - - # Get the db path - my $strDbPath = $self->manifestDbPathGet($oManifestRef, $strTarget, $strPath); - - # Create the path - testPathRemove($strDbPath); - - return $strDbPath; -} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostDbTest.pm b/test/lib/pgBackRestTest/Env/Host/HostDbTest.pm deleted file mode 100644 index 10a292d4ff..0000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostDbTest.pm +++ /dev/null @@ -1,528 +0,0 @@ -#################################################################################################################################### -# HostDbTest.pm - Database host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostDbTest; -use parent 'pgBackRestTest::Env::Host::HostDbCommonTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use DBI; -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(basename); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::Common::String; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Host::HostDbCommonTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# Db defaults -#################################################################################################################################### -use constant HOST_DB_DEFAULT => 'postgres'; -use constant HOST_DB_TIMEOUT => 30; - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'oParam', required => false, trace => true}, - ); - - # Get db version - my $strDbVersion = testRunGet()->pgVersion(); - - my $self = $class->SUPER::new( - { - strImage => containerRepo() . ':' . testRunGet()->vm() . "-test", - bTls => $oParam->{bTls}, - strBackupDestination => $$oParam{strBackupDestination}, - bStandby => $$oParam{bStandby}, - bRepoLocal => $oParam->{bRepoLocal}, - bRepoEncrypt => $oParam->{bRepoEncrypt}, - }); - bless $self, $class; - - # Set parameters - $self->{strPgSocketPath} = $self->dbPath(); - $self->{iPgPort} = defined($$oParam{bStandby}) && $$oParam{bStandby} ? 6544 : 6543; - - $self->{strPgLogPath} = $self->testPath(); - $self->{strPgLogFile} = $self->pgLogPath() . '/postgresql.log'; - - # Get Db version - if (defined($strDbVersion)) - { - my $strOutLog = $self->executeSimple($self->pgBinPath() . '/postgres --version'); - - my @stryVersionToken = split(/ /, $strOutLog); - @stryVersionToken = split(/\./, $stryVersionToken[2]); - my $strDbVersionActual = - trim($stryVersionToken[0]) . - (defined($stryVersionToken[1]) && trim($stryVersionToken[0]) < 10 ? '.' . trim($stryVersionToken[1]) : ''); - - # Warn if this is a devel/alpha/beta version - my $strVersionRegExp = '(devel|((alpha|beta|rc)[0-9]+))$'; - - if ($strDbVersionActual =~ /$strVersionRegExp/) - { - my $strDevVersion = $strDbVersionActual; - $strDbVersionActual =~ s/$strVersionRegExp//; - $strDevVersion = substr($strDevVersion, length($strDbVersionActual)); - - if (!defined($$oParam{bStandby}) || !$$oParam{bStandby}) - { - &log(WARN, 'Testing against ' . trim($strOutLog) . " ${strDevVersion}"); - } - } - elsif (!defined($$oParam{bStandby}) || !$$oParam{bStandby}) - { - &log(INFO, 'Testing against ' . trim($strOutLog)); - } - - # Don't run unit tests for unsupported versions - my @stryVersionSupport = versionSupport(); - - if ($strDbVersionActual < $stryVersionSupport[0]) - { - confess &log(ERROR, "only PostgreSQL version $stryVersionSupport[0] and up are supported"); - } - - if ($strDbVersion ne $strDbVersionActual) - { - confess &log(ERROR, "actual database version ${strDbVersionActual} does not match expected version ${strDbVersion}"); - } - } - - # Create wal directory - storageTest()->pathCreate($self->dbPath() . '/pg_' . $self->walId(), {strMode => '0700'}); - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -#################################################################################################################################### -# sqlConnect -#################################################################################################################################### -sub sqlConnect -{ - my $self = shift; - my $hParam = shift; - - # Set defaults - my $iTimeout = defined($$hParam{iTimeout}) ? $$hParam{iTimeout} : HOST_DB_TIMEOUT; - my $strDb = defined($$hParam{strDb}) ? $$hParam{strDb} : HOST_DB_DEFAULT; - - # If not connected - if (!defined($self->{db}{$strDb}{hDb})) - { - # Retry until connection is successful - my $oWait = waitInit($iTimeout); - - do - { - # Connect to the db (whether it is local or remote) - $self->{db}{$strDb}{hDb} = - DBI->connect( - "dbi:Pg:dbname=${strDb};port=" . $self->pgPort() . ';host=' . $self->pgSocketPath(), - $self->userGet(), undef, - {AutoCommit => 0, RaiseError => 0, PrintError => 0}); - - return $self->{db}{$strDb}{hDb} if $self->{db}{$strDb}{hDb}; - } - while (!defined($self->{db}{$strDb}{hDb}) && waitMore($oWait)); - - # Error if unable to connect - if (!defined($self->{db}{$strDb}{hDb})) - { - confess &log(ERROR, "unable to connect to PostgreSQL after ${iTimeout} second(s):\n" . $DBI::errstr, ERROR_DB_CONNECT); - } - } - - return $self->{db}{$strDb}{hDb}; -} - -#################################################################################################################################### -# sqlDisconnect -#################################################################################################################################### -sub sqlDisconnect -{ - my $self = shift; - my $hParam = shift; - - foreach my $strDb (keys(%{$self->{db}})) - { - if (defined($$hParam{$strDb}) && $$hParam{$strDb} ne $strDb) - { - next; - } - - if (defined($self->{db}{$strDb}{hDb})) - { - $self->{db}{$strDb}{hDb}->disconnect(); - undef($self->{db}{$strDb}{hDb}); - } - } -} - -#################################################################################################################################### -# sqlExecute -#################################################################################################################################### -sub sqlExecute -{ - my $self = shift; - my $strSql = shift; - my $hParam = shift; - - # Set defaults - my $bCheckPoint = defined($$hParam{bCheckPoint}) ? $$hParam{bCheckPoint} : false; - my $bCommit = defined($$hParam{bCommit}) ? $$hParam{bCommit} : true; - - # Get the db handle - my $hDb = $self->sqlConnect({strDb => $$hParam{strDb}}); - - # Set autocommit on/off - $hDb->{AutoCommit} = defined($$hParam{bAutoCommit}) ? ($$hParam{bAutoCommit} ? true : false) : false; - - # Log and execute the statement - &log(DETAIL, "SQL: ${strSql}"); - - my $hStatement = $hDb->prepare($strSql); - - $hStatement->execute() or - confess &log(ERROR, "Unable to execute: ${strSql}\n" . $DBI::errstr); - $hStatement->finish(); - - if ($bCommit && !$hDb->{AutoCommit}) - { - $self->sqlCommit(); - } - - # Perform a checkpoint if requested - if ($bCheckPoint) - { - $self->sqlExecute('checkpoint', {bCommit => false, bCheckPoint => false}); - } - - # Set autocommit off - $hDb->{AutoCommit} = 0; -} - -#################################################################################################################################### -# sqlSelect -#################################################################################################################################### -sub sqlSelect -{ - my $self = shift; - my $strSql = shift; - my $hParam = shift; - - # Get the db handle - my $hDb = $self->sqlConnect({strDb => $$hParam{strDb}}); - - # Log and execute the statement - &log(DEBUG, (defined($$hParam{strDb}) ? "DB: $$hParam{strDb}, " : "") . "SQL: ${strSql}"); - my $hStatement = $hDb->prepare($strSql); - - $hStatement = $hDb->prepare($strSql); - - $hStatement->execute() or - confess &log(ERROR, "Unable to execute: ${strSql}\n" . $DBI::errstr); - - my @oyRow = $hStatement->fetchrow_array(); - - $hStatement->finish(); - - return @oyRow; -} - -#################################################################################################################################### -# sqlSelectOne -#################################################################################################################################### -sub sqlSelectOne -{ - my $self = shift; - my $strSql = shift; - my $hParam = shift; - - return ($self->sqlSelect($strSql, $hParam))[0]; -} - -#################################################################################################################################### -# sqlSelectOneTest -#################################################################################################################################### -sub sqlSelectOneTest -{ - my $self = shift; - my $strSql = shift; - my $strExpectedValue = shift; - my $hParam = shift; - - # Set defaults - my $iTimeout = defined($$hParam{iTimeout}) ? $$hParam{iTimeout} : HOST_DB_TIMEOUT; - - my $lStartTime = time(); - my $strActualValue; - - do - { - $self->sqlConnect($hParam); - $strActualValue = $self->sqlSelectOne($strSql, $hParam); - - if (defined($strActualValue) && $strActualValue eq $strExpectedValue) - { - return; - } - - $self->sqlDisconnect(); - } - while (defined($iTimeout) && (time() - $lStartTime) <= $iTimeout); - - confess &log( - ERROR, "expected value '${strExpectedValue}' from '${strSql}' but actual was '" . - (defined($strActualValue) ? $strActualValue : '[undef]') . "'"); -} - -#################################################################################################################################### -# sqlCommit -#################################################################################################################################### -sub sqlCommit -{ - my $self = shift; - my $hParam = shift; - - my $bCheckPoint = defined($$hParam{bCheckPoint}) ? $$hParam{bCheckPoint} : false; - - $self->sqlExecute('commit', {bCommit => false, bCheckPoint => $bCheckPoint}); -} - -#################################################################################################################################### -# sqlWalRotate -#################################################################################################################################### -sub sqlWalRotate -{ - my $self = shift; - - $self->sqlExecute('select pg_switch_' . $self->walId() . '()', {bCommit => false, bCheckPoint => false}); -} - -#################################################################################################################################### -# clusterCreate -# -# Create the PostgreSQL cluster and start it. -#################################################################################################################################### -sub clusterCreate -{ - my $self = shift; - my $hParam = shift; - - # Set defaults - my $strWalPath = defined($$hParam{strWalPath}) ? $$hParam{strWalPath} : $self->dbPath() . '/pg_' . $self->walId(); - - $self->executeSimple( - $self->pgBinPath() . '/initdb -k' . - ($self->pgVersion() >= PG_VERSION_11 ? ' --wal-segsize=1' : '') . - ' --' . $self->walId() . "dir=${strWalPath}" . ' --pgdata=' . $self->dbBasePath() . ' --auth=trust'); - - if (!$self->standby()) - { - $self->executeSimple( - "echo 'host replication replicator db-standby trust' >> " . $self->dbBasePath() . '/pg_hba.conf'); - } - - $self->clusterStart( - {bHotStandby => $$hParam{bHotStandby}, bArchive => $$hParam{bArchive}, bArchiveAlways => $$hParam{bArchiveAlways}, - bArchiveInvalid => $$hParam{bArchiveInvalid}}); - - if (!$self->standby()) - { - $self->sqlExecute("create user replicator replication", {bCommit =>true}); - } -} - -#################################################################################################################################### -# clusterStart -# -# Start the PostgreSQL cluster with various test options. -#################################################################################################################################### -sub clusterStart -{ - my $self = shift; - my $hParam = shift; - - # Set defaults - my $bHotStandby = defined($$hParam{bHotStandby}) ? $$hParam{bHotStandby} : false; - my $bArchive = defined($$hParam{bArchive}) ? $$hParam{bArchive} : true; - my $bArchiveAlways = defined($$hParam{bArchiveAlways}) ? $$hParam{bArchiveAlways} : false; - my $bArchiveInvalid = defined($$hParam{bArchiveInvalid}) ? $$hParam{bArchiveInvalid} : false; - my $bArchiveEnabled = defined($$hParam{bArchiveEnabled}) ? $$hParam{bArchiveEnabled} : true; - - # Make sure postgres is not running - if (-e $self->dbBasePath() . '/' . DB_FILE_POSTMTRPID) - { - confess DB_FILE_POSTMTRPID . ' exists'; - } - - # Create the archive command - my $strArchive = - $self->backrestExe() . ' --stanza=' . ($bArchiveInvalid ? 'bogus' : $self->stanza()) . - ' --config=' . $self->backrestConfig() . ' archive-push %p'; - - # Start the cluster - my $strCommand = - $self->pgBinPath() . '/pg_ctl start -o "-c port=' . $self->pgPort() . - ($self->pgVersion() < PG_VERSION_95 ? ' -c checkpoint_segments=1' : ''); - - if ($bArchiveEnabled) - { - if ($self->pgVersion() >= PG_VERSION_95 && $bArchiveAlways) - { - $strCommand .= " -c archive_mode=always"; - } - else - { - $strCommand .= " -c archive_mode=on"; - } - } - else - { - $strCommand .= " -c archive_mode=off"; - } - - if ($bArchive) - { - $strCommand .= " -c archive_command='${strArchive}'"; - } - else - { - $strCommand .= " -c archive_command=true"; - } - - $strCommand .= ' -c wal_level=hot_standby -c hot_standby=' . ($bHotStandby ? 'on' : 'off'); - - # Force parallel mode on to make sure we are disabling it and there are no issues. This is important for testing that 9.6 - # works since pg_stop_backup() is marked parallel safe and will error if run in a worker. - if ($self->pgVersion() >= PG_VERSION_96) - { - if ($self->pgVersion() >= PG_VERSION_16) - { - $strCommand .= " -c debug_parallel_query='on'"; - } - else - { - $strCommand .= " -c force_parallel_mode='on'"; - } - - $strCommand .= " -c max_parallel_workers_per_gather=2"; - } - - $strCommand .= - ' -c max_wal_senders=3' . - ' -c listen_addresses=\'*\'' . - ' -c log_directory=\'' . $self->pgLogPath() . "'" . - ' -c log_filename=\'' . basename($self->pgLogFile()) . "'" . - ' -c log_rotation_age=0' . - ' -c log_rotation_size=0' . - ' -c log_error_verbosity=verbose' . - ' -c unix_socket_directories=\'' . $self->dbPath() . '\'"' . - ' -D ' . $self->dbBasePath() . ' -l ' . $self->pgLogFile() . ' -s'; - - $self->executeSimple($strCommand); - - # Connect user session - $self->sqlConnect(); -} - -#################################################################################################################################### -# clusterStop -# -# Stop the PostgreSQL cluster and optionally check for errors in the server log. -#################################################################################################################################### -sub clusterStop -{ - my $self = shift; - my $hParam = shift; - - # Set defaults - my $bIgnoreLogError = defined($$hParam{bIgnoreLogError}) ? $$hParam{bIgnoreLogError} : false; - my $bStop = defined($hParam->{bStop}) ? $$hParam{bStop} : true; - - # Disconnect user session - $self->sqlDisconnect(); - - # Grep for errors in postgresql.log - this is done first because we want to ignore any errors that happen during shutdown. - if (!$bIgnoreLogError && storageTest()->exists($self->pgLogFile())) - { - $self->executeSimple( - 'grep -v "FATAL\: 57P03\: the database system is (starting up|not yet accepting connections|" - "not accepting connections)" ' . $self->pgLogFile() . ' | grep "ERROR\|FATAL"', - {iExpectedExitStatus => 1}); - } - - # If pg process is running then stop the cluster - if ($bStop && -e $self->dbBasePath() . '/' . DB_FILE_POSTMTRPID) - { - $self->executeSimple($self->pgBinPath() . '/pg_ctl stop -D ' . $self->dbBasePath() . ' -w -s -m fast'); - } - - # Remove the log file - storageTest()->remove($self->pgLogFile(), {bIgnoreMissing => true}); -} - -#################################################################################################################################### -# clusterRestart -# -# Restart the PostgreSQL cluster. -#################################################################################################################################### -sub clusterRestart -{ - my $self = shift; - my $hParam = shift; - - $self->clusterStop($hParam); - $self->clusterStart($hParam); -} - -#################################################################################################################################### -# Getters -#################################################################################################################################### -sub walId {return shift->pgVersion() >= PG_VERSION_10 ? 'wal' : 'xlog'} -sub pgBinPath {return testRunGet()->pgBinPath()} -sub pgLogFile {return shift->{strPgLogFile}} -sub pgLogPath {return shift->{strPgLogPath}} -sub pgPort {return shift->{iPgPort}} -sub pgSocketPath {return shift->{strPgSocketPath}} -sub pgVersion {return testRunGet()->pgVersion()} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostGcsTest.pm b/test/lib/pgBackRestTest/Env/Host/HostGcsTest.pm deleted file mode 100644 index e9f89c1591..0000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostGcsTest.pm +++ /dev/null @@ -1,80 +0,0 @@ -#################################################################################################################################### -# GCS Test Host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostGcsTest; -use parent 'pgBackRestTest::Common::HostTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Cwd qw(abs_path); -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); -use Storable qw(dclone); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# GCS defaults -#################################################################################################################################### -use constant HOST_GCS_BUCKET => 'gcsbucket'; - push @EXPORT, qw(HOST_GCS_BUCKET); -use constant HOST_GCS_KEY => 'testkey'; - push @EXPORT, qw(HOST_GCS_KEY); -use constant HOST_GCS_KEY_TYPE => 'token'; - push @EXPORT, qw(HOST_GCS_KEY_TYPE); -use constant HOST_GCS_PORT => 4443; - push @EXPORT, qw(HOST_GCS_PORT); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - ); - - # Create the host - my $strProjectPath = dirname(dirname(abs_path($0))); - my $strFakeCertPath = "${strProjectPath}/doc/resource/fake-cert"; - - my $self = $class->SUPER::new( - HOST_GCS, 'test-' . testRunGet()->vmId() . '-' . HOST_GCS, 'fsouza/fake-gcs-server', 'root', undef, undef, undef, - false); - bless $self, $class; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostS3Test.pm b/test/lib/pgBackRestTest/Env/Host/HostS3Test.pm deleted file mode 100644 index 274619cdc1..0000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostS3Test.pm +++ /dev/null @@ -1,86 +0,0 @@ -#################################################################################################################################### -# S3 Test Host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostS3Test; -use parent 'pgBackRestTest::Common::HostTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Cwd qw(abs_path); -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); -use Storable qw(dclone); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# S3 defaults -#################################################################################################################################### -use constant HOST_S3_ACCESS_KEY => 'accessKey1'; - push @EXPORT, qw(HOST_S3_ACCESS_KEY); -use constant HOST_S3_ACCESS_SECRET_KEY => 'verySecretKey1'; - push @EXPORT, qw(HOST_S3_ACCESS_SECRET_KEY); -use constant HOST_S3_BUCKET => 'pgbackrest-dev'; - push @EXPORT, qw(HOST_S3_BUCKET); -use constant HOST_S3_ENDPOINT => 's3.amazonaws.com'; - push @EXPORT, qw(HOST_S3_ENDPOINT); -use constant HOST_S3_REGION => 'us-east-1'; - push @EXPORT, qw(HOST_S3_REGION); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - ); - - # Create the host - my $strProjectPath = dirname(dirname(abs_path($0))); - my $strFakeCertPath = "${strProjectPath}/doc/resource/fake-cert"; - - my $self = $class->SUPER::new( - HOST_S3, 'test-' . testRunGet()->vmId() . '-s3-server', 'minio/minio:RELEASE.2023-09-30T07-02-29Z', 'root', - ["${strFakeCertPath}/s3-server.crt:/root/.minio/certs/public.crt:ro", - "${strFakeCertPath}/s3-server.key:/root/.minio/certs/private.key:ro"], - '-e MINIO_REGION=' . HOST_S3_REGION . ' -e MINIO_DOMAIN=' . HOST_S3_ENDPOINT . ' -e MINIO_BROWSER=off' . - ' -e MINIO_ACCESS_KEY=' . HOST_S3_ACCESS_KEY . ' -e MINIO_SECRET_KEY=' . HOST_S3_ACCESS_SECRET_KEY, - 'server /data --address :443', false); - bless $self, $class; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -1; diff --git a/test/lib/pgBackRestTest/Env/Host/HostSftpTest.pm b/test/lib/pgBackRestTest/Env/Host/HostSftpTest.pm deleted file mode 100644 index 83efedc6cf..0000000000 --- a/test/lib/pgBackRestTest/Env/Host/HostSftpTest.pm +++ /dev/null @@ -1,72 +0,0 @@ -#################################################################################################################################### -# SFTP Test Host -#################################################################################################################################### -package pgBackRestTest::Env::Host::HostSftpTest; -use parent 'pgBackRestTest::Common::HostTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Cwd qw(abs_path); -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname); -use Storable qw(dclone); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# SFTP defaults -#################################################################################################################################### -use constant HOST_SFTP_ACCOUNT => TEST_USER; - push @EXPORT, qw(HOST_SFTP_ACCOUNT); -use constant HOST_SFTP_HOSTKEY_HASH_TYPE => 'sha1'; - push @EXPORT, qw(HOST_SFTP_HOSTKEY_HASH_TYPE); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; # Class name - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - ); - - # Create the host - my $self = $class->SUPER::new( - HOST_SFTP, 'test-' . testRunGet()->vmId() . '-' . HOST_SFTP, containerRepo() . ':' . testRunGet()->vm() . '-test', 'root'); - bless $self, $class; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self, trace => true} - ); -} - -1; diff --git a/test/lib/pgBackRestTest/Env/HostEnvTest.pm b/test/lib/pgBackRestTest/Env/HostEnvTest.pm deleted file mode 100644 index e7ca012d98..0000000000 --- a/test/lib/pgBackRestTest/Env/HostEnvTest.pm +++ /dev/null @@ -1,543 +0,0 @@ -#################################################################################################################################### -# FullCommonTest.pm - Common code for backup tests -#################################################################################################################################### -package pgBackRestTest::Env::HostEnvTest; -use parent 'pgBackRestTest::Common::RunTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Digest::SHA qw(sha1_hex); -use Exporter qw(import); - our @EXPORT = qw(); -use Storable qw(dclone); - -use pgBackRestDoc::Common::Log; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::StorageBase; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Env::ArchiveInfo; -use pgBackRestTest::Env::Host::HostAzureTest; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Host::HostDbCommonTest; -use pgBackRestTest::Env::Host::HostDbTest; -use pgBackRestTest::Env::Host::HostDbSyntheticTest; -use pgBackRestTest::Env::Host::HostGcsTest; -use pgBackRestTest::Env::Host::HostS3Test; - -#################################################################################################################################### -# Constants -#################################################################################################################################### -use constant ENCRYPTION_KEY_ARCHIVE => 'archive'; - push @EXPORT, qw(ENCRYPTION_KEY_ARCHIVE); -use constant ENCRYPTION_KEY_MANIFEST => 'manifest'; - push @EXPORT, qw(ENCRYPTION_KEY_MANIFEST); -use constant ENCRYPTION_KEY_BACKUPSET => 'backupset'; - push @EXPORT, qw(ENCRYPTION_KEY_BACKUPSET); - -#################################################################################################################################### -# setup -#################################################################################################################################### -sub setup -{ - my $self = shift; - my $bSynthetic = shift; - my $oConfigParam = shift; - - # Start object server first since it takes the longest - #------------------------------------------------------------------------------------------------------------------------------- - my $oHostObject; - - if ($oConfigParam->{strStorage} eq S3) - { - $oHostObject = new pgBackRestTest::Env::Host::HostS3Test(); - } - elsif ($oConfigParam->{strStorage} eq AZURE) - { - $oHostObject = new pgBackRestTest::Env::Host::HostAzureTest(); - } - elsif ($oConfigParam->{strStorage} eq GCS) - { - $oHostObject = new pgBackRestTest::Env::Host::HostGcsTest(); - } - elsif ($oConfigParam->{strStorage} eq SFTP) - { - $oHostObject = new pgBackRestTest::Env::Host::HostSftpTest(); - } - - # Get host group - my $oHostGroup = hostGroupGet(); - - # Create the backup host - my $strBackupDestination; - my $bHostBackup = defined($$oConfigParam{bHostBackup}) ? $$oConfigParam{bHostBackup} : false; - my $oHostBackup = undef; - - my $bRepoEncrypt = defined($$oConfigParam{bRepoEncrypt}) ? $$oConfigParam{bRepoEncrypt} : false; - - if ($bHostBackup) - { - $strBackupDestination = defined($$oConfigParam{strBackupDestination}) ? $$oConfigParam{strBackupDestination} : HOST_BACKUP; - - $oHostBackup = new pgBackRestTest::Env::Host::HostBackupTest( - {strBackupDestination => $strBackupDestination, bSynthetic => $bSynthetic, - bRepoLocal => $oConfigParam->{strStorage} eq POSIX, bRepoEncrypt => $bRepoEncrypt, bTls => $oConfigParam->{bTls}}); - $oHostGroup->hostAdd($oHostBackup); - } - else - { - $strBackupDestination = - defined($$oConfigParam{strBackupDestination}) ? $$oConfigParam{strBackupDestination} : HOST_DB_PRIMARY; - } - - # Create the db-primary host - my $oHostDbPrimary = undef; - - if ($bSynthetic) - { - $oHostDbPrimary = new pgBackRestTest::Env::Host::HostDbSyntheticTest( - {strBackupDestination => $strBackupDestination, - bRepoLocal => $oConfigParam->{strStorage} eq POSIX, bRepoEncrypt => $bRepoEncrypt, bTls => $oConfigParam->{bTls}}); - } - else - { - $oHostDbPrimary = new pgBackRestTest::Env::Host::HostDbTest( - {strBackupDestination => $strBackupDestination, bRepoLocal => $oConfigParam->{strStorage} eq POSIX, - bRepoEncrypt => $bRepoEncrypt, bTls => $oConfigParam->{bTls}}); - } - - $oHostGroup->hostAdd($oHostDbPrimary); - - # Create the db-standby host - my $oHostDbStandby = undef; - - if (defined($$oConfigParam{bStandby}) && $$oConfigParam{bStandby}) - { - $oHostDbStandby = new pgBackRestTest::Env::Host::HostDbTest( - {strBackupDestination => $strBackupDestination, bStandby => true, bRepoLocal => $oConfigParam->{strStorage} eq POSIX, - bTls => $oConfigParam->{bTls}}); - - $oHostGroup->hostAdd($oHostDbStandby); - } - - # Finalize object server - #------------------------------------------------------------------------------------------------------------------------------- - if ($oConfigParam->{strStorage} eq S3) - { - $oHostGroup->hostAdd($oHostObject, {rstryHostName => ['pgbackrest-dev.s3.amazonaws.com', 's3.amazonaws.com']}); - } - elsif ($oConfigParam->{strStorage} eq AZURE || $oConfigParam->{strStorage} eq GCS || $oConfigParam->{strStorage} eq SFTP) - { - $oHostGroup->hostAdd($oHostObject); - } - - # Create db-primary config - $oHostDbPrimary->configCreate({ - bTls => $oConfigParam->{bTls}, - strBackupSource => $$oConfigParam{strBackupSource}, - strCompressType => $$oConfigParam{strCompressType}, - bHardlink => $bHostBackup ? undef : $$oConfigParam{bHardLink}, - bArchiveAsync => $$oConfigParam{bArchiveAsync}, - strStorage => $oConfigParam->{strStorage}, - iRepoTotal => $oConfigParam->{iRepoTotal}, - bBundle => $oConfigParam->{bBundle}, - bBlockIncr => $oConfigParam->{bBlockIncr}}); - - # Create backup config if backup host exists - if (defined($oHostBackup)) - { - $oHostBackup->configCreate({ - bTls => $oConfigParam->{bTls}, - strCompressType => $$oConfigParam{strCompressType}, - bHardlink => $$oConfigParam{bHardLink}, - strStorage => $oConfigParam->{strStorage}, - iRepoTotal => $oConfigParam->{iRepoTotal}, - bBundle => $oConfigParam->{bBundle}, - bBlockIncr => $oConfigParam->{bBlockIncr}}); - } - # If backup host is not defined set it to db-primary - else - { - $oHostBackup = $strBackupDestination eq HOST_DB_PRIMARY || $strBackupDestination eq HOST_SFTP ? $oHostDbPrimary : - $oHostDbStandby; - } - - storageRepoCommandSet( - $self->backrestExeHelper() . - ' --config=' . $oHostBackup->backrestConfig() . ' --stanza=' . $self->stanza() . ' --log-level-console=off' . - ' --log-level-stderr=error' . - ($oConfigParam->{strStorage} ne POSIX ? - ($oConfigParam->{strStorage} ne SFTP ? " --no-repo1-storage-verify-tls" : '') . - " --repo1-$oConfigParam->{strStorage}-" . - ($oConfigParam->{strStorage} eq GCS ? 'endpoint' : 'host') . "=" . $oHostObject->ipGet() : '') . - ($oConfigParam->{strStorage} eq GCS ? ':' . HOST_GCS_PORT : ''), - $oConfigParam->{strStorage} eq POSIX ? STORAGE_POSIX : STORAGE_OBJECT); - - # Create db-standby config - if (defined($oHostDbStandby)) - { - $oHostDbStandby->configCreate({ - bTls => $oConfigParam->{bTls}, - strBackupSource => $$oConfigParam{strBackupSource}, - strCompressType => $$oConfigParam{strCompressType}, - bHardlink => $bHostBackup ? undef : $$oConfigParam{bHardLink}, - bArchiveAsync => $$oConfigParam{bArchiveAsync}, - strStorage => $oConfigParam->{strStorage}, - iRepoTotal => $oConfigParam->{iRepoTotal}, - bBundle => $oConfigParam->{bBundle}, - bBlockIncr => $oConfigParam->{bBlockIncr}}); - } - - # Create object storage - if (defined($oHostObject)) - { - storageRepo()->create(); - } - - return $oHostDbPrimary, $oHostDbStandby, $oHostBackup; -} - -#################################################################################################################################### -# Generate database system id for the db version -#################################################################################################################################### -sub dbSysId -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPgVersion, - ) = - logDebugParam - ( - __PACKAGE__ . '->dbSysId', \@_, - {name => 'strPgVersion', trace => true}, - ); - - return (1000000000000000000 + ($strPgVersion * 10)); -} - -#################################################################################################################################### -# Get database catalog version for the db version -#################################################################################################################################### -sub dbCatalogVersion -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPgVersion, - ) = - logDebugParam - ( - __PACKAGE__ . '->sysId', \@_, - {name => 'strPgVersion', trace => true}, - ); - - my $hCatalogVersion = - { - &PG_VERSION_94 => 201409291, - &PG_VERSION_95 => 201510051, - &PG_VERSION_96 => 201608131, - &PG_VERSION_10 => 201707211, - &PG_VERSION_11 => 201806231, - &PG_VERSION_12 => 201909212, - &PG_VERSION_13 => 202007201, - &PG_VERSION_14 => 202105121, - &PG_VERSION_15 => 202209061, - &PG_VERSION_16 => 202307071, - }; - - if (!defined($hCatalogVersion->{$strPgVersion})) - { - confess &log(ASSERT, "no catalog version defined for pg version ${strPgVersion}"); - } - - return $hCatalogVersion->{$strPgVersion}; -} - -#################################################################################################################################### -# Generate control file content -#################################################################################################################################### -sub controlGenerateContent -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPgVersion, - ) = - logDebugParam - ( - __PACKAGE__ . '->controlGenerateContent', \@_, - {name => 'strPgVersion', trace => true}, - ); - - my $hControlContent = - { - 32 => - { - &PG_VERSION_94 => - "5e0064a7b3b6e00dae0300000b43010c00000000000000000000000001000000000000000000000000000000000000000000000001000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000002000000000000000000000000000010000000000000000000000000000000000000000" . - "000000006b0756c8", - &PG_VERSION_95 => - "5f0064a7b3b6e00dae030000a3cc020c00000000000000000000000001000000000000000000000000000000000000000000000001000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" . - "000000000000000000000000000000003bfe413a", - &PG_VERSION_96 => - "600064a7b3b6e00dc0030000c34b040c00000000000000000000000001000000000000000000000000000000000000000000000001000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" . - "000000000000000000000000000000005d135da6", - &PG_VERSION_10 => - "640064a7b3b6e00dea030000cbce050c00000000000000000000000001000000000000000000000000000000000000000000000001000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" . - "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f8556c34", - }, - 64 => - { - &PG_VERSION_94 => - "5e0064a7b3b6e00dae0300000b43010c00000000000000000000000000000000010000000000000000000000000000000000000000000000" . - "0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" . - "00000000000000000000000000000000ee6cf996", - &PG_VERSION_95 => - "5f0064a7b3b6e00dae030000a3cc020c00000000000000000000000000000000010000000000000000000000000000000000000000000000" . - "0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000" . - "0000000000000001000000000000000000000000000000000000000000000000381ec2de", - &PG_VERSION_96 => - "600064a7b3b6e00dc0030000c34b040c00000000000000000000000000000000010000000000000000000000000000000000000000000000" . - "0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000" . - "00000000000000010000000000000000000000000000000000000000000000002d96a4c0", - &PG_VERSION_10 => - "640064a7b3b6e00dea030000cbce050c00000000000000000000000000000000010000000000000000000000000000000000000000000000" . - "0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000" . - "0000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" . - "00000000000000008d543cdf", - }, - }; - - my $strControlContent = $hControlContent->{$self->archBits()}{$strPgVersion}; - - if (!defined($strControlContent)) - { - confess &log(ASSERT, "no control content defined for pg version ${strPgVersion}"); - } - - my $tControlContent = ''; - - for (my $iIdx = 0; $iIdx < length($strControlContent) / 2; $iIdx++) - { - my $iChar = hex(substr($strControlContent, $iIdx * 2, 2)); - $tControlContent .= pack('C', $iChar); - } - - - # Pad bytes - for (my $iIdx = length($tControlContent); $iIdx < 8192; $iIdx++) - { - $tControlContent .= pack('C', 0); - } - - return \$tControlContent; -} - -#################################################################################################################################### -# Generate control file and write to disk -#################################################################################################################################### -sub controlGenerate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strDbPath, - $strPgVersion, - ) = - logDebugParam - ( - __PACKAGE__ . '->controlGenerate', \@_, - {name => 'strDbPath', trace => true}, - {name => 'strPgVersion', trace => true}, - ); - - storageTest()->put("${strDbPath}/global/pg_control", $self->controlGenerateContent($strPgVersion)); -} - -#################################################################################################################################### -# walSegment -# -# Generate name of WAL segment from component parts. -#################################################################################################################################### -sub walSegment -{ - my $self = shift; - my $iTimeline = shift; - my $iMajor = shift; - my $iMinor = shift; - - return uc(sprintf('%08x%08x%08x', $iTimeline, $iMajor, $iMinor)); -} - -#################################################################################################################################### -# Generate WAL file content -#################################################################################################################################### -sub walGenerateContent -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPgVersion, - $iSourceNo, - ) = - logDebugParam - ( - __PACKAGE__ . '->walGenerateContent', \@_, - {name => 'strPgVersion', trace => true}, - {name => 'iSourceNo', optional => true, default => 1, trace => true}, - ); - - # Get WAL magic for the PG version - my $hWalMagic = - { - &PG_VERSION_94 => hex('0xD07E'), - &PG_VERSION_95 => hex('0xD087'), - &PG_VERSION_96 => hex('0xD093'), - &PG_VERSION_10 => hex('0xD097'), - &PG_VERSION_11 => hex('0xD098'), - &PG_VERSION_12 => hex('0xD101'), - &PG_VERSION_13 => hex('0xD106'), - }; - - my $tWalContent = pack('S', $hWalMagic->{$strPgVersion}); - - # Indicate that the long header is present - $tWalContent .= pack('S', 2); - - # Add junk (H for header) for the bytes that won't be read by the tests - my $iOffset = 12 + (testRunGet()->archBits() / 8); - $tWalContent .= ('H' x $iOffset); - - # Add the system identifier - $tWalContent .= pack('Q', $self->dbSysId($strPgVersion)); - - # Add segment size - $tWalContent .= pack('L', PG_WAL_SEGMENT_SIZE); - - # Add the source number to produce WAL segments with different checksums - $tWalContent .= pack('S', $iSourceNo); - - # Pad out to the required size (B for body) - $tWalContent .= ('B' x (PG_WAL_SEGMENT_SIZE - length($tWalContent))); - - return \$tWalContent; -} - -#################################################################################################################################### -# Generate WAL file content checksum -#################################################################################################################################### -sub walGenerateContentChecksum -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strPgVersion, - $hParam, - ) = - logDebugParam - ( - __PACKAGE__ . '->walGenerateContent', \@_, - {name => 'strPgVersion', trace => true}, - {name => 'hParam', required => false, trace => true}, - ); - - return sha1_hex(${$self->walGenerateContent($strPgVersion, $hParam)}); -} - -#################################################################################################################################### -# walGenerate -# -# Generate a WAL segment and ready file for testing. -#################################################################################################################################### -sub walGenerate -{ - my $self = shift; - my $strWalPath = shift; - my $strPgVersion = shift; - my $iSourceNo = shift; - my $strWalSegment = shift; - my $bPartial = shift; - my $bChecksum = shift; - my $bReady = shift; - - my $rtWalContent = $self->walGenerateContent($strPgVersion, {iSourceNo => $iSourceNo}); - my $strWalFile = - "${strWalPath}/${strWalSegment}" . ($bChecksum ? '-' . sha1_hex($rtWalContent) : '') . - (defined($bPartial) && $bPartial ? '.partial' : ''); - - # Put the WAL segment and the ready file - storageTest()->put($strWalFile, $rtWalContent); - - if (!defined($bReady) || $bReady) - { - storageTest()->put("${strWalPath}/archive_status/${strWalSegment}.ready"); - } - - return $strWalFile; -} - -#################################################################################################################################### -# walRemove -# -# Remove WAL file and ready file. -#################################################################################################################################### -sub walRemove -{ - my $self = shift; - my $strWalPath = shift; - my $strWalFile = shift; - - storageTest()->remove("$self->{strWalPath}/${strWalFile}"); - storageTest()->remove("$self->{strWalPath}/archive_status/${strWalFile}.ready"); -} - -1; diff --git a/test/lib/pgBackRestTest/Env/InfoCommon.pm b/test/lib/pgBackRestTest/Env/InfoCommon.pm deleted file mode 100644 index 22990a81ec..0000000000 --- a/test/lib/pgBackRestTest/Env/InfoCommon.pm +++ /dev/null @@ -1,32 +0,0 @@ -#################################################################################################################################### -# INFO MODULE -# Constants, variables and functions common to the info files -#################################################################################################################################### -package pgBackRestTest::Env::InfoCommon; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Exporter qw(import); - our @EXPORT = qw(); - -#################################################################################################################################### -# DB section constants -#################################################################################################################################### -use constant INFO_BACKUP_SECTION_DB => 'db'; - push @EXPORT, qw(INFO_BACKUP_SECTION_DB); -use constant INFO_BACKUP_SECTION_DB_HISTORY => INFO_BACKUP_SECTION_DB . ':history'; - push @EXPORT, qw(INFO_BACKUP_SECTION_DB_HISTORY); - -#################################################################################################################################### -# History section constants -#################################################################################################################################### -use constant INFO_HISTORY_ID => 'id'; - push @EXPORT, qw(INFO_HISTORY_ID); -use constant INFO_DB_VERSION => 'version'; - push @EXPORT, qw(INFO_DB_VERSION); -use constant INFO_SYSTEM_ID => 'system-id'; - push @EXPORT, qw(INFO_SYSTEM_ID); - -1; diff --git a/test/lib/pgBackRestTest/Env/Manifest.pm b/test/lib/pgBackRestTest/Env/Manifest.pm deleted file mode 100644 index 564e6db657..0000000000 --- a/test/lib/pgBackRestTest/Env/Manifest.pm +++ /dev/null @@ -1,1446 +0,0 @@ -#################################################################################################################################### -# MANIFEST MODULE -#################################################################################################################################### -package pgBackRestTest::Env::Manifest; -use parent 'pgBackRestDoc::Common::Ini'; - -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use Exporter qw(import); - our @EXPORT = qw(); -use File::Basename qw(dirname basename); -use Fcntl qw(:mode); -use Time::Local qw(timelocal); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; - -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; - -#################################################################################################################################### -# File/path constants -#################################################################################################################################### -use constant PATH_BACKUP_HISTORY => 'backup.history'; - push @EXPORT, qw(PATH_BACKUP_HISTORY); -use constant FILE_MANIFEST => 'backup.manifest'; - push @EXPORT, qw(FILE_MANIFEST); -use constant FILE_MANIFEST_COPY => FILE_MANIFEST . INI_COPY_EXT; - push @EXPORT, qw(FILE_MANIFEST_COPY); - -#################################################################################################################################### -# Default match factor -#################################################################################################################################### -use constant MANIFEST_DEFAULT_MATCH_FACTOR => 0.1; - push @EXPORT, qw(MANIFEST_DEFAULT_MATCH_FACTOR); - -#################################################################################################################################### -# MANIFEST Constants -#################################################################################################################################### -use constant MANIFEST_TARGET_PGDATA => 'pg_data'; - push @EXPORT, qw(MANIFEST_TARGET_PGDATA); -use constant MANIFEST_TARGET_PGTBLSPC => 'pg_tblspc'; - push @EXPORT, qw(MANIFEST_TARGET_PGTBLSPC); - -use constant MANIFEST_VALUE_PATH => 'path'; - push @EXPORT, qw(MANIFEST_VALUE_PATH); -use constant MANIFEST_VALUE_LINK => 'link'; - push @EXPORT, qw(MANIFEST_VALUE_LINK); - -# Manifest sections -use constant MANIFEST_SECTION_BACKUP => 'backup'; - push @EXPORT, qw(MANIFEST_SECTION_BACKUP); -use constant MANIFEST_SECTION_BACKUP_DB => 'backup:db'; - push @EXPORT, qw(MANIFEST_SECTION_BACKUP_DB); -use constant MANIFEST_SECTION_BACKUP_INFO => 'backup:info'; - push @EXPORT, qw(MANIFEST_SECTION_BACKUP_INFO); -use constant MANIFEST_SECTION_BACKUP_OPTION => 'backup:option'; - push @EXPORT, qw(MANIFEST_SECTION_BACKUP_OPTION); -use constant MANIFEST_SECTION_BACKUP_TARGET => 'backup:target'; - push @EXPORT, qw(MANIFEST_SECTION_BACKUP_TARGET); -use constant MANIFEST_SECTION_DB => 'db'; - push @EXPORT, qw(MANIFEST_SECTION_DB); -use constant MANIFEST_SECTION_TARGET_PATH => 'target:path'; - push @EXPORT, qw(MANIFEST_SECTION_TARGET_PATH); -use constant MANIFEST_SECTION_TARGET_FILE => 'target:file'; - push @EXPORT, qw(MANIFEST_SECTION_TARGET_FILE); -use constant MANIFEST_SECTION_TARGET_LINK => 'target:link'; - push @EXPORT, qw(MANIFEST_SECTION_TARGET_LINK); - -# Backup metadata required for restores -use constant MANIFEST_KEY_ARCHIVE_START => 'backup-archive-start'; - push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_START); -use constant MANIFEST_KEY_ARCHIVE_STOP => 'backup-archive-stop'; - push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_STOP); -use constant MANIFEST_KEY_LABEL => 'backup-label'; - push @EXPORT, qw(MANIFEST_KEY_LABEL); -use constant MANIFEST_KEY_LSN_START => 'backup-lsn-start'; - push @EXPORT, qw(MANIFEST_KEY_LSN_START); -use constant MANIFEST_KEY_LSN_STOP => 'backup-lsn-stop'; - push @EXPORT, qw(MANIFEST_KEY_LSN_STOP); -use constant MANIFEST_KEY_PRIOR => 'backup-prior'; - push @EXPORT, qw(MANIFEST_KEY_PRIOR); -use constant MANIFEST_KEY_TIMESTAMP_COPY_START => 'backup-timestamp-copy-start'; - push @EXPORT, qw(MANIFEST_KEY_TIMESTAMP_COPY_START); -use constant MANIFEST_KEY_TIMESTAMP_START => 'backup-timestamp-start'; - push @EXPORT, qw(MANIFEST_KEY_TIMESTAMP_START); -use constant MANIFEST_KEY_TIMESTAMP_STOP => 'backup-timestamp-stop'; - push @EXPORT, qw(MANIFEST_KEY_TIMESTAMP_STOP); -use constant MANIFEST_KEY_TYPE => 'backup-type'; - push @EXPORT, qw(MANIFEST_KEY_TYPE); - -# Options that were set when the backup was made -use constant MANIFEST_KEY_BACKUP_STANDBY => 'option-backup-standby'; - push @EXPORT, qw(MANIFEST_KEY_BACKUP_STANDBY); -use constant MANIFEST_KEY_HARDLINK => 'option-hardlink'; - push @EXPORT, qw(MANIFEST_KEY_HARDLINK); -use constant MANIFEST_KEY_ARCHIVE_CHECK => 'option-archive-check'; - push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_CHECK); -use constant MANIFEST_KEY_ARCHIVE_COPY => 'option-archive-copy'; - push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_COPY); -use constant MANIFEST_KEY_BUFFER_SIZE => 'option-buffer-size'; - push @EXPORT, qw(MANIFEST_KEY_BUFFER_SIZE); -use constant MANIFEST_KEY_CHECKSUM_PAGE => 'option-checksum-page'; - push @EXPORT, qw(MANIFEST_KEY_CHECKSUM_PAGE); -use constant MANIFEST_KEY_COMPRESS => 'option-compress'; - push @EXPORT, qw(MANIFEST_KEY_COMPRESS); -use constant MANIFEST_KEY_COMPRESS_TYPE => 'option-compress-type'; - push @EXPORT, qw(MANIFEST_KEY_COMPRESS_TYPE); -use constant MANIFEST_KEY_COMPRESS_LEVEL => 'option-compress-level'; - push @EXPORT, qw(MANIFEST_KEY_COMPRESS_LEVEL); -use constant MANIFEST_KEY_COMPRESS_LEVEL_NETWORK => 'option-compress-level-network'; - push @EXPORT, qw(MANIFEST_KEY_COMPRESS_LEVEL_NETWORK); -use constant MANIFEST_KEY_ONLINE => 'option-online'; - push @EXPORT, qw(MANIFEST_KEY_ONLINE); -use constant MANIFEST_KEY_DELTA => 'option-delta'; - push @EXPORT, qw(MANIFEST_KEY_DELTA); -use constant MANIFEST_KEY_PROCESS_MAX => 'option-process-max'; - push @EXPORT, qw(MANIFEST_KEY_PROCESS_MAX); - -# Information about the database that was backed up -use constant MANIFEST_KEY_DB_ID => 'db-id'; - push @EXPORT, qw(MANIFEST_KEY_DB_ID); -use constant MANIFEST_KEY_SYSTEM_ID => 'db-system-id'; - push @EXPORT, qw(MANIFEST_KEY_SYSTEM_ID); -use constant MANIFEST_KEY_CATALOG => 'db-catalog-version'; - push @EXPORT, qw(MANIFEST_KEY_CATALOG); -use constant MANIFEST_KEY_CONTROL => 'db-control-version'; - push @EXPORT, qw(MANIFEST_KEY_CONTROL); -use constant MANIFEST_KEY_DB_LAST_SYSTEM_ID => 'db-last-system-id'; - push @EXPORT, qw(MANIFEST_KEY_DB_LAST_SYSTEM_ID); -use constant MANIFEST_KEY_DB_VERSION => 'db-version'; - push @EXPORT, qw(MANIFEST_KEY_DB_VERSION); - -# Subkeys used for path/file/link info -use constant MANIFEST_SUBKEY_CHECKSUM => 'checksum'; - push @EXPORT, qw(MANIFEST_SUBKEY_CHECKSUM); -use constant MANIFEST_SUBKEY_CHECKSUM_PAGE => 'checksum-page'; - push @EXPORT, qw(MANIFEST_SUBKEY_CHECKSUM_PAGE); -use constant MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR => 'checksum-page-error'; - push @EXPORT, qw(MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR); -use constant MANIFEST_SUBKEY_DESTINATION => 'destination'; - push @EXPORT, qw(MANIFEST_SUBKEY_DESTINATION); -use constant MANIFEST_SUBKEY_FILE => 'file'; - push @EXPORT, qw(MANIFEST_SUBKEY_FILE); -use constant MANIFEST_SUBKEY_FUTURE => 'future'; - push @EXPORT, qw(MANIFEST_SUBKEY_FUTURE); -use constant MANIFEST_SUBKEY_GROUP => 'group'; - push @EXPORT, qw(MANIFEST_SUBKEY_GROUP); -use constant MANIFEST_SUBKEY_MODE => 'mode'; - push @EXPORT, qw(MANIFEST_SUBKEY_MODE); -use constant MANIFEST_SUBKEY_TIMESTAMP => 'timestamp'; - push @EXPORT, qw(MANIFEST_SUBKEY_TIMESTAMP); -use constant MANIFEST_SUBKEY_TYPE => 'type'; - push @EXPORT, qw(MANIFEST_SUBKEY_TYPE); -use constant MANIFEST_SUBKEY_PATH => 'path'; - push @EXPORT, qw(MANIFEST_SUBKEY_PATH); -use constant MANIFEST_SUBKEY_REFERENCE => 'reference'; - push @EXPORT, qw(MANIFEST_SUBKEY_REFERENCE); -use constant MANIFEST_SUBKEY_REPO_SIZE => 'repo-size'; - push @EXPORT, qw(MANIFEST_SUBKEY_REPO_SIZE); -use constant MANIFEST_SUBKEY_SIZE => 'size'; - push @EXPORT, qw(MANIFEST_SUBKEY_SIZE); -use constant MANIFEST_SUBKEY_TABLESPACE_ID => 'tablespace-id'; - push @EXPORT, qw(MANIFEST_SUBKEY_TABLESPACE_ID); -use constant MANIFEST_SUBKEY_TABLESPACE_NAME => 'tablespace-name'; - push @EXPORT, qw(MANIFEST_SUBKEY_TABLESPACE_NAME); -use constant MANIFEST_SUBKEY_USER => 'user'; - push @EXPORT, qw(MANIFEST_SUBKEY_USER); - -#################################################################################################################################### -# Database locations for important files/paths -#################################################################################################################################### -use constant DB_PATH_ARCHIVESTATUS => 'archive_status'; - push @EXPORT, qw(DB_PATH_ARCHIVESTATUS); -use constant DB_PATH_BASE => 'base'; - push @EXPORT, qw(DB_PATH_BASE); -use constant DB_PATH_GLOBAL => 'global'; - push @EXPORT, qw(DB_PATH_GLOBAL); -use constant DB_PATH_PGDYNSHMEM => 'pg_dynshmem'; - push @EXPORT, qw(DB_PATH_PGDYNSHMEM); -use constant DB_PATH_PGMULTIXACT => 'pg_multixact'; - push @EXPORT, qw(DB_PATH_PGMULTIXACT); -use constant DB_PATH_PGNOTIFY => 'pg_notify'; - push @EXPORT, qw(DB_PATH_PGNOTIFY); -use constant DB_PATH_PGREPLSLOT => 'pg_replslot'; - push @EXPORT, qw(DB_PATH_PGREPLSLOT); -use constant DB_PATH_PGSERIAL => 'pg_serial'; - push @EXPORT, qw(DB_PATH_PGSERIAL); -use constant DB_PATH_PGSNAPSHOTS => 'pg_snapshots'; - push @EXPORT, qw(DB_PATH_PGSNAPSHOTS); -use constant DB_PATH_PGSTATTMP => 'pg_stat_tmp'; - push @EXPORT, qw(DB_PATH_PGSTATTMP); -use constant DB_PATH_PGSUBTRANS => 'pg_subtrans'; - push @EXPORT, qw(DB_PATH_PGSUBTRANS); -use constant DB_PATH_PGTBLSPC => 'pg_tblspc'; - push @EXPORT, qw(DB_PATH_PGTBLSPC); - -use constant DB_FILE_BACKUPLABEL => 'backup_label'; - push @EXPORT, qw(DB_FILE_BACKUPLABEL); -use constant DB_FILE_BACKUPLABELOLD => DB_FILE_BACKUPLABEL . '.old'; - push @EXPORT, qw(DB_FILE_BACKUPLABELOLD); -use constant DB_FILE_PGCONTROL => DB_PATH_GLOBAL . '/pg_control'; - push @EXPORT, qw(DB_FILE_PGCONTROL); -use constant DB_FILE_PGFILENODEMAP => 'pg_filenode.map'; - push @EXPORT, qw(DB_FILE_PGFILENODEMAP); -use constant DB_FILE_PGINTERNALINIT => 'pg_internal.init'; - push @EXPORT, qw(DB_FILE_PGINTERNALINIT); -use constant DB_FILE_PGVERSION => 'PG_VERSION'; - push @EXPORT, qw(DB_FILE_PGVERSION); -use constant DB_FILE_POSTGRESQLAUTOCONFTMP => 'postgresql.auto.conf.tmp'; - push @EXPORT, qw(DB_FILE_POSTGRESQLAUTOCONFTMP); -use constant DB_FILE_POSTMTROPTS => 'postmas'.'ter.opts'; - push @EXPORT, qw(DB_FILE_POSTMTROPTS); -use constant DB_FILE_POSTMTRPID => 'postmas'.'ter.pid'; - push @EXPORT, qw(DB_FILE_POSTMTRPID); -use constant DB_FILE_RECOVERYCONF => 'recovery.conf'; - push @EXPORT, qw(DB_FILE_RECOVERYCONF); -use constant DB_FILE_RECOVERYSIGNAL => 'recovery.signal'; - push @EXPORT, qw(DB_FILE_RECOVERYSIGNAL); -use constant DB_FILE_RECOVERYDONE => 'recovery.done'; - push @EXPORT, qw(DB_FILE_RECOVERYDONE); -use constant DB_FILE_STANDBYSIGNAL => 'standby.signal'; - push @EXPORT, qw(DB_FILE_STANDBYSIGNAL); -use constant DB_FILE_TABLESPACEMAP => 'tablespace_map'; - push @EXPORT, qw(DB_FILE_TABLESPACEMAP); - -use constant DB_FILE_PREFIX_TMP => 'pgsql_tmp'; - push @EXPORT, qw(DB_FILE_PREFIX_TMP); - -#################################################################################################################################### -# Manifest locations for important files/paths -#################################################################################################################################### -use constant MANIFEST_PATH_BASE => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_BASE; - push @EXPORT, qw(MANIFEST_PATH_BASE); -use constant MANIFEST_PATH_GLOBAL => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_GLOBAL; - push @EXPORT, qw(MANIFEST_PATH_GLOBAL); -use constant MANIFEST_PATH_PGDYNSHMEM => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGDYNSHMEM; - push @EXPORT, qw(MANIFEST_PATH_PGDYNSHMEM); -use constant MANIFEST_PATH_PGMULTIXACT => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGMULTIXACT; - push @EXPORT, qw(MANIFEST_PATH_PGMULTIXACT); -use constant MANIFEST_PATH_PGNOTIFY => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGNOTIFY; - push @EXPORT, qw(MANIFEST_PATH_PGNOTIFY); -use constant MANIFEST_PATH_PGREPLSLOT => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGREPLSLOT; - push @EXPORT, qw(MANIFEST_PATH_PGREPLSLOT); -use constant MANIFEST_PATH_PGSERIAL => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSERIAL; - push @EXPORT, qw(MANIFEST_PATH_PGSERIAL); -use constant MANIFEST_PATH_PGSNAPSHOTS => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSNAPSHOTS; - push @EXPORT, qw(MANIFEST_PATH_PGSNAPSHOTS); -use constant MANIFEST_PATH_PGSTATTMP => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSTATTMP; - push @EXPORT, qw(MANIFEST_PATH_PGSTATTMP); -use constant MANIFEST_PATH_PGSUBTRANS => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSUBTRANS; - push @EXPORT, qw(MANIFEST_PATH_PGSUBTRANS); -use constant MANIFEST_PATH_PGTBLSPC => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC; - push @EXPORT, qw(MANIFEST_PATH_PGTBLSPC); - -use constant MANIFEST_FILE_BACKUPLABEL => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_BACKUPLABEL; - push @EXPORT, qw(MANIFEST_FILE_BACKUPLABEL); -use constant MANIFEST_FILE_BACKUPLABELOLD => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_BACKUPLABELOLD; - push @EXPORT, qw(MANIFEST_FILE_BACKUPLABELOLD); -use constant MANIFEST_FILE_PGCONTROL => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_PGCONTROL; - push @EXPORT, qw(MANIFEST_FILE_PGCONTROL); -use constant MANIFEST_FILE_POSTGRESQLAUTOCONFTMP => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_POSTGRESQLAUTOCONFTMP; - push @EXPORT, qw(MANIFEST_FILE_PGCONTROL); -use constant MANIFEST_FILE_POSTMTROPTS => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_POSTMTROPTS; - push @EXPORT, qw(MANIFEST_FILE_POSTMTROPTS); -use constant MANIFEST_FILE_POSTMTRPID => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_POSTMTRPID; - push @EXPORT, qw(MANIFEST_FILE_POSTMTRPID); -use constant MANIFEST_FILE_RECOVERYCONF => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_RECOVERYCONF; - push @EXPORT, qw(MANIFEST_FILE_RECOVERYCONF); -use constant MANIFEST_FILE_RECOVERYSIGNAL => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_RECOVERYSIGNAL; - push @EXPORT, qw(MANIFEST_FILE_RECOVERYSIGNAL); -use constant MANIFEST_FILE_RECOVERYDONE => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_RECOVERYDONE; - push @EXPORT, qw(MANIFEST_FILE_RECOVERYDONE); -use constant MANIFEST_FILE_STANDBYSIGNAL => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_STANDBYSIGNAL; - push @EXPORT, qw(MANIFEST_FILE_STANDBYSIGNAL); -use constant MANIFEST_FILE_TABLESPACEMAP => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_TABLESPACEMAP; - push @EXPORT, qw(MANIFEST_FILE_TABLESPACEMAP); - -#################################################################################################################################### -# Minimum ID for a user object in postgres -#################################################################################################################################### -use constant DB_USER_OBJECT_MINIMUM_ID => 16384; - push @EXPORT, qw(DB_USER_OBJECT_MINIMUM_ID); - -#################################################################################################################################### -# new -#################################################################################################################################### -sub new -{ - my $class = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strFileName, - $bLoad, - $oStorage, - $strDbVersion, - $iDbCatalogVersion, - $strCipherPass, # Passphrase to open the manifest if encrypted - $strCipherPassSub, # Passphrase to encrypt the backup files - ) = - logDebugParam - ( - __PACKAGE__ . '->new', \@_, - {name => 'strFileName', trace => true}, - {name => 'bLoad', optional => true, default => true, trace => true}, - {name => 'oStorage', optional => true, default => storageRepo(), trace => true}, - {name => 'strDbVersion', optional => true, trace => true}, - {name => 'iDbCatalogVersion', optional => true, trace => true}, - {name => 'strCipherPass', optional => true, redact => true}, - {name => 'strCipherPassSub', optional => true, redact => true}, - ); - - # Init object and store variables - my $self = $class->SUPER::new( - $oStorage, $strFileName, {bLoad => $bLoad, strCipherPass => $strCipherPass, strCipherPassSub => $strCipherPassSub}); - - # If manifest not loaded from a file then the db version and catalog version must be set - if (!$bLoad) - { - if (!(defined($strDbVersion) && defined($iDbCatalogVersion))) - { - confess &log(ASSERT, 'strDbVersion and iDbCatalogVersion must be provided with bLoad = false'); - } - - # Force the version to a string since newer versions of JSON::PP lose track of the fact that it is one - $self->set(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_VERSION, undef, $strDbVersion . ''); - $self->numericSet(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_CATALOG, undef, $iDbCatalogVersion); - } - - # Mark the manifest as built if it was loaded from a file - $self->{bBuilt} = $bLoad; - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'self', value => $self} - ); -} - -#################################################################################################################################### -# save -# -# Save the manifest. -#################################################################################################################################### -sub save -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '->save'); - - # Call inherited save - $self->SUPER::save(); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# get -# -# Get a value. -#################################################################################################################################### -sub get -{ - my $self = shift; - my $strSection = shift; - my $strKey = shift; - my $strSubKey = shift; - my $bRequired = shift; - my $oDefault = shift; - - my $oValue = $self->SUPER::get($strSection, $strKey, $strSubKey, false); - - if (!defined($oValue) && defined($strKey) && defined($strSubKey) && - ($strSection eq MANIFEST_SECTION_TARGET_FILE || $strSection eq MANIFEST_SECTION_TARGET_PATH || - $strSection eq MANIFEST_SECTION_TARGET_LINK) && - ($strSubKey eq MANIFEST_SUBKEY_USER || $strSubKey eq MANIFEST_SUBKEY_GROUP || $strSubKey eq MANIFEST_SUBKEY_MODE) && - $self->test($strSection, $strKey)) - { - $oValue = $self->SUPER::get("${strSection}:default", $strSubKey, undef, $bRequired, $oDefault); - } - else - { - $oValue = $self->SUPER::get($strSection, $strKey, $strSubKey, $bRequired, $oDefault); - } - - return $oValue; -} - -#################################################################################################################################### -# boolGet -# -# Get a numeric value. -#################################################################################################################################### -sub boolGet -{ - my $self = shift; - my $strSection = shift; - my $strValue = shift; - my $strSubValue = shift; - my $bRequired = shift; - my $bDefault = shift; - - return $self->get($strSection, $strValue, $strSubValue, $bRequired, - defined($bDefault) ? ($bDefault ? INI_TRUE : INI_FALSE) : undef) ? true : false; -} - -#################################################################################################################################### -# numericGet -# -# Get a numeric value. -#################################################################################################################################### -sub numericGet -{ - my $self = shift; - my $strSection = shift; - my $strValue = shift; - my $strSubValue = shift; - my $bRequired = shift; - my $nDefault = shift; - - return $self->get($strSection, $strValue, $strSubValue, $bRequired, - defined($nDefault) ? $nDefault + 0 : undef) + 0; -} - -#################################################################################################################################### -# tablespacePathGet -# -# Get the unique path assigned by Postgres for the tablespace. -#################################################################################################################################### -sub tablespacePathGet -{ - my $self = shift; - - return('PG_' . $self->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_VERSION) . - '_' . $self->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_CATALOG)); -} - -#################################################################################################################################### -# dbPathGet -# -# Convert a repo path to where the file actually belongs in the db. -#################################################################################################################################### -sub dbPathGet -{ - my $self = shift; - my $strDbPath = shift; - my $strFile = shift; - - my $strDbFile = defined($strDbPath) ? "${strDbPath}/" : ''; - - if (index($strFile, MANIFEST_TARGET_PGDATA . '/') == 0) - { - $strDbFile .= substr($strFile, length(MANIFEST_TARGET_PGDATA) + 1); - } - else - { - $strDbFile .= $strFile; - } - - return $strDbFile; -} - -#################################################################################################################################### -# repoPathGet -# -# Convert a database path to where to file is located in the repo. -#################################################################################################################################### -sub repoPathGet -{ - my $self = shift; - my $strTarget = shift; - my $strFile = shift; - - my $strRepoFile = $strTarget; - - if ($self->isTargetTablespace($strTarget)) - { - $strRepoFile .= '/' . $self->tablespacePathGet(); - } - - if (defined($strFile)) - { - $strRepoFile .= "/${strFile}"; - } - - return $strRepoFile; -} - -#################################################################################################################################### -# isTargetValid -# -# Determine if a target is valid. -#################################################################################################################################### -sub isTargetValid -{ - my $self = shift; - my $strTarget = shift; - my $bError = shift; - - if (!defined($strTarget)) - { - confess &log(ASSERT, 'target is not defined'); - } - - if (!$self->test(MANIFEST_SECTION_BACKUP_TARGET, $strTarget)) - { - if (defined($bError) && $bError) - { - confess &log(ASSERT, "${strTarget} is not a valid target"); - } - - return false; - } - - return true; -} - -#################################################################################################################################### -# isTargetLink -# -# Determine if a target is a link. -#################################################################################################################################### -sub isTargetLink -{ - my $self = shift; - my $strTarget = shift; - - $self->isTargetValid($strTarget, true); - - return $self->test(MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_TYPE, MANIFEST_VALUE_LINK); -} - -#################################################################################################################################### -# isTargetFile -# -# Determine if a target is a file link. -#################################################################################################################################### -sub isTargetFile -{ - my $self = shift; - my $strTarget = shift; - - $self->isTargetValid($strTarget, true); - - return $self->test(MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_FILE); -} - -#################################################################################################################################### -# isTargetTablespace -# -# Determine if a target is a tablespace. -#################################################################################################################################### -sub isTargetTablespace -{ - my $self = shift; - my $strTarget = shift; - - $self->isTargetValid($strTarget, true); - - return $self->test(MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_TABLESPACE_ID); -} - -#################################################################################################################################### -# checkDelta -# -# Determine if the delta option should be enabled. Only called if delta has not yet been enabled. -#################################################################################################################################### -sub checkDelta -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strLastBackupSource, - $bOnlineSame, - $strTimelineCurrent, - $strTimelineLast, - ) = - logDebugParam - ( - __PACKAGE__ . '->checkDelta', \@_, - {name => 'strLastBackupSource'}, - {name => 'bOnlineSame'}, - {name => 'strTimelineCurrent', required => false}, - {name => 'strTimelineLast', required => false}, - ); - - my $bDelta = false; - - # Determine if a timeline switch has occurred - if (defined($strTimelineLast) && defined($strTimelineCurrent)) - { - # If there is a prior backup, check if a timeline switch has occurred since then - if ($strTimelineLast ne $strTimelineCurrent) - { - &log(WARN, "a timeline switch has occurred since the ${strLastBackupSource} backup, enabling delta checksum"); - $bDelta = true; - } - } - - # If delta was not set above and there is a change in the online option, then set delta option - if (!$bDelta && !$bOnlineSame) - { - &log(WARN, "the online option has changed since the ${strLastBackupSource} backup, enabling delta checksum"); - $bDelta = true; - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bDelta', value => $bDelta, trace => true}, - ); -} - -#################################################################################################################################### -# checkDeltaFile -# -# Determine if the delta option should be enabled. Only called if delta has not yet been enabled. -#################################################################################################################################### -sub checkDeltaFile -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $stryFileList, - $oPriorManifest, - $lTimeBegin, - ) = - logDebugParam - ( - __PACKAGE__ . '->checkDeltaFile', \@_, - {name => 'stryFileList'}, - {name => 'oPriorManifest', required => false}, - {name => 'lTimeBegin', required => false}, - ); - - my $bDelta = false; - - # Loop though all files - foreach my $strName (@{$stryFileList}) - { - # If $lTimeBegin is defined, then this is not an aborted manifest so check if modification time is in the future (in this - # backup OR the last backup) then enable delta and exit - if (defined($lTimeBegin) && - ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) > $lTimeBegin || - (defined($oPriorManifest) && - $oPriorManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_FUTURE, 'y')))) - { - &log(WARN, "file $strName has timestamp in the future, enabling delta checksum"); - $bDelta = true; - last; - } - - # If the time on the file is earlier than the last manifest time or the size is different but the timestamp is the - # same, then enable delta and exit - if (defined($oPriorManifest) && $oPriorManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName) && - ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) < - $oPriorManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) || - ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) != - $oPriorManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) && - $self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) == - $oPriorManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP)))) - { - &log(WARN, "file $strName timestamp in the past or size changed but timestamp did not, enabling delta checksum"); - $bDelta = true; - last; - } - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bDelta', value => $bDelta, trace => true}, - ); -} - -#################################################################################################################################### -# build -# -# Build the manifest object. -#################################################################################################################################### -sub build -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $oStorageDbPrimary, - $strPath, - $oLastManifest, - $bOnline, - $bDelta, - $hTablespaceMap, - $hDatabaseMap, - $rhExclude, - $strTimelineCurrent, - $strTimelineLast, - $strLevel, - $bTablespace, - $strParentPath, - $strFilter, - $iLevel, - ) = - logDebugParam - ( - __PACKAGE__ . '->build', \@_, - {name => 'oStorageDbPrimary'}, - {name => 'strPath'}, - {name => 'oLastManifest', required => false}, - {name => 'bOnline'}, - {name => 'bDelta'}, - {name => 'hTablespaceMap', required => false}, - {name => 'hDatabaseMap', required => false}, - {name => 'rhExclude', required => false}, - {name => 'strTimelineCurrent', required => false}, - {name => 'strTimelineLast', required => false}, - {name => 'strLevel', required => false}, - {name => 'bTablespace', required => false}, - {name => 'strParentPath', required => false}, - {name => 'strFilter', required => false}, - {name => 'iLevel', required => false, default => 0}, - ); - - # Limit recursion to something reasonable (if more then we are very likely in a link loop) - if ($iLevel >= 16) - { - confess &log( - ERROR, - "recursion in manifest build exceeds depth of ${iLevel}: ${strLevel}\n" . - 'HINT: is there a link loop in $PGDATA?', - ERROR_FORMAT); - } - - if (!defined($strLevel)) - { - # Don't allow the manifest to be built more than once - if ($self->{bBuilt}) - { - confess &log(ASSERT, "manifest has already been built"); - } - - $self->{bBuilt} = true; - - # Set initial level - $strLevel = MANIFEST_TARGET_PGDATA; - - # If not online then build the tablespace map from pg_tblspc path - if (!$bOnline && !defined($hTablespaceMap)) - { - my $hTablespaceManifest = $oStorageDbPrimary->manifest($strPath . '/' . DB_PATH_PGTBLSPC); - $hTablespaceMap = {}; - - foreach my $strOid (sort(CORE::keys(%{$hTablespaceManifest}))) - { - if ($strOid eq '.' or $strOid eq '..') - { - next; - } - - logDebugMisc($strOperation, "found tablespace ${strOid} in offline mode"); - - $hTablespaceMap->{$strOid} = "ts${strOid}"; - } - } - - # If there is a last manifest, then check to see if delta checksum should be enabled - if (defined($oLastManifest) && !$bDelta) - { - $bDelta = $self->checkDelta( - 'last', $oLastManifest->boolTest(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ONLINE, undef, $bOnline), - $strTimelineCurrent, $strTimelineLast); - } - } - - $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_PATH, $strPath); - $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_TYPE, - $strLevel eq MANIFEST_TARGET_PGDATA ? MANIFEST_VALUE_PATH : MANIFEST_VALUE_LINK); - - if ($bTablespace) - { - my $iTablespaceId = (split('\/', $strLevel))[1]; - - if (!defined($hTablespaceMap->{$iTablespaceId})) - { - confess &log(ASSERT, "tablespace with oid ${iTablespaceId} not found in tablespace map\n" . - "HINT: was a tablespace created or dropped during the backup?"); - } - - $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_TABLESPACE_ID, $iTablespaceId); - $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_TABLESPACE_NAME, - $hTablespaceMap->{$iTablespaceId}); - } - - if (index($strPath, '/') != 0) - { - if (!defined($strParentPath)) - { - confess &log(ASSERT, "cannot get manifest for '${strPath}' when no parent path is specified"); - } - - $strPath = $oStorageDbPrimary->pathAbsolute($strParentPath, $strPath); - } - - # Get the manifest for this level - my $hManifest = $oStorageDbPrimary->manifest($strPath, {strFilter => $strFilter}); - my $strManifestType = MANIFEST_VALUE_LINK; - - # Loop though all paths/files/links in the manifest - foreach my $strName (sort(CORE::keys(%{$hManifest}))) - { - my $strFile = $strLevel; - - if ($strName ne '.') - { - if ($strManifestType eq MANIFEST_VALUE_LINK && $hManifest->{$strName}{type} eq 'l') - { - confess &log(ERROR, 'link \'' . - $self->dbPathGet( - $self->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH), $strLevel) . - '\' -> \'' . $self->get(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_PATH) . - '\' cannot reference another link', ERROR_LINK_DESTINATION); - } - - if ($strManifestType eq MANIFEST_VALUE_LINK) - { - $strFile = dirname($strFile); - $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_PATH, - dirname($self->get(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_PATH))); - $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_FILE, $strName); - } - - $strFile .= "/${strName}"; - } - else - { - $strManifestType = MANIFEST_VALUE_PATH; - } - - # Skip wal directory when doing an online backup. WAL will be restored from the archive or stored in the wal directory at - # the end of the backup if the archive-copy option is set. - next if ($bOnline && $strFile =~ (qw{^} . MANIFEST_TARGET_PGDATA . qw{/} . $self->walPath() . '\/') && - $strFile !~ ('^' . MANIFEST_TARGET_PGDATA . qw{/} . $self->walPath() . qw{/} . DB_PATH_ARCHIVESTATUS . '$')); - - # Skip all directories and files that start with pgsql_tmp. The files are removed when the server is restarted and the - # directories are recreated. - next if $strName =~ ('(^|\/)' . DB_FILE_PREFIX_TMP); - - # Skip pg_dynshmem/* since these files cannot be reused on recovery - next if $strFile =~ ('^' . MANIFEST_PATH_PGDYNSHMEM . '\/') && $self->dbVersion() >= PG_VERSION_94; - - # Skip pg_notify/* since these files cannot be reused on recovery - next if $strFile =~ ('^' . MANIFEST_PATH_PGNOTIFY . '\/'); - - # Skip pg_replslot/* since these files are generally not useful after a restore - next if $strFile =~ ('^' . MANIFEST_PATH_PGREPLSLOT . '\/') && $self->dbVersion() >= PG_VERSION_94; - - # Skip pg_serial/* since these files are reset - next if $strFile =~ ('^' . MANIFEST_PATH_PGSERIAL . '\/'); - - # Skip pg_snapshots/* since these files cannot be reused on recovery - next if $strFile =~ ('^' . MANIFEST_PATH_PGSNAPSHOTS . '\/'); - - # Skip temporary statistics in pg_stat_tmp even when stats_temp_directory is set because PGSS_TEXT_FILE is always created - # there. - next if $strFile =~ ('^' . MANIFEST_PATH_PGSTATTMP . '\/'); - - # Skip pg_subtrans/* since these files are reset - next if $strFile =~ ('^' . MANIFEST_PATH_PGSUBTRANS . '\/'); - - # Skip pg_internal.init since it is recreated on startup - next if $strFile =~ (DB_FILE_PGINTERNALINIT . '$'); - - # Skip recovery files - if ($self->dbVersion() >= PG_VERSION_12) - { - next if ($strFile eq MANIFEST_FILE_RECOVERYSIGNAL || $strFile eq MANIFEST_FILE_STANDBYSIGNAL); - } - else - { - next if ($strFile eq MANIFEST_FILE_RECOVERYDONE || $strFile eq MANIFEST_FILE_RECOVERYCONF); - } - - # Skip ignored files - if ($strFile eq MANIFEST_FILE_POSTGRESQLAUTOCONFTMP || # postgresql.auto.conf.tmp - temp file for safe writes - $strFile eq MANIFEST_FILE_BACKUPLABELOLD || # backup_label.old - old backup labels are not useful - $strFile eq MANIFEST_FILE_POSTMTROPTS || # not useful for backup - $strFile eq MANIFEST_FILE_POSTMTRPID) # to avoid confusing postgres after restore - { - next; - } - - # Check for files to exclude - if ($hManifest->{$strName}{type} eq 'f') - { - # Get the directory name from the manifest; it will be used later to search for existence in the keys - my $strDir = dirname($strName); - - # If it is a database data directory (base or tablespace) then check for files to skip - if ($strDir =~ '^base\/[0-9]+$' || - $strDir =~ ('^' . $self->tablespacePathGet() . '\/[0-9]+$')) - { - # Get just the filename - my $strBaseName = basename($strName); - - # Skip temp tables (lower t followed by numbers underscore numbers and a dot (segment) or underscore (fork) and/or - # segment, e.g. t1234_123, t1234_123.1, t1234_123_vm, t1234_123_fsm.1 - if ($strBaseName =~ '^t[0-9]+\_[0-9]+(|\_(fsm|vm)){0,1}(\.[0-9]+){0,1}$') - { - next; - } - - # Check for unlogged tables to skip - # Exclude all forks for unlogged tables except the init fork (numbers underscore init and optional dot segment) - if ($strBaseName =~ '^[0-9]+(|\_(fsm|vm)){0,1}(\.[0-9]+){0,1}$') - { - # Get the filenode (OID) - my ($strFileNode) = $strBaseName =~ '^(\d+)'; - - # Add _init to the OID to see if this is an unlogged object - $strFileNode = $strDir. "/" . $strFileNode . "_init"; - - # If key exists in manifest then skip - if (exists($hManifest->{$strFileNode}) && $hManifest->{$strFileNode}{type} eq 'f') - { - next; - } - } - } - } - - # Exclude files requested by the user - if (defined($rhExclude)) - { - # Exclusions are based on the name of the file relative to PGDATA - my $strPgFile = $self->dbPathGet(undef, $strFile); - my $bExclude = false; - - # Iterate through exclusions - foreach my $strExclude (sort(keys(%{$rhExclude}))) - { - # If the exclusion ends in / then we must do a prefix match - if ($strExclude =~ /\/$/) - { - if (index($strPgFile, $strExclude) == 0) - { - $bExclude = true; - } - } - # Else an exact match or a prefix match with / appended is required - elsif ($strPgFile eq $strExclude || index($strPgFile, "${strExclude}/") == 0) - { - $bExclude = true; - } - - # Log everything that gets excluded at a high level so it will hopefully be seen if wrong - if ($bExclude) - { - &log(INFO, "exclude ${strPgFile} from backup using '${strExclude}' exclusion"); - last; - } - } - - # Skip the file if it was excluded - next if $bExclude; - } - - my $cType = $hManifest->{$strName}{type}; - my $strSection = MANIFEST_SECTION_TARGET_PATH; - - if ($cType eq 'f') - { - $strSection = MANIFEST_SECTION_TARGET_FILE; - } - elsif ($cType eq 'l') - { - $strSection = MANIFEST_SECTION_TARGET_LINK; - } - elsif ($cType ne 'd') - { - &log(WARN, "exclude special file '" . $self->dbPathGet(undef, $strFile) . "' from backup"); - next; - } - - # Make sure that DB_PATH_PGTBLSPC contains only absolute links that do not point inside PGDATA - my $bTablespace = false; - - if (index($strName, DB_PATH_PGTBLSPC . '/') == 0 && $strLevel eq MANIFEST_TARGET_PGDATA) - { - $bTablespace = true; - $strFile = MANIFEST_TARGET_PGDATA . '/' . $strName; - - # Check for files in DB_PATH_PGTBLSPC that are not links - if ($hManifest->{$strName}{type} ne 'l') - { - confess &log(ERROR, "${strName} is not a symlink - " . DB_PATH_PGTBLSPC . ' should contain only symlinks', - ERROR_LINK_EXPECTED); - } - - # Check for tablespaces in PGDATA - if (index($hManifest->{$strName}{link_destination}, "${strPath}/") == 0 || - (index($hManifest->{$strName}{link_destination}, '/') != 0 && - index($oStorageDbPrimary->pathAbsolute($strPath . '/' . DB_PATH_PGTBLSPC, - $hManifest->{$strName}{link_destination}) . '/', "${strPath}/") == 0)) - { - confess &log(ERROR, 'tablespace symlink ' . $hManifest->{$strName}{link_destination} . - ' destination must not be in $PGDATA', ERROR_LINK_DESTINATION); - } - } - - # User and group required for all types - if (defined($hManifest->{$strName}{user})) - { - $self->set($strSection, $strFile, MANIFEST_SUBKEY_USER, $hManifest->{$strName}{user}); - } - else - { - $self->boolSet($strSection, $strFile, MANIFEST_SUBKEY_USER, false); - } - - if (defined($hManifest->{$strName}{group})) - { - $self->set($strSection, $strFile, MANIFEST_SUBKEY_GROUP, $hManifest->{$strName}{group}); - } - else - { - $self->boolSet($strSection, $strFile, MANIFEST_SUBKEY_GROUP, false); - } - - # Mode for required file and path type only - if ($cType eq 'f' || $cType eq 'd') - { - $self->set($strSection, $strFile, MANIFEST_SUBKEY_MODE, $hManifest->{$strName}{mode}); - } - - # Modification time and size required for file type only - if ($cType eq 'f') - { - $self->set($strSection, $strFile, MANIFEST_SUBKEY_TIMESTAMP, - $hManifest->{$strName}{modification_time} + 0); - $self->set($strSection, $strFile, MANIFEST_SUBKEY_SIZE, $hManifest->{$strName}{size} + 0); - } - - # Link destination required for link type only - if ($cType eq 'l') - { - my $strLinkDestination = $hManifest->{$strName}{link_destination}; - $self->set($strSection, $strFile, MANIFEST_SUBKEY_DESTINATION, $strLinkDestination); - - # If this is a tablespace then set the filter to use for the next level - my $strFilter; - - if ($bTablespace) - { - $strFilter = $self->tablespacePathGet(); - - $self->set(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGTBLSPC, undef, - $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA)); - - # PGDATA prefix was only needed for the link so strip it off before recursing - $strFile = substr($strFile, length(MANIFEST_TARGET_PGDATA) + 1); - } - - $bDelta = $self->build( - $oStorageDbPrimary, $strLinkDestination, undef, $bOnline, $bDelta, $hTablespaceMap, $hDatabaseMap, $rhExclude, - undef, undef, $strFile, $bTablespace, dirname("${strPath}/${strName}"), $strFilter, $iLevel + 1); - } - } - - # If this is the base level then do post-processing - if ($strLevel eq MANIFEST_TARGET_PGDATA) - { - my $bTimeInFuture = false; - - # Wait for the remainder of the second when doing online backups. This is done because most filesystems only have a one - # second resolution and Postgres will still be modifying files during the second that the manifest is built and this could - # lead to an invalid diff/incr backup later when using timestamps to determine which files have changed. Offline backups do - # not wait because it makes testing much faster and Postgres should not be running (if it is the backup will not be - # consistent anyway and the one-second resolution problem is the least of our worries). - my $lTimeBegin = waitRemainder($bOnline); - - if (defined($oLastManifest)) - { - $self->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR, undef, - $oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL)); - } - - # Store database map information when provided during an online backup. - foreach my $strDbName (sort(keys(%{$hDatabaseMap}))) - { - $self->numericSet(MANIFEST_SECTION_DB, $strDbName, MANIFEST_KEY_DB_ID, - $hDatabaseMap->{$strDbName}{&MANIFEST_KEY_DB_ID}); - $self->numericSet(MANIFEST_SECTION_DB, $strDbName, MANIFEST_KEY_DB_LAST_SYSTEM_ID, - $hDatabaseMap->{$strDbName}{&MANIFEST_KEY_DB_LAST_SYSTEM_ID}); - } - - # Determine if delta checksum should be enabled - if (!$bDelta) - { - my @stryFileList = $self->keys(MANIFEST_SECTION_TARGET_FILE); - - if (@stryFileList) - { - $bDelta = $self->checkDeltaFile(\@stryFileList, $oLastManifest, $lTimeBegin); - } - } - - # Loop though all files - foreach my $strName ($self->keys(MANIFEST_SECTION_TARGET_FILE)) - { - # If modification time is in the future (in this backup OR the last backup) set warning flag and do not - # allow a reference - if ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) > $lTimeBegin || - (defined($oLastManifest) && - $oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_FUTURE, 'y'))) - { - $bTimeInFuture = true; - - # Only mark as future if still in the future in the current backup - if ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) > $lTimeBegin) - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_FUTURE, 'y'); - } - } - # Else check if the size and timestamp match OR if the size matches and the delta option is set, then keep the file. - # In the latter case, if there had been a timestamp change then rather than removing and recopying the file, the file - # will be tested in backupFile to see if the db/repo checksum still matches: if so, it is not necessary to recopy, - # else it will need to be copied to the new backup. For zero sized files, the reference will be set and copying - # will be skipped later. - elsif (defined($oLastManifest) && $oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName) && - $self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) == - $oLastManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) && - ($bDelta || ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) == 0 || - $self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) == - $oLastManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP)))) - { - # Copy reference from previous backup if possible - if ($oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE)) - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE, - $oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE)); - } - # Otherwise the reference is to the previous backup - else - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE, - $oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL)); - } - - # Copy the checksum from previous manifest (if it exists - zero sized files don't have checksums) - if ($oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM)) - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM, - $oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM)); - } - - # Copy repo size from the previous manifest (if it exists) - if ($oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE)) - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE, - $oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE)); - } - - # Copy checksum page from the previous manifest (if it exists) - my $bChecksumPage = $oLastManifest->get( - MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE, false); - - if (defined($bChecksumPage)) - { - $self->boolSet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE, $bChecksumPage); - - if (!$bChecksumPage && - $oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR)) - { - $self->set( - MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR, - $oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR)); - } - } - } - } - - # Warn if any files in the current backup are in the future - if ($bTimeInFuture) - { - &log(WARN, "some files have timestamps in the future - they will be copied to prevent possible race conditions"); - } - - # Record the time when copying will start - $self->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START, undef, $lTimeBegin + ($bOnline ? 1 : 0)); - - # Build default sections - $self->buildDefault(); - } - - # Return from function and log return values if any - return logDebugReturn - ( - $strOperation, - {name => 'bDelta', value => $bDelta, trace => true}, - ); -} - -#################################################################################################################################### -# fileAdd -# -# Add files to the manifest that were generated after the initial manifest build, e.g. backup_label, tablespace_map, and copied WAL -# files. Since the files were not in the original cluster the user, group, and mode must be defaulted. -#################################################################################################################################### -sub fileAdd -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my - ( - $strOperation, - $strManifestFile, - $lModificationTime, - $lSize, - $strChecksum, - $bPrimary, - ) = - logDebugParam - ( - __PACKAGE__ . '->fileAdd', \@_, - {name => 'strManifestFile'}, - {name => 'lModificationTime'}, - {name => 'lSize'}, - {name => 'lChecksum'}, - {name => 'bPrimary'}, - ); - - # Set manifest values - if (!$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_USER) || - !$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_USER, undef, - $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_USER))) - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_USER, - $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_USER)); - } - - if (!$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_GROUP) || - !$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_GROUP, undef, - $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_GROUP))) - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_GROUP, - $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_GROUP)); - } - - if (!$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_MODE) || - !$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_MODE, undef, '0600')) - { - $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_MODE, '0600'); - } - - $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_TIMESTAMP, $lModificationTime); - $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_SIZE, $lSize); - $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_CHECKSUM, $strChecksum); -} - -#################################################################################################################################### -# buildDefault -# -# Builds the default section. -#################################################################################################################################### -sub buildDefault -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . '->buildDefault'); - - # Defaults for subkeys that tend to repeat - my $strDefaultUser = $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_USER); - my $strDefaultGroup = $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_GROUP); - my $strDefaultPathMode = $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_MODE); - my $strDefaultFileMode = sprintf('%04o', oct($strDefaultPathMode) & (S_IRUSR | S_IWUSR | S_IRGRP)); - - # Remove subkeys that match the defaults - foreach my $strSection (&MANIFEST_SECTION_TARGET_FILE, &MANIFEST_SECTION_TARGET_PATH, &MANIFEST_SECTION_TARGET_LINK) - { - next if !$self->test($strSection); - - foreach my $strFile ($self->keys($strSection)) - { - if ($self->test($strSection, $strFile, MANIFEST_SUBKEY_USER, $strDefaultUser)) - { - $self->remove($strSection, $strFile, MANIFEST_SUBKEY_USER); - } - - if ($self->test($strSection, $strFile, MANIFEST_SUBKEY_GROUP, $strDefaultGroup)) - { - $self->remove($strSection, $strFile, MANIFEST_SUBKEY_GROUP); - } - - if ($self->test( - $strSection, $strFile, MANIFEST_SUBKEY_MODE, - $strSection eq MANIFEST_SECTION_TARGET_PATH ? $strDefaultPathMode : $strDefaultFileMode)) - { - $self->remove($strSection, $strFile, MANIFEST_SUBKEY_MODE); - } - } - } - - # Write defaults - $self->set(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_USER, undef, $strDefaultUser); - $self->set(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_GROUP, undef, $strDefaultGroup); - $self->set(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_MODE, undef, $strDefaultFileMode); - - if ($self->test(MANIFEST_SECTION_TARGET_LINK)) - { - $self->set(MANIFEST_SECTION_TARGET_LINK . ':default', MANIFEST_SUBKEY_USER, undef, $strDefaultUser); - $self->set(MANIFEST_SECTION_TARGET_LINK . ':default', MANIFEST_SUBKEY_GROUP, undef, $strDefaultGroup); - } - - $self->set(MANIFEST_SECTION_TARGET_PATH . ':default', MANIFEST_SUBKEY_USER, undef, $strDefaultUser); - $self->set(MANIFEST_SECTION_TARGET_PATH . ':default', MANIFEST_SUBKEY_GROUP, undef, $strDefaultGroup); - $self->set(MANIFEST_SECTION_TARGET_PATH . ':default', MANIFEST_SUBKEY_MODE, undef, $strDefaultPathMode); - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# validate -# -# Checks for any missing values or inconsistencies in the manifest. -#################################################################################################################################### -sub validate -{ - my $self = shift; - - # Assign function parameters, defaults, and log debug info - my ($strOperation) = logDebugParam(__PACKAGE__ . 'validate'); - - # Make sure that all files have size and checksum (when size > 0). Since these values are removed before the backup file copy - # starts this ensures that all files had results stored in the manifest during the file copy. - foreach my $strFile ($self->keys(MANIFEST_SECTION_TARGET_FILE)) - { - # Ensure size is set - if (!$self->test(MANIFEST_SECTION_TARGET_FILE, $strFile, MANIFEST_SUBKEY_SIZE)) - { - confess &log(ASSERT, "manifest subvalue 'size' not set for file '${strFile}'"); - } - - # If size > 0 then checksum must also be set - if ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strFile, MANIFEST_SUBKEY_SIZE) > 0 && - !$self->test(MANIFEST_SECTION_TARGET_FILE, $strFile, MANIFEST_SUBKEY_CHECKSUM)) - { - confess &log(ASSERT, "manifest subvalue 'checksum' not set for file '${strFile}'"); - } - } - - # Return from function and log return values if any - return logDebugReturn($strOperation); -} - -#################################################################################################################################### -# dbVersion - version of PostgreSQL that the manifest is being built for -#################################################################################################################################### -sub dbVersion -{ - my $self = shift; - - return $self->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_VERSION); -} - -#################################################################################################################################### -# xactPath - return the transaction directory based on the PostgreSQL version -#################################################################################################################################### -sub xactPath -{ - my $self = shift; - - return $self->dbVersion() >= PG_VERSION_10 ? 'pg_xact' : 'pg_clog'; -} - -#################################################################################################################################### -# walPath - return the wal directory based on the PostgreSQL version -#################################################################################################################################### -sub walPath -{ - my $self = shift; - - return $self->dbVersion() >= PG_VERSION_10 ? 'pg_wal' : 'pg_xlog'; -} - -#################################################################################################################################### -# isPrimaryFile -# -# Is this file required to be copied from the primary? -#################################################################################################################################### -sub isPrimaryFile -{ - my $self = shift; - my $strFile = shift; - - return - $strFile !~ ('^(' . MANIFEST_TARGET_PGDATA . '\/' . '(' . DB_PATH_BASE . '|' . DB_PATH_GLOBAL . '|' . - $self->xactPath() . '|' . DB_PATH_PGMULTIXACT . ')|' . DB_PATH_PGTBLSPC . ')\/'); -} - -#################################################################################################################################### -# isChecksumPage -# -# Can this file have page checksums? -#################################################################################################################################### -sub isChecksumPage -{ - my $strFile = shift; - - if (($strFile =~ ('^' . MANIFEST_TARGET_PGDATA . '\/' . DB_PATH_BASE . '\/[0-9]+\/|^' . MANIFEST_TARGET_PGTBLSPC . - '\/[0-9]+\/[^\/]+\/[0-9]+\/') && - $strFile !~ ('(' . DB_FILE_PGFILENODEMAP . '|' . DB_FILE_PGINTERNALINIT . '|' . DB_FILE_PGVERSION . ')$')) || - ($strFile =~ ('^' . MANIFEST_TARGET_PGDATA . '\/' . DB_PATH_GLOBAL . '\/') && - $strFile !~ ('(' . DB_FILE_PGFILENODEMAP . '|' . DB_FILE_PGINTERNALINIT . '|' . DB_FILE_PGVERSION . '|' . - DB_FILE_PGCONTROL . ')$'))) - { - return true; - } - - return false; -} - -push @EXPORT, qw(isChecksumPage); - -1; diff --git a/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm b/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm deleted file mode 100644 index 911c42a444..0000000000 --- a/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm +++ /dev/null @@ -1,747 +0,0 @@ -#################################################################################################################################### -# Test All Commands On PostgreSQL Clusters -#################################################################################################################################### -package pgBackRestTest::Module::Real::RealAllTest; -use parent 'pgBackRestTest::Env::HostEnvTest'; - -#################################################################################################################################### -# Perl includes -#################################################################################################################################### -use strict; -use warnings FATAL => qw(all); -use Carp qw(confess); - -use File::Basename qw(dirname); - -use pgBackRestDoc::Common::Exception; -use pgBackRestDoc::Common::Ini; -use pgBackRestDoc::Common::Log; -use pgBackRestDoc::ProjectInfo; - -use pgBackRestTest::Common::ContainerTest; -use pgBackRestTest::Common::DbVersion; -use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::FileTest; -use pgBackRestTest::Common::RunTest; -use pgBackRestTest::Common::VmTest; -use pgBackRestTest::Common::Storage; -use pgBackRestTest::Common::StoragePosix; -use pgBackRestTest::Common::StorageRepo; -use pgBackRestTest::Common::Wait; -use pgBackRestTest::Env::ArchiveInfo; -use pgBackRestTest::Env::BackupInfo; -use pgBackRestTest::Env::InfoCommon; -use pgBackRestTest::Env::Host::HostBaseTest; -use pgBackRestTest::Env::Host::HostBackupTest; -use pgBackRestTest::Env::Host::HostDbTest; -use pgBackRestTest::Env::Host::HostDbTest; -use pgBackRestTest::Env::HostEnvTest; -use pgBackRestTest::Env::Manifest; - -#################################################################################################################################### -# Backup advisory lock -#################################################################################################################################### -use constant DB_BACKUP_ADVISORY_LOCK => '12340078987004321'; - -#################################################################################################################################### -# run -#################################################################################################################################### -sub run -{ - my $self = shift; - - foreach my $rhRun - ( - {pg => '9.4', dst => 'db-standby', tls => 0, stg => POSIX, enc => 1, cmp => LZ4, rt => 1, bnd => 1, bi => 0}, - {pg => '9.5', dst => 'backup', tls => 1, stg => GCS, enc => 0, cmp => BZ2, rt => 1, bnd => 0, bi => 1}, - {pg => '9.6', dst => 'backup', tls => 0, stg => POSIX, enc => 0, cmp => NONE, rt => 2, bnd => 1, bi => 1}, - {pg => '10', dst => 'sftp-srvr', tls => 0, stg => SFTP, enc => 1, cmp => GZ, rt => 1, bnd => 1, bi => 0}, - {pg => '11', dst => 'backup', tls => 1, stg => AZURE, enc => 0, cmp => ZST, rt => 2, bnd => 0, bi => 0}, - {pg => '12', dst => 'backup', tls => 0, stg => S3, enc => 1, cmp => LZ4, rt => 1, bnd => 0, bi => 1}, - {pg => '13', dst => 'db-standby', tls => 1, stg => GCS, enc => 0, cmp => ZST, rt => 1, bnd => 1, bi => 1}, - {pg => '14', dst => 'sftp-srvr', tls => 0, stg => SFTP, enc => 0, cmp => LZ4, rt => 1, bnd => 1, bi => 0}, - {pg => '15', dst => 'db-standby', tls => 0, stg => AZURE, enc => 0, cmp => NONE, rt => 2, bnd => 1, bi => 1}, - {pg => '16', dst => 'backup', tls => 0, stg => S3, enc => 1, cmp => NONE, rt => 1, bnd => 0, bi => 0}, - ) - { - # Only run tests for this pg version - next if ($rhRun->{pg} ne $self->pgVersion()); - - # Get run parameters - my $bHostBackup = $rhRun->{dst} eq HOST_BACKUP ? true : false; - my $bTls = $rhRun->{tls}; - my $strBackupDestination = $rhRun->{dst}; - my $strStorage = $rhRun->{stg}; - my $bRepoEncrypt = $rhRun->{enc}; - my $strCompressType = $rhRun->{cmp}; - my $iRepoTotal = $rhRun->{rt}; - my $bBundle = $rhRun->{bnd}; - my $bBlockIncr = $rhRun->{bi}; - - # Some tests are not version specific so only run them on a single version of PostgreSQL - my $bNonVersionSpecific = $self->pgVersion() eq PG_VERSION_96; - - # Increment the run, log, and decide whether this unit test should be run - next if !$self->begin( - "bkp ${bHostBackup}, tls ${bTls}, dst ${strBackupDestination}, cmp ${strCompressType}, storage ${strStorage}" . - ", enc ${bRepoEncrypt}, bi ${bBlockIncr}"); - - # Create hosts, file object, and config - my ($oHostDbPrimary, $oHostDbStandby, $oHostBackup) = $self->setup( - false, - {bHostBackup => $bHostBackup, bStandby => true, bTls => $bTls, strBackupDestination => $strBackupDestination, - strCompressType => $strCompressType, bArchiveAsync => false, strStorage => $strStorage, - bRepoEncrypt => $bRepoEncrypt, iRepoTotal => $iRepoTotal, bBundle => $bBundle, bBlockIncr => $bBlockIncr}); - - # Some commands will fail because of the bogus host created when a standby is present. These options reset the bogus host - # so it won't interfere with commands that won't tolerate a connection failure. - my $strBogusReset = $oHostBackup->bogusHost() ? - ' --reset-pg2-host --reset-pg2-host-type --reset-pg2-host-cmd --reset-pg2-host-config --reset-pg2-host-user' . - ' --reset-pg2-path' : - ''; - - # If S3 set process max to 2. This seems like the best place for parallel testing since it will help speed S3 processing - # without slowing down the other tests too much. - if ($strStorage eq S3) - { - $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}}); - $oHostDbPrimary->configUpdate({&CFGDEF_SECTION_GLOBAL => {'process-max' => 2}}); - } - - $oHostDbPrimary->clusterCreate(); - - # Create the stanza - $oHostDbPrimary->stanzaCreate('main create stanza info files'); - - # Get passphrase to access the Manifest file from backup.info - returns undefined if repo not encrypted - my $strCipherPass = - (new pgBackRestTest::Env::BackupInfo($oHostBackup->repoBackupPath()))->cipherPassSub(); - - # Create a manifest with the pg version to get version-specific paths - my $oManifest = new pgBackRestTest::Env::Manifest(BOGUS, {bLoad => false, strDbVersion => $self->pgVersion(), - iDbCatalogVersion => $self->dbCatalogVersion($self->pgVersion()), - strCipherPass => $strCipherPass, strCipherPassSub => $bRepoEncrypt ? ENCRYPTION_KEY_BACKUPSET : undef}); - - # Static backup parameters - my $fTestDelay = 1; - - # Restore test string - my $strDefaultMessage = 'default'; - my $strFullMessage = 'full'; - my $strStandbyMessage = 'standby'; - my $strIncrMessage = 'incr'; - my $strTimeMessage = 'time'; - my $strXidMessage = 'xid'; - my $strNameMessage = 'name'; - my $strTimelineMessage = 'timeline'; - - # Create two new databases - $oHostDbPrimary->sqlExecute('create database test1', {bAutoCommit => true}); - $oHostDbPrimary->sqlExecute('create database test2', {bAutoCommit => true}); - - # ??? Removed temporarily until manifest build can be brought back into the check command - # Create a directory in pg_data location that is only readable by root to ensure manifest->build is called by check - # -------------------------------------------------------------------------------------------------------------------------- - # my $strDir = $oHostDbPrimary->dbBasePath() . '/rootreaddir'; - # executeTest('sudo mkdir ' . $strDir); - # executeTest("sudo chown root:root ${strDir}"); - # executeTest("sudo chmod 400 ${strDir}"); - # - # $strComment = 'confirm primary manifest->build executed'; - # $oHostDbPrimary->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_PATH_OPEN}); - # executeTest("sudo rmdir ${strDir}"); - - # -------------------------------------------------------------------------------------------------------------------------- - my $strComment = 'verify check command runs successfully'; - - $oHostDbPrimary->check($strComment, {iTimeout => 10, bStanza => false}); - - # Also run check on the backup host when present - if ($bHostBackup) - { - $oHostBackup->check($strComment, {iTimeout => 10, strOptionalParam => $strBogusReset}); - } - - # Restart the cluster ignoring any errors in the postgresql log - $oHostDbPrimary->clusterRestart({bIgnoreLogError => true}); - - # Full backup - #--------------------------------------------------------------------------------------------------------------------------- - # Create the table where test messages will be stored - $oHostDbPrimary->sqlExecute("create table test (message text not null)"); - $oHostDbPrimary->sqlWalRotate(); - $oHostDbPrimary->sqlExecute("insert into test values ('$strDefaultMessage')"); - - # Acquire the backup advisory lock so it looks like a backup is running - if (!$oHostDbPrimary->sqlSelectOne('select pg_try_advisory_lock(' . DB_BACKUP_ADVISORY_LOCK . ')')) - { - confess 'unable to acquire advisory lock for testing'; - } - - $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_FULL, 'fail on backup lock exists', {iExpectedExitStatus => ERROR_LOCK_ACQUIRE}); - - # Release the backup advisory lock so the next backup will succeed - if (!$oHostDbPrimary->sqlSelectOne('select pg_advisory_unlock(' . DB_BACKUP_ADVISORY_LOCK . ')')) - { - confess 'unable to release advisory lock'; - } - - $oHostDbPrimary->sqlExecute("update test set message = '$strFullMessage'"); - - # Required to set hint bits to be sent to the standby to make the heap match on both sides - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strFullMessage); - - # Backup to repo1 - my $strFullBackup = $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_FULL, 'repo1', - {strOptionalParam => ' --buffer-size=16384'}); - - # Backup to repo2 if it exists - if ($iRepoTotal == 2) - { - $oHostBackup->backup(CFGOPTVAL_BACKUP_TYPE_FULL, 'repo2', {iRepo => 2}); - } - - # Make a new backup with expire-auto disabled then run the expire command and compare backup numbers to ensure that expire - # was really disabled. This test is not version specific so is run on only one version. - #--------------------------------------------------------------------------------------------------------------------------- - if ($bNonVersionSpecific) - { - my $oBackupInfo = new pgBackRestTest::Env::BackupInfo($oHostBackup->repoBackupPath()); - push(my @backupLst1, $oBackupInfo->list()); - - $strFullBackup = $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_FULL, 'with disabled expire-auto', - {strOptionalParam => ' --repo1-retention-full='.scalar(@backupLst1). ' --no-expire-auto'}); - - $oBackupInfo = new pgBackRestTest::Env::BackupInfo($oHostBackup->repoBackupPath()); - push(my @backupLst2, $oBackupInfo->list()); - - &log(INFO, " run the expire command"); - $oHostBackup->expire({iRetentionFull => scalar(@backupLst1)}); - $oBackupInfo = new pgBackRestTest::Env::BackupInfo($oHostBackup->repoBackupPath()); - push(my @backupLst3, $oBackupInfo->list()); - - unless (scalar(@backupLst2) == scalar(@backupLst1) + 1 && scalar(@backupLst1) == scalar(@backupLst3)) - { - confess "expire-auto option didn't work as expected"; - } - } - - # Enabled async archiving - $oHostBackup->configUpdate({&CFGDEF_SECTION_GLOBAL => {'archive-async' => 'y'}}); - - # Kick out a bunch of archive logs to exercise async archiving. Only do this when compressed and remote to slow it down - # enough to make it evident that the async process is working. - if ($strCompressType ne NONE && $strBackupDestination eq HOST_BACKUP) - { - &log(INFO, ' multiple wal switches to exercise async archiving'); - $oHostDbPrimary->sqlExecute("create table wal_activity (id int)"); - $oHostDbPrimary->sqlWalRotate(); - $oHostDbPrimary->sqlExecute("insert into wal_activity values (1)"); - $oHostDbPrimary->sqlWalRotate(); - $oHostDbPrimary->sqlExecute("insert into wal_activity values (2)"); - $oHostDbPrimary->sqlWalRotate(); - $oHostDbPrimary->sqlExecute("insert into wal_activity values (3)"); - $oHostDbPrimary->sqlWalRotate(); - $oHostDbPrimary->sqlExecute("insert into wal_activity values (4)"); - $oHostDbPrimary->sqlWalRotate(); - } - - # Setup replica - #--------------------------------------------------------------------------------------------------------------------------- - my %oRemapHash; - $oRemapHash{&MANIFEST_TARGET_PGDATA} = $oHostDbStandby->dbBasePath(); - - $oHostDbStandby->linkRemap($oManifest->walPath(), $oHostDbStandby->dbPath() . '/' . $oManifest->walPath()); - - $oHostDbStandby->restore( - 'restore backup on replica', 'latest', - {rhRemapHash => \%oRemapHash, strType => CFGOPTVAL_RESTORE_TYPE_STANDBY, - strOptionalParam => - ' --recovery-option="primary_conninfo=host=' . HOST_DB_PRIMARY . - ' port=' . $oHostDbPrimary->pgPort() . ' user=replicator"'}); - - $oHostDbStandby->clusterStart({bHotStandby => true}); - - # Make sure streaming replication is on - $oHostDbPrimary->sqlSelectOneTest( - "select client_addr || '-' || state from pg_stat_replication", $oHostDbStandby->ipGet() . '/32-streaming'); - - # Check that the cluster was restored properly - $oHostDbStandby->sqlSelectOneTest('select message from test', $strFullMessage); - - # Update message for standby - $oHostDbPrimary->sqlExecute("update test set message = '$strStandbyMessage'"); - - if (!$bTls) - { - # If there is only a primary and a replica and the replica is the backup destination, then if pg2-host and - # pg256-host are BOGUS, confirm failure to reach the primary - if (!$bHostBackup && $strBackupDestination eq HOST_DB_STANDBY) - { - my $strStandbyBackup = $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_FULL, 'backup from standby, failure to reach primary', - {bStandby => true, iExpectedExitStatus => ERROR_DB_CONNECT, strOptionalParam => '--pg256-host=' . BOGUS}); - } - else - { - my $strStandbyBackup = $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_FULL, 'backup from standby, failure to access at least one standby', - {bStandby => true, iExpectedExitStatus => ERROR_DB_CONNECT, strOptionalParam => '--pg256-host=' . BOGUS}); - } - } - - my $strStandbyBackup = $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_FULL, 'backup from standby', - {bStandby => true, iExpectedExitStatus => undef, strOptionalParam => '--repo1-retention-full=1'}); - - $strFullBackup = $strStandbyBackup; - - # ??? Removed temporarily until manifest build can be brought back into the check command - # # Create a directory in pg_data location that is only readable by root to ensure manifest->build is called by check - # my $strDir = $oHostDbStandby->dbBasePath() . '/rootreaddir'; - # executeTest('sudo mkdir ' . $strDir); - # executeTest("sudo chown root:root ${strDir}"); - # executeTest("sudo chmod 400 ${strDir}"); - # - # my $strComment = 'confirm standby manifest->build executed'; - # - # # If there is an invalid host, the final error returned from check will be the inability to resolve the name which is - # # an open error instead of a read error - # if (!$oHostDbStandby->bogusHost()) - # { - # $oHostDbStandby->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_PATH_OPEN}); - # } - # else - # { - # $oHostDbStandby->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_FILE_READ}); - # } - # - # # Remove the directory in pg_data location that is only readable by root - # executeTest("sudo rmdir ${strDir}"); - - # Confirm the check command runs without error on a standby (when a bogus host is not configured) - $oHostDbStandby->check('verify check command on standby', {strOptionalParam => $strBogusReset}); - - # Shutdown the standby before creating tablespaces (this will error since paths are different) - $oHostDbStandby->clusterStop({bIgnoreLogError => true}); - - my $strAdhocBackup; - - # Execute stop and make sure the backup fails - #--------------------------------------------------------------------------------------------------------------------------- - # Restart the cluster to check for any errors before continuing since the stop tests will definitely create errors and the - # logs will need to be deleted to avoid causing issues further down the line. This test is not version specific so is run on - # only one version. - if ($bNonVersionSpecific) - { - confess "test must be performed on posix storage" if $strStorage ne POSIX; - - $oHostDbPrimary->clusterRestart(); - - # Add backup for adhoc expire - $strAdhocBackup = $oHostBackup->backup(CFGOPTVAL_BACKUP_TYPE_DIFF, 'backup for adhoc expire'); - - $oHostDbPrimary->stop(); - - $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_INCR, 'attempt backup when stopped', - {iExpectedExitStatus => $oHostBackup == $oHostDbPrimary ? ERROR_STOP : ERROR_DB_CONNECT}); - - $oHostDbPrimary->start(); - } - - # Setup the time targets - #--------------------------------------------------------------------------------------------------------------------------- - # If the tests are running quickly then the time target might end up the same as the end time of the prior full backup. That - # means restore auto-select will not pick it as a candidate and restore the last backup instead causing the restore compare - # to fail. So, sleep one second. - sleep(1); - - $oHostDbPrimary->sqlExecute("update test set message = '$strTimeMessage'"); - $oHostDbPrimary->sqlWalRotate(); - my $strTimeTarget = $oHostDbPrimary->sqlSelectOne("select current_timestamp"); - &log(INFO, " time target is ${strTimeTarget}"); - - # Incr backup - fail on archive_mode=always when version >= 9.5 - #--------------------------------------------------------------------------------------------------------------------------- - if ($oHostDbPrimary->pgVersion() >= PG_VERSION_95) - { - # Set archive_mode=always - $oHostDbPrimary->clusterRestart({bArchiveAlways => true}); - - $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_INCR, 'fail on archive_mode=always', {iExpectedExitStatus => ERROR_FEATURE_NOT_SUPPORTED}); - - # Reset the cluster to a normal state so the next test will work - $oHostDbPrimary->clusterRestart(); - } - - # Incr backup - #--------------------------------------------------------------------------------------------------------------------------- - # Create a tablespace directory - storageTest()->pathCreate($oHostDbPrimary->tablespacePath(1), {strMode => '0700', bCreateParent => true}); - - # Also create it on the standby so replay won't fail - if (defined($oHostDbStandby)) - { - storageTest()->pathCreate($oHostDbStandby->tablespacePath(1), {strMode => '0700', bCreateParent => true}); - } - - $oHostDbPrimary->sqlExecute( - "create tablespace ts1 location '" . $oHostDbPrimary->tablespacePath(1) . "'", {bAutoCommit => true}); - $oHostDbPrimary->sqlExecute("alter table test set tablespace ts1"); - - # Create a table in the tablespace that will not be modified again to be sure it does get full page writes in the WAL later - $oHostDbPrimary->sqlExecute("create table test_exists (id int) tablespace ts1", {bCommit => true, bCheckPoint => true}); - - # Create a table in the tablespace - $oHostDbPrimary->sqlExecute("create table test_remove (id int)"); - $oHostDbPrimary->sqlWalRotate(); - $oHostDbPrimary->sqlExecute("update test set message = '$strDefaultMessage'"); - $oHostDbPrimary->sqlWalRotate(); - - # Create a database in the tablespace and a table to check - $oHostDbPrimary->sqlExecute("create database test3 with tablespace ts1", {bAutoCommit => true}); - $oHostDbPrimary->sqlExecute( - 'create table test3_exists (id int);' . - 'insert into test3_exists values (1);', - {strDb => 'test3', bAutoCommit => true}); - - # Create a table in test1 to check - test1 will not be restored - $oHostDbPrimary->sqlExecute( - 'create table test1_zeroed (id int);' . - 'insert into test1_zeroed values (1);', - {strDb => 'test1', bAutoCommit => true}); - - # Start a backup so the next backup has to restart it. This test is not required for PostgreSQL >= 9.6 since backups are run - # in non-exclusive mode. - if ($oHostDbPrimary->pgVersion() < PG_VERSION_96) - { - $oHostDbPrimary->sqlSelectOne("select pg_start_backup('test backup that will cause an error', true)"); - - # Verify that an error is returned if the backup is already running - $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_INCR, 'fail on backup already running', {iExpectedExitStatus => ERROR_DB_QUERY}); - - # Restart the cluster ignoring any errors in the postgresql log - $oHostDbPrimary->clusterRestart({bIgnoreLogError => true}); - - # Start a new backup to make the next test restarts it - $oHostDbPrimary->sqlSelectOne("select pg_start_backup('test backup that will be restarted', true)"); - } - - if (defined($strAdhocBackup)) - { - # Adhoc expire the latest backup - no other tests should be affected - $oHostBackup->expire({strOptionalParam => '--set=' . $strAdhocBackup}); - } - - # Drop a table - $oHostDbPrimary->sqlExecute('drop table test_remove'); - $oHostDbPrimary->sqlWalRotate(); - $oHostDbPrimary->sqlExecute("update test set message = '$strIncrMessage'", {bCommit => true}); - - # Exercise --delta checksum option - my $strIncrBackup = $oHostBackup->backup( - CFGOPTVAL_BACKUP_TYPE_INCR, 'delta', - {strOptionalParam => '--stop-auto --buffer-size=32768 --delta', iRepo => $iRepoTotal}); - - # Ensure the check command runs properly with a tablespace - $oHostBackup->check( 'check command with tablespace', {iTimeout => 10, strOptionalParam => $strBogusReset}); - - # Setup the xid target - #--------------------------------------------------------------------------------------------------------------------------- - my $strXidTarget = undef; - - $oHostDbPrimary->sqlExecute("update test set message = '$strXidMessage'", {bCommit => false}); - $oHostDbPrimary->sqlWalRotate(); - $strXidTarget = $oHostDbPrimary->sqlSelectOne("select txid_current()"); - $oHostDbPrimary->sqlCommit(); - &log(INFO, " xid target is ${strXidTarget}"); - - # Setup the name target - #--------------------------------------------------------------------------------------------------------------------------- - my $strNameTarget = 'backrest'; - - $oHostDbPrimary->sqlExecute("update test set message = '$strNameMessage'", {bCommit => true}); - $oHostDbPrimary->sqlWalRotate(); - - $oHostDbPrimary->sqlExecute("select pg_create_restore_point('${strNameTarget}')"); - - &log(INFO, " name target is ${strNameTarget}"); - - # Create a table and data in database test2 - #--------------------------------------------------------------------------------------------------------------------------- - # Initialize variables for SHA1 and path of the pg_filenode.map for the database that will not be restored - my $strDb1TablePath; - my $strDb1TableSha1; - - $oHostDbPrimary->sqlExecute( - 'create table test (id int);' . - 'insert into test values (1);' . - 'create table test_ts1 (id int) tablespace ts1;' . - 'insert into test_ts1 values (2);', - {strDb => 'test2', bAutoCommit => true}); - - $oHostDbPrimary->sqlWalRotate(); - - # Get the SHA1 and path of the table for the database that will not be restored - $strDb1TablePath = $oHostDbPrimary->dbBasePath(). "/base/" . - $oHostDbPrimary->sqlSelectOne("select oid from pg_database where datname='test1'") . "/" . - $oHostDbPrimary->sqlSelectOne("select relfilenode from pg_class where relname='test1_zeroed'", {strDb => 'test1'}); - $strDb1TableSha1 = storageTest()->hashSize($strDb1TablePath); - - # Restore (type = default) - #--------------------------------------------------------------------------------------------------------------------------- - # Expect failure because pg (appears to be) running - $oHostDbPrimary->restore('pg running', 'latest', {iExpectedExitStatus => ERROR_PG_RUNNING}); - - $oHostDbPrimary->clusterStop(); - - # Expect failure because db path is not empty - $oHostDbPrimary->restore('path not empty', 'latest', {iExpectedExitStatus => ERROR_PATH_NOT_EMPTY}); - - # Drop and recreate db path - testPathRemove($oHostDbPrimary->dbBasePath()); - storageTest()->pathCreate($oHostDbPrimary->dbBasePath(), {strMode => '0700'}); - testPathRemove($oHostDbPrimary->dbPath() . qw{/} . $oManifest->walPath()); - storageTest()->pathCreate($oHostDbPrimary->dbPath() . qw{/} . $oManifest->walPath(), {strMode => '0700'}); - testPathRemove($oHostDbPrimary->tablespacePath(1)); - storageTest()->pathCreate($oHostDbPrimary->tablespacePath(1), {strMode => '0700'}); - - # Now the restore should work - $oHostDbPrimary->restore( - undef, 'latest', - {strOptionalParam => ' --db-include=test2 --db-include=test3 --buffer-size=16384', iRepo => $iRepoTotal}); - - # Test that the first database has not been restored since --db-include did not include test1 - my ($strSHA1, $lSize) = storageTest()->hashSize($strDb1TablePath); - - # Create a zeroed sparse file in the test directory that is the same size as the filenode.map. We need to use the posix - # driver directly to do this because handles cannot be passed back from the C code. - my $oStorageTrunc = new pgBackRestTest::Common::Storage($self->testPath(), new pgBackRestTest::Common::StoragePosix()); - - my $strTestTable = $self->testPath() . "/testtable"; - my $oDestinationFileIo = $oStorageTrunc->openWrite($strTestTable); - $oDestinationFileIo->open(); - - # Truncate to the original size which will create a sparse file. - if (!truncate($oDestinationFileIo->handle(), $lSize)) - { - confess "unable to truncate '$strTestTable' with handle " . $oDestinationFileIo->handle(); - } - $oDestinationFileIo->close(); - - # Confirm the test filenode.map and the database test1 filenode.map are zeroed - my ($strSHA1Test, $lSizeTest) = storageTest()->hashSize($strTestTable); - $self->testResult(sub {($strSHA1Test eq $strSHA1) && ($lSizeTest == $lSize) && ($strSHA1 ne $strDb1TableSha1)}, - true, 'database test1 not restored'); - - $oHostDbPrimary->clusterStart(); - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strNameMessage); - - # Once the cluster is back online, make sure the database & table in the tablespace exists properly - $oHostDbPrimary->sqlSelectOneTest('select id from test_ts1', 2, {strDb => 'test2'}); - $oHostDbPrimary->sqlDisconnect({strDb => 'test2'}); - - $oHostDbPrimary->sqlSelectOneTest('select id from test3_exists', 1, {strDb => 'test3'}); - $oHostDbPrimary->sqlDisconnect({strDb => 'test3'}); - - # The tablespace path should exist and have files in it - my $strTablespacePath = $oHostDbPrimary->tablespacePath(1); - - # Backup info will have the catalog number - my $oBackupInfo = new pgBackRestDoc::Common::Ini( - storageRepo(), $oHostBackup->repoBackupPath(FILE_BACKUP_INFO), - {bLoad => false, strContent => ${storageRepo()->get($oHostBackup->repoBackupPath(FILE_BACKUP_INFO))}}); - - # Construct the special path - $strTablespacePath .= - '/PG_' . $oHostDbPrimary->pgVersion() . qw{_} . $oBackupInfo->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CATALOG); - - # Check that path exists - if (!storageTest()->pathExists($strTablespacePath)) - { - confess &log(ASSERT, "unable to find tablespace path '${strTablespacePath}'"); - } - - # Make sure there are some files in the tablespace path - if (grep(!/^PG\_VERSION$/i, storageTest()->list($strTablespacePath)) == 0) - { - confess &log(ASSERT, "no files found in tablespace path '${strTablespacePath}'"); - } - - # This table should exist to prove that the tablespace was restored. It has not been updated since it was created so it - # should not be created by any full page writes. Once it is verified to exist it can be dropped. - $oHostDbPrimary->sqlSelectOneTest("select count(*) from test_exists", 0); - $oHostDbPrimary->sqlExecute('drop table test_exists'); - - # Now it should be OK to drop database test2 and test3 - $oHostDbPrimary->sqlExecute('drop database test2', {bAutoCommit => true}); - - # The test table lives in ts1 so it needs to be moved or dropped - $oHostDbPrimary->sqlExecute('alter table test set tablespace pg_default'); - - # And drop the tablespace - $oHostDbPrimary->sqlExecute('drop database test3', {bAutoCommit => true}); - $oHostDbPrimary->sqlExecute("drop tablespace ts1", {bAutoCommit => true}); - - # Restore (restore type = immediate, inclusive) - #--------------------------------------------------------------------------------------------------------------------------- - if ($oHostDbPrimary->pgVersion() >= PG_VERSION_94) - { - &log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_IMMEDIATE); - - $oHostDbPrimary->clusterStop(); - - $oHostDbPrimary->restore( - undef, $strFullBackup, {bForce => true, strType => CFGOPTVAL_RESTORE_TYPE_IMMEDIATE, strTargetAction => 'promote'}); - - $oHostDbPrimary->clusterStart(); - $oHostDbPrimary->sqlSelectOneTest('select message from test', ($strStandbyMessage)); - } - - # Restore (restore type = xid, inclusive) - #--------------------------------------------------------------------------------------------------------------------------- - my $strRecoveryFile = undef; - - &log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_XID); - - $oHostDbPrimary->clusterStop(); - - executeTest('rm -rf ' . $oHostDbPrimary->dbBasePath() . "/*"); - executeTest('rm -rf ' . $oHostDbPrimary->dbPath() . qw{/} . $oManifest->walPath() . '/*'); - - $oHostDbPrimary->restore( - undef, $strIncrBackup, - {bForce => true, strType => CFGOPTVAL_RESTORE_TYPE_XID, strTarget => $strXidTarget, strTargetAction => 'promote', - strTargetTimeline => $oHostDbPrimary->pgVersion() >= PG_VERSION_12 ? 'current' : undef, - strOptionalParam => '--tablespace-map-all=../../tablespace', bTablespace => false, - iRepo => $iRepoTotal}); - - # Save recovery file to test so we can use it in the next test - $strRecoveryFile = $oHostDbPrimary->pgVersion() >= PG_VERSION_12 ? 'postgresql.auto.conf' : DB_FILE_RECOVERYCONF; - - storageTest()->copy( - $oHostDbPrimary->dbBasePath() . qw{/} . $strRecoveryFile, $self->testPath() . qw{/} . $strRecoveryFile); - - $oHostDbPrimary->clusterStart(); - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strXidMessage); - - $oHostDbPrimary->sqlExecute("update test set message = '$strTimelineMessage'"); - - # Restore (restore type = preserve, inclusive) - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_PRESERVE); - - $oHostDbPrimary->clusterStop(); - - executeTest('rm -rf ' . $oHostDbPrimary->dbBasePath() . "/*"); - executeTest('rm -rf ' . $oHostDbPrimary->dbPath() . qw{/} . $oManifest->walPath() . '/*'); - executeTest('rm -rf ' . $oHostDbPrimary->tablespacePath(1) . "/*"); - - # Restore recovery file that was saved in last test - storageTest()->move($self->testPath . "/${strRecoveryFile}", $oHostDbPrimary->dbBasePath() . "/${strRecoveryFile}"); - - # Also touch recovery.signal when required - if ($oHostDbPrimary->pgVersion() >= PG_VERSION_12) - { - storageTest()->put($oHostDbPrimary->dbBasePath() . "/" . DB_FILE_RECOVERYSIGNAL); - } - - $oHostDbPrimary->restore(undef, 'latest', {strType => CFGOPTVAL_RESTORE_TYPE_PRESERVE}); - - $oHostDbPrimary->clusterStart(); - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strXidMessage); - - $oHostDbPrimary->sqlExecute("update test set message = '$strTimelineMessage'"); - - # Restore (restore type = time, inclusive, automatically select backup) - there is no exclusive time test because I can't - # find a way to find the exact commit time of a transaction. - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_TIME); - - $oHostDbPrimary->clusterStop(); - - $oHostDbPrimary->restore( - undef, 'latest', - {bDelta => true, strType => CFGOPTVAL_RESTORE_TYPE_TIME, strTarget => $strTimeTarget, strTargetAction => 'promote', - strTargetTimeline => $oHostDbPrimary->pgVersion() >= PG_VERSION_12 ? 'current' : undef, - strBackupExpected => $strFullBackup}); - - $oHostDbPrimary->clusterStart(); - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strTimeMessage); - - # Restore (restore type = xid, exclusive) - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_XID); - - $oHostDbPrimary->clusterStop(); - - $oHostDbPrimary->restore( - undef, $strIncrBackup, - {bDelta => true, strType => CFGOPTVAL_RESTORE_TYPE_XID, strTarget => $strXidTarget, bTargetExclusive => true, - strTargetAction => 'promote', - strTargetTimeline => $oHostDbPrimary->pgVersion() >= PG_VERSION_12 ? 'current' : undef, - iRepo => $iRepoTotal}); - - $oHostDbPrimary->clusterStart(); - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strIncrMessage); - - # Restore (restore type = name) - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_NAME); - - $oHostDbPrimary->clusterStop(); - - $oHostDbPrimary->restore( - undef, 'latest', - {bDelta => true, bForce => true, strType => CFGOPTVAL_RESTORE_TYPE_NAME, strTarget => $strNameTarget, - strTargetAction => 'promote', - strTargetTimeline => $oHostDbPrimary->pgVersion() >= PG_VERSION_12 ? 'current' : undef}); - - $oHostDbPrimary->clusterStart(); - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strNameMessage); - - # Restore (restore type = default, timeline = created by type = xid, inclusive recovery) - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, ' testing recovery type = ' . CFGOPTVAL_RESTORE_TYPE_DEFAULT); - - $oHostDbPrimary->clusterStop(); - - # The timeline to use for this test is subject to change based on tests being added or removed above. The best thing would - # be to automatically grab the timeline after the restore, but since this test has been stable for a long time it does not - # seem worth the effort to automate. - $oHostDbPrimary->restore( - undef, $strIncrBackup, - {bDelta => true, strType => CFGOPTVAL_RESTORE_TYPE_STANDBY, strTargetTimeline => 4, iRepo => $iRepoTotal}); - - $oHostDbPrimary->clusterStart({bHotStandby => true}); - $oHostDbPrimary->sqlSelectOneTest('select message from test', $strTimelineMessage, {iTimeout => 120}); - - # Stop clusters to catch any errors in the postgres log - #--------------------------------------------------------------------------------------------------------------------------- - $oHostDbPrimary->clusterStop(); - - # Stanza-delete --force without access to pgbackrest on database host. This test is not version specific so is run on only - # one version. - #--------------------------------------------------------------------------------------------------------------------------- - if ($bNonVersionSpecific) - { - # Make sure this test has a backup host to work with - confess "test must run with backup dst = " . HOST_BACKUP if !$bHostBackup; - - $oHostDbPrimary->stop(); - $oHostBackup->stop({strStanza => $self->stanza}); - $oHostBackup->stanzaDelete( - "delete stanza with --force when pgbackrest on pg host not accessible", {strOptionalParam => ' --force'}); - $oHostDbPrimary->start(); - $oHostBackup->start(); - } - } -} - -1; diff --git a/test/src/build/config/config.yaml b/test/src/build/config/config.yaml index a12fc79cd5..8006823c4f 100644 --- a/test/src/build/config/config.yaml +++ b/test/src/build/config/config.yaml @@ -70,6 +70,12 @@ option: command: test: {} + pg-version: + type: string + default: invalid + command: + test: {} + profile: type: boolean default: false diff --git a/test/src/build/help/help.xml b/test/src/build/help/help.xml index 1901798953..b9b93b01a7 100644 --- a/test/src/build/help/help.xml +++ b/test/src/build/help/help.xml @@ -120,6 +120,16 @@ n + + version for integration test. + + + Version of to use for the integration test. + + + 16 + + Generate profile report. diff --git a/test/src/command/test/build.c b/test/src/command/test/build.c index c485703a83..7e7240c92e 100644 --- a/test/src/command/test/build.c +++ b/test/src/command/test/build.c @@ -29,15 +29,18 @@ Constants /**********************************************************************************************************************************/ TestBuild * testBldNew( - const String *const pathRepo, const String *const pathTest, const String *const vm, const unsigned int vmId, - const TestDefModule *const module, const unsigned int test, const uint64_t scale, const LogLevel logLevel, const bool logTime, - const String *const timeZone, const bool coverage, const bool profile, const bool optimize, const bool backTrace) + const String *const pathRepo, const String *const pathTest, const String *const vm, const String *const vmInt, + const unsigned int vmId, const String *const pgVersion, const TestDefModule *const module, const unsigned int test, + const uint64_t scale, const LogLevel logLevel, const bool logTime, const String *const timeZone, const bool coverage, + const bool profile, const bool optimize, const bool backTrace) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STRING, pathRepo); FUNCTION_LOG_PARAM(STRING, pathTest); FUNCTION_LOG_PARAM(STRING, vm); + FUNCTION_LOG_PARAM(STRING, vmInt); FUNCTION_LOG_PARAM(UINT, vmId); + FUNCTION_LOG_PARAM(STRING, pgVersion); FUNCTION_LOG_PARAM_P(VOID, module); FUNCTION_LOG_PARAM(UINT, test); FUNCTION_LOG_PARAM(UINT64, scale); @@ -53,6 +56,7 @@ testBldNew( ASSERT(pathRepo != NULL); ASSERT(pathTest != NULL); ASSERT(vm != NULL); + ASSERT(vmInt != NULL); ASSERT(module != NULL); ASSERT(scale != 0); @@ -65,7 +69,9 @@ testBldNew( .pathRepo = strDup(pathRepo), .pathTest = strDup(pathTest), .vm = strDup(vm), + .vmInt = strDup(vmInt), .vmId = vmId, + .pgVersion = strDup(pgVersion), .module = module, .test = test, .scale = scale, @@ -315,6 +321,11 @@ testBldUnit(TestBuild *const this) for (unsigned int shimIdx = 0; shimIdx < lstSize(module->shimList); shimIdx++) { const TestDefShim *const shim = lstGet(module->shimList, shimIdx); + + // Skip this shim for integration tests + if (module->type == testDefTypeIntegration && !shim->integration) + continue; + const String *const shimFile = strNewFmt("%s.c", strZ(cmdBldPathModule(shim->name))); String *const shimC = strCatBuf( @@ -332,6 +343,11 @@ testBldUnit(TestBuild *const this) for (unsigned int harnessIdx = 0; harnessIdx < lstSize(module->harnessList); harnessIdx++) { const TestDefHarness *const harness = lstGet(module->harnessList, harnessIdx); + + // Skip this harness for integration tests + if (module->type == testDefTypeIntegration && !harness->integration) + continue; + const String *const harnessFile = strNewFmt("test/src/common/%s.c", strZ(bldEnum("harness", harness->name))); const String *harnessPath = strNewFmt("%s/%s", strZ(pathRepo), strZ(harnessFile)); @@ -634,8 +650,8 @@ testBldUnit(TestBuild *const this) const String *const pathProjectExe = storagePathP( testBldStorageTest(this), strNewFmt( - "%s/%s%s/" PROJECT_BIN, strEqZ(testBldVm(this), "none") ? "build" : "bin", strZ(testBldVm(this)), - strEqZ(testBldVm(this), "none") ? "/src" : "")); + "%s/%s%s/" PROJECT_BIN, strEqZ(testBldVmInt(this), "none") ? "build" : "bin", strZ(testBldVmInt(this)), + strEqZ(testBldVmInt(this), "none") ? "/src" : "")); strReplace(testC, STRDEF("{[C_TEST_PROJECT_EXE]}"), pathProjectExe); // Path to source -- used to construct __FILENAME__ tests @@ -671,6 +687,12 @@ testBldUnit(TestBuild *const this) strReplace(testC, STRDEF("{[C_TEST_USER_LEN]}"), strNewFmt("%zu", strSize(userName()))); strReplace(testC, STRDEF("{[C_TEST_USER_ID]}"), strNewFmt("%u", userId())); + // VM for integration testing + strReplace(testC, STRDEF("{[C_TEST_VM]}"), testBldVmInt(this)); + + // PostgreSQL version for integration testing + strReplace(testC, STRDEF("{[C_TEST_PG_VERSION]}"), testBldPgVersion(this)); + // Test id strReplace(testC, STRDEF("{[C_TEST_IDX]}"), strNewFmt("%u", testBldVmId(this))); diff --git a/test/src/command/test/build.h b/test/src/command/test/build.h index 3ccf52c3b8..7ad0918458 100644 --- a/test/src/command/test/build.h +++ b/test/src/command/test/build.h @@ -19,9 +19,9 @@ typedef struct TestBuild TestBuild; Constructors ***********************************************************************************************************************************/ TestBuild *testBldNew( - const String *pathRepo, const String *pathTest, const String *const vm, unsigned int vmId, const TestDefModule *module, - unsigned int test, uint64_t scale, LogLevel logLevel, bool logTime, const String *timeZone, bool coverage, bool profile, - bool optimize, bool backTrace); + const String *pathRepo, const String *pathTest, const String *const vm, const String *const vmInt, unsigned int vmId, + const String *pgVersion, const TestDefModule *module, unsigned int test, uint64_t scale, LogLevel logLevel, bool logTime, + const String *timeZone, bool coverage, bool profile, bool optimize, bool backTrace); /*********************************************************************************************************************************** Getters/Setters @@ -33,7 +33,9 @@ typedef struct TestBuildPub const Storage *storageRepo; // Repository storage const Storage *storageTest; // Test storage const String *vm; // Vm to run the test on + const String *vmInt; // Vm to run the test on for integration unsigned int vmId; // Vm id (0-based) to run the test on + const String *pgVersion; // Pg version to run the test on const TestDefModule *module; // Module definition unsigned int test; // Specific test to run (0 if all) LogLevel logLevel; // Test log level @@ -82,6 +84,13 @@ testBldVm(const TestBuild *const this) return THIS_PUB(TestBuild)->vm; } +// Vm integration +FN_INLINE_ALWAYS const String * +testBldVmInt(const TestBuild *const this) +{ + return THIS_PUB(TestBuild)->vmInt; +} + // Vm id FN_INLINE_ALWAYS unsigned int testBldVmId(const TestBuild *const this) @@ -89,6 +98,13 @@ testBldVmId(const TestBuild *const this) return THIS_PUB(TestBuild)->vmId; } +// Pg version +FN_INLINE_ALWAYS const String * +testBldPgVersion(const TestBuild *const this) +{ + return THIS_PUB(TestBuild)->pgVersion; +} + // Test Definition FN_INLINE_ALWAYS const TestDefModule * testBldModule(const TestBuild *const this) diff --git a/test/src/command/test/define.c b/test/src/command/test/define.c index 5f949a8057..a3b89966a5 100644 --- a/test/src/command/test/define.c +++ b/test/src/command/test/define.c @@ -137,7 +137,7 @@ testDefParseModuleList(Yaml *const yaml, List *const moduleList) } else if (strEqZ(subModuleDef.value, "harness")) { - TestDefHarness testDefHarness = {0}; + TestDefHarness testDefHarness = {.integration = true}; StringList *harnessIncludeList = strLstNew(); if (yamlEventPeek(yaml).type == yamlEventTypeScalar) @@ -148,44 +148,59 @@ testDefParseModuleList(Yaml *const yaml, List *const moduleList) { YAML_MAP_BEGIN(yaml) { - yamlScalarNextCheckZ(yaml, "name"); - testDefHarness.name = yamlScalarNext(yaml).value; + const String *const type = yamlScalarNext(yaml).value; - yamlScalarNextCheckZ(yaml, "shim"); - - YAML_MAP_BEGIN(yaml) + if (strEqZ(type, "name")) { - const String *const shim = yamlScalarNext(yaml).value; - strLstAdd(harnessIncludeList, shim); + testDefHarness.name = yamlScalarNext(yaml).value; + } + else if (strEqZ(type, "integration")) + { + testDefHarness.integration = yamlBoolParse(yamlScalarNext(yaml)); + } + else + { + CHECK_FMT(AssertError, strEqZ(type, "shim"), "invalid key '%s'", strZ(type)); - if (yamlEventPeek(yaml).type == yamlEventTypeScalar) - { - yamlScalarNext(yaml); - } - else + YAML_MAP_BEGIN(yaml) { - TestDefShim testDefShim = {.name = shim, .functionList = strLstNew()}; + const String *const shim = yamlScalarNext(yaml).value; + strLstAdd(harnessIncludeList, shim); - YAML_MAP_BEGIN(yaml) + if (yamlEventPeek(yaml).type == yamlEventTypeScalar) { - yamlScalarNextCheckZ(yaml, "function"); - - StringList *const functionList = strLstNew(); + yamlScalarNext(yaml); + } + else + { + TestDefShim testDefShim = + { + .name = shim, + .integration = testDefHarness.integration, + .functionList = strLstNew() + }; - YAML_SEQ_BEGIN(yaml) + YAML_MAP_BEGIN(yaml) { - strLstAdd(functionList, yamlScalarNext(yaml).value); + yamlScalarNextCheckZ(yaml, "function"); + + StringList *const functionList = strLstNew(); + + YAML_SEQ_BEGIN(yaml) + { + strLstAdd(functionList, yamlScalarNext(yaml).value); + } + YAML_SEQ_END(); + + testDefShim.functionList = functionList; } - YAML_SEQ_END(); + YAML_MAP_END(); - testDefShim.functionList = functionList; + lstAdd(globalShimList, &testDefShim); } - YAML_MAP_END(); - - lstAdd(globalShimList, &testDefShim); } + YAML_MAP_END(); } - YAML_MAP_END(); } YAML_MAP_END(); } @@ -256,6 +271,7 @@ testDefParseModuleList(Yaml *const yaml, List *const moduleList) const TestDefHarness testDefHarness = { .name = strDup(globalHarness->name), + .integration = globalHarness->integration, .includeList = strLstDup(globalHarness->includeList), }; @@ -277,6 +293,7 @@ testDefParseModuleList(Yaml *const yaml, List *const moduleList) const TestDefShim testDefShim = { .name = strDup(globalShim->name), + .integration = globalShim->integration, .functionList = strLstDup(globalShim->functionList), }; diff --git a/test/src/command/test/define.h b/test/src/command/test/define.h index 3a1a3d4160..b4bf0f1a09 100644 --- a/test/src/command/test/define.h +++ b/test/src/command/test/define.h @@ -28,6 +28,7 @@ typedef struct TestDefCoverage typedef struct TestDefHarness { const String *name; // Harness module name + bool integration; // Include in integration tests? const StringList *includeList; // List of modules to include directly in harness } TestDefHarness; @@ -35,6 +36,7 @@ typedef struct TestDefHarness typedef struct TestDefShim { const String *name; // Shim module name + bool integration; // Include in integration tests? const StringList *functionList; // List of functions to shim } TestDefShim; diff --git a/test/src/command/test/test.c b/test/src/command/test/test.c index b6c677eb08..5aa2d2de72 100644 --- a/test/src/command/test/test.c +++ b/test/src/command/test/test.c @@ -49,15 +49,17 @@ cmdTestPathCreate(const Storage *const storage, const String *const path) /**********************************************************************************************************************************/ void cmdTest( - const String *const pathRepo, const String *const pathTest, const String *const vm, const unsigned int vmId, - const String *moduleName, const unsigned int test, const uint64_t scale, const LogLevel logLevel, const bool logTime, - const String *const timeZone, const bool coverage, const bool profile, const bool optimize, const bool backTrace) + const String *const pathRepo, const String *const pathTest, const String *vm, const unsigned int vmId, + const String *const pgVersion, const String *moduleName, const unsigned int test, const uint64_t scale, const LogLevel logLevel, + const bool logTime, const String *const timeZone, const bool coverage, const bool profile, const bool optimize, + const bool backTrace) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STRING, pathRepo); FUNCTION_LOG_PARAM(STRING, pathTest); FUNCTION_LOG_PARAM(STRING, vm); FUNCTION_LOG_PARAM(UINT, vmId); + FUNCTION_LOG_PARAM(STRING, pgVersion); FUNCTION_LOG_PARAM(STRING, moduleName); FUNCTION_LOG_PARAM(UINT, test); FUNCTION_LOG_PARAM(UINT64, scale); @@ -80,6 +82,13 @@ cmdTest( CHECK_FMT(ParamInvalidError, module != NULL, "'%s' is not a valid test", strZ(moduleName)); + // Vm used for integration and the pgbackrest binary + const String *const vmInt = vm; + + // If integration then the vm should be none. Integration tests do not run in containers but instead spawn their own. + if (module->type == testDefTypeIntegration) + vm = strNewZ("none"); + // Build test bool buildRetry = false; const String *const pathUnit = strNewFmt("%s/unit-%u/%s", strZ(pathTest), vmId, strZ(vm)); @@ -92,8 +101,8 @@ cmdTest( { // Build unit TestBuild *const testBld = testBldNew( - pathRepo, pathTest, vm, vmId, module, test, scale, logLevel, logTime, timeZone, coverage, profile, optimize, - backTrace); + pathRepo, pathTest, vm, vmInt, vmId, pgVersion, module, test, scale, logLevel, logTime, timeZone, coverage, + profile, optimize, backTrace); testBldUnit(testBld); // Meson setup diff --git a/test/src/command/test/test.h b/test/src/command/test/test.h index 961d642639..b4c3e47f34 100644 --- a/test/src/command/test/test.h +++ b/test/src/command/test/test.h @@ -13,8 +13,8 @@ Perform a test. Functions ***********************************************************************************************************************************/ void cmdTest( - const String *pathRepo, const String *pathTest, const String *const vm, unsigned int vmId, const String *moduleName, - unsigned int test, uint64_t scale, LogLevel logLevel, bool logTime, const String *timeZone, bool coverage, bool profile, - bool optimize, bool backTrace); + const String *pathRepo, const String *pathTest, const String *const vm, unsigned int vmId, const String *pgVersion, + const String *moduleName, unsigned int test, uint64_t scale, LogLevel logLevel, bool logTime, const String *timeZone, + bool coverage, bool profile, bool optimize, bool backTrace); #endif diff --git a/test/src/common/harnessHost.c b/test/src/common/harnessHost.c new file mode 100644 index 0000000000..0453a504c8 --- /dev/null +++ b/test/src/common/harnessHost.c @@ -0,0 +1,1301 @@ +/*********************************************************************************************************************************** +Host Harness +***********************************************************************************************************************************/ +#include "build.auto.h" + +#include "build/common/exec.h" +#include "common/compress/helper.h" +#include "common/crypto/common.h" +#include "common/error/retry.h" +#include "common/io/io.h" +#include "common/wait.h" +#include "config/config.h" +#include "postgres/interface.h" +#include "postgres/version.h" +#include "storage/azure/storage.h" +#include "storage/gcs/storage.h" +#include "storage/posix/storage.h" +#include "storage/s3/storage.h" +#include "storage/sftp/storage.h" + +#include "common/harnessDebug.h" +#include "common/harnessHost.h" + +/*********************************************************************************************************************************** +Object type +***********************************************************************************************************************************/ +struct HrnHost +{ + HrnHostPub pub; // Publicly accessible variables + const String *pgBinPath; // Pg bin path + PgClient *pgClient; // Pg client +}; + +/*********************************************************************************************************************************** +Constants +***********************************************************************************************************************************/ +// Azure constants +#define HRN_HOST_AZURE_ACCOUNT "azaccount" +#define HRN_HOST_AZURE_KEY "YXpLZXk=" +#define HRN_HOST_AZURE_CONTAINER "azcontainer" + +// GCS constants +#define HRN_HOST_GCS_BUCKET "gcsbucket" +#define HRN_HOST_GCS_KEY "testkey" +#define HRN_HOST_GCS_KEY_TYPE "token" +#define HRN_HOST_GCS_PORT 4443 + +// S3 constants +#define HRN_HOST_S3_ACCESS_KEY "accessKey1" +#define HRN_HOST_S3_ACCESS_SECRET_KEY "verySecretKey1" +#define HRN_HOST_S3_BUCKET "pgbackrest-dev" +#define HRN_HOST_S3_ENDPOINT "s3.amazonaws.com" +#define HRN_HOST_S3_REGION "us-east-1" + +// SFTP constants +#define HRN_HOST_SFTP_HOSTKEY_HASH_TYPE "sha1" +#define HRN_HOST_SFTP_KEY_PATH "test/certificate/ssh" +#define HRN_HOST_SFTP_KEY_PRIVATE HRN_HOST_SFTP_KEY_PATH "/id_rsa" +#define HRN_HOST_SFTP_KEY_PUBLIC HRN_HOST_SFTP_KEY_PATH "/id_rsa.pub" + +// TLS constants +#define HRN_HOST_TLS_CERT_PATH "test/certificate" +#define HRN_HOST_TLS_CLIENT_CERT HRN_HOST_TLS_CERT_PATH "/pgbackrest-test-client.crt" +#define HRN_HOST_TLS_CLIENT_KEY HRN_HOST_TLS_CERT_PATH "/pgbackrest-test-client.key" +#define HRN_HOST_TLS_SERVER_CA HRN_HOST_TLS_CERT_PATH "/pgbackrest-test-ca.crt" +#define HRN_HOST_TLS_SERVER_CERT HRN_HOST_TLS_CERT_PATH "/pgbackrest-test-server.crt" +#define HRN_HOST_TLS_SERVER_KEY HRN_HOST_TLS_CERT_PATH "/pgbackrest-test-server.key" + +// Cipher passphrase +#define HRN_CIPHER_PASSPHRASE "x" + +/*********************************************************************************************************************************** +Local variables +***********************************************************************************************************************************/ +static struct HrnHostLocal +{ + MemContext *memContext; // Mem context for variables + + String *stanza; // Stanza + unsigned int pgVersion; // Pg version + StringId repoHost; // Host acting as repository + StringId storage; // Storage type + CompressType compressType; // Compress type + CipherType cipherType; // Cipher type + const String *cipherPass; // Cipher passphrase + unsigned int repoTotal; // Repository total + bool tls; // Use TLS instead of SSH? + bool bundle; // Bundling enabled? + bool blockIncr; // Block incremental enabled? + bool archiveAsync; // Async archiving enabled? + bool nonVersionSpecific; // Run non version-specific tests? + + List *hostList; // List of hosts +} hrnHostLocal; + +/**********************************************************************************************************************************/ +HrnHost * +hrnHostNew(const StringId id, const String *const container, const String *const image, const HrnHostNewParam param) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(STRING_ID, id); + FUNCTION_HARNESS_PARAM(STRING, container); + FUNCTION_HARNESS_PARAM(STRING, image); + FUNCTION_HARNESS_PARAM(STRING, param.user); + FUNCTION_HARNESS_PARAM(STRING, param.option); + FUNCTION_HARNESS_PARAM(BOOL, param.noUpdateHosts); + FUNCTION_HARNESS_PARAM(STRING, param.dataPath); + FUNCTION_HARNESS_PARAM(BOOL, param.pgStandby); + FUNCTION_HARNESS_PARAM(BOOL, param.isPg); + FUNCTION_HARNESS_PARAM(BOOL, param.isRepo); + FUNCTION_HARNESS_END(); + + OBJ_NEW_BEGIN(HrnHost, .childQty = MEM_CONTEXT_QTY_MAX) + { + *this = (HrnHost) + { + .pub = + { + .id = id, + .name = strIdToStr(id), + .container = strDup(container), + .image = strDup(image), + .user = param.user == NULL ? strNewZ("root") : strDup(param.user), + .brBin = strNewZ("pgbackrest"), + .pgStandby = param.pgStandby, + .updateHosts = !param.noUpdateHosts, + .isPg = param.isPg, + .isRepo = param.isRepo, + }, + }; + + // Initializes paths and storage + if (param.dataPath != NULL) + { + this->pub.dataPath = strDup(param.dataPath); + this->pub.dataStorage = storagePosixNewP(hrnHostDataPath(this), .write = true); + this->pub.logPath = strNewFmt("%s/log", strZ(hrnHostDataPath(this))); + this->pub.pgPath = strNewFmt("%s/pg", strZ(hrnHostDataPath(this))); + this->pub.pgDataPath = strNewFmt("%s/data", strZ(hrnHostPgPath(this))); + this->pub.pgTsPath = strNewFmt("%s/ts", strZ(hrnHostPgPath(this))); + this->pub.pgStorage = storagePosixNewP(hrnHostPgDataPath(this), .write = true); + this->pub.pgLogFile = strNewFmt("%s/postgresql.log", strZ(hrnHostLogPath(this))); + this->pub.repo1Path = strNewFmt("%s/repo1", strZ(hrnHostDataPath(this))); + this->pub.repo2Path = strNewFmt("%s/repo2", strZ(hrnHostDataPath(this))); + this->pub.spoolPath = strNewFmt("%s/spool", strZ(hrnHostDataPath(this))); + } + + MEM_CONTEXT_TEMP_BEGIN() + { + // Add host to list + if (hrnHostGet(hrnHostId(this)) != NULL) + THROW_FMT(AssertError, "host '%s' already exists", strZ(hrnHostName(this))); + + lstAdd(hrnHostLocal.hostList, &this); + + // Remove prior container with same name if it exists + execOneP(strNewFmt("docker rm -f %s", strZ(hrnHostContainer(this)))); + + // Run container + String *const command = strCatFmt( + strNew(), "docker run -itd -h %s --name=%s", strZ(hrnHostName(this)), strZ(hrnHostContainer(this))); + + if (hrnHostDataPath(this) != NULL) + { + Storage *const storageData = storagePosixNewP(STR(testPath()), .write = true); + storagePathCreateP(storageData, hrnHostLogPath(this), .mode = 0700); + + strCatFmt(command, " -v '%s:%s'", strZ(hrnHostDataPath(this)), strZ(hrnHostDataPath(this))); + } + + if (param.option != NULL) + strCatFmt(command, " %s", strZ(param.option)); + + if (param.entryPoint != NULL) + strCatFmt(command, " --entrypoint=%s --user=%s", strZ(param.entryPoint), strZ(hrnHostUser(this))); + + strCatFmt(command, " %s", strZ(hrnHostImage(this))); + + if (param.param != NULL) + strCatFmt(command, " %s", strZ(param.param)); + + execOneP(command); + + // Get IP address + const String *const ip = strTrim( + execOneP(strNewFmt("docker inspect --format '{{ .NetworkSettings.IPAddress }}' %s", strZ(hrnHostContainer(this))))); + + MEM_CONTEXT_PRIOR_BEGIN() + { + this->pub.ip = strDup(ip); + } + MEM_CONTEXT_PRIOR_END(); + + // Update host IP addresses + for (unsigned int hostIdx = 0; hostIdx < lstSize(hrnHostLocal.hostList); hostIdx++) + { + HrnHost *const host = *(HrnHost **)lstGet(hrnHostLocal.hostList, hostIdx); + + // Skip if current host + if (hrnHostId(this) != hrnHostId(host)) + { + // Add this IP to other hosts + if (hrnHostUpdateHosts(host)) + { + hrnHostExecP( + host, + strNewFmt("echo \"%s\t%s\" >> /etc/hosts", strZ(hrnHostIp(this)), strZ(hrnHostName(this))), + .user = STRDEF("root")); + } + + // Add other hosts IP to this host + if (hrnHostUpdateHosts(this)) + { + hrnHostExecP( + this, + strNewFmt("echo \"%s\t%s\" >> /etc/hosts", strZ(hrnHostIp(host)), strZ(hrnHostName(host))), + .user = STRDEF("root")); + } + } + } + } + MEM_CONTEXT_TEMP_END(); + } + OBJ_NEW_END(); + + FUNCTION_HARNESS_RETURN(HRN_HOST, this); +} + +/**********************************************************************************************************************************/ +String * +hrnHostExec(HrnHost *const this, const String *const command, const HrnHostExecParam param) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(HRN_HOST, this); + FUNCTION_HARNESS_PARAM(STRING, command); + FUNCTION_HARNESS_PARAM(STRING, param.user); + FUNCTION_HARNESS_PARAM(INT, param.resultExpect); + FUNCTION_HARNESS_END(); + + ASSERT(this != NULL); + ASSERT(command != NULL); + + String *const result = strNew(); + + MEM_CONTEXT_TEMP_BEGIN() + { + strCat( + result, + execOneP( + command, + .shell = strNewFmt( + "docker exec -u %s %s sh -c", param.user == NULL ? strZ(hrnHostUser(this)) : strZ(param.user), + strZ(hrnHostContainer(this))), + .resultExpect = param.resultExpect)); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_HARNESS_RETURN(STRING, result); +} + +/**********************************************************************************************************************************/ +String * +hrnHostExecBr(HrnHost *const this, const char *const command, const HrnHostExecBrParam param) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(HRN_HOST, this); + FUNCTION_HARNESS_PARAM(STRINGZ, command); + FUNCTION_HARNESS_PARAM(STRINGZ, param.option); + FUNCTION_HARNESS_PARAM(STRINGZ, param.param); + FUNCTION_HARNESS_PARAM(INT, param.resultExpect); + FUNCTION_HARNESS_END(); + + ASSERT(this != NULL); + ASSERT(command != NULL); + + String *const result = strNew(); + + MEM_CONTEXT_TEMP_BEGIN() + { + String *const commandStr = strCatFmt(strNew(), "pgbackrest --stanza=" HRN_STANZA); + + if (param.option != NULL) + strCatFmt(commandStr, " %s", param.option); + + strCatFmt(commandStr, " %s", command); + + if (param.param != NULL) + strCatFmt(commandStr, " %s", param.param); + + strCat(result, hrnHostExecP(this, commandStr, .resultExpect = param.resultExpect)); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_HARNESS_RETURN(STRING, result); +} + +/**********************************************************************************************************************************/ +void +hrnHostPgConf(HrnHost *const this) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(HRN_HOST, this); + FUNCTION_HARNESS_END(); + + ASSERT(this != NULL); + + MEM_CONTEXT_TEMP_BEGIN() + { + String *const config = strNew(); + + // Connection options + strCatZ(config, "listen_addresses = '*'\n"); + strCatFmt(config, "unix_socket_directories = '%s'\n", strZ(hrnHostPgPath(this))); + + // Archive options + strCatZ(config, "\n"); + strCatZ(config, "archive_mode = on\n"); + strCatZ(config, "wal_level = hot_standby\n"); + strCatZ(config, "max_wal_senders = 3\n"); + strCatZ(config, "hot_standby = on\n"); + strCatZ(config, "archive_command = 'pgbackrest --stanza=" HRN_STANZA " archive-push \"%p\"'\n"); + + // Log options + strCatZ(config, "\n"); + strCatFmt(config, "log_directory = '%s'\n", strZ(hrnHostLogPath(this))); + strCatFmt(config, "log_filename = '%s'\n", strZ(strBase(hrnHostPgLogFile(this)))); + strCatZ(config, "log_rotation_age = 0\n"); + strCatZ(config, "log_rotation_size = 0\n"); + strCatZ(config, "log_error_verbosity = verbose\n"); + + // Force parallel mode on to make sure we are disabling it and there are no issues. This is important for testing that 9.6 + // works since pg_stop_backup() is marked parallel safe and will error if run in a worker. + if (hrnHostPgVersion() >= PG_VERSION_96) + { + strCatZ(config, "\n"); + + if (hrnHostPgVersion() >= PG_VERSION_16) + strCatZ(config, "debug_parallel_query = on\n"); + else + strCatZ(config, "force_parallel_mode = on\n"); + + strCatZ(config, "max_parallel_workers_per_gather = 2\n"); + } + + // Write postgresql.conf + storagePutP( + storageNewWriteP( + hrnHostDataStorage(this), strNewFmt("%s/postgresql.conf", strZ(hrnHostPgDataPath(this))), .modeFile = 0600), + BUFSTR(config)); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_HARNESS_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +void +hrnHostPgCreate(HrnHost *const this) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(HRN_HOST, this); + FUNCTION_HARNESS_END(); + + ASSERT(this != NULL); + ASSERT(hrnHostIsPg(this)); + + MEM_CONTEXT_TEMP_BEGIN() + { + // Create data directory + storagePathCreateP(hrnHostDataStorage(this), hrnHostPgDataPath(this), .mode = 0700); + + // Init database + String *const command = strCatFmt( + strNew(), "%s/initdb -k --pgdata='%s' --auth=trust", strZ(hrnHostPgBinPath(this)), strZ(hrnHostPgDataPath(this))); + + if (hrnHostPgVersion() >= PG_VERSION_11) + strCatZ(command, " --wal-segsize=1"); + + hrnHostExecP(this, command); + + // Add replication to pg_hba + hrnHostExecP( + this, + strNewFmt( + "echo 'host\treplication\treplicator\tpg2\t\t\ttrust' >> '%s/pg_hba.conf'", strZ(hrnHostPgDataPath(this)))); + + // Start Pg + hrnHostPgStart(this); + + // Add replication user + hrnHostSqlExec(this, STRDEF("create user replicator replication")); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_HARNESS_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +void +hrnHostPgStart(HrnHost *const this) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(HRN_HOST, this); + FUNCTION_HARNESS_END(); + + ASSERT(this != NULL); + ASSERT(hrnHostIsPg(this)); + + MEM_CONTEXT_TEMP_BEGIN() + { + // Create postgresql.conf + hrnHostPgConf(this); + + // Start Pg + const String *const command = strNewFmt( + "%s/pg_ctl start -D '%s' -l '%s' -s -w", strZ(hrnHostPgBinPath(this)), strZ(hrnHostPgDataPath(this)), + strZ(hrnHostPgLogFile(this))); + + hrnHostExecP(this, command); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_HARNESS_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +void +hrnHostPgStop(HrnHost *const this) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(HRN_HOST, this); + FUNCTION_HARNESS_END(); + + ASSERT(this != NULL); + ASSERT(hrnHostIsPg(this)); + + MEM_CONTEXT_TEMP_BEGIN() + { + // Free old client + pgClientFree(this->pgClient); + this->pgClient = NULL; + + // Stop pg + const String *const command = strNewFmt( + "%s/pg_ctl stop -D '%s' -l '%s' -s -w", strZ(hrnHostPgBinPath(this)), strZ(hrnHostPgDataPath(this)), + strZ(hrnHostPgLogFile(this))); + + hrnHostExecP(this, command); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_HARNESS_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +PackRead * +hrnHostSql(HrnHost *const this, const String *const statement, const PgClientQueryResult resultType) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(HRN_HOST, this); + FUNCTION_HARNESS_PARAM(STRING, statement); + FUNCTION_HARNESS_PARAM(STRING_ID, resultType); + FUNCTION_HARNESS_END(); + + ASSERT(this != NULL); + ASSERT(hrnHostIsPg(this)); + ASSERT(statement != NULL); + + PackRead *result = NULL; + + MEM_CONTEXT_TEMP_BEGIN() + { + Wait *const wait = waitNew(10000); + ErrorRetry *const errRetry = errRetryNew(); + bool done = false; + + do + { + TRY_BEGIN() + { + Pack *const pack = pgClientQuery( + hrnHostPgClient(this), statement, resultType == pgClientQueryResultNone ? pgClientQueryResultAny : resultType); + + if (pack != NULL && resultType != pgClientQueryResultNone) + { + MEM_CONTEXT_PRIOR_BEGIN() + { + result = pckReadNew(pack); + pckMove(pack, objMemContext(result)); + } + MEM_CONTEXT_PRIOR_END(); + } + + done = true; + } + CATCH_ANY() + { + errRetryAddP(errRetry); + + if (!waitMore(wait)) + THROWP(errRetryType(errRetry), strZ(errRetryMessage(errRetry))); + } + TRY_END(); + } + while (!done); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_HARNESS_RETURN(PACK_READ, result); +} + +/**********************************************************************************************************************************/ +void +hrnHostSqlExec(HrnHost *const this, const String *const statement) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(HRN_HOST, this); + FUNCTION_HARNESS_PARAM(STRING, statement); + FUNCTION_HARNESS_END(); + + ASSERT(this != NULL); + ASSERT(hrnHostIsPg(this)); + ASSERT(statement != NULL); + + MEM_CONTEXT_TEMP_BEGIN() + { + const String *const statementTrim = strTrim(strDup(statement)); + + if (strBeginsWithZ(statementTrim, "perform")) + hrnHostSql(this, strNewFmt("do $$begin %s; end$$", strZ(statement)), pgClientQueryResultNone); + else + hrnHostSql(this, statement, pgClientQueryResultNone); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_HARNESS_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +void +hrnHostSqlTest(HrnHost *const this, const String *const statement, const String *const expected) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(HRN_HOST, this); + FUNCTION_HARNESS_PARAM(STRING, statement); + FUNCTION_HARNESS_PARAM(STRING, expected); + FUNCTION_HARNESS_END(); + + ASSERT(this != NULL); + ASSERT(hrnHostIsPg(this)); + ASSERT(statement != NULL); + ASSERT(expected != NULL); + + MEM_CONTEXT_TEMP_BEGIN() + { + Wait *const wait = waitNew(10000); + bool done = false; + + do + { + const String *const actual = pckReadStrP(hrnHostSqlValue(this, strZ(statement))); + done = strEq(actual, expected); + + if (done || !waitMore(wait)) + hrnTestResultZ(strZ(actual), strZ(expected), harnessTestResultOperationEq); + } + while (!done); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_HARNESS_RETURN_VOID(); +} + +/*********************************************************************************************************************************** +Host pgBackRest configuration +***********************************************************************************************************************************/ +// Helper to configure TLS +static void +hrnHostConfigTls(HrnHost *const this, String *const config, const char *const host) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(HRN_HOST, this); + FUNCTION_HARNESS_PARAM(STRING, config); + FUNCTION_HARNESS_PARAM(STRINGZ, host); + FUNCTION_HARNESS_END(); + + if (hrnHostLocal.tls) + { + strCatFmt(config, "%s-host-type=tls\n", host); + strCatFmt(config, "%s-host-cert-file=%s/" HRN_HOST_TLS_CLIENT_CERT "\n", host, hrnPathRepo()); + strCatFmt(config, "%s-host-key-file=%s/" HRN_HOST_TLS_CLIENT_KEY "\n", host, hrnPathRepo()); + } + + FUNCTION_HARNESS_RETURN_VOID(); +} + +static void +hrnHostConfig(HrnHost *const this) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(HRN_HOST, this); + FUNCTION_HARNESS_END(); + + ASSERT(this != NULL); + + MEM_CONTEXT_TEMP_BEGIN() + { + String *const config = strNew(); + + // General options + strCatZ(config, "[global]\n"); + strCatZ(config, "beta=y\n"); + strCatZ(config, "buffer-size=128KiB\n"); + strCatZ(config, "job-retry=0\n"); + strCatZ(config, "archive-timeout=20\n"); + strCatZ(config, "protocol-timeout=60\n"); + strCatZ(config, "db-timeout=45\n"); + strCatZ(config, "start-fast=y\n"); + + // Increase process-max for non-posix storage since there will be more latency + if (hrnHostLocal.storage == STORAGE_POSIX_TYPE) + strCatZ(config, "process-max=2\n"); + + // Log options + strCatZ(config, "\n"); + strCatFmt(config, "log-path=%s\n", strZ(hrnHostLogPath(this))); + strCatZ(config, "log-level-stderr=off\n"); + strCatZ(config, "log-level-console=warn\n"); + strCatZ(config, "log-level-file=info\n"); + strCatZ(config, "log-subprocess=n\n"); + + // Compress options + strCatZ(config, "\n"); + strCatFmt(config, "compress-type=%s\n", strZ(compressTypeStr(hrnHostLocal.compressType))); + strCatFmt(config, "compress-level=1\n"); + strCatFmt(config, "compress-level-network=1\n"); + + if (hrnHostIsPg(this)) + { + // Enable archive async + if (hrnHostLocal.archiveAsync) + { + strCatZ(config, "\n"); + strCatZ(config, "archive-async=y\n"); + strCatFmt(config, "spool-path=%s\n", strZ(hrnHostSpoolPath(this))); + } + + // Recovery options + const char *const primary = hrnHostId(this) == HRN_HOST_PG1 ? HRN_HOST_PG2_Z : HRN_HOST_PG1_Z; + + strCatZ(config, "\n"); + strCatFmt(config, "recovery-option=primary_conninfo=host=%s port=5432 user=replicator\n", primary); + } + + // Repo options + if (hrnHostIsRepo(this)) + { + objFree(this->pub.repo1Storage); + this->pub.repo1Storage = NULL; + objFree(this->pub.repo2Storage); + this->pub.repo2Storage = NULL; + + strCatZ(config, "\n"); + strCatFmt(config, "repo1-type=%s\n", strZ(strIdToStr(hrnHostLocal.storage))); + strCatFmt(config, "repo1-path=%s\n", strZ(hrnHostRepo1Path(this))); + strCatZ(config, "repo1-retention-full=2\n"); + + if (hrnHostLocal.cipherType != cipherTypeNone) + { + strCatFmt(config, "repo1-cipher-type=%s\n", strZ(strIdToStr(hrnHostLocal.cipherType))); + strCatFmt(config, "repo1-cipher-pass=%s\n", strZ(hrnHostLocal.cipherPass)); + } + + if (hrnHostLocal.bundle) + { + strCatZ(config, "repo1-bundle=y\n"); + // Set bundle size/limit smaller for testing + strCatZ(config, "repo1-bundle-size=1MiB\n"); + strCatZ(config, "repo1-bundle-limit=64KiB\n"); + } + + if (hrnHostLocal.blockIncr) + { + ASSERT(hrnHostLocal.bundle); + strCatZ(config, "repo1-block=y\n"); + } + + switch (hrnHostLocal.storage) + { + case STORAGE_AZURE_TYPE: + { + strCatZ(config, "repo1-azure-account=" HRN_HOST_AZURE_ACCOUNT "\n"); + strCatZ(config, "repo1-azure-key=" HRN_HOST_AZURE_KEY "\n"); + strCatZ(config, "repo1-azure-container=" HRN_HOST_AZURE_CONTAINER "\n"); + strCatZ(config, "repo1-azure-host=" HRN_HOST_AZURE_Z "\n"); + strCatZ(config, "repo1-azure-uri-style=path\n"); + strCatZ(config, "repo1-storage-verify-tls=n\n"); + + MEM_CONTEXT_OBJ_BEGIN(this) + { + HrnHost *const azure = hrnHostGet(HRN_HOST_AZURE); + + this->pub.repo1Storage = storageAzureNew( + hrnHostRepo1Path(this), true, NULL, STRDEF(HRN_HOST_AZURE_CONTAINER), STRDEF(HRN_HOST_AZURE_ACCOUNT), + storageAzureKeyTypeShared, STRDEF(HRN_HOST_AZURE_KEY), 4 * 1024 * 1024, NULL, hrnHostIp(azure), + storageAzureUriStylePath, 443, ioTimeoutMs(), false, NULL, NULL); + } + MEM_CONTEXT_OBJ_END(); + + break; + } + + case STORAGE_GCS_TYPE: + { + strCatZ(config, "repo1-gcs-bucket=" HRN_HOST_GCS_BUCKET "\n"); + strCatZ(config, "repo1-gcs-key-type=" HRN_HOST_GCS_KEY_TYPE "\n"); + strCatZ(config, "repo1-gcs-key=" HRN_HOST_GCS_KEY "\n"); + strCatZ(config, "repo1-gcs-endpoint=" HRN_HOST_GCS_Z ":" STRINGIFY(HRN_HOST_GCS_PORT) "\n"); + strCatZ(config, "repo1-storage-verify-tls=n\n"); + + MEM_CONTEXT_OBJ_BEGIN(this) + { + HrnHost *const gcs = hrnHostGet(HRN_HOST_GCS); + + this->pub.repo1Storage = storageGcsNew( + hrnHostRepo1Path(this), true, NULL, STRDEF(HRN_HOST_GCS_BUCKET), storageGcsKeyTypeToken, + STRDEF(HRN_HOST_GCS_KEY), 4 * 1024 * 1024, NULL, + strNewFmt("%s:%d", strZ(hrnHostIp(gcs)), HRN_HOST_GCS_PORT), ioTimeoutMs(), false, NULL, NULL); + } + MEM_CONTEXT_OBJ_END(); + + break; + } + + case STORAGE_S3_TYPE: + { + strCatZ(config, "repo1-s3-key=" HRN_HOST_S3_ACCESS_KEY "\n"); + strCatZ(config, "repo1-s3-key-secret=" HRN_HOST_S3_ACCESS_SECRET_KEY "\n"); + strCatZ(config, "repo1-s3-bucket=" HRN_HOST_S3_BUCKET "\n"); + strCatZ(config, "repo1-s3-endpoint=" HRN_HOST_S3_ENDPOINT "\n"); + strCatZ(config, "repo1-s3-region=" HRN_HOST_S3_REGION "\n"); + strCatZ(config, "repo1-storage-verify-tls=n\n"); + + MEM_CONTEXT_OBJ_BEGIN(this) + { + HrnHost *const s3 = hrnHostGet(HRN_HOST_S3); + + this->pub.repo1Storage = storageS3New( + hrnHostRepo1Path(this), true, NULL, STRDEF(HRN_HOST_S3_BUCKET), STRDEF(HRN_HOST_S3_ENDPOINT), + storageS3UriStyleHost, STR(HRN_HOST_S3_REGION), storageS3KeyTypeShared, STRDEF(HRN_HOST_S3_ACCESS_KEY), + STRDEF(HRN_HOST_S3_ACCESS_SECRET_KEY), NULL, NULL, NULL, NULL, 5 * 1024 * 1024, NULL, hrnHostIp(s3), + 443, ioTimeoutMs(), false, NULL, NULL); + } + MEM_CONTEXT_OBJ_END(); + + break; + } + + case STORAGE_SFTP_TYPE: + { + const String *const keyPrivate = strNewFmt("%s/" HRN_HOST_SFTP_KEY_PRIVATE, hrnPathRepo()); + const String *const keyPublic = strNewFmt("%s/" HRN_HOST_SFTP_KEY_PUBLIC, hrnPathRepo()); + + strCatZ(config, "repo1-sftp-host=" HRN_HOST_SFTP_Z "\n"); + strCatZ(config, "repo1-sftp-host-key-hash-type=" HRN_HOST_SFTP_HOSTKEY_HASH_TYPE "\n"); + strCatFmt(config, "repo1-sftp-host-user=%s\n", strZ(hrnHostUser(this))); + strCatFmt(config, "repo1-sftp-private-key-file=%s\n", strZ(keyPrivate)); + strCatFmt(config, "repo1-sftp-public-key-file=%s\n", strZ(keyPublic)); + strCatZ(config, "repo1-sftp-host-key-check-type=none\n"); + + MEM_CONTEXT_OBJ_BEGIN(this) + { + HrnHost *const sftp = hrnHostGet(HRN_HOST_SFTP); + + this->pub.repo1Storage = storageSftpNewP( + hrnHostRepo1Path(this), hrnHostIp(sftp), 22, hrnHostUser(this), ioTimeoutMs(), keyPrivate, + strIdFromZ(HRN_HOST_SFTP_HOSTKEY_HASH_TYPE), .write = true, .modeFile = STORAGE_MODE_FILE_DEFAULT, + .modePath = STORAGE_MODE_PATH_DEFAULT, .keyPub = keyPublic, + .hostKeyCheckType = SFTP_STRICT_HOSTKEY_CHECKING_NONE); + } + MEM_CONTEXT_OBJ_END(); + + break; + } + + default: + { + ASSERT(hrnHostLocal.storage == STORAGE_POSIX_TYPE); + + MEM_CONTEXT_OBJ_BEGIN(this) + { + this->pub.repo1Storage = storagePosixNewP(hrnHostRepo1Path(this), .write = true); + } + MEM_CONTEXT_OBJ_END(); + + break; + } + } + + if (hrnHostLocal.repoTotal > 1) + { + strCatFmt(config, "repo2-path=%s\n", strZ(hrnHostRepo2Path(this))); + strCatZ(config, "repo2-retention-full=2\n"); + + MEM_CONTEXT_OBJ_BEGIN(this) + { + this->pub.repo2Storage = storagePosixNewP(hrnHostRepo2Path(this), .write = true); + } + MEM_CONTEXT_OBJ_END(); + } + } + // Pg options + else + { + const HrnHost *const repo = hrnHostRepo(); + + strCatZ(config, "\n"); + strCatFmt(config, "repo1-host=%s\n", strZ(hrnHostName(repo))); + strCatFmt(config, "repo1-host-user=%s\n", strZ(hrnHostUser(repo))); + hrnHostConfigTls(this, config, "repo1"); + + if (hrnHostLocal.repoTotal > 1) + { + strCatFmt(config, "repo2-host=%s\n", strZ(hrnHostName(repo))); + strCatFmt(config, "repo2-host-user=%s\n", strZ(hrnHostUser(repo))); + hrnHostConfigTls(this, config, "repo2"); + } + } + + // Stanza options + strCatZ(config, "\n[" HRN_STANZA "]\n"); + + switch (hrnHostId(this)) + { + case HRN_HOST_PG1: + { + strCatFmt(config, "pg1-path=%s\n", strZ(hrnHostPgDataPath(this))); + strCatFmt(config, "pg1-socket-path=%s/pg1/pg\n", testPath()); + + break; + } + + case HRN_HOST_PG2: + { + strCatFmt(config, "pg1-path=%s\n", strZ(hrnHostPgDataPath(this))); + strCatFmt(config, "pg1-socket-path=%s/pg2/pg\n", testPath()); + + if (hrnHostLocal.repoHost == HRN_HOST_PG2) + { + strCatFmt(config, "pg2-path=%s/pg1/pg/data\n", testPath()); + strCatZ(config, "pg2-host=pg1\n"); + strCatFmt(config, "pg2-host-user=%s\n", strZ(hrnHostUser(this))); + hrnHostConfigTls(this, config, "pg2"); + } + + break; + } + + default: + { + ASSERT(hrnHostId(this) == HRN_HOST_REPO); + + strCatFmt(config, "pg1-path=%s/pg1/pg/data\n", testPath()); + strCatZ(config, "pg1-host=pg1\n"); + strCatFmt(config, "pg1-host-user=%s\n", strZ(hrnHostUser(this))); + hrnHostConfigTls(this, config, "pg1"); + + strCatFmt(config, "pg2-path=%s/pg2/pg/data\n", testPath()); + strCatZ(config, "pg2-host=pg2\n"); + strCatFmt(config, "pg2-host-user=%s\n", strZ(hrnHostUser(this))); + hrnHostConfigTls(this, config, "pg2"); + + break; + } + } + + storagePutP( + storageNewWriteP(hrnHostDataStorage(this), STRDEF("cfg/pgbackrest.conf"), .modeFile = 0600), BUFSTR(config)); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_HARNESS_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +const String * +hrnHostPgBinPath(HrnHost *const this) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(HRN_HOST, this); + FUNCTION_HARNESS_END(); + + ASSERT(this != NULL); + ASSERT(hrnHostIsPg(this)); + + if (this->pgBinPath == NULL) + { + MEM_CONTEXT_TEMP_BEGIN() + { + const String *pgPath = strNewFmt("/usr/lib/postgresql/%s/bin", strZ(pgVersionToStr(hrnHostPgVersion()))); + + TRY_BEGIN() + { + hrnHostExecP(this, strNewFmt("ls %s/initdb", strZ(pgPath))); + } + CATCH_ANY() + { + pgPath = strNewFmt("/usr/pgsql-%s/bin", strZ(pgVersionToStr(hrnHostPgVersion()))); + } + TRY_END(); + + MEM_CONTEXT_OBJ_BEGIN(this) + { + this->pgBinPath = strDup(pgPath); + } + MEM_CONTEXT_OBJ_END(); + } + MEM_CONTEXT_TEMP_END(); + } + + FUNCTION_HARNESS_RETURN(STRING, this->pgBinPath); +} + +/**********************************************************************************************************************************/ +PgClient * +hrnHostPgClient(HrnHost *const this) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(HRN_HOST, this); + FUNCTION_HARNESS_END(); + + ASSERT(this != NULL); + ASSERT(hrnHostIsPg(this)); + + if (this->pgClient == NULL) + { + MEM_CONTEXT_OBJ_BEGIN(this) + { + this->pgClient = pgClientNew(hrnHostPgPath(this), 5432, STRDEF("postgres"), hrnHostUser(this), ioTimeoutMs()); + pgClientOpen(this->pgClient); + } + MEM_CONTEXT_OBJ_END(); + } + + FUNCTION_HARNESS_RETURN(PG_CLIENT, this->pgClient); +} + +/**********************************************************************************************************************************/ +HrnHost * +hrnHostGet(const StringId id) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(STRING_ID, id); + FUNCTION_HARNESS_END(); + + ASSERT(id != 0); + + HrnHost *result = NULL; + + for (unsigned int hostIdx = 0; hostIdx < lstSize(hrnHostLocal.hostList); hostIdx++) + { + HrnHost *const host = *(HrnHost **)lstGet(hrnHostLocal.hostList, hostIdx); + + if (hrnHostId(host) == id) + { + result = host; + break; + } + } + + FUNCTION_HARNESS_RETURN(HRN_HOST, result); +} + +/**********************************************************************************************************************************/ +CipherType +hrnHostCipherType(void) +{ + FUNCTION_HARNESS_VOID(); + FUNCTION_HARNESS_RETURN(STRING_ID, hrnHostLocal.cipherType); +} + +const String * +hrnHostCipherPass(void) +{ + FUNCTION_HARNESS_VOID(); + FUNCTION_HARNESS_RETURN(UINT, hrnHostLocal.cipherPass); +} + +CompressType +hrnHostCompressType(void) +{ + FUNCTION_HARNESS_VOID(); + FUNCTION_HARNESS_RETURN(ENUM, hrnHostLocal.compressType); +} + +bool +hrnHostNonVersionSpecific(void) +{ + FUNCTION_HARNESS_VOID(); + FUNCTION_HARNESS_RETURN(BOOL, hrnHostLocal.nonVersionSpecific); +} + +HrnHost * +hrnHostPg1(void) +{ + FUNCTION_HARNESS_VOID(); + FUNCTION_HARNESS_RETURN(HRN_HOST, hrnHostGet(HRN_HOST_PG1)); +} + +HrnHost * +hrnHostPg2(void) +{ + FUNCTION_HARNESS_VOID(); + FUNCTION_HARNESS_RETURN(HRN_HOST, hrnHostGet(HRN_HOST_PG2)); +} + +unsigned int +hrnHostPgVersion(void) +{ + FUNCTION_HARNESS_VOID(); + FUNCTION_HARNESS_RETURN(UINT, hrnHostLocal.pgVersion); +} + +HrnHost * +hrnHostRepo(void) +{ + FUNCTION_HARNESS_VOID(); + FUNCTION_HARNESS_RETURN(HRN_HOST, hrnHostLocal.repoHost == HRN_HOST_PG2 ? hrnHostPg2() : hrnHostGet(HRN_HOST_REPO)); +} + +unsigned int +hrnHostRepoTotal(void) +{ + FUNCTION_HARNESS_VOID(); + FUNCTION_HARNESS_RETURN(UINT, hrnHostLocal.repoTotal); +} + +/**********************************************************************************************************************************/ +void +hrnHostConfigUpdate(const HrnHostConfigUpdateParam param) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(VARIANT, param.archiveAsync); + FUNCTION_HARNESS_END(); + + // Update config + if (param.archiveAsync != NULL) + hrnHostLocal.archiveAsync = varBool(param.archiveAsync); + + // Write pgBackRest configuration for hosts + hrnHostConfig(hrnHostPg1()); + hrnHostConfig(hrnHostPg2()); + + if (hrnHostLocal.repoHost == HRN_HOST_REPO) + hrnHostConfig(hrnHostRepo()); + + FUNCTION_HARNESS_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +// Helper to run a host +static HrnHost * +hrnHostBuildRun(const int line, const StringId id) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(INT, line); + FUNCTION_HARNESS_PARAM(STRING_ID, id); + FUNCTION_HARNESS_END(); + + HrnHost *result; + + MEM_CONTEXT_TEMP_BEGIN() + { + const String *const name = strIdToStr(id); + const bool isPg = strBeginsWithZ(name, "pg"); + const bool isRepo = id == hrnHostLocal.repoHost; + const String *const container = strNewFmt("test-%u-%s", testIdx(), strZ(name)); + const String *const image = strNewFmt("pgbackrest/test:%s-test", testVm()); + const String *const dataPath = strNewFmt("%s/%s", testPath(), strZ(name)); + String *const option = strNewFmt( + "-v '%s/cfg:/etc/pgbackrest:ro' -v '%s:/usr/bin/pgbackrest:ro' -v '%s:%s:ro'", strZ(dataPath), testProjectExe(), + hrnPathRepo(), hrnPathRepo()); + const String *param = NULL; + const String *entryPoint = NULL; + + if (hrnHostLocal.tls) + { + THROW_ON_SYS_ERROR_FMT( + chmod(zNewFmt("%s/" HRN_HOST_TLS_SERVER_KEY, hrnPathRepo()), 0600) == -1, FileModeError, + "unable to set mode on '%s/" HRN_HOST_TLS_SERVER_KEY "'", hrnPathRepo()); + THROW_ON_SYS_ERROR_FMT( + chmod(zNewFmt("%s/" HRN_HOST_TLS_CLIENT_KEY, hrnPathRepo()), 0600) == -1, FileModeError, + "unable to set mode on '%s/" HRN_HOST_TLS_CLIENT_KEY "'", hrnPathRepo()); + + param = strNewFmt( + "server --log-level-console=info --tls-server-ca-file=%s/" HRN_HOST_TLS_SERVER_CA " --tls-server-cert-file=%s/" + HRN_HOST_TLS_SERVER_CERT " --tls-server-key-file=%s/" HRN_HOST_TLS_SERVER_KEY + " --tls-server-auth=pgbackrest-client=* --tls-server-address=0.0.0.0", hrnPathRepo(), hrnPathRepo(), hrnPathRepo()); + entryPoint = strNewZ("/usr/bin/pgbackrest"); + } + + MEM_CONTEXT_PRIOR_BEGIN() + { + Storage *const storageData = storagePosixNewP(dataPath, .write = true); + storagePutP(storageNewWriteP(storageData, STRDEF("cfg/pgbackrest.conf"), .modeFile = 0600), BUFSTRDEF("")); + + result = hrnHostNewP( + id, container, image, .user = STR(testUser()), .dataPath = dataPath, .option = option, .param = param, + .entryPoint = entryPoint, .isPg = isPg, .isRepo = isRepo); + } + MEM_CONTEXT_PRIOR_END(); + + TEST_RESULT_INFO_LINE_FMT(line, "host %s (ip %s, container %s)", strZ(name), strZ(hrnHostIp(result)), strZ(container)); + + if (isPg) + { + TEST_RESULT_INFO_LINE_FMT( + line, " psql: docker exec -it %s psql -U %s -h %s postgres", strZ(container), strZ(hrnHostUser(result)), + strZ(hrnHostPgPath(result))); + } + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_HARNESS_RETURN(HRN_HOST, result); +} + +void +hrnHostBuild(const int line, const HrnHostTestDefine *const testMatrix, const size_t testMatrixSize) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(INT, line); + FUNCTION_HARNESS_PARAM_P(VOID, testMatrix); + FUNCTION_HARNESS_PARAM(SIZE, testMatrixSize); + FUNCTION_HARNESS_END(); + + // Find test using pg version + const HrnHostTestDefine *testDef = NULL; + + for (unsigned int matrixIdx = 0; matrixIdx < testMatrixSize; matrixIdx++) + { + if (strcmp(testMatrix[matrixIdx].pg, testPgVersion()) == 0) + { + testDef = &testMatrix[matrixIdx]; + break; + } + } + + if (testDef == NULL) + THROW_FMT(AssertError, "unable to find PostgreSQL %s in test matrix", testPgVersion()); + + // Initialize mem context + if (hrnHostLocal.memContext == NULL) + { + MEM_CONTEXT_BEGIN(memContextTop()) + { + MEM_CONTEXT_NEW_BEGIN(HrnHostLocal, .childQty = MEM_CONTEXT_QTY_MAX) + { + hrnHostLocal.memContext = MEM_CONTEXT_NEW(); + hrnHostLocal.hostList = lstNewP(sizeof(HrnHost *), .comparator = lstComparatorStr); + } + MEM_CONTEXT_NEW_END(); + } + MEM_CONTEXT_END(); + } + + // Set local variables + MEM_CONTEXT_BEGIN(hrnHostLocal.memContext) + { + hrnHostLocal.pgVersion = pgVersionFromStr(STR(testDef->pg)); + hrnHostLocal.repoHost = strIdFromZ(testDef->repo); + hrnHostLocal.storage = strIdFromZ(testDef->stg); + hrnHostLocal.compressType = compressTypeFromName(STR(testDef->cmp)); + hrnHostLocal.cipherType = testDef->enc ? cipherTypeAes256Cbc : cipherTypeNone; + hrnHostLocal.cipherPass = testDef->enc ? strNewZ(HRN_CIPHER_PASSPHRASE) : NULL; + hrnHostLocal.repoTotal = testDef->rt; + hrnHostLocal.tls = testDef->tls; + hrnHostLocal.bundle = testDef->bnd; + hrnHostLocal.blockIncr = testDef->bi; + hrnHostLocal.nonVersionSpecific = strcmp(testDef->pg, testMatrix[testMatrixSize - 1].pg) == 0; + } + MEM_CONTEXT_END(); + + ASSERT(hrnHostLocal.repoTotal >= 1 && hrnHostLocal.repoTotal <= 2); + ASSERT(hrnHostLocal.repoHost == HRN_HOST_PG2 || hrnHostLocal.repoHost == HRN_HOST_REPO); + + TEST_RESULT_INFO_LINE_FMT( + line, "pg = %s, repo = %s, .tls = %d, stg = %s, enc = %d, cmp = %s, rt = %u, bnd = %d, bi = %d, nv = %d", testDef->pg, + testDef->repo, testDef->tls, testDef->stg, testDef->enc, testDef->cmp, testDef->rt, testDef->bnd, testDef->bi, + hrnHostLocal.nonVersionSpecific); + + // Create pg hosts + hrnHostBuildRun(line, HRN_HOST_PG1); + HrnHost *const pg2 = hrnHostBuildRun(line, HRN_HOST_PG2); + + // Create repo host if the destination is repo + HrnHost *const repo = hrnHostLocal.repoHost == HRN_HOST_REPO ? hrnHostBuildRun(line, HRN_HOST_REPO) : pg2; + + MEM_CONTEXT_TEMP_BEGIN() + { + // Create storage host if not posix + if (hrnHostLocal.storage != STORAGE_POSIX_TYPE) + { + const char *const fakeCertPath = zNewFmt("%s/doc/resource/fake-cert", hrnPathRepo()); + const String *const containerName = strNewFmt("test-%u-%s", testIdx(), strZ(strIdToStr(hrnHostLocal.storage))); + + switch (hrnHostLocal.storage) + { + case STORAGE_AZURE_TYPE: + { + String *const option = strNewFmt( + "-v '%s/s3-server.crt:/root/public.crt:ro' -v '%s/s3-server.key:/root/private.key:ro'" + " -e AZURITE_ACCOUNTS='" HRN_HOST_AZURE_ACCOUNT ":" HRN_HOST_AZURE_KEY "'", + fakeCertPath, fakeCertPath); + String *const param = strNewZ( + "azurite-blob --blobPort 443 --blobHost 0.0.0.0 --cert=/root/public.crt --key=/root/private.key -d" + " debug.log --disableProductStyleUrl"); + + MEM_CONTEXT_PRIOR_BEGIN() + { + hrnHostNewP( + HRN_HOST_AZURE, containerName, STRDEF("mcr.microsoft.com/azure-storage/azurite"), .option = option, + .param = param, .noUpdateHosts = true); + } + MEM_CONTEXT_PRIOR_END(); + + break; + } + + case STORAGE_GCS_TYPE: + { + MEM_CONTEXT_PRIOR_BEGIN() + { + hrnHostNewP(HRN_HOST_GCS, containerName, STRDEF("fsouza/fake-gcs-server"), .noUpdateHosts = true); + } + MEM_CONTEXT_PRIOR_END(); + + break; + } + + case STORAGE_S3_TYPE: + { + String *const option = strNewFmt( + "-v '%s/s3-server.crt:/root/.minio/certs/public.crt:ro'" + " -v '%s/s3-server.key:/root/.minio/certs/private.key:ro' -e MINIO_REGION=" HRN_HOST_S3_REGION + " -e MINIO_DOMAIN=" HRN_HOST_S3_ENDPOINT " -e MINIO_BROWSER=off" + " -e MINIO_ACCESS_KEY=" HRN_HOST_S3_ACCESS_KEY " -e MINIO_SECRET_KEY=" HRN_HOST_S3_ACCESS_SECRET_KEY, + fakeCertPath, fakeCertPath); + String *const param = strNewZ("server /data --address :443"); + + MEM_CONTEXT_PRIOR_BEGIN() + { + hrnHostNewP( + HRN_HOST_S3, containerName, STRDEF("minio/minio:RELEASE.2023-09-30T07-02-29Z"), .option = option, + .param = param, .noUpdateHosts = true); + } + MEM_CONTEXT_PRIOR_END(); + + HrnHost *const s3 = hrnHostGet(HRN_HOST_S3); + + hrnHostExecP( + repo, strNewFmt("echo \"%s\t" HRN_HOST_S3_ENDPOINT "\" >> /etc/hosts", strZ(hrnHostIp(s3))), + .user = STRDEF("root")); + hrnHostExecP( + repo, + strNewFmt( + "echo \"%s\t" HRN_HOST_S3_BUCKET "." HRN_HOST_S3_ENDPOINT "\" >> /etc/hosts", strZ(hrnHostIp(s3))), + .user = STRDEF("root")); + + break; + } + + default: + { + ASSERT(hrnHostLocal.storage == STORAGE_SFTP_TYPE); + + MEM_CONTEXT_PRIOR_BEGIN() + { + hrnHostNewP( + HRN_HOST_SFTP, containerName, strNewFmt("pgbackrest/test:%s-test", testVm()), .noUpdateHosts = true); + } + MEM_CONTEXT_PRIOR_END(); + + break; + } + } + } + } + MEM_CONTEXT_TEMP_END(); + + // Write pgBackRest configuration for hosts + hrnHostConfigUpdateP(); + + // Create the repo for object stores + if (hrnHostLocal.storage == STORAGE_AZURE_TYPE || hrnHostLocal.storage == STORAGE_GCS_TYPE || + hrnHostLocal.storage == STORAGE_S3_TYPE) + { + hrnHostExecBrP(repo, CFGCMD_REPO_CREATE, .option = "--repo=1"); + } + + FUNCTION_HARNESS_RETURN_VOID(); +} diff --git a/test/src/common/harnessHost.h b/test/src/common/harnessHost.h new file mode 100644 index 0000000000..44f03f0a88 --- /dev/null +++ b/test/src/common/harnessHost.h @@ -0,0 +1,490 @@ +/*********************************************************************************************************************************** +Host Harness +***********************************************************************************************************************************/ +#ifndef TEST_COMMON_HARNESS_HOST_H +#define TEST_COMMON_HARNESS_HOST_H + +/*********************************************************************************************************************************** +Object type +***********************************************************************************************************************************/ +typedef struct HrnHost HrnHost; + +#include "common/type/object.h" +#include "common/type/string.h" +#include "postgres/client.h" + +#include "common/harnessTest.h" + +/*********************************************************************************************************************************** +Constants +***********************************************************************************************************************************/ +// Host constants +#define HRN_HOST_AZURE STORAGE_AZURE_TYPE +#define HRN_HOST_AZURE_Z "azure" +#define HRN_HOST_GCS STORAGE_GCS_TYPE +#define HRN_HOST_GCS_Z "gcs" +#define HRN_HOST_POSIX STORAGE_POSIX_TYPE +#define HRN_HOST_POSIX_Z "posix" +#define HRN_HOST_PG1 STRID6("pg1", 0x1d1d01) +#define HRN_HOST_PG1_Z "pg1" +#define HRN_HOST_PG2 STRID5("pg2", 0x70f00) +#define HRN_HOST_PG2_Z "pg2" +#define HRN_HOST_REPO STRID5("repo", 0x7c0b20) +#define HRN_HOST_REPO_Z "repo" +#define HRN_HOST_S3 STORAGE_S3_TYPE +#define HRN_HOST_S3_Z "s3" +#define HRN_HOST_SFTP STORAGE_SFTP_TYPE +#define HRN_HOST_SFTP_Z "sftp" + +// Test stanza +#define HRN_STANZA "test" + +/*********************************************************************************************************************************** +Test +***********************************************************************************************************************************/ +typedef struct HrnHostTestDefine +{ + const char *pg; // PostgreSQL version + const char *repo; // Host acting as repository + bool tls; // Use TLS instead of SSH for remote protocol? + const char *stg; // Storage type + bool enc; // Encryption + const char *cmp; // Compression type + unsigned int rt; // Repository total + bool bnd; // Bundling enabled? + bool bi; // Block incremental enabled? +} HrnHostTestDefine; + +/*********************************************************************************************************************************** +Constructor +***********************************************************************************************************************************/ +typedef struct HrnHostNewParam +{ + VAR_PARAM_HEADER; + const String *entryPoint; // Container entry point + const String *user; // Container user + const String *option; // Options (e.g. -v) + const String *param; // Parameters (passed to the container command) + bool noUpdateHosts; // Do not update /etc/hosts file on this host + const String *dataPath; // Path where container data is stored + bool pgStandby; // Pg standby? + bool isPg; // Is a pg host? + bool isRepo; // Is a repo host? +} HrnHostNewParam; + +#define hrnHostNewP(name, container, image, ...) \ + hrnHostNew(name, container, image, (HrnHostNewParam){VAR_PARAM_INIT, __VA_ARGS__}) + +HrnHost *hrnHostNew(StringId id, const String *container, const String *image, HrnHostNewParam param); + +/*********************************************************************************************************************************** +Getters/Setters +***********************************************************************************************************************************/ +typedef struct HrnHostPub +{ + StringId id; // Host id + const String *name; // Host name + const String *container; // Container name + const String *image; // Container image + const String *ip; // IP address + const String *user; // Container user + const String *dataPath; // Data path + const Storage *dataStorage; // Data storage + const String *logPath; // Log path + const String *brBin; // pgBackRest binary + bool pgStandby; // Pg Standby + const String *pgPath; // Pg path + const String *pgDataPath; // Pg data path + const String *pgTsPath; // Pg tablespace path + const String *pgLogFile; // Pg log path + const Storage *pgStorage; // Pg storage + const String *repo1Path; // Repo1 path + Storage *repo1Storage; // Repo1 storage + const String *repo2Path; // Repo2 path + Storage *repo2Storage; // Repo2 storage + const String *spoolPath; // Spool path + bool isPg; // Is a pg host? + bool isRepo; // Is a repo host? + bool updateHosts; // Is /etc/hosts file updated on this host? +} HrnHostPub; + +// Container name +FN_INLINE_ALWAYS const String * +hrnHostContainer(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->container; +} + +// Host data path +FN_INLINE_ALWAYS const String * +hrnHostDataPath(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->dataPath; +} + +// Host data storage +FN_INLINE_ALWAYS const Storage * +hrnHostDataStorage(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->dataStorage; +} + +// Container image +FN_INLINE_ALWAYS const String * +hrnHostImage(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->image; +} + +// Host id +FN_INLINE_ALWAYS StringId +hrnHostId(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->id; +} + +// Container IP address +FN_INLINE_ALWAYS const String * +hrnHostIp(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->ip; +} + +// Host name +FN_INLINE_ALWAYS const String * +hrnHostName(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->name; +} + +// Is a pg host? +FN_INLINE_ALWAYS bool +hrnHostIsPg(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->isPg; +} + +// Is a repo host? +FN_INLINE_ALWAYS bool +hrnHostIsRepo(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->isRepo; +} + +// Log path +FN_INLINE_ALWAYS const String * +hrnHostLogPath(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->logPath; +} + +// pgBackRest binary +FN_INLINE_ALWAYS const String * +hrnHostBrBin(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->brBin; +} + +// Pg bin path +const String *hrnHostPgBinPath(HrnHost *this); + +// Pg client +PgClient *hrnHostPgClient(HrnHost *this); + +// Pg path +FN_INLINE_ALWAYS const String * +hrnHostPgPath(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->pgPath; +} + +// Pg log file +FN_INLINE_ALWAYS const String * +hrnHostPgLogFile(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->pgLogFile; +} + +// Pg data path +FN_INLINE_ALWAYS const String * +hrnHostPgDataPath(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->pgDataPath; +} + +// Pg standby? +FN_INLINE_ALWAYS bool +hrnHostPgStandby(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->pgStandby; +} + +// Host data storage +FN_INLINE_ALWAYS const Storage * +hrnHostPgStorage(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->pgStorage; +} + +// Pg tablespace path +FN_INLINE_ALWAYS const String * +hrnHostPgTsPath(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->pgTsPath; +} + +// Repo1 path +FN_INLINE_ALWAYS const String * +hrnHostRepo1Path(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->repo1Path; +} + +// Repo1 storage +FN_INLINE_ALWAYS const Storage * +hrnHostRepo1Storage(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->repo1Storage; +} + +// Repo2 path +FN_INLINE_ALWAYS const String * +hrnHostRepo2Path(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->repo2Path; +} + +// Repo2 storage +FN_INLINE_ALWAYS const Storage * +hrnHostRepo2Storage(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->repo2Storage; +} + +// Spool path +FN_INLINE_ALWAYS const String * +hrnHostSpoolPath(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->spoolPath; +} + +// Host name +FN_INLINE_ALWAYS bool +hrnHostUpdateHosts(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->updateHosts; +} + +// Container user +FN_INLINE_ALWAYS const String * +hrnHostUser(const HrnHost *const this) +{ + return THIS_PUB(HrnHost)->user; +} + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +// Execute a command +typedef struct HrnHostExecParam +{ + VAR_PARAM_HEADER; + const String *user; // User to execute command + int resultExpect; // Expected result, if not 0 +} HrnHostExecParam; + +#define hrnHostExecP(this, command, ...) \ + hrnHostExec(this, command, (HrnHostExecParam){VAR_PARAM_INIT, __VA_ARGS__}) + +String *hrnHostExec(HrnHost *this, const String *command, HrnHostExecParam param); + +// Execute pgbackrest +#define TEST_HOST_BR(this, command, ...) \ + do \ + { \ + const HrnHostExecBrParam param = {VAR_PARAM_INIT, __VA_ARGS__}; \ + \ + TEST_RESULT_INFO_FMT( \ + "%s: pgbackrest%s %s%s%s", strZ(hrnHostName(this)), param.option == NULL ? "" : zNewFmt(" %s", param.option), command, \ + param.param == NULL ? "" : zNewFmt(" %s", param.param), \ + param.resultExpect != 0 ? zNewFmt(" -- result %d", param.resultExpect) : ""); \ + hrnHostExecBrP(this, command , __VA_ARGS__); \ + } \ + while (0); + +typedef struct HrnHostExecBrParam +{ + VAR_PARAM_HEADER; + const char *option; // Options + const char *param; // Parameters + int resultExpect; // Expected result, if not 0 +} HrnHostExecBrParam; + +#define hrnHostExecBrP(this, command, ...) \ + hrnHostExecBr(this, command, (HrnHostExecBrParam){VAR_PARAM_INIT, __VA_ARGS__}) + +String *hrnHostExecBr(HrnHost *this, const char *command, HrnHostExecBrParam param); + +// Create/start/stop Pg cluster +#define HRN_HOST_PG_CREATE(this) \ + do \ + { \ + TEST_RESULT_INFO_FMT("%s: create pg cluster", strZ(hrnHostName(this))); \ + hrnHostPgCreate(this); \ + } \ + while (0) + +void hrnHostPgCreate(HrnHost *this); + +#define HRN_HOST_PG_START(this) \ + do \ + { \ + TEST_RESULT_INFO_FMT("%s: start pg cluster", strZ(hrnHostName(this))); \ + hrnHostPgStart(this); \ + } \ + while (0) + +void hrnHostPgStart(HrnHost *this); + +#define HRN_HOST_PG_STOP(this) \ + do \ + { \ + TEST_RESULT_INFO_FMT("%s: stop pg cluster", strZ(hrnHostName(this))); \ + hrnHostPgStop(this); \ + } \ + while (0) + +void hrnHostPgStop(HrnHost *this); + +// Query +PackRead *hrnHostSql(HrnHost *this, const String *statement, const PgClientQueryResult resultType); + +FN_INLINE_ALWAYS PackRead * +hrnHostSqlLog(HrnHost *const this, int line, const char *const statement, const PgClientQueryResult resultType) +{ + TEST_RESULT_INFO_LINE_FMT(line, "%s: %s", strZ(hrnHostName(this)), statement); + return hrnHostSql(this, STR(statement), resultType); +} + +#define HRN_HOST_SQL(this, statement, resultType) \ + hrnHostSqlLog(this, __LINE__, statement, resultType) + +// Exec statement +void hrnHostSqlExec(HrnHost *this, const String *statement); + +#define HRN_HOST_SQL_EXEC(this, statement) \ + do \ + { \ + TEST_RESULT_INFO_FMT("%s: %s", strZ(hrnHostName(this)), statement); \ + hrnHostSqlExec(this, STR(statement)); \ + } while (0) + +// Query a single value +FN_INLINE_ALWAYS PackRead * +hrnHostSqlValue(HrnHost *const this, const char *const statement) +{ + return hrnHostSql(this, STR(statement), pgClientQueryResultColumn); +} + +#define HRN_HOST_SQL_VALUE(this, statement) \ + HRN_HOST_SQL(this, statement, pgClientQueryResulColumn) + +// Test a single value +void hrnHostSqlTest(HrnHost *this, const String *statement, const String *expected); + +#define TEST_HOST_SQL_ONE_STR_Z(this, statement, expected) \ + do \ + { \ + TEST_RESULT_INFO_FMT("%s: %s == '%s'", strZ(hrnHostName(this)), statement, expected); \ + hrnTestResultBegin(#statement, true); \ + hrnHostSqlTest(this, STR(statement), STR(expected)); \ + } \ + while (0) + +// Begin transaction +FN_INLINE_ALWAYS void +hrnHostSqlBegin(HrnHost *const this) +{ + hrnHostSqlExec(this, STRDEF("begin")); +} + +// Commit transaction +FN_INLINE_ALWAYS void +hrnHostSqlCommit(HrnHost *const this) +{ + hrnHostSqlExec(this, STRDEF("commit")); +} + +// Switch WAL segment +#define HRN_HOST_WAL_SWITCH(this) \ + HRN_HOST_SQL_EXEC(this, zNewFmt("select pg_switch_%s()::text", strZ(pgWalName(hrnHostPgVersion())))) + +/*********************************************************************************************************************************** +Helper functions +***********************************************************************************************************************************/ +// Cipher Type +CipherType hrnHostCipherType(void); + +// Cipher Passphrase +const String *hrnHostCipherPass(void); + +// Compress Type +CompressType hrnHostCompressType(void); + +// Non version-specific testing enabled +bool hrnHostNonVersionSpecific(void); + +// Pg version +unsigned int hrnHostPgVersion(void); + +// Pg1 host +HrnHost *hrnHostPg1(void); + +// Pg2 host +HrnHost *hrnHostPg2(void); + +// Repo host +HrnHost *hrnHostRepo(void); + +// Repo total +unsigned int hrnHostRepoTotal(void); + +// Get a host by name +HrnHost *hrnHostGet(StringId id); + +// Run hosts and build configuration based on test matrix +void hrnHostBuild(int line, const HrnHostTestDefine *testMatrix, size_t testMatrixSize); + +#define HRN_HOST_BUILD(testMatrix) \ + hrnHostBuild(__LINE__, testMatrix, LENGTH_OF(testMatrix)) + +// Update configuration +typedef struct HrnHostConfigUpdateParam +{ + VAR_PARAM_HEADER; + const Variant *archiveAsync; // Update async archiving? +} HrnHostConfigUpdateParam; + +#define hrnHostConfigUpdateP(...) \ + hrnHostConfigUpdate((HrnHostConfigUpdateParam){VAR_PARAM_INIT, __VA_ARGS__}) + +void hrnHostConfigUpdate(const HrnHostConfigUpdateParam param); + +/*********************************************************************************************************************************** +Destructor +***********************************************************************************************************************************/ +FN_INLINE_ALWAYS void +hrnHostFree(HrnHost *const this) +{ + objFree(this); +} + +/*********************************************************************************************************************************** +Macros for function logging +***********************************************************************************************************************************/ +#define FUNCTION_LOG_HRN_HOST_TYPE \ + HrnHost * +#define FUNCTION_LOG_HRN_HOST_FORMAT(value, buffer, bufferSize) \ + objNameToLog(value, "HrnHost", buffer, bufferSize) + +#endif diff --git a/test/src/common/harnessTest.c b/test/src/common/harnessTest.c index 4171ed1221..c48e747ddb 100644 --- a/test/src/common/harnessTest.c +++ b/test/src/common/harnessTest.c @@ -37,6 +37,9 @@ static bool testLogExpectData = false; static unsigned int testIdxData = 0; static bool testTiming = true; static const char *testPathData = NULL; +static const char *testUserData = NULL; +static const char *testVmData = NULL; +static const char *testPgVersionData = NULL; static const char *testDataPathData = NULL; static const char *testRepoPathData = NULL; @@ -67,8 +70,9 @@ Initialize harness ***********************************************************************************************************************************/ void hrnInit( - const char *testExe, const char *testProjectExe, bool testContainer, bool testLogExpect, unsigned int testIdx, bool timing, - const char *testPath, const char *testDataPath, const char *testRepoPath) + const char *const testExe, const char *const testProjectExe, const bool testContainer, const bool testLogExpect, + const unsigned int testIdx, const bool timing, const char *const testPath, const char *const testUser, const char *const testVm, + const char *const testPgVersion,const char *const testDataPath, const char *const testRepoPath) { FUNCTION_HARNESS_VOID(); @@ -81,6 +85,9 @@ hrnInit( testIdxData = testIdx; testTiming = timing; testPathData = testPath; + testUserData = testUser; + testVmData = testVm; + testPgVersionData = testPgVersion; testDataPathData = testDataPath; testRepoPathData = testRepoPath; @@ -727,6 +734,30 @@ testPath(void) FUNCTION_HARNESS_RETURN(STRINGZ, testPathData); } +/**********************************************************************************************************************************/ +const char * +testPgVersion(void) +{ + FUNCTION_HARNESS_VOID(); + FUNCTION_HARNESS_RETURN(STRINGZ, testPgVersionData); +} + +/**********************************************************************************************************************************/ +const char * +testUser(void) +{ + FUNCTION_HARNESS_VOID(); + FUNCTION_HARNESS_RETURN(STRINGZ, testUserData); +} + +/**********************************************************************************************************************************/ +const char * +testVm(void) +{ + FUNCTION_HARNESS_VOID(); + FUNCTION_HARNESS_RETURN(STRINGZ, testVmData); +} + /**********************************************************************************************************************************/ const char * hrnPath(void) diff --git a/test/src/common/harnessTest.h b/test/src/common/harnessTest.h index f9488f94b2..60da568b03 100644 --- a/test/src/common/harnessTest.h +++ b/test/src/common/harnessTest.h @@ -67,6 +67,15 @@ bool testContainer(void); // parallel. unsigned int testIdx(void); +// PostgreSQL version for integration testing +const char *testPgVersion(void); + +// User running the test +const char *testUser(void); + +// VM for integration testing +const char *testVm(void); + /*********************************************************************************************************************************** Test that an expected error is actually thrown and error when it isn't ***********************************************************************************************************************************/ @@ -142,6 +151,14 @@ Output information about the test puts(comment); \ fflush(stdout); +#define TEST_RESULT_INFO_LINE_FMT(line, comment, ...) \ + hrnTestLogPrefix(line); \ + printf(comment "\n", __VA_ARGS__); \ + fflush(stdout) + +#define TEST_RESULT_INFO_FMT(comment, ...) \ + TEST_RESULT_INFO_LINE_FMT(__LINE__, comment, __VA_ARGS__) + /*********************************************************************************************************************************** Test that a void statement returns and does not throw an error ***********************************************************************************************************************************/ diff --git a/test/src/common/harnessTest.intern.h b/test/src/common/harnessTest.intern.h index 1de00a0b29..35dce36eec 100644 --- a/test/src/common/harnessTest.intern.h +++ b/test/src/common/harnessTest.intern.h @@ -41,7 +41,8 @@ Functions ***********************************************************************************************************************************/ void hrnInit( const char *testExe, const char *testProjectExe, bool testContainer, bool testLogExpect, unsigned int testIdx, bool timing, - const char *testPath, const char *testDataPath, const char *testRepoPath); + const char *testPath, const char *testUser, const char *testVm, const char *testPgVersion, const char *testDataPath, + const char *testRepoPath); void hrnAdd(int run, bool selected); void hrnComplete(void); diff --git a/test/src/main.c b/test/src/main.c index aea78f42ce..e439236d7b 100644 --- a/test/src/main.c +++ b/test/src/main.c @@ -71,7 +71,7 @@ main(int argListSize, const char *argList[]) { cmdTest( cfgOptionStr(cfgOptRepoPath), cfgOptionStr(cfgOptTestPath), cfgOptionStr(cfgOptVm), - cfgOptionUInt(cfgOptVmId), strLstGet(cfgCommandParam(), 0), + cfgOptionUInt(cfgOptVmId), cfgOptionStr(cfgOptPgVersion), strLstGet(cfgCommandParam(), 0), cfgOptionTest(cfgOptTest) ? cfgOptionUInt(cfgOptTest) : 0, cfgOptionUInt64(cfgOptScale), logLevelEnum(cfgOptionStrId(cfgOptLogLevelTest)), cfgOptionBool(cfgOptLogTimestamp), cfgOptionStrNull(cfgOptTz), cfgOptionBool(cfgOptCoverage), cfgOptionBool(cfgOptProfile), diff --git a/test/src/module/common/execTest.c b/test/src/module/common/execTest.c index b56dd283dd..bda349e6b5 100644 --- a/test/src/module/common/execTest.c +++ b/test/src/module/common/execTest.c @@ -135,7 +135,7 @@ testRun(void) TEST_RESULT_VOID(execOpen(exec), "open cat exec"); kill(exec->processId, SIGKILL); - TEST_ERROR(execProcess(exec), ExecuteError, "cat terminated unexpectedly on signal 9"); + TEST_ERROR(execProcess(exec, (ExecOneParam){0}), ExecuteError, "cat terminated unexpectedly on signal 9"); TEST_RESULT_VOID(execFree(exec), "free exec"); // ------------------------------------------------------------------------------------------------------------------------- @@ -144,6 +144,13 @@ testRun(void) TEST_ERROR( execOneP(STRDEF("cat missing.txt")), UnknownError, "cat missing.txt terminated unexpectedly [1]: cat: missing.txt: No such file or directory"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("exec ignores error"); + + TEST_RESULT_STR_Z( + execOneP(STRDEF("cat missing.txt"), .resultExpect = 1), "cat: missing.txt: No such file or directory\n", + "ignore error"); } FUNCTION_HARNESS_RETURN_VOID(); diff --git a/test/src/module/integration/allTest.c b/test/src/module/integration/allTest.c new file mode 100644 index 0000000000..d749731fad --- /dev/null +++ b/test/src/module/integration/allTest.c @@ -0,0 +1,404 @@ +/*********************************************************************************************************************************** +Real Integration Test +***********************************************************************************************************************************/ +#include "common/crypto/common.h" +#include "config/config.h" +#include "info/infoBackup.h" +#include "postgres/interface.h" +#include "postgres/version.h" + +#include "common/harnessErrorRetry.h" +#include "common/harnessHost.h" +#include "common/harnessPostgres.h" +#include "common/harnessStorage.h" + +/*********************************************************************************************************************************** +Test definition +***********************************************************************************************************************************/ +static HrnHostTestDefine testMatrix[] = +{ + // {uncrustify_off - struct alignment} + {.pg = "9.4", .repo = "pg2", .tls = 0, .stg = "azure", .enc = 1, .cmp = "lz4", .rt = 1, .bnd = 1, .bi = 0}, + {.pg = "9.5", .repo = "repo", .tls = 1, .stg = "s3", .enc = 0, .cmp = "bz2", .rt = 1, .bnd = 1, .bi = 1}, + {.pg = "9.6", .repo = "repo", .tls = 0, .stg = "posix", .enc = 0, .cmp = "none", .rt = 2, .bnd = 1, .bi = 1}, + {.pg = "10", .repo = "pg2", .tls = 0, .stg = "sftp", .enc = 1, .cmp = "gz", .rt = 1, .bnd = 1, .bi = 0}, + {.pg = "11", .repo = "repo", .tls = 1, .stg = "gcs", .enc = 0, .cmp = "zst", .rt = 2, .bnd = 0, .bi = 0}, + {.pg = "12", .repo = "repo", .tls = 0, .stg = "s3", .enc = 1, .cmp = "lz4", .rt = 1, .bnd = 1, .bi = 1}, + {.pg = "13", .repo = "pg2", .tls = 1, .stg = "sftp", .enc = 0, .cmp = "zst", .rt = 1, .bnd = 1, .bi = 1}, + {.pg = "14", .repo = "repo", .tls = 0, .stg = "gcs", .enc = 0, .cmp = "lz4", .rt = 1, .bnd = 1, .bi = 0}, + {.pg = "15", .repo = "pg2", .tls = 0, .stg = "azure", .enc = 0, .cmp = "none", .rt = 2, .bnd = 1, .bi = 1}, + {.pg = "16", .repo = "repo", .tls = 0, .stg = "posix", .enc = 0, .cmp = "none", .rt = 1, .bnd = 0, .bi = 0}, + // {uncrustify_on} +}; + +/*********************************************************************************************************************************** +Test statuses +***********************************************************************************************************************************/ +#define TEST_STATUS_FULL "full" +#define TEST_STATUS_INCR "incr" +#define TEST_STATUS_NAME "name" +#define TEST_STATUS_STANDBY "standby" +#define TEST_STATUS_TIME "time" +#define TEST_STATUS_TIMELINE "timeline" +#define TEST_STATUS_XID "xid" + +#define TEST_RESTORE_POINT "pgbackrest" + +/*********************************************************************************************************************************** +Test Run +***********************************************************************************************************************************/ +static void +testRun(void) +{ + FUNCTION_HARNESS_VOID(); + + // Enable error retry detail. This works because we are not trying to check exact error messages. + hrnErrorRetryDetailEnable(); + + // ***************************************************************************************************************************** + if (testBegin("integration")) + { + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("build hosts"); + + HRN_HOST_BUILD(testMatrix); + + HrnHost *const pg1 = hrnHostPg1(); + HrnHost *const pg2 = hrnHostPg2(); + HrnHost *const repo = hrnHostRepo(); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("create pg cluster"); + { + HRN_HOST_PG_CREATE(pg1); + TEST_HOST_BR(repo, CFGCMD_STANZA_CREATE); + + // Create tablespace + HRN_STORAGE_PATH_CREATE(hrnHostDataStorage(pg1), strZ(hrnHostPgTsPath(pg1)), .mode = 0700); + HRN_HOST_SQL_EXEC(pg1, zNewFmt("create tablespace ts1 location '%s'", strZ(hrnHostPgTsPath(pg1)))); + + // Init status table + HRN_HOST_SQL_EXEC(pg1, "create table status (message text not null) tablespace ts1"); + HRN_HOST_WAL_SWITCH(pg1); + HRN_HOST_SQL_EXEC(pg1, "insert into status values ('" TEST_STATUS_FULL "')"); + } + + // Get ts1 tablespace oid + const unsigned int ts1Oid = pckReadU32P(hrnHostSqlValue(pg1, "select oid from pg_tablespace where spcname = 'ts1'")); + TEST_LOG_FMT("ts1 tablespace oid = %u", ts1Oid); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("check hosts (skip pg2 for now)"); + { + if (pg1 != repo) + TEST_HOST_BR(pg1, CFGCMD_CHECK); + + if (pg2 != repo) + TEST_HOST_BR(repo, CFGCMD_CHECK); + } + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("primary full backup"); + { + // Full backup to repo 1 + TEST_HOST_BR(repo, CFGCMD_BACKUP, .option = "--type=full --buffer-size=16KiB"); + + // Full backup to repo 2 + if (hrnHostRepoTotal() == 2) + TEST_HOST_BR(repo, CFGCMD_BACKUP, .option = "--type=full --repo=2"); + } + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("standby restore"); + { + TEST_HOST_BR( + pg2, CFGCMD_RESTORE, .option = zNewFmt("--type=standby --tablespace-map=ts1='%s'", strZ(hrnHostPgTsPath(pg2)))); + HRN_HOST_PG_START(pg2); + + // Check standby + TEST_HOST_BR(pg2, CFGCMD_CHECK); + + // Check that backup recovered completely + TEST_HOST_SQL_ONE_STR_Z(pg2, "select message from status", TEST_STATUS_FULL); + + // Update message for standby + HRN_HOST_SQL_EXEC(pg1, "update status set message = '" TEST_STATUS_STANDBY "'"); + + // Check that standby is streaming from the primary + TEST_HOST_SQL_ONE_STR_Z( + pg1, + zNewFmt( + "select client_addr || '-' || state from pg_stat_replication where client_addr = '%s'", + strZ(hrnHostIp(pg2))), + zNewFmt("%s/32-streaming", strZ(hrnHostIp(pg2)))); + } + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("standby full backup and expire"); + { + // Check backups before the backup so we know how many will exist after + const InfoBackup *infoBackup = infoBackupLoadFile( + hrnHostRepo1Storage(repo), STRDEF("backup/" HRN_STANZA "/backup.info"), hrnHostCipherType(), hrnHostCipherPass()); + TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 1, "backup total = 1"); + + TEST_HOST_BR(repo, CFGCMD_BACKUP, .option = "--type=full --backup-standby --repo1-retention-full=1 --no-expire-auto"); + + // Expire was disabled so the backup total has increased + infoBackup = infoBackupLoadFile( + hrnHostRepo1Storage(repo), STRDEF("backup/" HRN_STANZA "/backup.info"), hrnHostCipherType(), hrnHostCipherPass()); + TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 2, "backup total = 2"); + + // Now force an expire + TEST_HOST_BR(repo, CFGCMD_EXPIRE, .option = "--repo1-retention-full=1"); + + // Backup has been expired + infoBackup = infoBackupLoadFile( + hrnHostRepo1Storage(repo), STRDEF("backup/" HRN_STANZA "/backup.info"), hrnHostCipherType(), hrnHostCipherPass()); + TEST_RESULT_UINT(infoBackupDataTotal(infoBackup), 1, "backup total = 1"); + + // Stop the standby since restores to primary will break it + HRN_HOST_PG_STOP(pg2); + } + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("exercise async archiving"); + + hrnHostConfigUpdateP(.archiveAsync = BOOL_TRUE_VAR); + + HRN_HOST_SQL_EXEC(pg1, "create table wal_activity (id int)"); + HRN_HOST_WAL_SWITCH(pg1); + + HRN_HOST_SQL_EXEC(pg1, "insert into wal_activity values (1)"); + HRN_HOST_WAL_SWITCH(pg1); + HRN_HOST_SQL_EXEC(pg1, "insert into wal_activity values (2)"); + HRN_HOST_WAL_SWITCH(pg1); + HRN_HOST_SQL_EXEC(pg1, "insert into wal_activity values (3)"); + HRN_HOST_WAL_SWITCH(pg1); + HRN_HOST_SQL_EXEC(pg1, "insert into wal_activity values (4)"); + HRN_HOST_WAL_SWITCH(pg1); + + TEST_STORAGE_EXISTS( + hrnHostDataStorage(pg1), zNewFmt("%s/" HRN_STANZA "-archive-push-async.log", strZ(hrnHostLogPath(pg1)))); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("setup time target"); + + // If the tests are running quickly then the time target might end up the same as the end time of the prior full backup. + // That means restore auto-select will not pick it as a candidate and restore the last backup instead causing the restore + // compare to fail. So, sleep one second. + sleepMSec(1000); + + HRN_HOST_SQL_EXEC(pg1, "update status set message = '" TEST_STATUS_TIME "'"); + HRN_HOST_WAL_SWITCH(pg1); + + const char *const targetTime = strZ(pckReadStrP(hrnHostSqlValue(pg1, "select current_timestamp::text"))); + TEST_LOG_FMT("time target = %s", targetTime); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("primary incr delta backup"); + { + // Create a database that can be excluded from restores + HRN_HOST_SQL_EXEC(pg1, "create database exclude_me with tablespace ts1"); + + // Check that backup fails for <= 9.5 when another backup is already running + if (hrnHostPgVersion() <= PG_VERSION_95) + { + HRN_HOST_SQL_EXEC(pg1, "perform pg_start_backup('test backup that will be restarted', true)"); + TEST_HOST_BR(repo, CFGCMD_BACKUP, .resultExpect = errorTypeCode(&DbQueryError)); + } + + // Include stop auto here so backups for <= 9.5 will stop the prior backup + HRN_HOST_SQL_EXEC(pg1, "update status set message = '" TEST_STATUS_INCR "'"); + TEST_HOST_BR(repo, CFGCMD_BACKUP, .option = "--type=incr --delta --stop-auto"); + } + + // Get exclude_me database oid + const unsigned int excludeMeOid = pckReadU32P( + hrnHostSqlValue(pg1, "select oid from pg_database where datname = 'exclude_me'")); + TEST_LOG_FMT("exclude_me database oid = %u", excludeMeOid); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("setup xid target"); + + hrnHostSqlBegin(pg1); + HRN_HOST_SQL_EXEC(pg1, "update status set message = '" TEST_STATUS_XID "'"); + HRN_HOST_WAL_SWITCH(pg1); + + const char *const targetXid = strZ(pckReadStrP(hrnHostSqlValue(pg1, "select txid_current()::text"))); + TEST_LOG_FMT("xid target = %s", targetXid); + hrnHostSqlCommit(pg1); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("setup name target"); + + HRN_HOST_SQL_EXEC(pg1, "update status set message = '" TEST_STATUS_NAME "'"); + HRN_HOST_WAL_SWITCH(pg1); + + HRN_HOST_SQL_EXEC(pg1, "perform pg_create_restore_point('" TEST_RESTORE_POINT "')"); + TEST_LOG("name target = " TEST_RESTORE_POINT); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("primary restore (default target)"); + { + // Stop the cluster + HRN_HOST_PG_STOP(pg1); + + // Restore + TEST_HOST_BR(pg1, CFGCMD_RESTORE, .option = zNewFmt("--force --repo=%u", hrnHostRepoTotal())); + HRN_HOST_PG_START(pg1); + + // Check that backup recovered to the expected target + TEST_HOST_SQL_ONE_STR_Z(pg1, "select message from status", TEST_STATUS_NAME); + } + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("primary restore (immediate target)"); + { + // Expect failure when pg is running + TEST_HOST_BR(pg1, CFGCMD_RESTORE, .resultExpect = errorTypeCode(&PgRunningError)); + + // Stop the cluster and try again + HRN_HOST_PG_STOP(pg1); + + // Restore + TEST_HOST_BR(pg1, CFGCMD_RESTORE, .option = "--delta --type=immediate --target-action=promote --db-exclude=exclude_me"); + HRN_HOST_PG_START(pg1); + + // Test that the exclude_me database has a zeroed pg_filenode.map + const Buffer *const pgFileNodeMap = storageGetP( + storageNewReadP( + hrnHostPgStorage(pg1), + strNewFmt( + PG_PATH_PGTBLSPC "/%u/%s/%u/" PG_FILE_PGFILENODEMAP, ts1Oid, + strZ(pgTablespaceId(hrnHostPgVersion(), hrnPgCatalogVersion(hrnHostPgVersion()))), excludeMeOid))); + + Buffer *const pgFileNodeMapZero = bufNew(bufUsed(pgFileNodeMap)); + memset(bufPtr(pgFileNodeMapZero), 0, bufSize(pgFileNodeMapZero)); + bufUsedSet(pgFileNodeMapZero, bufSize(pgFileNodeMapZero)); + + TEST_RESULT_BOOL(bufEq(pgFileNodeMap, pgFileNodeMapZero), true, "exclude_me db " PG_FILE_PGFILENODEMAP " is zeroed"); + + // Check that backup recovered to the expected target + TEST_HOST_SQL_ONE_STR_Z(pg1, "select message from status", TEST_STATUS_INCR); + + // It should be possible to drop the exclude_me database even though it was not restored + HRN_HOST_SQL_EXEC(pg1, "drop database exclude_me"); + } + + // From here on restores need to specify the current timeline for > 11 for recovery to be reliable + const char *const targetTimeline = hrnHostPgVersion() <= PG_VERSION_11 ? "" : " --target-timeline=current"; + + // Be careful about moving the order of this restore since the timeline created is used in a later test + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("primary restore (xid target)"); + { + // Stop the cluster + HRN_HOST_PG_STOP(pg1); + + // Restore + TEST_HOST_BR( + pg1, CFGCMD_RESTORE, + .option = zNewFmt( + "--delta --type=xid --target=%s --target-action=promote --repo=%u%s", targetXid, hrnHostRepoTotal(), + targetTimeline)); + HRN_HOST_PG_START(pg1); + + // Check that backup recovered to the expected target + TEST_HOST_SQL_ONE_STR_Z(pg1, "select message from status", TEST_STATUS_XID); + + // Update status to test following a specified timeline + HRN_HOST_SQL_EXEC(pg1, "update status set message = '" TEST_STATUS_TIMELINE "'"); + HRN_HOST_WAL_SWITCH(pg1); + } + + // Store the time so it can be used in a later test + const char *const xidTimeline = strZ( + pckReadStrP( + hrnHostSqlValue( + pg1, + zNewFmt( + "select trim(leading '0' from substring(pg_%sfile_name('1/1'), 1, 8))", + strZ(pgWalName(hrnHostPgVersion())))))); + TEST_LOG_FMT("xid timeline = %s", xidTimeline); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("primary restore (time target, auto-select backup)"); + { + // Stop the cluster + HRN_HOST_PG_STOP(pg1); + + // Restore + TEST_HOST_BR( + pg1, CFGCMD_RESTORE, + .option = zNewFmt("--delta --type=time --target='%s'%s", targetTime, targetTimeline)); + HRN_HOST_PG_START(pg1); + + // Check that backup recovered to the expected target + TEST_HOST_SQL_ONE_STR_Z(pg1, "select message from status", TEST_STATUS_TIME); + } + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("primary restore (time xid, exclusive)"); + { + // Stop the cluster + HRN_HOST_PG_STOP(pg1); + + // Restore + TEST_HOST_BR( + pg1, CFGCMD_RESTORE, + .option = zNewFmt( + "--delta --type=xid --target='%s' --target-exclusive --repo=%u%s", targetXid, hrnHostRepoTotal(), + targetTimeline)); + HRN_HOST_PG_START(pg1); + + // Check that backup recovered to the expected target + TEST_HOST_SQL_ONE_STR_Z(pg1, "select message from status", TEST_STATUS_INCR); + } + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("primary restore (name)"); + { + // Stop the cluster + HRN_HOST_PG_STOP(pg1); + + // Restore + TEST_HOST_BR( + pg1, CFGCMD_RESTORE, .option = zNewFmt("--delta --type=name --target='" TEST_RESTORE_POINT "'%s", targetTimeline)); + HRN_HOST_PG_START(pg1); + + // Check that backup recovered to the expected target + TEST_HOST_SQL_ONE_STR_Z(pg1, "select message from status", TEST_STATUS_NAME); + } + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("primary restore (default, timeline created by type = xid)"); + { + // Stop the cluster + HRN_HOST_PG_STOP(pg1); + + TEST_HOST_BR(pg1, CFGCMD_RESTORE, .option = zNewFmt("--delta --type=standby --target-timeline=%s", xidTimeline)); + HRN_HOST_PG_START(pg1); + + // Check that backup recovered to the expected target + TEST_HOST_SQL_ONE_STR_Z(pg1, "select message from status", TEST_STATUS_TIMELINE); + } + + // ------------------------------------------------------------------------------------------------------------------------- + if (hrnHostNonVersionSpecific()) + { + TEST_TITLE("stanza-delete --force with pgbackrest stopped"); + + // Stop the cluster + HRN_HOST_PG_STOP(pg1); + + // Stop pgbackrest + TEST_HOST_BR(pg1, CFGCMD_STOP); + TEST_HOST_BR(repo, CFGCMD_STOP); + + // Delete stanza + TEST_HOST_BR(repo, CFGCMD_STANZA_DELETE, .option = "--force"); + } + } + + FUNCTION_HARNESS_RETURN_VOID(); +} diff --git a/test/src/module/test/testTest.c b/test/src/module/test/testTest.c index 00336511c0..0fec684bb1 100644 --- a/test/src/module/test/testTest.c +++ b/test/src/module/test/testTest.c @@ -152,6 +152,7 @@ testRun(void) " feature: stackTrace\n" " harness:\n" " name: stackTrace\n" + " integration: false\n" " shim:\n" " common/stackTrace:\n" " function:\n" @@ -196,7 +197,7 @@ testRun(void) " db: true\n" " test:\n" " - name: all\n" - " total: 2\n" + " total: 1\n" "\n" "performance:\n" " - name: performance\n" @@ -241,7 +242,7 @@ testRun(void) TEST_RESULT_VOID( cmdTest( - STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("none"), 3, + STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("none"), 3, STRDEF("invalid"), STRDEF("common/stack-trace"), 0, 1, logLevelDebug, true, NULL, false, false, false, true), "new build"); @@ -326,6 +327,8 @@ testRun(void) String *const testCDup = strCat(strNew(), testC); strReplace(testCDup, STRDEF("{[C_TEST_CONTAINER]}"), STRDEF("false")); + strReplace(testCDup, STRDEF("{[C_TEST_VM]}"), STRDEF("none")); + strReplace(testCDup, STRDEF("{[C_TEST_PG_VERSION]}"), STRDEF("invalid")); strReplace(testCDup, STRDEF("{[C_TEST_LOG_EXPECT]}"), STRDEF("true")); strReplace(testCDup, STRDEF("{[C_TEST_DEBUG_TEST_TRACE]}"), STRDEF("#define DEBUG_TEST_TRACE")); strReplace(testCDup, STRDEF("{[C_TEST_PATH_BUILD]}"), STRDEF(TEST_PATH "/test/unit-3/none/build")); @@ -366,7 +369,7 @@ testRun(void) TEST_RESULT_VOID( cmdTest( - STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("none"), 3, + STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("none"), 3, STRDEF("invalid"), STRDEF("common/error"), 5, 1, logLevelDebug, true, NULL, false, false, false, true), "new build"); @@ -443,6 +446,8 @@ testRun(void) String *const testCDup = strCat(strNew(), testC); strReplace(testCDup, STRDEF("{[C_TEST_CONTAINER]}"), STRDEF("false")); + strReplace(testCDup, STRDEF("{[C_TEST_VM]}"), STRDEF("none")); + strReplace(testCDup, STRDEF("{[C_TEST_PG_VERSION]}"), STRDEF("invalid")); strReplace(testCDup, STRDEF("{[C_TEST_LOG_EXPECT]}"), STRDEF("true")); strReplace(testCDup, STRDEF("{[C_TEST_DEBUG_TEST_TRACE]}"), STRDEF("#define DEBUG_TEST_TRACE")); strReplace(testCDup, STRDEF("{[C_TEST_PATH_BUILD]}"), STRDEF(TEST_PATH "/test/unit-3/none/build")); @@ -561,7 +566,7 @@ testRun(void) TEST_RESULT_VOID( cmdTest( - STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3, + STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3, STRDEF("invalid"), STRDEF("test/shim"), 0, 1, logLevelDebug, true, NULL, true, true, true, true), "new build"); @@ -671,6 +676,8 @@ testRun(void) String *const testCDup = strCat(strNew(), testC); strReplace(testCDup, STRDEF("{[C_TEST_CONTAINER]}"), STRDEF("true")); + strReplace(testCDup, STRDEF("{[C_TEST_VM]}"), STRDEF("uXX")); + strReplace(testCDup, STRDEF("{[C_TEST_PG_VERSION]}"), STRDEF("invalid")); strReplace(testCDup, STRDEF("{[C_TEST_LOG_EXPECT]}"), STRDEF("true")); strReplace(testCDup, STRDEF("{[C_TEST_DEBUG_TEST_TRACE]}"), STRDEF("// Debug test trace not enabled")); strReplace(testCDup, STRDEF("{[C_TEST_PATH_BUILD]}"), STRDEF(TEST_PATH "/test/unit-3/uXX/build")); @@ -706,10 +713,157 @@ testRun(void) TEST_RESULT_VOID( cmdTest( - STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3, + STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3, STRDEF("invalid"), STRDEF("test/shim"), 0, 1, logLevelDebug, true, NULL, true, true, true, true), "new build"); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("Test real/all"); + + HRN_STORAGE_PUT_Z( + storageTest, "repo/test/src/module/real/allTest.c", + "static void\n" + "testRun(void)\n" + "{\n" + " FUNCTION_HARNESS_VOID();\n" + "\n" + " if (testBegin(\"all\"))\n" + " {\n" + " }\n" + "\n" + " FUNCTION_HARNESS_RETURN_VOID();\n" + "}\n"); + + TEST_RESULT_VOID( + cmdTest( + STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3, STRDEF("invalid"), + STRDEF("real/all"), 0, 1, logLevelDebug, true, STRDEF("America/New_York"), false, true, false, true), + "new build"); + + storageUnit = storagePosixNewP(STRDEF(TEST_PATH "/test/unit-3/none")); + fileList = testStorageList(storageUnit); + + TEST_RESULT_STRLST_Z( + fileList, + "meson.build\n" + "meson_options.txt\n" + "test/src/common/harnessError.c\n" + "test/src/common/harnessShim.c\n" + "test/src/common/shim.c\n" + "test.c\n", + "check files"); + + for (unsigned int fileIdx = 0; fileIdx < strLstSize(fileList); fileIdx++) + { + const String *const file = strLstGet(fileList, fileIdx); + + if (strEqZ(file, "meson.build")) + { + TEST_STORAGE_GET( + storageUnit, strZ(file), + zNewFmt( + "%s" + "add_global_arguments('-DHRN_FEATURE_ERROR', language : 'c')\n" + "add_global_arguments('-DHRN_FEATURE_STACKTRACE', language : 'c')\n" + "\n" + MESON_COMMENT_BLOCK "\n" + "# Unit test\n" + MESON_COMMENT_BLOCK "\n" + "src_unit = files(\n" + " '../../../repo/src/common/stackTrace.c',\n" + " '../../../repo/src/common/type/stringStatic.c',\n" + " '../../../repo/src/common/debug.c',\n" + " 'test/src/common/harnessError.c',\n" + " '../../../repo/test/src/common/harnessNoShim.c',\n" + " 'test/src/common/harnessShim.c',\n" + " '../../../repo/test/src/common/harnessTest.c',\n" + " 'test.c',\n" + ")\n" + "\n" + "executable(\n" + " 'test-unit',\n" + " sources: src_unit,\n" + " c_args: [\n" + " '-pg',\n" + " '-no-pie',\n" + " ],\n" + " link_args: [\n" + " '-pg',\n" + " '-no-pie',\n" + " ],\n" + " include_directories:\n" + " include_directories(\n" + " '.',\n" + " '../../../repo/src',\n" + " '../../../repo/doc/src',\n" + " '../../../repo/test/src',\n" + " ),\n" + " dependencies: [\n" + " lib_backtrace,\n" + " lib_bz2,\n" + " lib_openssl,\n" + " lib_lz4,\n" + " lib_pq,\n" + " lib_ssh2,\n" + " lib_xml,\n" + " lib_yaml,\n" + " lib_z,\n" + " lib_zstd,\n" + " ],\n" + ")\n", + strZ(mesonBuildRoot))); + } + else if (strEqZ(file, "meson_options.txt")) + { + TEST_STORAGE_GET(storageUnit, strZ(file), mesonOption); + } + else if (strEqZ(file, "src/common/stackTrace.c") || strEqZ(file, "test/src/common/harnessStackTrace.c")) + { + // No test needed + } + else if (strEqZ(file, "test/src/common/harnessError.c")) + { + TEST_STORAGE_GET(storageUnit, strZ(file), strZ(harnessErrorC)); + } + else if (strEqZ(file, "test/src/common/harnessShim.c")) + { + String *const harnessShimCInt = strCat(strNew(), harnessShimC); + strReplace(harnessShimCInt, STRDEF("uXX"), STRDEF("none")); + + TEST_STORAGE_GET(storageUnit, strZ(file), strZ(harnessShimCInt)); + } + else if (strEqZ(file, "test/src/common/shim.c")) + { + TEST_STORAGE_GET(storageUnit, strZ(file), strZ(shimC)); + } + else if (strEqZ(file, "test.c")) + { + String *const testCDup = strCat(strNew(), testC); + + strReplace(testCDup, STRDEF("{[C_TEST_CONTAINER]}"), STRDEF("false")); + strReplace(testCDup, STRDEF("{[C_TEST_VM]}"), STRDEF("uXX")); + strReplace(testCDup, STRDEF("{[C_TEST_PG_VERSION]}"), STRDEF("invalid")); + strReplace(testCDup, STRDEF("{[C_TEST_LOG_EXPECT]}"), STRDEF("false")); + strReplace(testCDup, STRDEF("{[C_TEST_DEBUG_TEST_TRACE]}"), STRDEF("// Debug test trace not enabled")); + strReplace(testCDup, STRDEF("{[C_TEST_PATH_BUILD]}"), STRDEF(TEST_PATH "/test/unit-3/none/build")); + strReplace(testCDup, STRDEF("{[C_TEST_PROFILE]}"), STRDEF("true")); + strReplace(testCDup, STRDEF("{[C_TEST_PROJECT_EXE]}"), STRDEF(TEST_PATH "/test/bin/uXX/pgbackrest")); + strReplace(testCDup, STRDEF("{[C_TEST_TZ]}"), STRDEF("setenv(\"TZ\", \"America/New_York\", true);")); + + strReplace(testCDup, STRDEF("{[C_INCLUDE]}"), STRDEF("")); + strReplace( + testCDup, STRDEF("{[C_TEST_INCLUDE]}"), STRDEF("#include \"../../../repo/test/src/module/real/allTest.c\"")); + strReplace( + testCDup, STRDEF("{[C_TEST_LIST]}"), + STRDEF( + "hrnAdd( 1, true);")); + + TEST_STORAGE_GET(storageUnit, strZ(file), strZ(testCDup)); + } + else + THROW_FMT(TestError, "no test for '%s'", strZ(file)); + } + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("Test performance/type"); @@ -735,7 +889,7 @@ testRun(void) TEST_RESULT_VOID( cmdTest( - STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3, + STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3, STRDEF("invalid"), STRDEF("performance/type"), 0, 1, logLevelDebug, true, STRDEF("America/New_York"), false, true, false, false), "new build"); @@ -845,6 +999,8 @@ testRun(void) String *const testCDup = strCat(strNew(), testC); strReplace(testCDup, STRDEF("{[C_TEST_CONTAINER]}"), STRDEF("true")); + strReplace(testCDup, STRDEF("{[C_TEST_VM]}"), STRDEF("uXX")); + strReplace(testCDup, STRDEF("{[C_TEST_PG_VERSION]}"), STRDEF("invalid")); strReplace(testCDup, STRDEF("{[C_TEST_LOG_EXPECT]}"), STRDEF("false")); strReplace(testCDup, STRDEF("{[C_TEST_DEBUG_TEST_TRACE]}"), STRDEF("// Debug test trace not enabled")); strReplace(testCDup, STRDEF("{[C_TEST_PATH_BUILD]}"), STRDEF(TEST_PATH "/test/unit-3/uXX/build")); @@ -872,7 +1028,7 @@ testRun(void) TEST_RESULT_VOID( cmdTest( - STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3, + STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3, STRDEF("invalid"), STRDEF("performance/type"), 0, 1, logLevelDebug, true, STRDEF("America/New_York"), false, false, false, false), "new build"); @@ -885,6 +1041,8 @@ testRun(void) String *const testCDup = strCat(strNew(), testC); strReplace(testCDup, STRDEF("{[C_TEST_CONTAINER]}"), STRDEF("true")); + strReplace(testCDup, STRDEF("{[C_TEST_VM]}"), STRDEF("uXX")); + strReplace(testCDup, STRDEF("{[C_TEST_PG_VERSION]}"), STRDEF("invalid")); strReplace(testCDup, STRDEF("{[C_TEST_LOG_EXPECT]}"), STRDEF("false")); strReplace(testCDup, STRDEF("{[C_TEST_DEBUG_TEST_TRACE]}"), STRDEF("// Debug test trace not enabled")); strReplace(testCDup, STRDEF("{[C_TEST_PATH_BUILD]}"), STRDEF(TEST_PATH "/test/unit-3/uXX/build")); @@ -910,7 +1068,7 @@ testRun(void) TEST_ERROR( cmdTest( - STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3, + STRDEF(TEST_PATH "/repo"), storagePathP(storageTest, STRDEF("test")), STRDEF("uXX"), 3, STRDEF("invalid"), STRDEF("performance/type"), 0, 1, logLevelDebug, true, STRDEF("America/New_York"), false, false, false, false), FileOpenError, "build failed for unit performance/type: unable to open file '" TEST_PATH "/repo/meson.build' for read: [13] Permission" diff --git a/test/src/test.c b/test/src/test.c index 93dde44dac..04939ee062 100644 --- a/test/src/test.c +++ b/test/src/test.c @@ -79,6 +79,12 @@ STRING_EXTERN(HRN_PATH_STR, HRN_PATH); #define TEST_USER_ID_Z "{[C_TEST_USER_ID]}" #define TEST_USER_LEN "{[C_TEST_USER_LEN]}" +// VM for integration testing +#define TEST_VM "{[C_TEST_VM]}" + +// PostgreSQL version for integration testing +#define TEST_PG_VERSION "{[C_TEST_PG_VERSION]}" + #ifdef HRN_FEATURE_STRING STRING_EXTERN(TEST_USER_STR, TEST_USER); #endif @@ -203,6 +209,9 @@ main(int argListSize, const char *argList[]) {[C_TEST_IDX]}, // The 0-based index of this test {[C_TEST_TIMING]}, // Is timing enabled (may be disabled for reproducible documentation) TEST_PATH, // Path where tests write data + TEST_USER, // User running the test + TEST_VM, // VM for integration testing + TEST_PG_VERSION, // PostgreSQL version for integration testing HRN_PATH, // Path where the harness stores temp files (expect, diff, etc.) HRN_PATH_REPO); // Path with a copy of the repository diff --git a/test/src/valgrind.suppress.none b/test/src/valgrind.suppress.none new file mode 100644 index 0000000000..4534477941 --- /dev/null +++ b/test/src/valgrind.suppress.none @@ -0,0 +1,16 @@ +# Suppress issues in libssh2 on Ubuntu 20.04 +{ + libssh2_ubuntu_20_04 + Memcheck:Addr1 + ... + obj:/usr/lib/x86_64-linux-gnu/libssh2.so.1.0.1 + ... +} +{ + libssh2_ubuntu_20_04 + Memcheck:Leak + match-leak-kinds: definite + ... + obj:/usr/lib/x86_64-linux-gnu/libssh2.so.1.0.1 + ... +} diff --git a/test/test.pl b/test/test.pl index 7ad184b0e3..86fa6a81f7 100755 --- a/test/test.pl +++ b/test/test.pl @@ -40,10 +40,8 @@ use pgBackRestTest::Common::CoverageTest; use pgBackRestTest::Common::DefineTest; use pgBackRestTest::Common::ExecuteTest; -use pgBackRestTest::Common::HostGroupTest; use pgBackRestTest::Common::JobTest; use pgBackRestTest::Common::ListTest; -use pgBackRestTest::Common::RunTest; use pgBackRestTest::Common::Storage; use pgBackRestTest::Common::StoragePosix; use pgBackRestTest::Common::VmTest; @@ -135,7 +133,6 @@ =head1 SYNOPSIS my @stryModuleTest; my @iyModuleTestRun; my $iVmMax = 1; -my $iVmId = undef; my $bDryRun = false; my $bCodeFormat = false; my $bCodeFormatCheck = false; @@ -195,7 +192,6 @@ =head1 SYNOPSIS 'module=s@' => \@stryModule, 'test=s@' => \@stryModuleTest, 'run=s@' => \@iyModuleTestRun, - 'vm-id=s' => \$iVmId, 'vm-max=s' => \$iVmMax, 'dry-run' => \$bDryRun, 'no-cleanup' => \$bNoCleanup, @@ -387,832 +383,787 @@ =head1 SYNOPSIS ################################################################################################################################ # Start VM and run ################################################################################################################################ - if (!defined($iVmId)) + # Clean any existing files in the src path that might interfere with the vpath build. This is kosher because the user should + # be expecting us to do builds in the src path during testing. Instead we clean the src path and do the builds elsewhere. + #------------------------------------------------------------------------------------------------------------------------------- + executeTest("make -C ${strBackRestBase}/src -f Makefile.in clean-all"); + + # Clean up + #------------------------------------------------------------------------------------------------------------------------------- + my $iTestFail = 0; + my $iTestRetry = 0; + my $oyProcess = []; + my $strCodePath = "${strBackRestBase}/test/result/coverage/raw"; + + if (!$bDryRun || $bVmOut) { - # Clean any existing files in the src path that might interfere with the vpath build. This is kosher because the user should - # be expecting us to do builds in the src path during testing. Instead we clean the src path and do the builds elsewhere. - #--------------------------------------------------------------------------------------------------------------------------- - executeTest("make -C ${strBackRestBase}/src -f Makefile.in clean-all"); - - # Clean up - #--------------------------------------------------------------------------------------------------------------------------- - my $iTestFail = 0; - my $iTestRetry = 0; - my $oyProcess = []; - my $strCodePath = "${strBackRestBase}/test/result/coverage/raw"; - - if (!$bDryRun || $bVmOut) - { - &log(INFO, "cleanup old data" . ($strVm ne VM_NONE ? " and containers" : '')); - - if ($strVm ne VM_NONE) - { - containerRemove('test-([0-9]+|build)'); - } - - for (my $iVmIdx = 0; $iVmIdx < 8; $iVmIdx++) - { - push(@{$oyProcess}, undef); - } - - executeTest( - "chmod 700 -R ${strTestPath}/test-* 2>&1 || true && rm -rf ${strTestPath}/temp ${strTestPath}/test-*" . - " ${strTestPath}/data-*"); - $oStorageTest->pathCreate("${strTestPath}/temp", {strMode => '0770', bIgnoreExists => true, bCreateParent => true}); - - # Remove old lcov dirs -- do it this way so the dirs stay open in finder/explorer, etc. - executeTest("rm -rf ${strBackRestBase}/test/result/coverage/lcov/*"); + &log(INFO, "cleanup old data" . ($strVm ne VM_NONE ? " and containers" : '')); - # Overwrite the C coverage report so it will load but not show old coverage - $oStorageTest->pathCreate( - "${strBackRestBase}/test/result/coverage", {strMode => '0770', bIgnoreExists => true, bCreateParent => true}); - $oStorageBackRest->put( - "${strBackRestBase}/test/result/coverage/coverage.html", - "[ " . ($bNoCoverage ? "No Coverage Testing" : "Generating Coverage Report") . " ]"); - executeTest("rm -rf ${strBackRestBase}/test/result/coverage/lcov"); - - # Copy C code for coverage tests - if (vmCoverageC($strVm) && !$bDryRun) - { - executeTest("rm -rf ${strBackRestBase}/test/result/coverage/raw/*"); - $oStorageTest->pathCreate("${strCodePath}", {strMode => '0770', bIgnoreExists => true, bCreateParent => true}); - } + if ($strVm ne VM_NONE) + { + containerRemove('test-([0-9]+|build)'); } - # Auto-generate configure files unless --min-gen specified - #--------------------------------------------------------------------------------------------------------------------------- - if (!$bMinGen) + for (my $iVmIdx = 0; $iVmIdx < 8; $iVmIdx++) { - &log(INFO, "autogenerate configure"); - - # Auto-generate version for configure.ac script - #----------------------------------------------------------------------------------------------------------------------- - my $strConfigureAcOld = ${$oStorageTest->get("${strBackRestBase}/src/build/configure.ac")}; - my $strConfigureAcNew; - - foreach my $strLine (split("\n", $strConfigureAcOld)) - { - if ($strLine =~ /^AC_INIT\(/) - { - $strLine = 'AC_INIT([' . PROJECT_NAME . '], [' . PROJECT_VERSION . '])'; - } + push(@{$oyProcess}, undef); + } - $strConfigureAcNew .= "${strLine}\n"; - } + executeTest( + "chmod 700 -R ${strTestPath}/test-* 2>&1 || true && rm -rf ${strTestPath}/temp ${strTestPath}/test-*" . + " ${strTestPath}/data-*"); + $oStorageTest->pathCreate("${strTestPath}/temp", {strMode => '0770', bIgnoreExists => true, bCreateParent => true}); + + # Remove old lcov dirs -- do it this way so the dirs stay open in finder/explorer, etc. + executeTest("rm -rf ${strBackRestBase}/test/result/coverage/lcov/*"); + + # Overwrite the C coverage report so it will load but not show old coverage + $oStorageTest->pathCreate( + "${strBackRestBase}/test/result/coverage", {strMode => '0770', bIgnoreExists => true, bCreateParent => true}); + $oStorageBackRest->put( + "${strBackRestBase}/test/result/coverage/coverage.html", + "[ " . ($bNoCoverage ? "No Coverage Testing" : "Generating Coverage Report") . " ]"); + executeTest("rm -rf ${strBackRestBase}/test/result/coverage/lcov"); + + # Copy C code for coverage tests + if (vmCoverageC($strVm) && !$bDryRun) + { + executeTest("rm -rf ${strBackRestBase}/test/result/coverage/raw/*"); + $oStorageTest->pathCreate("${strCodePath}", {strMode => '0770', bIgnoreExists => true, bCreateParent => true}); + } + } - # Save into the src dir - my @stryBuilt; - my $strBuilt = 'src/build/configure.ac'; + # Auto-generate configure files unless --min-gen specified + #------------------------------------------------------------------------------------------------------------------------------- + if (!$bMinGen) + { + &log(INFO, "autogenerate configure"); - if (buildPutDiffers($oStorageBackRest, "${strBackRestBase}/${strBuilt}", $strConfigureAcNew)) - { - push(@stryBuilt, $strBuilt); - } + # Auto-generate version for configure.ac script + #----------------------------------------------------------------------------------------------------------------------- + my $strConfigureAcOld = ${$oStorageTest->get("${strBackRestBase}/src/build/configure.ac")}; + my $strConfigureAcNew; - # Error when checking that files have already been generated but they change - if ($bGenCheck && @stryBuilt) + foreach my $strLine (split("\n", $strConfigureAcOld)) + { + if ($strLine =~ /^AC_INIT\(/) { - confess &log( - ERROR, - "unexpected autogeneration of version in configure.ac script: " . join(', ', @stryBuilt) . ":\n" . - trim(executeTest("git -C ${strBackRestBase} diff"))); + $strLine = 'AC_INIT([' . PROJECT_NAME . '], [' . PROJECT_VERSION . '])'; } - &log(INFO, - " autogenerated version in configure.ac script: " . (@stryBuilt ? join(', ', @stryBuilt) : 'no changes')); - - # Auto-generate configure script - #----------------------------------------------------------------------------------------------------------------------- - # Set build file - @stryBuilt = (); - $strBuilt = 'src/configure'; + $strConfigureAcNew .= "${strLine}\n"; + } - # Get configure.ac and configure to see if anything has changed - my $strConfigureAc = ${$oStorageBackRest->get('src/build/configure.ac')}; - my $strConfigureAcHash = sha1_hex($strConfigureAc); - my $rstrConfigure = $oStorageBackRest->get($oStorageBackRest->openRead($strBuilt, {bIgnoreMissing => true})); + # Save into the src dir + my @stryBuilt; + my $strBuilt = 'src/build/configure.ac'; - # Check if configure needs to be regenerated - if (!defined($rstrConfigure) || !defined($$rstrConfigure) || - $strConfigureAcHash ne substr($$rstrConfigure, length($$rstrConfigure) - 41, 40)) - { - # Generate aclocal.m4 - my $strAcLocal = executeTest("cd ${strBackRestBase}/src/build && aclocal --OUT=-"); - $strAcLocal = trim($strAcLocal) . "\n"; + if (buildPutDiffers($oStorageBackRest, "${strBackRestBase}/${strBuilt}", $strConfigureAcNew)) + { + push(@stryBuilt, $strBuilt); + } - buildPutDiffers($oStorageBackRest, "${strBackRestBase}/src/build/aclocal.m4", $strAcLocal); + # Error when checking that files have already been generated but they change + if ($bGenCheck && @stryBuilt) + { + confess &log( + ERROR, + "unexpected autogeneration of version in configure.ac script: " . join(', ', @stryBuilt) . ":\n" . + trim(executeTest("git -C ${strBackRestBase} diff"))); + } - # Generate configure - my $strConfigure = executeTest("cd ${strBackRestBase}/src/build && autoconf --output=-"); - $strConfigure = - trim($strConfigure) . "\n\n# Generated from src/build/configure.ac sha1 ${strConfigureAcHash}\n"; + &log(INFO, + " autogenerated version in configure.ac script: " . (@stryBuilt ? join(', ', @stryBuilt) : 'no changes')); + + # Auto-generate configure script + #----------------------------------------------------------------------------------------------------------------------- + # Set build file + @stryBuilt = (); + $strBuilt = 'src/configure'; + + # Get configure.ac and configure to see if anything has changed + my $strConfigureAc = ${$oStorageBackRest->get('src/build/configure.ac')}; + my $strConfigureAcHash = sha1_hex($strConfigureAc); + my $rstrConfigure = $oStorageBackRest->get($oStorageBackRest->openRead($strBuilt, {bIgnoreMissing => true})); + + # Check if configure needs to be regenerated + if (!defined($rstrConfigure) || !defined($$rstrConfigure) || + $strConfigureAcHash ne substr($$rstrConfigure, length($$rstrConfigure) - 41, 40)) + { + # Generate aclocal.m4 + my $strAcLocal = executeTest("cd ${strBackRestBase}/src/build && aclocal --OUT=-"); + $strAcLocal = trim($strAcLocal) . "\n"; - # Remove cache created by autconf - executeTest("rm -rf ${strBackRestBase}/src/build/autom4te.cache"); + buildPutDiffers($oStorageBackRest, "${strBackRestBase}/src/build/aclocal.m4", $strAcLocal); - # Remove unused options from help - my $strDirList = - "sbin|libexec|sysconf|sharedstate|localstate|runstate|lib|include|oldinclude|dataroot|data|info" . - "|locale|man|doc|html|dvi|pdf|ps"; + # Generate configure + my $strConfigure = executeTest("cd ${strBackRestBase}/src/build && autoconf --output=-"); + $strConfigure = + trim($strConfigure) . "\n\n# Generated from src/build/configure.ac sha1 ${strConfigureAcHash}\n"; - $strConfigure =~ s/^ --(${strDirList})*dir=DIR.*\n//mg; + # Remove cache created by autconf + executeTest("rm -rf ${strBackRestBase}/src/build/autom4te.cache"); - # Save into the src dir - $oStorageBackRest->put( - $oStorageBackRest->openWrite("${strBackRestBase}/${strBuilt}", {strMode => '0755'}), $strConfigure); + # Remove unused options from help + my $strDirList = + "sbin|libexec|sysconf|sharedstate|localstate|runstate|lib|include|oldinclude|dataroot|data|info" . + "|locale|man|doc|html|dvi|pdf|ps"; - # Add to built list - push(@stryBuilt, $strBuilt); - } + $strConfigure =~ s/^ --(${strDirList})*dir=DIR.*\n//mg; - # Error when checking that files have already been generated but they change - if ($bGenCheck && @stryBuilt) - { - confess &log( - ERROR, - "unexpected autogeneration of configure script: " . join(', ', @stryBuilt) . ":\n" . - trim(executeTest("git -C ${strBackRestBase} diff"))); - } + # Save into the src dir + $oStorageBackRest->put( + $oStorageBackRest->openWrite("${strBackRestBase}/${strBuilt}", {strMode => '0755'}), $strConfigure); - &log(INFO, " autogenerated configure script: " . (@stryBuilt ? join(', ', @stryBuilt) : 'no changes')); + # Add to built list + push(@stryBuilt, $strBuilt); } - # Auto-generate version for root meson.build script - #--------------------------------------------------------------------------------------------------------------------------- - my $strMesonBuildOld = ${$oStorageTest->get("${strBackRestBase}/meson.build")}; - my $strMesonBuildNew; - - foreach my $strLine (split("\n", $strMesonBuildOld)) + # Error when checking that files have already been generated but they change + if ($bGenCheck && @stryBuilt) { - if ($strLine =~ /^ version\: '/) - { - $strLine = " version: '" . PROJECT_VERSION . "',"; - } - - $strMesonBuildNew .= "${strLine}\n"; + confess &log( + ERROR, + "unexpected autogeneration of configure script: " . join(', ', @stryBuilt) . ":\n" . + trim(executeTest("git -C ${strBackRestBase} diff"))); } - buildPutDiffers($oStorageBackRest, "${strBackRestBase}/meson.build", $strMesonBuildNew); - - # Start build container if vm is not none - #--------------------------------------------------------------------------------------------------------------------------- - if ($strVm ne VM_NONE) - { - my $strCCachePath = "${strTestPath}/ccache-0/${strVm}"; + &log(INFO, " autogenerated configure script: " . (@stryBuilt ? join(', ', @stryBuilt) : 'no changes')); + } - if (!$oStorageTest->pathExists($strCCachePath)) - { - $oStorageTest->pathCreate($strCCachePath, {strMode => '0770', bCreateParent => true}); - } + # Auto-generate version for root meson.build script + #------------------------------------------------------------------------------------------------------------------------------- + my $strMesonBuildOld = ${$oStorageTest->get("${strBackRestBase}/meson.build")}; + my $strMesonBuildNew; - executeTest( - "docker run -itd -h test-build --name=test-build" . - " -v ${strBackRestBase}:${strBackRestBase} -v ${strTestPath}:${strTestPath}" . - " -v ${strCCachePath}:/home/${\TEST_USER}/.ccache" . ' ' . containerRepo() . ":${strVm}-test", - {bSuppressStdErr => true}); + foreach my $strLine (split("\n", $strMesonBuildOld)) + { + if ($strLine =~ /^ version\: '/) + { + $strLine = " version: '" . PROJECT_VERSION . "',"; } - # Create path for repo cache - #--------------------------------------------------------------------------------------------------------------------------- - my $strRepoCachePath = "${strTestPath}/repo"; - - # Create the repo path -- this should hopefully prevent obvious rsync errors below - $oStorageTest->pathCreate($strRepoCachePath, {strMode => '0770', bIgnoreExists => true, bCreateParent => true}); - - # Auto-generate code files (if --min-gen specified then do minimum required) - #--------------------------------------------------------------------------------------------------------------------------- - my $strBuildPath = "${strTestPath}/build/${strVm}"; - my $strBuildNinja = "${strBuildPath}/build.ninja"; + $strMesonBuildNew .= "${strLine}\n"; + } - &log(INFO, (!-e $strBuildNinja ? 'clean ' : '') . 'autogenerate code'); + buildPutDiffers($oStorageBackRest, "${strBackRestBase}/meson.build", $strMesonBuildNew); - # Setup build if it does not exist - my $strGenerateCommand = - "ninja -C ${strBuildPath} src/build-code test/src/test-pgbackrest" . - ($bMinGen ? '' : " && \\\n${strBuildPath}/src/build-code config ${strBackRestBase}/src") . - ($bMinGen ? '' : " && \\\n${strBuildPath}/src/build-code error ${strBackRestBase}/src") . - ($bMinGen ? '' : " && \\\n${strBuildPath}/src/build-code postgres-version ${strBackRestBase}/src") . - " && \\\n${strBuildPath}/src/build-code postgres ${strBackRestBase}/src ${strRepoCachePath}"; + # Start build container if vm is not none + #------------------------------------------------------------------------------------------------------------------------------- + if ($strVm ne VM_NONE) + { + my $strCCachePath = "${strTestPath}/ccache-0/${strVm}"; - if (!-e $strBuildNinja) + if (!$oStorageTest->pathExists($strCCachePath)) { - $strGenerateCommand = - "meson setup -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug ${strBuildPath} ${strBackRestBase} && \\\n" . - $strGenerateCommand; + $oStorageTest->pathCreate($strCCachePath, {strMode => '0770', bCreateParent => true}); } - # Build code executeTest( - ($strVm ne VM_NONE ? "docker exec -i -u ${\TEST_USER} test-build bash -l -c ' \\\n" : '') . - $strGenerateCommand . ($strVm ne VM_NONE ? "'" : '')); - - if ($bGenOnly) - { - exit 0; - } + "docker run -itd -h test-build --name=test-build" . + " -v ${strBackRestBase}:${strBackRestBase} -v ${strTestPath}:${strTestPath}" . + " -v ${strCCachePath}:/home/${\TEST_USER}/.ccache" . ' ' . containerRepo() . ":${strVm}-test", + {bSuppressStdErr => true}); + } - # Make a copy of the repo to track which files have been changed - #--------------------------------------------------------------------------------------------------------------------------- - executeTest( - "git -C ${strBackRestBase} ls-files -c --others --exclude-standard |" . - " rsync -rLtW --delete --files-from=- --exclude=test/result" . - # This option is not supported on MacOS. The eventual plan is to remove the need for it. - (trim(`uname`) ne 'Darwin' ? ' --ignore-missing-args' : '') . - " ${strBackRestBase}/ ${strRepoCachePath}"); - - # Format code with uncrustify and check permissions - #--------------------------------------------------------------------------------------------------------------------------- - if ($bCodeFormat || $bCodeFormatCheck) - { - &log(INFO, 'code format' . ($bCodeFormatCheck ? ' check' : '')); + # Create path for repo cache + #------------------------------------------------------------------------------------------------------------------------------- + my $strRepoCachePath = "${strTestPath}/repo"; - my $hRepoManifest = $oStorageTest->manifest($strRepoCachePath); - my $hManifest = $oStorageBackRest->manifest(''); - my $strCommand = - "uncrustify -c ${strBackRestBase}/test/uncrustify.cfg" . - ($bCodeFormatCheck ? ' --check' : ' --replace --no-backup'); + # Create the repo path -- this should hopefully prevent obvious rsync errors below + $oStorageTest->pathCreate($strRepoCachePath, {strMode => '0770', bIgnoreExists => true, bCreateParent => true}); - foreach my $strFile (sort(keys(%{$hManifest}))) - { - # Skip non-C files - next if $hManifest->{$strFile}{type} ne 'f' || ($strFile !~ /\.c$/ && $strFile !~ /\.h$/); + # Auto-generate code files (if --min-gen specified then do minimum required) + #------------------------------------------------------------------------------------------------------------------------------- + my $strBuildPath = "${strTestPath}/build/${strVm}"; + my $strBuildNinja = "${strBuildPath}/build.ninja"; - # Skip files that do are not version controlled - next if !defined($hRepoManifest->{$strFile}); + &log(INFO, (!-e $strBuildNinja ? 'clean ' : '') . 'autogenerate code'); - # Skip specific file - next if - # Does not format correctly because it is a template - $strFile eq 'test/src/test.c' || - # Contains code copied directly from PostgreSQL - $strFile eq 'src/postgres/interface/static.vendor.h' || - $strFile eq 'src/postgres/interface/version.vendor.h'; + # Setup build if it does not exist + my $strGenerateCommand = + "ninja -C ${strBuildPath} src/build-code test/src/test-pgbackrest" . + ($bMinGen ? '' : " && \\\n${strBuildPath}/src/build-code config ${strBackRestBase}/src") . + ($bMinGen ? '' : " && \\\n${strBuildPath}/src/build-code error ${strBackRestBase}/src") . + ($bMinGen ? '' : " && \\\n${strBuildPath}/src/build-code postgres-version ${strBackRestBase}/src") . + " && \\\n${strBuildPath}/src/build-code postgres ${strBackRestBase}/src ${strRepoCachePath}"; - $strCommand .= " ${strBackRestBase}/${strFile}"; - } + if (!-e $strBuildNinja) + { + $strGenerateCommand = + "meson setup -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug ${strBuildPath} ${strBackRestBase} && \\\n" . + $strGenerateCommand; + } - executeTest($strCommand . " 2>&1"); + # Build code + executeTest( + ($strVm ne VM_NONE ? "docker exec -i -u ${\TEST_USER} test-build bash -l -c ' \\\n" : '') . + $strGenerateCommand . ($strVm ne VM_NONE ? "'" : '')); - # Check execute permissions to make sure nothing got munged - foreach my $strFile (sort(keys(%{$hManifest}))) - { - next if ($strFile eq '.') || !defined($hRepoManifest->{$strFile}); + if ($bGenOnly) + { + exit 0; + } - my $strExpectedMode = sprintf('%04o', oct($hManifest->{$strFile}{mode}) & 0666); + # Make a copy of the repo to track which files have been changed + #------------------------------------------------------------------------------------------------------------------------------- + executeTest( + "git -C ${strBackRestBase} ls-files -c --others --exclude-standard |" . + " rsync -rLtW --delete --files-from=- --exclude=test/result" . + # This option is not supported on MacOS. The eventual plan is to remove the need for it. + (trim(`uname`) ne 'Darwin' ? ' --ignore-missing-args' : '') . + " ${strBackRestBase}/ ${strRepoCachePath}"); + + # Format code with uncrustify and check permissions + #------------------------------------------------------------------------------------------------------------------------------- + if ($bCodeFormat || $bCodeFormatCheck) + { + &log(INFO, 'code format' . ($bCodeFormatCheck ? ' check' : '')); - if ($strFile eq 'doc/doc.pl' || - $strFile eq 'doc/release.pl' || - $strFile eq 'src/build/install-sh' || - $strFile eq 'src/configure' || - $strFile eq 'test/ci.pl' || - $strFile eq 'test/test.pl' || - $hManifest->{$strFile}{type} eq 'd') - { - $strExpectedMode = sprintf('%04o', oct($hManifest->{$strFile}{mode}) & 0777); - } + my $hRepoManifest = $oStorageTest->manifest($strRepoCachePath); + my $hManifest = $oStorageBackRest->manifest(''); + my $strCommand = + "uncrustify -c ${strBackRestBase}/test/uncrustify.cfg" . + ($bCodeFormatCheck ? ' --check' : ' --replace --no-backup'); - if ($hManifest->{$strFile}{mode} ne $strExpectedMode) - { - confess &log( - ERROR, - "expected mode for '${strExpectedMode}' for '${strFile}' but found '" . $hManifest->{$strFile}{mode} . "'"); - } - } + foreach my $strFile (sort(keys(%{$hManifest}))) + { + # Skip non-C files + next if $hManifest->{$strFile}{type} ne 'f' || ($strFile !~ /\.c$/ && $strFile !~ /\.h$/); - exit 0; - } + # Skip files that do are not version controlled + next if !defined($hRepoManifest->{$strFile}); - # Generate code counts - #--------------------------------------------------------------------------------------------------------------------------- - if ($bCodeCount) - { - &log(INFO, "classify code files"); + # Skip specific file + next if + # Does not format correctly because it is a template + $strFile eq 'test/src/test.c' || + # Contains code copied directly from PostgreSQL + $strFile eq 'src/postgres/interface/static.vendor.h' || + $strFile eq 'src/postgres/interface/version.vendor.h'; - codeCountScan($oStorageBackRest, $strBackRestBase); - exit 0; + $strCommand .= " ${strBackRestBase}/${strFile}"; } - # Determine which tests to run - #--------------------------------------------------------------------------------------------------------------------------- - my $oyTestRun; - my $bBinRequired = $bBuildOnly; - my $bHostBinRequired = $bBuildOnly; + executeTest($strCommand . " 2>&1"); - # Only get the test list when they can run - if (!$bBuildOnly) + # Check execute permissions to make sure nothing got munged + foreach my $strFile (sort(keys(%{$hManifest}))) { - # Get the test list - $oyTestRun = testListGet( - $strVm, \@stryModule, \@stryModuleTest, \@iyModuleTestRun, $strPgVersion, $bCoverageOnly, $bCOnly, $bContainerOnly, - $bNoPerformance); + next if ($strFile eq '.') || !defined($hRepoManifest->{$strFile}); + + my $strExpectedMode = sprintf('%04o', oct($hManifest->{$strFile}{mode}) & 0666); - # Determine if the C binary and test library need to be built - foreach my $hTest (@{$oyTestRun}) + if ($strFile eq 'doc/doc.pl' || + $strFile eq 'doc/release.pl' || + $strFile eq 'src/build/install-sh' || + $strFile eq 'src/configure' || + $strFile eq 'test/ci.pl' || + $strFile eq 'test/test.pl' || + $hManifest->{$strFile}{type} eq 'd') { - # Bin build required for all Perl tests or if a C unit test calls Perl - if (!$hTest->{&TEST_C} || $hTest->{&TEST_BIN_REQ}) - { - $bBinRequired = true; - } + $strExpectedMode = sprintf('%04o', oct($hManifest->{$strFile}{mode}) & 0777); + } - # Host bin required if a Perl test - if (!$hTest->{&TEST_C}) - { - $bHostBinRequired = true; - } + if ($hManifest->{$strFile}{mode} ne $strExpectedMode) + { + confess &log( + ERROR, + "expected mode for '${strExpectedMode}' for '${strFile}' but found '" . $hManifest->{$strFile}{mode} . "'"); } } - my $strBuildRequired; + exit 0; + } + + # Generate code counts + #------------------------------------------------------------------------------------------------------------------------------- + if ($bCodeCount) + { + &log(INFO, "classify code files"); + + codeCountScan($oStorageBackRest, $strBackRestBase); + exit 0; + } + + # Determine which tests to run + #------------------------------------------------------------------------------------------------------------------------------- + my $oyTestRun; + my $bBinRequired = $bBuildOnly; + my $bIntegrationRequired = $bBuildOnly; - if ($bBinRequired || $bHostBinRequired) + # Only get the test list when they can run + if (!$bBuildOnly) + { + # Get the test list + $oyTestRun = testListGet( + $strVm, \@stryModule, \@stryModuleTest, \@iyModuleTestRun, $strPgVersion, $bCoverageOnly, $bCOnly, $bContainerOnly, + $bNoPerformance); + + # Determine if the C binary and test library need to be built + foreach my $hTest (@{$oyTestRun}) { - if ($bBinRequired) + # Bin build required for all Perl tests or if a C unit test calls Perl + if (!$hTest->{&TEST_C} || $hTest->{&TEST_BIN_REQ}) { - $strBuildRequired = "bin"; + $bBinRequired = true; } - if ($bHostBinRequired) + # Host bin required if an integration test + if (!$hTest->{&TEST_C}) { - $strBuildRequired .= ", bin host"; + $bIntegrationRequired = true; } } - else + } + + my $strBuildRequired; + + if ($bBinRequired || $bIntegrationRequired) + { + if ($bBinRequired) { - $strBuildRequired = "none"; + $strBuildRequired = "bin"; } - &log(INFO, "builds required: ${strBuildRequired}"); - - # Build the binary and packages - #--------------------------------------------------------------------------------------------------------------------------- - if (!$bDryRun) + if ($bIntegrationRequired) { - my $oVm = vmGet(); - my $lTimestampLast; - my $rhBinBuild = {}; + $strBuildRequired .= ", integration"; + } + } + else + { + $strBuildRequired = "none"; + } - # Build the binary - #----------------------------------------------------------------------------------------------------------------------- - if ($bBinRequired) - { - # Find the lastest modified time for dirs that affect the bin build - $lTimestampLast = buildLastModTime($oStorageBackRest, $strBackRestBase, ['src']); + &log(INFO, "builds required: ${strBuildRequired}"); - # Loop through VMs to do the C bin builds - my $bLogDetail = $strLogLevel eq 'detail'; - my @stryBuildVm = $strVm eq VM_ALL ? VM_LIST : ($strVm); + # Build the binary and packages + #------------------------------------------------------------------------------------------------------------------------------- + if (!$bDryRun) + { + my $oVm = vmGet(); + my $lTimestampLast; + my $rhBinBuild = {}; - # Build binary for the host - if ($bHostBinRequired) - { - push(@stryBuildVm, VM_NONE); - } + # Build the binary + #----------------------------------------------------------------------------------------------------------------------- + if ($bBinRequired) + { + # Find the lastest modified time for dirs that affect the bin build + $lTimestampLast = buildLastModTime($oStorageBackRest, $strBackRestBase, ['src']); - foreach my $strBuildVM (@stryBuildVm) - { - if ($strBuildVM eq VM_NONE) - { - my $strBuildPath = "${strTestPath}/build/${strBuildVM}"; - my $strBuildNinja = "${strBuildPath}/build.ninja"; + # Loop through VMs to do the C bin builds + my $bLogDetail = $strLogLevel eq 'detail'; + my @stryBuildVm = $strVm eq VM_ALL ? VM_LIST : ($strVm); - &log(INFO, " " . (!-e $strBuildNinja ? 'clean ' : '') . "bin build for ${strBuildVM} (${strBuildPath})"); + # Build binary for the host + if ($bIntegrationRequired) + { + push(@stryBuildVm, VM_NONE); + } - # Setup build if it does not exist - my $strBuildCommand = "ninja -C ${strBuildPath} src/pgbackrest"; + foreach my $strBuildVM (@stryBuildVm) + { + if ($strBuildVM eq VM_NONE) + { + my $strBuildPath = "${strTestPath}/build/${strBuildVM}"; + my $strBuildNinja = "${strBuildPath}/build.ninja"; - if (!-e $strBuildNinja) - { - $strBuildCommand = - "meson setup -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug ${strBuildPath}" . - " ${strBackRestBase} && \\\n" . - $strBuildCommand; - } + &log( + INFO, + " " . (!-e $strBuildNinja ? 'clean ' : '') . "integration build for ${strBuildVM} (${strBuildPath})"); - # Build code - executeTest($strBuildCommand); - } - else + # Setup build if it does not exist + my $strBuildCommand = "ninja -C ${strBuildPath} test/src/test-pgbackrest src/pgbackrest"; + + if (!-e $strBuildNinja) { - my $strBinPath = "${strTestPath}/bin"; - my $strBuildPath = "${strBinPath}/${strBuildVM}"; + $strBuildCommand = + "meson setup -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug ${strBuildPath}" . + " ${strBackRestBase} && \\\n" . + $strBuildCommand; + } - &log(INFO, " bin build for ${strBuildVM} (${strBuildPath})"); + # Build code + executeTest($strBuildCommand); + } + else + { + my $strBinPath = "${strTestPath}/bin"; + my $strBuildPath = "${strBinPath}/${strBuildVM}"; - my $bRebuild = false; - $rhBinBuild->{$strBuildVM} = false; + &log(INFO, " bin build for ${strBuildVM} (${strBuildPath})"); - # Build configure/compile options and see if they have changed from the previous build - my $strCFlags = ($bDebugTestTrace ? ' -DDEBUG_TEST_TRACE' : ''); - my $strConfigOptions = (vmDebugIntegration($strBuildVM) ? ' --enable-test' : ''); - my $strBuildFlags = "CFLAGS_EXTRA=${strCFlags}\nCONFIGURE=${strConfigOptions}"; - my $strBuildFlagFile = "${strBinPath}/${strBuildVM}/build.flags"; + my $bRebuild = false; + $rhBinBuild->{$strBuildVM} = false; - my $bBuildOptionsDiffer = buildPutDiffers($oStorageBackRest, $strBuildFlagFile, $strBuildFlags); + # Build configure/compile options and see if they have changed from the previous build + my $strCFlags = ($bDebugTestTrace ? ' -DDEBUG_TEST_TRACE' : ''); + my $strConfigOptions = (vmDebugIntegration($strBuildVM) ? ' --enable-test' : ''); + my $strBuildFlags = "CFLAGS_EXTRA=${strCFlags}\nCONFIGURE=${strConfigOptions}"; + my $strBuildFlagFile = "${strBinPath}/${strBuildVM}/build.flags"; - if ($bBuildOptionsDiffer || - !-e "${strBuildPath}/Makefile" || - stat("${strBackRestBase}/src/Makefile.in")->mtime > stat("${strBuildPath}/Makefile")->mtime || - stat("${strBackRestBase}/src/configure")->mtime > stat("${strBuildPath}/Makefile")->mtime || - stat("${strBackRestBase}/src/build.auto.h.in")->mtime > stat("${strBuildPath}/Makefile")->mtime) - { - &log(INFO, ' bin dependencies have changed, rebuilding'); + my $bBuildOptionsDiffer = buildPutDiffers($oStorageBackRest, $strBuildFlagFile, $strBuildFlags); - # Remove old path if it exists and save the build flags - executeTest("rm -rf ${strBuildPath}"); - buildPutDiffers($oStorageBackRest, $strBuildFlagFile, $strBuildFlags); + if ($bBuildOptionsDiffer || + !-e "${strBuildPath}/Makefile" || + stat("${strBackRestBase}/src/Makefile.in")->mtime > stat("${strBuildPath}/Makefile")->mtime || + stat("${strBackRestBase}/src/configure")->mtime > stat("${strBuildPath}/Makefile")->mtime || + stat("${strBackRestBase}/src/build.auto.h.in")->mtime > stat("${strBuildPath}/Makefile")->mtime) + { + &log(INFO, ' bin dependencies have changed, rebuilding'); - executeTest( - 'docker exec -i -u ' . TEST_USER . ' test-build ' . - "bash -c 'cd ${strBuildPath} && ${strBackRestBase}/src/configure -q${strConfigOptions}'", - {bShowOutputAsync => $bLogDetail}); - } + # Remove old path if it exists and save the build flags + executeTest("rm -rf ${strBuildPath}"); + buildPutDiffers($oStorageBackRest, $strBuildFlagFile, $strBuildFlags); executeTest( 'docker exec -i -u ' . TEST_USER . ' test-build ' . - "${strMakeCmd} -s -j ${iBuildMax}" . ($bLogDetail ? '' : ' --silent') . - " --directory ${strBuildPath} CFLAGS_EXTRA='${strCFlags}'", + "bash -c 'cd ${strBuildPath} && ${strBackRestBase}/src/configure -q${strConfigOptions}'", {bShowOutputAsync => $bLogDetail}); } + + executeTest( + 'docker exec -i -u ' . TEST_USER . ' test-build ' . + "${strMakeCmd} -s -j ${iBuildMax}" . ($bLogDetail ? '' : ' --silent') . + " --directory ${strBuildPath} CFLAGS_EXTRA='${strCFlags}'", + {bShowOutputAsync => $bLogDetail}); } } + } - # Shut down the build vm - #----------------------------------------------------------------------------------------------------------------------- - if ($strVm ne VM_NONE) - { - executeTest("docker rm -f test-build"); - } + # Shut down the build vm + #----------------------------------------------------------------------------------------------------------------------- + if ($strVm ne VM_NONE) + { + executeTest("docker rm -f test-build"); + } - # Build the package - #----------------------------------------------------------------------------------------------------------------------- - if ($bBuildPackage && $strVm ne VM_NONE) - { - my $strPackagePath = "${strBackRestBase}/test/result/package"; + # Build the package + #----------------------------------------------------------------------------------------------------------------------- + if ($bBuildPackage && $strVm ne VM_NONE) + { + my $strPackagePath = "${strBackRestBase}/test/result/package"; - # Loop through VMs to do the package builds - my @stryBuildVm = $strVm eq VM_ALL ? VM_LIST : ($strVm); - $oStorageBackRest->pathCreate($strPackagePath, {bIgnoreExists => true, bCreateParent => true}); + # Loop through VMs to do the package builds + my @stryBuildVm = $strVm eq VM_ALL ? VM_LIST : ($strVm); + $oStorageBackRest->pathCreate($strPackagePath, {bIgnoreExists => true, bCreateParent => true}); + + foreach my $strBuildVM (@stryBuildVm) + { + my $strBuildPath = "${strPackagePath}/${strBuildVM}"; - foreach my $strBuildVM (@stryBuildVm) + if ($oVm->{$strBuildVM}{&VM_OS_BASE} eq VM_OS_BASE_DEBIAN) { - my $strBuildPath = "${strPackagePath}/${strBuildVM}"; + &log(INFO, "build package for ${strBuildVM} (${strBuildPath})"); - if ($oVm->{$strBuildVM}{&VM_OS_BASE} eq VM_OS_BASE_DEBIAN) + if ($strVm ne VM_NONE) { - &log(INFO, "build package for ${strBuildVM} (${strBuildPath})"); - - if ($strVm ne VM_NONE) - { - executeTest( - "docker run -itd -h test-build --name=test-build" . - " -v ${strBackRestBase}:${strBackRestBase} " . containerRepo() . ":${strBuildVM}-test", - {bSuppressStdErr => true}); - } - - $oStorageBackRest->pathCreate($strBuildPath, {bIgnoreExists => true, bCreateParent => true}); - - # Clone a copy of the debian package repo - executeTest( - ($strVm ne VM_NONE ? "docker exec -i test-build " : '') . - "bash -c 'git clone https://salsa.debian.org/postgresql/pgbackrest.git /root/package-src 2>&1'"); - - executeTest( - "rsync -rL --exclude=.vagrant --exclude=.git --exclude=test/result ${strBackRestBase}/" . - " ${strBuildPath}/"); executeTest( - ($strVm ne VM_NONE ? "docker exec -i test-build " : '') . - "bash -c 'cp -r /root/package-src/debian ${strBuildPath} && sudo chown -R " . TEST_USER . - " ${strBuildPath}'"); - - # Patch files in debian package builds - # - # Use these commands to create a new patch (may need to modify first line): - # BRDIR=/home/vagrant/pgbackrest;BRVM=u22;BRPATCHFILE=${BRDIR?}/test/patch/debian-package.patch - # DBDIR=${BRDIR?}/test/result/package/${BRVM}/debian - # diff -Naur ${DBDIR?}.old ${DBDIR}.new > ${BRPATCHFILE?} - my $strDebianPackagePatch = "${strBackRestBase}/test/patch/debian-package.patch"; - - if ($oStorageBackRest->exists($strDebianPackagePatch)) - { - executeTest("cp -r ${strBuildPath}/debian ${strBuildPath}/debian.old"); - executeTest("patch -d ${strBuildPath}/debian < ${strDebianPackagePatch}"); - executeTest("cp -r ${strBuildPath}/debian ${strBuildPath}/debian.new"); - } - - # If dev build then disable static release date used for reproducibility - my $bVersionDev = PROJECT_VERSION =~ /dev$/; - - if ($bVersionDev) - { - my $strRules = ${$oStorageBackRest->get("${strBuildPath}/debian/rules")}; - - $strRules =~ s/\-\-var\=release-date-static\=y/\-\-var\=release-date-static\=n/g; - $strRules =~ s/\-\-out\=html \-\-cache\-only/\-\-out\=html \-\-no\-exe/g; - - $oStorageBackRest->put("${strBuildPath}/debian/rules", $strRules); - } + "docker run -itd -h test-build --name=test-build" . + " -v ${strBackRestBase}:${strBackRestBase} " . containerRepo() . ":${strBuildVM}-test", + {bSuppressStdErr => true}); + } - # Remove patches that should be applied to core code - $oStorageBackRest->remove("${strBuildPath}/debian/patches", {bRecurse => true, bIgnoreExists => true}); + $oStorageBackRest->pathCreate($strBuildPath, {bIgnoreExists => true, bCreateParent => true}); + + # Clone a copy of the debian package repo + executeTest( + ($strVm ne VM_NONE ? "docker exec -i test-build " : '') . + "bash -c 'git clone https://salsa.debian.org/postgresql/pgbackrest.git /root/package-src 2>&1'"); + + executeTest( + "rsync -rL --exclude=.vagrant --exclude=.git --exclude=test/result ${strBackRestBase}/" . + " ${strBuildPath}/"); + executeTest( + ($strVm ne VM_NONE ? "docker exec -i test-build " : '') . + "bash -c 'cp -r /root/package-src/debian ${strBuildPath} && sudo chown -R " . TEST_USER . + " ${strBuildPath}'"); + + # Patch files in debian package builds + # + # Use these commands to create a new patch (may need to modify first line): + # BRDIR=/home/vagrant/pgbackrest;BRVM=u22;BRPATCHFILE=${BRDIR?}/test/patch/debian-package.patch + # DBDIR=${BRDIR?}/test/result/package/${BRVM}/debian + # diff -Naur ${DBDIR?}.old ${DBDIR}.new > ${BRPATCHFILE?} + my $strDebianPackagePatch = "${strBackRestBase}/test/patch/debian-package.patch"; + + if ($oStorageBackRest->exists($strDebianPackagePatch)) + { + executeTest("cp -r ${strBuildPath}/debian ${strBuildPath}/debian.old"); + executeTest("patch -d ${strBuildPath}/debian < ${strDebianPackagePatch}"); + executeTest("cp -r ${strBuildPath}/debian ${strBuildPath}/debian.new"); + } - # Update changelog to add experimental version - $oStorageBackRest->put("${strBuildPath}/debian/changelog", - "pgbackrest (${\PROJECT_VERSION}-0." . ($bVersionDev ? 'D' : 'P') . strftime("%Y%m%d%H%M%S", gmtime) . - ") experimental; urgency=medium\n" . - "\n" . - ' * Automated experimental ' . ($bVersionDev ? 'development' : 'production') . " build.\n" . - "\n" . - ' -- David Steele ' . strftime("%a, %e %b %Y %H:%M:%S %z", gmtime) . "\n\n" . - ${$oStorageBackRest->get("${strBuildPath}/debian/changelog")}); + # If dev build then disable static release date used for reproducibility + my $bVersionDev = PROJECT_VERSION =~ /dev$/; - executeTest( - ($strVm ne VM_NONE ? "docker exec -i test-build " : '') . - "bash -c 'cd ${strBuildPath} && debuild -d -i -us -uc -b'"); + if ($bVersionDev) + { + my $strRules = ${$oStorageBackRest->get("${strBuildPath}/debian/rules")}; - executeTest( - ($strVm ne VM_NONE ? "docker exec -i test-build " : '') . - "bash -c 'rm -f ${strPackagePath}/${strBuildVM}/*.build ${strPackagePath}/${strBuildVM}/*.changes" . - " ${strPackagePath}/${strBuildVM}/pgbackrest-doc*'"); + $strRules =~ s/\-\-var\=release-date-static\=y/\-\-var\=release-date-static\=n/g; + $strRules =~ s/\-\-out\=html \-\-cache\-only/\-\-out\=html \-\-no\-exe/g; - if ($strVm ne VM_NONE) - { - executeTest("docker rm -f test-build"); - } + $oStorageBackRest->put("${strBuildPath}/debian/rules", $strRules); } - if ($oVm->{$strBuildVM}{&VM_OS_BASE} eq VM_OS_BASE_RHEL) + # Remove patches that should be applied to core code + $oStorageBackRest->remove("${strBuildPath}/debian/patches", {bRecurse => true, bIgnoreExists => true}); + + # Update changelog to add experimental version + $oStorageBackRest->put("${strBuildPath}/debian/changelog", + "pgbackrest (${\PROJECT_VERSION}-0." . ($bVersionDev ? 'D' : 'P') . strftime("%Y%m%d%H%M%S", gmtime) . + ") experimental; urgency=medium\n" . + "\n" . + ' * Automated experimental ' . ($bVersionDev ? 'development' : 'production') . " build.\n" . + "\n" . + ' -- David Steele ' . strftime("%a, %e %b %Y %H:%M:%S %z", gmtime) . "\n\n" . + ${$oStorageBackRest->get("${strBuildPath}/debian/changelog")}); + + executeTest( + ($strVm ne VM_NONE ? "docker exec -i test-build " : '') . + "bash -c 'cd ${strBuildPath} && debuild -d -i -us -uc -b'"); + + executeTest( + ($strVm ne VM_NONE ? "docker exec -i test-build " : '') . + "bash -c 'rm -f ${strPackagePath}/${strBuildVM}/*.build ${strPackagePath}/${strBuildVM}/*.changes" . + " ${strPackagePath}/${strBuildVM}/pgbackrest-doc*'"); + + if ($strVm ne VM_NONE) { - &log(INFO, "build package for ${strBuildVM} (${strBuildPath})"); - - # Create build container - if ($strVm ne VM_NONE) - { - executeTest( - "docker run -itd -h test-build --name=test-build" . - " -v ${strBackRestBase}:${strBackRestBase} " . containerRepo() . ":${strBuildVM}-test", - {bSuppressStdErr => true}); - } + executeTest("docker rm -f test-build"); + } + } - # Fetching specific files is fragile but even a shallow clone of the entire pgrpms repo is very expensive. - # Using 'git archive' does not seem to work: access denied or repository not exported: /git/pgrpms.git. - executeTest( - ($strVm ne VM_NONE ? "docker exec -i test-build " : '') . - "bash -c \"" . - "mkdir /root/package-src && " . - "wget -q -O /root/package-src/pgbackrest-conf.patch " . - "'https://git.postgresql.org/gitweb/?p=pgrpms.git;a=blob_plain;hb=refs/heads/mas"."ter;" . - "f=rpm/redhat/mas"."ter/common/pgbackrest/mas"."ter/pgbackrest-conf.patch' && " . - "wget -q -O /root/package-src/pgbackrest.logrotate " . - "'https://git.postgresql.org/gitweb/?p=pgrpms.git;a=blob_plain;hb=refs/heads/mas"."ter;" . - "f=rpm/redhat/mas"."ter/common/pgbackrest/mas"."ter/pgbackrest.logrotate' && " . - "wget -q -O /root/package-src/pgbackrest.spec " . - "'https://git.postgresql.org/gitweb/?p=pgrpms.git;a=blob_plain;hb=refs/heads/mas"."ter;" . - "f=rpm/redhat/mas"."ter/common/pgbackrest/mas"."ter/pgbackrest.spec'\""); - - # Create build directories - $oStorageBackRest->pathCreate($strBuildPath, {bIgnoreExists => true, bCreateParent => true}); - $oStorageBackRest->pathCreate("${strBuildPath}/SOURCES", {bIgnoreExists => true, bCreateParent => true}); - $oStorageBackRest->pathCreate("${strBuildPath}/SPECS", {bIgnoreExists => true, bCreateParent => true}); - $oStorageBackRest->pathCreate("${strBuildPath}/RPMS", {bIgnoreExists => true, bCreateParent => true}); - $oStorageBackRest->pathCreate("${strBuildPath}/BUILD", {bIgnoreExists => true, bCreateParent => true}); - - # Install PostreSQL 11 development for package builds - executeTest( - ($strVm ne VM_NONE ? "docker exec -i test-build " : '') . - "bash -c 'yum install -y postgresql11-devel 2>&1'"); + if ($oVm->{$strBuildVM}{&VM_OS_BASE} eq VM_OS_BASE_RHEL) + { + &log(INFO, "build package for ${strBuildVM} (${strBuildPath})"); - # Copy source files + # Create build container + if ($strVm ne VM_NONE) + { executeTest( - "tar --transform='s_^_pgbackrest-release-${\PROJECT_VERSION}/_'" . - " -czf ${strBuildPath}/SOURCES/${\PROJECT_VERSION}.tar.gz -C ${strBackRestBase}" . - " src LICENSE"); + "docker run -itd -h test-build --name=test-build" . + " -v ${strBackRestBase}:${strBackRestBase} " . containerRepo() . ":${strBuildVM}-test", + {bSuppressStdErr => true}); + } - # Copy package files - executeTest( - ($strVm ne VM_NONE ? "docker exec -i test-build " : '') . "bash -c '" . - "ln -s ${strBuildPath} /root/rpmbuild && " . - "cp /root/package-src/pgbackrest.spec ${strBuildPath}/SPECS && " . - "cp /root/package-src/*.patch ${strBuildPath}/SOURCES && " . - "cp /root/package-src/pgbackrest.logrotate ${strBuildPath}/SOURCES && " . - "sudo chown -R " . TEST_USER . " ${strBuildPath}'"); - - # Patch files in RHEL package builds - # - # Use these commands to create a new patch (may need to modify first line): - # BRDIR=/home/vagrant/pgbackrest;BRVM=rh7;BRPATCHFILE=${BRDIR?}/test/patch/rhel-package.patch - # PKDIR=${BRDIR?}/test/result/package/${BRVM}/SPECS - # diff -Naur ${PKDIR?}.old ${PKDIR}.new > ${BRPATCHFILE?} - my $strPackagePatch = "${strBackRestBase}/test/patch/rhel-package.patch"; - - if ($oStorageBackRest->exists($strPackagePatch)) - { - executeTest("cp -r ${strBuildPath}/SPECS ${strBuildPath}/SPECS.old"); - executeTest("patch -d ${strBuildPath}/SPECS < ${strPackagePatch}"); - executeTest("cp -r ${strBuildPath}/SPECS ${strBuildPath}/SPECS.new"); - } + # Fetching specific files is fragile but even a shallow clone of the entire pgrpms repo is very expensive. + # Using 'git archive' does not seem to work: access denied or repository not exported: /git/pgrpms.git. + executeTest( + ($strVm ne VM_NONE ? "docker exec -i test-build " : '') . + "bash -c \"" . + "mkdir /root/package-src && " . + "wget -q -O /root/package-src/pgbackrest-conf.patch " . + "'https://git.postgresql.org/gitweb/?p=pgrpms.git;a=blob_plain;hb=refs/heads/mas"."ter;" . + "f=rpm/redhat/mas"."ter/common/pgbackrest/mas"."ter/pgbackrest-conf.patch' && " . + "wget -q -O /root/package-src/pgbackrest.logrotate " . + "'https://git.postgresql.org/gitweb/?p=pgrpms.git;a=blob_plain;hb=refs/heads/mas"."ter;" . + "f=rpm/redhat/mas"."ter/common/pgbackrest/mas"."ter/pgbackrest.logrotate' && " . + "wget -q -O /root/package-src/pgbackrest.spec " . + "'https://git.postgresql.org/gitweb/?p=pgrpms.git;a=blob_plain;hb=refs/heads/mas"."ter;" . + "f=rpm/redhat/mas"."ter/common/pgbackrest/mas"."ter/pgbackrest.spec'\""); + + # Create build directories + $oStorageBackRest->pathCreate($strBuildPath, {bIgnoreExists => true, bCreateParent => true}); + $oStorageBackRest->pathCreate("${strBuildPath}/SOURCES", {bIgnoreExists => true, bCreateParent => true}); + $oStorageBackRest->pathCreate("${strBuildPath}/SPECS", {bIgnoreExists => true, bCreateParent => true}); + $oStorageBackRest->pathCreate("${strBuildPath}/RPMS", {bIgnoreExists => true, bCreateParent => true}); + $oStorageBackRest->pathCreate("${strBuildPath}/BUILD", {bIgnoreExists => true, bCreateParent => true}); + + # Install PostreSQL 11 development for package builds + executeTest( + ($strVm ne VM_NONE ? "docker exec -i test-build " : '') . + "bash -c 'yum install -y postgresql11-devel 2>&1'"); + + # Copy source files + executeTest( + "tar --transform='s_^_pgbackrest-release-${\PROJECT_VERSION}/_'" . + " -czf ${strBuildPath}/SOURCES/${\PROJECT_VERSION}.tar.gz -C ${strBackRestBase}" . + " src LICENSE"); + + # Copy package files + executeTest( + ($strVm ne VM_NONE ? "docker exec -i test-build " : '') . "bash -c '" . + "ln -s ${strBuildPath} /root/rpmbuild && " . + "cp /root/package-src/pgbackrest.spec ${strBuildPath}/SPECS && " . + "cp /root/package-src/*.patch ${strBuildPath}/SOURCES && " . + "cp /root/package-src/pgbackrest.logrotate ${strBuildPath}/SOURCES && " . + "sudo chown -R " . TEST_USER . " ${strBuildPath}'"); + + # Patch files in RHEL package builds + # + # Use these commands to create a new patch (may need to modify first line): + # BRDIR=/home/vagrant/pgbackrest;BRVM=rh7;BRPATCHFILE=${BRDIR?}/test/patch/rhel-package.patch + # PKDIR=${BRDIR?}/test/result/package/${BRVM}/SPECS + # diff -Naur ${PKDIR?}.old ${PKDIR}.new > ${BRPATCHFILE?} + my $strPackagePatch = "${strBackRestBase}/test/patch/rhel-package.patch"; + + if ($oStorageBackRest->exists($strPackagePatch)) + { + executeTest("cp -r ${strBuildPath}/SPECS ${strBuildPath}/SPECS.old"); + executeTest("patch -d ${strBuildPath}/SPECS < ${strPackagePatch}"); + executeTest("cp -r ${strBuildPath}/SPECS ${strBuildPath}/SPECS.new"); + } - # Update version number to match current version - my $strSpec = ${$oStorageBackRest->get("${strBuildPath}/SPECS/pgbackrest.spec")}; - $strSpec =~ s/^Version\:.*$/Version\:\t${\PROJECT_VERSION}/gm; - $oStorageBackRest->put("${strBuildPath}/SPECS/pgbackrest.spec", $strSpec); + # Update version number to match current version + my $strSpec = ${$oStorageBackRest->get("${strBuildPath}/SPECS/pgbackrest.spec")}; + $strSpec =~ s/^Version\:.*$/Version\:\t${\PROJECT_VERSION}/gm; + $oStorageBackRest->put("${strBuildPath}/SPECS/pgbackrest.spec", $strSpec); - # Build package - executeTest( - ($strVm ne VM_NONE ? "docker exec -i test-build " : '') . - "rpmbuild --define 'pgmajorversion %{nil}' -v -bb --clean root/rpmbuild/SPECS/pgbackrest.spec", - {bSuppressStdErr => true}); + # Build package + executeTest( + ($strVm ne VM_NONE ? "docker exec -i test-build " : '') . + "rpmbuild --define 'pgmajorversion %{nil}' -v -bb --clean root/rpmbuild/SPECS/pgbackrest.spec", + {bSuppressStdErr => true}); - # Remove build container - if ($strVm ne VM_NONE) - { - executeTest("docker rm -f test-build"); - } + # Remove build container + if ($strVm ne VM_NONE) + { + executeTest("docker rm -f test-build"); } } } - - # Exit if only testing builds - exit 0 if $bBuildOnly; } - # Perform static source code analysis - #--------------------------------------------------------------------------------------------------------------------------- - if (!$bDryRun) - { - logFileSet($oStorageTest, cwd() . "/test"); - } + # Exit if only testing builds + exit 0 if $bBuildOnly; + } - # Run the tests - #--------------------------------------------------------------------------------------------------------------------------- - if (@{$oyTestRun} == 0) - { - confess &log(ERROR, 'no tests were selected'); - } + # Perform static source code analysis + #------------------------------------------------------------------------------------------------------------------------------- + if (!$bDryRun) + { + logFileSet($oStorageTest, cwd() . "/test"); + } - &log(INFO, @{$oyTestRun} . ' test' . (@{$oyTestRun} > 1 ? 's': '') . " selected\n"); + # Run the tests + #------------------------------------------------------------------------------------------------------------------------------- + if (@{$oyTestRun} == 0) + { + confess &log(ERROR, 'no tests were selected'); + } - # Don't allow --no-cleanup when more than one test will run. How would the prior results be preserved? - if ($bNoCleanup && @{$oyTestRun} > 1) - { - confess &log(ERROR, '--no-cleanup is not valid when more than one test will run') - } + &log(INFO, @{$oyTestRun} . ' test' . (@{$oyTestRun} > 1 ? 's': '') . " selected\n"); - # Disable file logging for integration tests when there is more than one test since it will be overwritten - if (@{$oyTestRun} > 1) - { - $strLogLevelTestFile = lc(OFF); - } + # Don't allow --no-cleanup when more than one test will run. How would the prior results be preserved? + if ($bNoCleanup && @{$oyTestRun} > 1) + { + confess &log(ERROR, '--no-cleanup is not valid when more than one test will run') + } + + # Disable file logging for integration tests when there is more than one test since it will be overwritten + if (@{$oyTestRun} > 1) + { + $strLogLevelTestFile = lc(OFF); + } - # Don't allow --no-cleanup when more than one test will run. How would the prior results be preserved? + # Don't allow --no-cleanup when more than one test will run. How would the prior results be preserved? - # Only use one vm for dry run so results are printed in order - if ($bDryRun) - { - $iVmMax = 1; - } + # Only use one vm for dry run so results are printed in order + if ($bDryRun) + { + $iVmMax = 1; + } - my $iTestIdx = 0; - my $iVmTotal; - my $iTestMax = @{$oyTestRun}; - my $bShowOutputAsync = $bVmOut && (@{$oyTestRun} == 1 || $iVmMax == 1) && ! $bDryRun ? true : false; + my $iTestIdx = 0; + my $iVmTotal; + my $iTestMax = @{$oyTestRun}; + my $bShowOutputAsync = $bVmOut && (@{$oyTestRun} == 1 || $iVmMax == 1) && ! $bDryRun ? true : false; + do + { do { - do - { - $iVmTotal = 0; + $iVmTotal = 0; - for (my $iVmIdx = 0; $iVmIdx < $iVmMax; $iVmIdx++) + for (my $iVmIdx = 0; $iVmIdx < $iVmMax; $iVmIdx++) + { + if (defined($$oyProcess[$iVmIdx])) { - if (defined($$oyProcess[$iVmIdx])) - { - my ($bDone, $bFail) = $$oyProcess[$iVmIdx]->end(); + my ($bDone, $bFail) = $$oyProcess[$iVmIdx]->end(); - if ($bDone) + if ($bDone) + { + if ($bFail) { - if ($bFail) + if ($oyProcess->[$iVmIdx]->run()) { - if ($oyProcess->[$iVmIdx]->run()) - { - $iTestRetry++; - $iVmTotal++; - } - else - { - $iTestFail++; - $$oyProcess[$iVmIdx] = undef; - } + $iTestRetry++; + $iVmTotal++; } else { + $iTestFail++; $$oyProcess[$iVmIdx] = undef; } } else { - $iVmTotal++; + $$oyProcess[$iVmIdx] = undef; } } - } - - # Only wait when all VMs are running or all tests have been assigned. Otherwise, there is something to do. - if ($iVmTotal == $iVmMax || $iTestIdx == @{$oyTestRun}) - { - waitHiRes(.05); + else + { + $iVmTotal++; + } } } - while ($iVmTotal == $iVmMax); - for (my $iVmIdx = 0; $iVmIdx < $iVmMax; $iVmIdx++) + # Only wait when all VMs are running or all tests have been assigned. Otherwise, there is something to do. + if ($iVmTotal == $iVmMax || $iTestIdx == @{$oyTestRun}) { - if (!defined($$oyProcess[$iVmIdx]) && $iTestIdx < @{$oyTestRun}) - { - my $oJob = new pgBackRestTest::Common::JobTest( - $oStorageTest, $strBackRestBase, $strTestPath, $$oyTestRun[$iTestIdx], $bDryRun, $bVmOut, $iVmIdx, $iVmMax, - $strMakeCmd, $iTestIdx, $iTestMax, $strLogLevel, $strLogLevelTest, $strLogLevelTestFile, !$bNoLogTimestamp, - $bShowOutputAsync, $bNoCleanup, $iRetry, !$bNoBackTrace, !$bNoValgrind, !$bNoCoverage, $bCoverageSummary, - !$bNoOptimize, $bProfile, $iScale, $strTimeZone, !$bNoDebug, $bDebugTestTrace, - $iBuildMax / $iVmMax < 1 ? 1 : int($iBuildMax / $iVmMax)); - $iTestIdx++; - - if ($oJob->run()) - { - $$oyProcess[$iVmIdx] = $oJob; - } - - $iVmTotal++; - } + waitHiRes(.05); } } - while ($iVmTotal > 0); + while ($iVmTotal == $iVmMax); - # Write out coverage info and test coverage - #--------------------------------------------------------------------------------------------------------------------------- - my $iUncoveredCodeModuleTotal = 0; - - if (vmCoverageC($strVm) && !$bNoCoverage && !$bDryRun && $iTestFail == 0) + for (my $iVmIdx = 0; $iVmIdx < $iVmMax; $iVmIdx++) { - $iUncoveredCodeModuleTotal = coverageValidateAndGenerate( - $oyTestRun, $oStorageBackRest, !$bNoCoverageReport, $bCoverageSummary, $strTestPath, "${strTestPath}/temp", - "${strBackRestBase}/test/result", "${strBackRestBase}/doc/xml/auto"); - } - - # Print test info and exit - #--------------------------------------------------------------------------------------------------------------------------- - &log(INFO, - ($bDryRun ? 'DRY RUN COMPLETED' : 'TESTS COMPLETED') . ($iTestFail == 0 ? ' SUCCESSFULLY' . - ($iUncoveredCodeModuleTotal == 0 ? '' : " WITH ${iUncoveredCodeModuleTotal} MODULE(S) MISSING COVERAGE") : - " WITH ${iTestFail} FAILURE(S)") . ($iTestRetry == 0 ? '' : ", ${iTestRetry} RETRY(IES)") . - ($bNoLogTimestamp ? '' : ' (' . (time() - $lStartTime) . 's)')); - - exit 1 if ($iTestFail > 0 || ($iUncoveredCodeModuleTotal > 0 && !$bCoverageSummary)); + if (!defined($$oyProcess[$iVmIdx]) && $iTestIdx < @{$oyTestRun}) + { + my $oJob = new pgBackRestTest::Common::JobTest( + $oStorageTest, $strBackRestBase, $strTestPath, $$oyTestRun[$iTestIdx], $bDryRun, $bVmOut, $iVmIdx, $iVmMax, + $strMakeCmd, $iTestIdx, $iTestMax, $strLogLevel, $strLogLevelTest, $strLogLevelTestFile, !$bNoLogTimestamp, + $bShowOutputAsync, $bNoCleanup, $iRetry, !$bNoBackTrace, !$bNoValgrind, !$bNoCoverage, $bCoverageSummary, + !$bNoOptimize, $bProfile, $iScale, $strTimeZone, !$bNoDebug, $bDebugTestTrace, + $iBuildMax / $iVmMax < 1 ? 1 : int($iBuildMax / $iVmMax)); + $iTestIdx++; + + if ($oJob->run()) + { + $$oyProcess[$iVmIdx] = $oJob; + } - exit 0; + $iVmTotal++; + } + } } + while ($iVmTotal > 0); - ################################################################################################################################ - # Runs tests - ################################################################################################################################ - my $iRun = 0; - - # Create host group for containers - my $oHostGroup = hostGroupGet(); + # Write out coverage info and test coverage + #------------------------------------------------------------------------------------------------------------------------------- + my $iUncoveredCodeModuleTotal = 0; - # Set timezone - if (defined($strTimeZone)) + if (vmCoverageC($strVm) && !$bNoCoverage && !$bDryRun && $iTestFail == 0) { - $ENV{TZ} = $strTimeZone; + $iUncoveredCodeModuleTotal = coverageValidateAndGenerate( + $oyTestRun, $oStorageBackRest, !$bNoCoverageReport, $bCoverageSummary, $strTestPath, "${strTestPath}/temp", + "${strBackRestBase}/test/result", "${strBackRestBase}/doc/xml/auto"); } - # Run the test - testRun($stryModule[0], $stryModuleTest[0])->process( - $strVm, $iVmId, # Vm info - $strBackRestBase, # Base backrest directory - $strTestPath, # Path where the tests will run - dirname($strTestPath) . "/bin/${strVm}/" . PROJECT_EXE, # Path to the pgbackrest binary - dirname($strTestPath) . "/build/" . VM_NONE . '/src/' . PROJECT_EXE, # Path to the pgbackrest storage helper - $strPgVersion ne 'minimal' ? $strPgSqlBin: undef, # Pg bin path - $strPgVersion ne 'minimal' ? $strPgVersion: undef, # Pg version - $stryModule[0], $stryModuleTest[0], \@iyModuleTestRun, # Module info - $bVmOut, $bDryRun, $bNoCleanup, # Test options - $strLogLevelTestFile, # Log options - TEST_USER, TEST_GROUP); # User/group info - - if (!$bNoCleanup) - { - if ($oHostGroup->removeAll() > 0) - { - executeTest("rm -rf ${strTestPath}"); - } - } + # Print test info and exit + #------------------------------------------------------------------------------------------------------------------------------- + &log(INFO, + ($bDryRun ? 'DRY RUN COMPLETED' : 'TESTS COMPLETED') . ($iTestFail == 0 ? ' SUCCESSFULLY' . + ($iUncoveredCodeModuleTotal == 0 ? '' : " WITH ${iUncoveredCodeModuleTotal} MODULE(S) MISSING COVERAGE") : + " WITH ${iTestFail} FAILURE(S)") . ($iTestRetry == 0 ? '' : ", ${iTestRetry} RETRY(IES)") . + ($bNoLogTimestamp ? '' : ' (' . (time() - $lStartTime) . 's)')); - if (!$bDryRun && !$bVmOut) - { - &log(INFO, 'TESTS COMPLETED SUCCESSFULLY (DESPITE ANY ERROR MESSAGES YOU SAW)'); - } + exit 1 if ($iTestFail > 0 || ($iUncoveredCodeModuleTotal > 0 && !$bCoverageSummary)); - # Exit with success exit 0; }
Version of to use for the integration test.