Skip to content

Commit

Permalink
Read segment size from WAL headers.
Browse files Browse the repository at this point in the history 
This allows validation of the WAL segment size for PostgreSQL versions <= 10.
  • Loading branch information
@dwsteele
dwsteele committed Jul 9, 2020
1 parent 2f7823c commit d5df397
Show file tree
Hide file tree
Showing 8 changed files with 65 additions and 26 deletions.
35 changes: 29 additions & 6 deletions src/postgres/interface.c
View file
Expand Up @@ -386,6 +386,27 @@ pgCatalogVersion(unsigned int pgVersion)
FUNCTION_TEST_RETURN(pgInterfaceVersion(pgVersion)->catalogVersion());
}

/***********************************************************************************************************************************
Check expected WAL segment size for older PostgreSQL versions
***********************************************************************************************************************************/
static void
pgWalSegmentSizeCheck(unsigned int pgVersion, unsigned int walSegmentSize)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(UINT, pgVersion);
FUNCTION_TEST_PARAM(UINT, walSegmentSize);
FUNCTION_TEST_END();

if (pgVersion < PG_VERSION_11 && walSegmentSize != PG_WAL_SEGMENT_SIZE_DEFAULT)
{
THROW_FMT(
FormatError, "wal segment size is %u but must be %u for " PG_NAME " <= " PG_VERSION_10_STR, walSegmentSize,
PG_WAL_SEGMENT_SIZE_DEFAULT);
}

FUNCTION_TEST_RETURN_VOID();
}

/**********************************************************************************************************************************/
PgControl
pgControlFromBuffer(const Buffer *controlFile)
Expand Down Expand Up @@ -425,12 +446,7 @@ pgControlFromBuffer(const Buffer *controlFile)
result.version = interface->version;

// Check the segment size
if (result.version < PG_VERSION_11 && result.walSegmentSize != PG_WAL_SEGMENT_SIZE_DEFAULT)
{
THROW_FMT(
FormatError, "wal segment size is %u but must be %u for " PG_NAME " <= " PG_VERSION_10_STR, result.walSegmentSize,
PG_WAL_SEGMENT_SIZE_DEFAULT);
}
pgWalSegmentSizeCheck(result.version, result.walSegmentSize);

// Check the page size
if (result.pageSize != PG_PAGE_SIZE_DEFAULT)
Expand Down Expand Up @@ -526,6 +542,9 @@ pgWalFromBuffer(const Buffer *walBuffer)
PgWal result = interface->wal(bufPtrConst(walBuffer));
result.version = interface->version;

// Check the segment size
pgWalSegmentSizeCheck(result.version, result.size);

FUNCTION_LOG_RETURN(PG_WAL, result);
}

Expand Down Expand Up @@ -790,6 +809,10 @@ pgWalTestToBuffer(PgWal pgWal, Buffer *walBuffer)

ASSERT(walBuffer != NULL);

// Set default WAL segment size if not specified
if (pgWal.size == 0)
pgWal.size = PG_WAL_SEGMENT_SIZE_DEFAULT;

// Generate WAL
pgInterfaceVersion(pgWal.version)->walTest(pgWal, bufPtr(walBuffer));

Expand Down
1 change: 1 addition & 0 deletions src/postgres/interface.h
View file
Expand Up @@ -106,6 +106,7 @@ PostgreSQL WAL Info
typedef struct PgWal
{
unsigned int version;
unsigned int size;
uint64_t systemId;
} PgWal;

Expand Down
2 changes: 2 additions & 0 deletions src/postgres/interface/version.intern.h
View file
Expand Up @@ -191,6 +191,7 @@ Read the version specific WAL header into a general data structure
return (PgWal) \
{ \
.systemId = ((XLogLongPageHeaderData *)walFile)->xlp_sysid, \
.size = ((XLogLongPageHeaderData *)walFile)->xlp_seg_size, \
}; \
}

Expand All @@ -210,6 +211,7 @@ Create a WAL file file for testing
((XLogLongPageHeaderData *)buffer)->std.xlp_magic = XLOG_PAGE_MAGIC; \
((XLogLongPageHeaderData *)buffer)->std.xlp_info = XLP_LONG_HEADER; \
((XLogLongPageHeaderData *)buffer)->xlp_sysid = pgWal.systemId; \
((XLogLongPageHeaderData *)buffer)->xlp_seg_size = pgWal.size; \
}

#endif
Expand Down
12 changes: 6 additions & 6 deletions test/expect/mock-stanza-001.log
View file
Expand Up @@ -296,9 +296,9 @@ full backup - create first full backup (db-primary host)
> [CONTAINER-EXEC] db-primary [BACKREST-BIN] --config=[TEST_PATH]/db-primary/pgbackrest.conf --repo1-retention-full=2 --no-online --type=full --stanza=db backup
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=zst --config=[TEST_PATH]/db-primary/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-primary/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-primary/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-primary/repo --repo1-retention-full=2 --stanza=db --start-fast --type=full
P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum e40de8cea99dd469c3efb47f5f33a73c7390fb9c
P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c
P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c
P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum f92539dea1f9482e2946c1138eeeecdea29d7f19
P01 INFO: backup file [TEST_PATH]/db-primary/db/base/global/pg_control (8KB, 100%) checksum 4c77c900f7af0d9ab13fa9982051a42e0b637f6c
P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000002.ready (0B, 100%)
P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000001.ready (0B, 100%)
Expand Down Expand Up @@ -441,9 +441,9 @@ diff backup - diff changed to full backup (db-primary host)
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=zst --config=[TEST_PATH]/db-primary/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-primary/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-primary/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-primary/repo --repo1-retention-full=2 --stanza=db --start-fast --type=diff
P00 WARN: no prior backup exists, diff backup has been changed to full
P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum 65b64687e7bfd1a918c6144f84a133f03e50ac57
P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c
P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c
P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum 762dae884fdccb805c5f3283662ea0f8da55f228
P01 INFO: backup file [TEST_PATH]/db-primary/db/base/global/pg_control (8KB, 100%) checksum 4969435f3b36bfaa0f5a486bef97f1988a135520
P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000002.ready (0B, 100%)
P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000001.ready (0B, 100%)
Expand Down
12 changes: 6 additions & 6 deletions test/expect/mock-stanza-002.log
View file
Expand Up @@ -320,9 +320,9 @@ full backup - create first full backup (backup host)
> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --repo1-retention-full=2 --no-online --type=full --stanza=db backup
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --compress-type=lz4 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --no-online --pg1-host=db-primary --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-primary/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo1-azure-account=<redacted> --repo1-azure-container=azContainer --repo1-azure-host=azure --repo1-azure-key=<redacted> --no-repo1-azure-verify-tls --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-retention-full=2 --repo1-type=azure --stanza=db --start-fast --type=full
P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum e40de8cea99dd469c3efb47f5f33a73c7390fb9c
P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c
P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c
P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum f92539dea1f9482e2946c1138eeeecdea29d7f19
P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/global/pg_control (8KB, 100%) checksum 4c77c900f7af0d9ab13fa9982051a42e0b637f6c
P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000002.ready (0B, 100%)
P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000001.ready (0B, 100%)
Expand Down Expand Up @@ -526,9 +526,9 @@ diff backup - diff changed to full backup (backup host)
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --compress-type=lz4 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --no-online --pg1-host=db-primary --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-primary/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo1-azure-account=<redacted> --repo1-azure-container=azContainer --repo1-azure-host=azure --repo1-azure-key=<redacted> --no-repo1-azure-verify-tls --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-retention-full=2 --repo1-type=azure --stanza=db --start-fast --type=diff
P00 WARN: no prior backup exists, diff backup has been changed to full
P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum 65b64687e7bfd1a918c6144f84a133f03e50ac57
P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c
P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c
P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum 762dae884fdccb805c5f3283662ea0f8da55f228
P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/global/pg_control (8KB, 100%) checksum 4969435f3b36bfaa0f5a486bef97f1988a135520
P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000002.ready (0B, 100%)
P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000001.ready (0B, 100%)
Expand Down
3 changes: 3 additions & 0 deletions test/lib/pgBackRestTest/Env/HostEnvTest.pm
View file
Expand Up @@ -458,6 +458,9 @@ sub walGenerateContent
# Add the system identifier
$tWalContent .= pack('Q', $self->dbSysId($strPgVersion));

# Add segment size
$tWalContent .= pack('L', PG_WAL_SEGMENT_SIZE);

# Add the source number to produce WAL segments with different checksums
$tWalContent .= pack('S', $iSourceNo);

Expand Down
12 changes: 6 additions & 6 deletions test/src/module/command/archivePushTest.c
View file
Expand Up @@ -271,7 +271,7 @@ testRun(void)
storageTest,
strNewFmt(
"repo/archive/test/11-1/0000000100000001/000000010000000100000001-%s.gz",
TEST_64BIT() ? "3e5ecd22712f319b2420d5b901fd29f4f6be2336" : "6903dce7e3cd64ba9a6134056405eaeb8dedcd37")),
TEST_64BIT() ? "aae7591a1dbc58f21d0d004886075094f622e6dd" : "28a13fd8cf6fcd9f9a8108aed4c8bcc58040863a")),
true, "check repo for WAL file");

TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment again");
Expand Down Expand Up @@ -309,7 +309,7 @@ testRun(void)
storageTest,
strNewFmt(
"repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s.gz",
TEST_64BIT() ? "edad2f5a9d8a03ee3c09e8ce92c771e0d20232f5" : "e7c81f5513e0c6e3f19b9dbfc450019165994dda")),
TEST_64BIT() ? "755defa48a0a0872767b6dea49bdd3b64902f147" : "9c2a6ec4491a2118bcdc9b653366581d8821c982")),
true, "check repo for WAL file");

// Push a history file
Expand Down Expand Up @@ -416,7 +416,7 @@ testRun(void)
storageTest,
strNewFmt(
"repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s",
TEST_64BIT() ? "edad2f5a9d8a03ee3c09e8ce92c771e0d20232f5" : "e7c81f5513e0c6e3f19b9dbfc450019165994dda")),
TEST_64BIT() ? "755defa48a0a0872767b6dea49bdd3b64902f147" : "9c2a6ec4491a2118bcdc9b653366581d8821c982")),
true, "check repo for WAL file");
}

Expand Down Expand Up @@ -585,7 +585,7 @@ testRun(void)
storageTest,
strNewFmt(
"repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s",
TEST_64BIT() ? "f81d63dd5e258cd607534f3531bbd71442797e37" : "02d228126281e8e102b35a2737e45a0527946296")),
TEST_64BIT() ? "ce58ec898a080e59d71dce49051843b03a497d99" : "765ee94e4f65d081f2dc3b8e77556da95bc31cbf")),
true, "check repo for WAL file");

// Direct tests of the async function
Expand Down Expand Up @@ -650,7 +650,7 @@ testRun(void)
storageTest,
strNewFmt(
"repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s",
TEST_64BIT() ? "f81d63dd5e258cd607534f3531bbd71442797e37" : "02d228126281e8e102b35a2737e45a0527946296")),
TEST_64BIT() ? "ce58ec898a080e59d71dce49051843b03a497d99" : "765ee94e4f65d081f2dc3b8e77556da95bc31cbf")),
true, "check repo for WAL 1 file");

TEST_RESULT_STR_Z(
Expand Down Expand Up @@ -679,7 +679,7 @@ testRun(void)
storageTest,
strNewFmt(
"repo/archive/test/9.4-1/0000000100000001/000000010000000100000002-%s",
TEST_64BIT() ? "0aea6fa5d53500ce548b84a86bc3a29ae77fa048" : "408822a89ef44ef6740e785743bf1b870d8024a2")),
TEST_64BIT() ? "251820f85b31554bbf33061fce7dabf18c16afe0" : "ae8cb1fe45d40bc364df6bc6e8803774f0e3eaeb")),
true, "check repo for WAL 2 file");

TEST_RESULT_STR_Z(
Expand Down
14 changes: 12 additions & 2 deletions test/src/module/postgres/interfaceTest.c
View file
Expand Up @@ -224,22 +224,32 @@ testRun(void)

//--------------------------------------------------------------------------------------------------------------------------
memset(bufPtr(result), 0, bufSize(result));
pgWalTestToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 0xECAFECAF}, result);
pgWalTestToBuffer(
(PgWal){.version = PG_VERSION_11, .systemId = 0xECAFECAF, .size = PG_WAL_SEGMENT_SIZE_DEFAULT * 2}, result);
storagePutP(storageNewWriteP(storageTest, walFile), result);

PgWal info = {0};
TEST_ASSIGN(info, pgWalFromFile(walFile), "get wal info v11");
TEST_RESULT_UINT(info.systemId, 0xECAFECAF, " check system id");
TEST_RESULT_UINT(info.version, PG_VERSION_11, " check version");
TEST_RESULT_UINT(info.size, PG_WAL_SEGMENT_SIZE_DEFAULT * 2, " check size");

//--------------------------------------------------------------------------------------------------------------------------
memset(bufPtr(result), 0, bufSize(result));
pgWalTestToBuffer((PgWal){.version = PG_VERSION_83, .systemId = 0xEAEAEAEA}, result);
pgWalTestToBuffer(
(PgWal){.version = PG_VERSION_83, .systemId = 0xEAEAEAEA, .size = PG_WAL_SEGMENT_SIZE_DEFAULT * 2}, result);

TEST_ERROR(pgWalFromBuffer(result), FormatError, "wal segment size is 33554432 but must be 16777216 for PostgreSQL <= 10");

//--------------------------------------------------------------------------------------------------------------------------
memset(bufPtr(result), 0, bufSize(result));
pgWalTestToBuffer((PgWal){.version = PG_VERSION_83, .systemId = 0xEAEAEAEA, .size = PG_WAL_SEGMENT_SIZE_DEFAULT}, result);
storagePutP(storageNewWriteP(storageTest, walFile), result);

TEST_ASSIGN(info, pgWalFromFile(walFile), "get wal info v8.3");
TEST_RESULT_UINT(info.systemId, 0xEAEAEAEA, " check system id");
TEST_RESULT_UINT(info.version, PG_VERSION_83, " check version");
TEST_RESULT_UINT(info.size, PG_WAL_SEGMENT_SIZE_DEFAULT, " check size");
}

// *****************************************************************************************************************************
Expand Down

0 comments on commit d5df397

Please sign in to comment.