diff --git a/doc/xml/release/2024/2.50.xml b/doc/xml/release/2024/2.50.xml
index 19b31ea0c0..90be558512 100644
--- a/doc/xml/release/2024/2.50.xml
+++ b/doc/xml/release/2024/2.50.xml
@@ -11,6 +11,17 @@
Add support for alternate compile-time page sizes.
+
+
+
+
+
+
+
+
+
+ Skip files truncated during backup when bundling.
+
diff --git a/src/command/backup/backup.c b/src/command/backup/backup.c
index 61aacf533e..c29633eb15 100644
--- a/src/command/backup/backup.c
+++ b/src/command/backup/backup.c
@@ -1439,7 +1439,7 @@ backupJobResult(
// Format log progress
String *const logProgress = strNew();
- if (bundleId != 0 && copyResult != backupCopyResultNoOp)
+ if (bundleId != 0 && copyResult != backupCopyResultNoOp && copyResult != backupCopyResultTruncate)
strCatFmt(logProgress, "bundle %" PRIu64 "/%" PRIu64 ", ", bundleId, bundleOffset);
// Log original manifest size if copy size differs
@@ -1492,7 +1492,9 @@ backupJobResult(
strZ(file.name), strZ(strNewEncode(encodingHex, BUF(file.checksumSha1, HASH_TYPE_SHA1_SIZE))));
}
- LOG_DETAIL_PID_FMT(processId, "backup file %s (%s)%s", strZ(fileLog), strZ(logProgress), strZ(logChecksum));
+ LOG_DETAIL_PID_FMT(
+ processId, "%s file %s (%s)%s", copyResult != backupCopyResultTruncate ? "backup" : "store truncated",
+ strZ(fileLog), strZ(logProgress), strZ(logChecksum));
// If the file had page checksums calculated during the copy
ASSERT((!file.checksumPage && checksumPageResult == NULL) || (file.checksumPage && checksumPageResult != NULL));
@@ -1576,7 +1578,8 @@ backupJobResult(
file.checksumPageError = checksumPageError;
file.checksumPageErrorList =
checksumPageErrorList != NULL ? jsonFromVar(varNewVarLst(checksumPageErrorList)) : NULL;
- file.bundleId = bundleId;
+ // Truncated file is not put in bundle
+ file.bundleId = copyResult != backupCopyResultTruncate ? bundleId : 0;
file.bundleOffset = bundleOffset;
file.blockIncrMapSize = blockIncrMapSize;
diff --git a/src/command/backup/file.c b/src/command/backup/file.c
index 9dcd825a20..97ff2c7c69 100644
--- a/src/command/backup/file.c
+++ b/src/command/backup/file.c
@@ -295,61 +295,99 @@ backupFile(
// Add size filter last to calculate repo size
ioFilterGroupAdd(ioReadFilterGroup(readIo), ioSizeNew());
- // Open the source and destination and copy the file
+ // Open the source
if (ioReadOpen(readIo))
{
- // Setup the repo file for write. There is no need to write the file atomically (e.g. via a temp file on
- // Posix) because checksums are tested on resume after a failed backup. The path does not need to be synced
- // for each file because all paths are synced at the end of the backup. It needs to be created in the prior
- // context because it will live longer than a single loop when more than one file is being written.
- if (write == NULL)
+ Buffer *const buffer = bufNew(ioBufferSize());
+ bool readEof = false;
+
+ // Read the first buffer to determine if the file was truncated. Detecting truncation matters only when
+ // bundling is enabled as otherwise the file will be stored anyway.
+ ioRead(readIo, buffer);
+
+ if (ioReadEof(readIo) && bundleId != 0)
{
- MEM_CONTEXT_PRIOR_BEGIN()
+ // Close the source and set eof
+ ioReadClose(readIo);
+ readEof = true;
+
+ // If the file is zero-length then it was truncated during the backup
+ if (pckReadU64P(ioFilterGroupResultP(ioReadFilterGroup(readIo), SIZE_FILTER_TYPE, .idx = 0)) == 0)
+ fileResult->backupCopyResult = backupCopyResultTruncate;
+ }
+
+ // Copy the file in non-bundling mode or if the file is not zero-length
+ if (fileResult->backupCopyResult != backupCopyResultTruncate)
+ {
+ // Setup the repo file for write. There is no need to write the file atomically (e.g. via a temp file on
+ // Posix) because checksums are tested on resume after a failed backup. The path does not need to be
+ // synced for each file because all paths are synced at the end of the backup. It needs to be created in
+ // the prior context because it will live longer than a single loop when more than one file is being
+ // written.
+ if (write == NULL)
{
- write = storageNewWriteP(
- storageRepoWrite(), repoFile, .compressible = compressible, .noAtomic = true,
- .noSyncPath = true);
- ioWriteOpen(storageWriteIo(write));
+ MEM_CONTEXT_PRIOR_BEGIN()
+ {
+ write = storageNewWriteP(
+ storageRepoWrite(), repoFile, .compressible = compressible, .noAtomic = true,
+ .noSyncPath = true);
+ ioWriteOpen(storageWriteIo(write));
+ }
+ MEM_CONTEXT_PRIOR_END();
}
- MEM_CONTEXT_PRIOR_END();
- }
- // Copy data from source to destination
- ioCopyP(readIo, storageWriteIo(write));
+ // Write the first buffer
+ ioWrite(storageWriteIo(write), buffer);
+ bufFree(buffer);
+
+ // Copy remainder of the file if not eof
+ if (!readEof)
+ {
+ ioCopyP(readIo, storageWriteIo(write));
- // Close the source
- ioReadClose(readIo);
+ // Close the source
+ ioReadClose(readIo);
+ }
+ }
MEM_CONTEXT_BEGIN(lstMemContext(result))
{
- // Get sizes and checksum
+ // Get size and checksum
fileResult->copySize = pckReadU64P(
ioFilterGroupResultP(ioReadFilterGroup(readIo), SIZE_FILTER_TYPE, .idx = 0));
- fileResult->bundleOffset = bundleOffset;
fileResult->copyChecksum = pckReadBinP(
ioFilterGroupResultP(ioReadFilterGroup(readIo), CRYPTO_HASH_FILTER_TYPE, .idx = 0));
- fileResult->repoSize = pckReadU64P(
- ioFilterGroupResultP(ioReadFilterGroup(readIo), SIZE_FILTER_TYPE, .idx = 1));
- // Get results of page checksum validation
- if (file->pgFileChecksumPage)
+ // If the file is not bundled or not zero-length then it was copied
+ if (fileResult->backupCopyResult != backupCopyResultTruncate)
{
- fileResult->pageChecksumResult = pckDup(
- ioFilterGroupResultPackP(ioReadFilterGroup(readIo), PAGE_CHECKSUM_FILTER_TYPE));
- }
+ // Get bundle offset
+ fileResult->bundleOffset = bundleOffset;
- // Get results of block incremental
- if (file->blockIncrSize != 0)
- {
- fileResult->blockIncrMapSize = pckReadU64P(
- ioFilterGroupResultP(ioReadFilterGroup(readIo), BLOCK_INCR_FILTER_TYPE));
- }
+ // Get repo size
+ fileResult->repoSize = pckReadU64P(
+ ioFilterGroupResultP(ioReadFilterGroup(readIo), SIZE_FILTER_TYPE, .idx = 1));
- // Get repo checksum
- if (repoChecksum)
- {
- fileResult->repoChecksum = pckReadBinP(
- ioFilterGroupResultP(ioReadFilterGroup(readIo), CRYPTO_HASH_FILTER_TYPE, .idx = 1));
+ // Get results of page checksum validation
+ if (file->pgFileChecksumPage)
+ {
+ fileResult->pageChecksumResult = pckDup(
+ ioFilterGroupResultPackP(ioReadFilterGroup(readIo), PAGE_CHECKSUM_FILTER_TYPE));
+ }
+
+ // Get results of block incremental
+ if (file->blockIncrSize != 0)
+ {
+ fileResult->blockIncrMapSize = pckReadU64P(
+ ioFilterGroupResultP(ioReadFilterGroup(readIo), BLOCK_INCR_FILTER_TYPE));
+ }
+
+ // Get repo checksum
+ if (repoChecksum)
+ {
+ fileResult->repoChecksum = pckReadBinP(
+ ioFilterGroupResultP(ioReadFilterGroup(readIo), CRYPTO_HASH_FILTER_TYPE, .idx = 1));
+ }
}
}
MEM_CONTEXT_END();
diff --git a/src/command/backup/file.h b/src/command/backup/file.h
index 2685b53884..68e50fd66f 100644
--- a/src/command/backup/file.h
+++ b/src/command/backup/file.h
@@ -19,6 +19,7 @@ typedef enum
backupCopyResultReCopy,
backupCopyResultSkip,
backupCopyResultNoOp,
+ backupCopyResultTruncate,
} BackupCopyResult;
/***********************************************************************************************************************************
diff --git a/src/info/manifest.c b/src/info/manifest.c
index caf6d0a83c..891ea3a583 100644
--- a/src/info/manifest.c
+++ b/src/info/manifest.c
@@ -3050,6 +3050,7 @@ manifestFileUpdate(Manifest *const this, const ManifestFile *const file)
(!file->checksumPage && !file->checksumPageError && file->checksumPageErrorList == NULL) ||
(file->checksumPage && !file->checksumPageError && file->checksumPageErrorList == NULL) ||
(file->checksumPage && file->checksumPageError));
+ ASSERT(file->size != 0 || (file->bundleId == 0 && file->bundleOffset == 0));
ManifestFilePack **const filePack = manifestFilePackFindInternal(this, file->name);
manifestFilePackUpdate(this, filePack, file);
diff --git a/test/src/module/command/backupTest.c b/test/src/module/command/backupTest.c
index e866ac7b4f..c6f0631b6f 100644
--- a/test/src/module/command/backupTest.c
+++ b/test/src/module/command/backupTest.c
@@ -3608,7 +3608,7 @@ testRun(void)
"P00 INFO: check archive for segment 0000000105DC213000000000\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-larger (1.4MB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-grow (128KB, [PCT]) checksum [SHA1]\n"
- "P01 DETAIL: backup file " TEST_PATH "/pg1/truncate-to-zero (bundle 1/0, 4B->0B, [PCT])\n"
+ "P01 DETAIL: store truncated file " TEST_PATH "/pg1/truncate-to-zero (4B->0B, [PCT])\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/grow-to-block-incr (bundle 1/0, 16KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (bundle 1/16411, 8KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink (bundle 1/24603, 16.0KB, [PCT]) checksum [SHA1]\n"
@@ -3628,7 +3628,6 @@ testRun(void)
"bundle/1/pg_data/block-incr-shrink {file, s=16383}\n"
"bundle/1/pg_data/global/pg_control {file, s=8192}\n"
"bundle/1/pg_data/grow-to-block-incr {file, m=1:{0,1,2}, s=16385}\n"
- "bundle/1/pg_data/truncate-to-zero {file, s=0}\n"
"pg_data {path}\n"
"pg_data/backup_label {file, s=17}\n"
"pg_data/block-incr-grow.pgbi {file, m=0:{0},1:{0},0:{2},1:{1,2,3,4,5,6,7,8,9,10,11,12,13}, s=131072}\n"