Block-level incremental backup super blocks.

Small blocks sizes can lead to reduced compression efficiency, so allow multiple blocks to be compressed together in a super block. The disadvantage is that the super block must be read sequentially to retrieve blocks. However, different super block sizes can be used for different backup types, so the full backup super block sizes are large for compression efficiency and diff/incr are smaller for retrieval efficiency.

doc/xml/release.xml

@@ -45,12 +45,16 @@
                         <commit subject="Rename block incremental manifest keys."/>
                         <commit subject="Add optional raw format for compression types."/>
                         <commit subject="Use raw compression/encryption to bundling and block incremental backup."/>
+                        <commit subject="Block-level incremental backup super blocks.">
+                            <github-pull-request id="2011"/>
+                        </commit>
 
                         <release-item-contributor-list>
                             <release-item-contributor id="david.steele"/>
+                            <release-item-reviewer id="john.morris"/>
                         </release-item-contributor-list>
 
-                        <p>Block-level incremental backup.</p>
+                        <p>Block-level incremental backup (BETA).</p>
                     </release-item>
                 </release-feature-list>
 

src/Makefile.in

@@ -82,6 +82,7 @@ SRCS = \
 	command/repo/ls.c \
 	command/repo/put.c \
 	command/repo/rm.c \
+	command/restore/blockDelta.c \
 	command/restore/blockHash.c \
 	command/restore/file.c \
 	command/restore/protocol.c \

src/build/config/config.yaml

@@ -1866,6 +1866,25 @@ option:
   repo-block-age-map:
     inherit: repo-block-size-map
 
+  repo-block-size-super:
+    section: global
+    group: repo
+    type: size
+    default: 1MiB
+    allow-range: [128KiB, 16MiB]
+    internal: true
+    command: repo-block
+    command-role:
+      main: {}
+    depend:
+      option: repo-block
+      list:
+        - true
+
+  repo-block-size-super-full:
+    inherit: repo-block-size-super
+    default: 4MiB
+
   repo-cipher-pass:
     section: global
     type: string

src/build/help/help.xml

@@ -542,6 +542,26 @@
                         <example>16KiB=8KiB</example>
                     </config-key>
 
+                    <config-key id="repo-block-size-super" name="Block Incremental Super Block Size">
+                        <summary>Block incremental super block size.</summary>
+
+                        <text>
+                            <p>A super block contains multiple blocks to improve compression efficiency. The disadvantage is that block reads must read from the beginning of the super block.</p>
+                       </text>
+
+                        <example>2MiB</example>
+                    </config-key>
+
+                    <config-key id="repo-block-size-super-full" name="Block Incremental Full Super Block Size">
+                        <summary>Block incremental full super block size.</summary>
+
+                        <text>
+                            <p>Super block size for full backups. This is generally larger than <br-option>repo-block-size-super</br-option> because it is likely that many blocks will need to be restored from the full backup so the larger super block size is not a disadvantage.</p>
+                       </text>
+
+                        <example>8MiB</example>
+                    </config-key>
+
                     <config-key id="repo-bundle" name="Repository Bundles">
                         <summary>Bundle files in repository.</summary>
 

src/command/backup/backup.c

@@ -274,12 +274,16 @@ Build block incremental maps
 static const ManifestBlockIncrSizeMap manifestBlockIncrSizeMapDefault[] =
 {
     {.fileSize = 1024 * 1024 * 1024, .blockSize = 1024 * 1024},
-    {.fileSize = 256 * 1024 * 1024, .blockSize = 768 * 1024},
-    {.fileSize = 64 * 1024 * 1024, .blockSize = 512 * 1024},
-    {.fileSize = 16 * 1024 * 1024, .blockSize = 384 * 1024},
-    {.fileSize = 4 * 1024 * 1024, .blockSize = 256 * 1024},
-    {.fileSize = 2 * 1024 * 1024, .blockSize = 192 * 1024},
-    {.fileSize = 128 * 1024, .blockSize = 128 * 1024},
+    {.fileSize = 512 * 1024 * 1024, .blockSize = 768 * 1024},
+    {.fileSize = 256 * 1024 * 1024, .blockSize = 512 * 1024},
+    {.fileSize = 64 * 1024 * 1024, .blockSize = 384 * 1024},
+    {.fileSize = 16 * 1024 * 1024, .blockSize = 256 * 1024},
+    {.fileSize = 4 * 1024 * 1024, .blockSize = 192 * 1024},
+    {.fileSize = 2 * 1024 * 1024, .blockSize = 128 * 1024},
+    {.fileSize = 1024 * 1024, .blockSize = 64 * 1024},
+    {.fileSize = 512 * 1024, .blockSize = 32 * 1024},
+    {.fileSize = 128 * 1024, .blockSize = 16 * 1024},
+    {.fileSize = 16 * 1024, .blockSize = 8 * 1024},
 };
 
 // Age map
@@ -1590,6 +1594,7 @@ typedef struct BackupJobData
     uint64_t bundleLimit;                                           // Limit on files to bundle
     uint64_t bundleId;                                              // Bundle id
     const bool blockIncr;                                           // Block incremental?
+    size_t blockIncrSizeSuper;                                      // Super block size
 
     List *queueList;                                                // List of processing queues
 } BackupJobData;
@@ -1909,6 +1914,7 @@ backupJobCallback(void *data, unsigned int clientIdx)
                 if (blockIncr)
                 {
                     pckWriteU64P(param, file.blockIncrSize);
+                    pckWriteU64P(param, jobData->blockIncrSizeSuper);
 
                     if (file.blockIncrMapSize != 0)
                     {
@@ -2021,6 +2027,14 @@ backupProcess(
             jobData.bundleLimit = cfgOptionUInt64(cfgOptRepoBundleLimit);
         }
 
+        if (jobData.blockIncr)
+        {
+            // Set super block size based on the backup type
+            jobData.blockIncrSizeSuper =
+                backupType == backupTypeFull ?
+                    (size_t)cfgOptionUInt64(cfgOptRepoBlockSizeSuperFull) : (size_t)cfgOptionUInt64(cfgOptRepoBlockSizeSuper);
+        }
+
         // If this is a full backup or hard-linked and paths are supported then create all paths explicitly so that empty paths will
         // exist in to repo. Also create tablespace symlinks when symlinks are available. This makes it possible for the user to
         // make a copy of the backup path and get a valid cluster.