WT-10633 Add a backup source verify function to the test utilities. (#…

…8840)

* WT-10633 Add a backup source verify function to the test utilities.

* Remove loop and just cmp the entire unchanged range. Other minor fixes.

* Add arg to specify a source ID and move where it is checked in the test.

* Flush any verbose output before verifying.

* Remove checking against the backup directory. Only check the copied dir
before restart.

Eviction could change the source during a run so we cannot compare.

* Rename variable and change comment text per review.

* Minor comment updates.

* Update comments.

* Add optional ID arg to source verify function.
Remove changes that apply to another test/ticket.

* Update comment

* Remove missed srctest changes.

test/csuite/incr_backup/main.c

@@ -661,6 +661,8 @@ incr_backup(WT_CONNECTION *conn, const char *home, const char *backup_home, TABL
                 testutil_check(file_cursor->get_key(file_cursor, &offset, &size, &type));
                 testutil_assert(type == WT_BACKUP_FILE || type == WT_BACKUP_RANGE);
                 if (type == WT_BACKUP_RANGE) {
+                    VERBOSE(3, "RANGE: %s: offset %" PRIu64 " size %" PRIu64 "\n", filename, offset,
+                      size);
                     nrange++;
                     tmp = dcalloc(1, size);
 
@@ -827,7 +829,8 @@ main(int argc, char *argv[])
     uint32_t file_max, iter, max_value_size, next_checkpoint, rough_size, slot;
     int ch, ncheckpoints, nreopens, status;
     const char *backup_verbose, *working_dir;
-    char conf[1024], home[1024], backup_check[1024], backup_dir[1024], command[4096];
+    char backup_check[1024], backup_dir[1024], backup_src[1024], command[4096], conf[1024],
+      home[1024];
     bool preserve;
 
     preserve = false;
@@ -867,8 +870,10 @@ main(int argc, char *argv[])
         rnd.v = seed;
 
     testutil_work_dir_from_path(home, sizeof(home), working_dir);
-    testutil_check(__wt_snprintf(backup_dir, sizeof(backup_dir), "../%s.BACKUP", home));
-    testutil_check(__wt_snprintf(backup_check, sizeof(backup_check), "../%s.CHECK", home));
+    /* Put the backup directories as the same level as the home directory. */
+    testutil_check(__wt_snprintf(backup_check, sizeof(backup_check), "./%s.CHECK", home));
+    testutil_check(__wt_snprintf(backup_dir, sizeof(backup_dir), "./%s.BACKUP", home));
+    testutil_check(__wt_snprintf(backup_src, sizeof(backup_src), "./%s.BACKUP.SRC", home));
     printf("Seed: %" PRIu64 "\n", seed);
 
     testutil_check(
@@ -955,11 +960,20 @@ main(int argc, char *argv[])
         }
 
         /* Close and reopen the connection once in a while. */
-        if (__wt_random(&rnd) % 10 == 0) {
+        if (iter != 0 && __wt_random(&rnd) % 5 == 0) {
             VERBOSE(2, "Close and reopen the connection %d\n", nreopens);
             testutil_check(conn->close(conn, NULL));
+            /* Check the source bitmap after restart. Copy while closed. */
+            testutil_check(__wt_snprintf(command, sizeof(command),
+              "rm -rf %s; mkdir %s; cp -rp %s/* %s", backup_src, backup_src, home, backup_src));
+            if ((status = system(command)) < 0)
+                testutil_die(status, "system: %s", command);
+
             testutil_check(wiredtiger_open(home, NULL, conf, &conn));
             testutil_check(conn->open_session(conn, NULL, NULL, &session));
+
+            /* Test against the copied directory. */
+            testutil_verify_src_backup(conn, backup_src, home, NULL);
             nreopens++;
         }
 

test/utility/misc.c

@@ -42,7 +42,7 @@ const char *progname = "program name not set";
  * Backup directory initialize command, remove and re-create the primary backup directory, plus a
  * copy we maintain for recovery testing.
  */
-#define HOME_BACKUP_INIT_CMD "rm -rf %s/BACKUP %s/BACKUP.copy && mkdir %s/BACKUP %s/BACKUP.copy"
+#define HOME_BACKUP_INIT_CMD "rm -rf %s/BACKUP %s/BACKUP.copy && mkdir %s/BACKUP %s/BACKUP.copy "
 
 /*
  * testutil_die --
@@ -330,6 +330,95 @@ testutil_create_backup_directory(const char *home)
     free(cmd);
 }
 
+/*
+ * testutil_verify_src_backup --
+ *     Verify a backup source home directory against a backup directory for changes to blocks that
+ *     are not marked as changed. If an ID is given, then the backup directory is only compared
+ *     against that ID, otherwise walk and compare against all IDs.
+ */
+void
+testutil_verify_src_backup(WT_CONNECTION *conn, const char *backup, const char *home, char *srcid)
+{
+    struct stat sb;
+    WT_CURSOR *cursor, *file_cursor;
+    WT_DECL_RET;
+    WT_SESSION *session;
+    uint64_t cmp_size, offset, prev_offset, size, type;
+    int i, j, status;
+    char buf[1024], *filename, *id[WT_BLKINCR_MAX];
+    const char *idstr;
+
+    WT_CLEAR(buf);
+    testutil_check(conn->open_session(conn, NULL, NULL, &session));
+    testutil_check(session->open_cursor(session, "backup:query_id", NULL, buf, &cursor));
+    /*
+     * If we are given a source ID, use it. Otherwise query the backup and check against all IDs
+     * that exist in the system.
+     */
+    if (srcid == NULL) {
+        i = 0;
+        while ((ret = cursor->next(cursor)) == 0) {
+            testutil_check(cursor->get_key(cursor, &idstr));
+            id[i++] = dstrdup(idstr);
+        }
+        testutil_check(cursor->close(cursor));
+    } else {
+        id[0] = srcid;
+        id[1] = NULL;
+        i = 1;
+    }
+    testutil_assert(i <= WT_BLKINCR_MAX);
+
+    /* Go through each id and open a backup cursor on it to test incremental values. */
+    for (j = 0; j < i; ++j) {
+        testutil_check(__wt_snprintf(buf, sizeof(buf), "incremental=(src_id=%s)", id[j]));
+        testutil_check(session->open_cursor(session, "backup:", NULL, buf, &cursor));
+        while ((ret = cursor->next(cursor)) == 0) {
+            testutil_check(cursor->get_key(cursor, &filename));
+            testutil_check(__wt_snprintf(buf, sizeof(buf), "incremental=(file=%s)", filename));
+            testutil_check(session->open_cursor(session, NULL, cursor, buf, &file_cursor));
+            prev_offset = 0;
+            while ((ret = file_cursor->next(file_cursor)) == 0) {
+                testutil_check(file_cursor->get_key(file_cursor, &offset, &size, &type));
+                /* We only want to check ranges for files. So if it is a full file copy, ignore. */
+                if (type != WT_BACKUP_RANGE)
+                    break;
+                testutil_check(__wt_snprintf(buf, sizeof(buf), "%s/%s", backup, filename));
+                ret = stat(buf, &sb);
+                /*
+                 * The file may not exist in the backup directory. If the stat call doesn't succeed
+                 * skip this file. If we're skipping changed blocks go to the next one.
+                 */
+                if (ret != 0)
+                    break;
+                /*
+                 * If the block is changed we cannot check it (for differences, for example). The
+                 * source id may be older and we've already copied the block, or not, so we don't
+                 * know if it should be different or not. But if a block is indicated as unchanged
+                 * then it better be identical.
+                 */
+                if (offset > prev_offset) {
+                    /* Compare the unchanged chunk. */
+                    cmp_size = offset - prev_offset;
+                    testutil_check(__wt_snprintf(buf, sizeof(buf),
+                      "cmp -n %" PRIu64 " %s/%s %s/%s %" PRIu64 " %" PRIu64, cmp_size, home,
+                      filename, backup, filename, prev_offset, prev_offset));
+                    status = system(buf);
+                    if (status != 0)
+                        fprintf(stderr, "FAIL: status %d ID %s from cmd: %s\n", status, id[j], buf);
+                    testutil_assert(status == 0);
+                }
+                prev_offset = offset + size;
+            }
+            testutil_check(file_cursor->close(file_cursor));
+        }
+        testutil_check(cursor->close(cursor));
+        if (srcid == NULL)
+            free(id[j]);
+    }
+    testutil_check(session->close(session, NULL));
+}
+
 /*
  * testutil_copy_file --
  *     Copy a single file into the backup directories.

test/utility/test_util.h

@@ -469,6 +469,7 @@ void testutil_tiered_begin(TEST_OPTS *);
 void testutil_tiered_flush_complete(TEST_OPTS *, WT_SESSION *, void *);
 void testutil_tiered_sleep(TEST_OPTS *, WT_SESSION *, uint64_t, bool *);
 uint64_t testutil_time_us(WT_SESSION *);
+void testutil_verify_src_backup(WT_CONNECTION *, const char *, const char *, char *);
 void testutil_work_dir_from_path(char *, size_t, const char *);
 WT_THREAD_RET thread_append(void *);