diff --git a/.github/workflows/ubuntu-22.04-build.yml b/.github/workflows/ubuntu-22.04-build.yml index af4ae6d51..dcde1e0f3 100644 --- a/.github/workflows/ubuntu-22.04-build.yml +++ b/.github/workflows/ubuntu-22.04-build.yml @@ -45,7 +45,7 @@ jobs: - name: check run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check - name: check30 - run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check30 + run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt,@testsuite/skiplist/proto30.txt make check30 - name: check29 run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt,@testsuite/skiplist/proto29.txt make check29 - name: check (TCP daemon transport) diff --git a/.github/workflows/ubuntu-build.yml b/.github/workflows/ubuntu-build.yml index 3531c518c..1a33b1dca 100644 --- a/.github/workflows/ubuntu-build.yml +++ b/.github/workflows/ubuntu-build.yml @@ -41,7 +41,7 @@ jobs: - name: check run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check - name: check30 - run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt make check30 + run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt,@testsuite/skiplist/proto30.txt make check30 - name: check29 run: sudo RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/common.txt,@testsuite/skiplist/linux.txt,@testsuite/skiplist/proto29.txt make check29 - name: check (TCP daemon transport) diff --git a/fileio.c b/fileio.c index c4cf57829..b8db29b85 100644 --- a/fileio.c +++ b/fileio.c @@ -34,12 +34,16 @@ #define ALIGNED_LENGTH(len) ((((len) - 1) | (ALIGN_BOUNDARY-1)) + 1) extern int sparse_files; +extern struct stats stats; OFF_T preallocated_len = 0; static OFF_T sparse_seek = 0; static OFF_T sparse_past_write = 0; +static int last_tracked_fd = -1; +static int64 last_touched_blk = -1; + int sparse_end(int f, OFF_T size, int updating_basis_or_equiv) { int ret = 0; @@ -161,6 +165,8 @@ static int write_sparse(int f, int use_seek, OFF_T offset, const char *buf, int continue; } if (i > start) { + if (!use_seek) + track_block_touches(f, offset + start, i - start); if (emit_sparse_span(f, use_seek, buf + start, i - start) < 0) return -1; sparse_past_write = offset + i; @@ -170,6 +176,8 @@ static int write_sparse(int f, int use_seek, OFF_T offset, const char *buf, int start = i; } if (end > start) { + if (!use_seek) + track_block_touches(f, offset + start, end - start); if (emit_sparse_span(f, use_seek, buf + start, end - start) < 0) return -1; } @@ -201,6 +209,38 @@ int flush_write_file(int f) return ret; } +void reset_block_tracker(void) +{ + last_tracked_fd = -1; + last_touched_blk = -1; +} + +void track_block_touches(int f, OFF_T offset, int32 len) +{ + int64 start_blk, end_blk, blocks_to_add = 0; + + if (len <= 0) + return; + if (f != last_tracked_fd) { + last_tracked_fd = f; + last_touched_blk = -1; + } + start_blk = offset / 4096; + end_blk = start_blk + (((offset % 4096) + len - 1) / 4096); + if (start_blk > last_touched_blk) + blocks_to_add = end_blk - start_blk + 1; + else if (end_blk > last_touched_blk) + blocks_to_add = end_blk - last_touched_blk; + if (blocks_to_add > 0) { + if (INT64_MAX - stats.touched_blocks_4k < blocks_to_add) + stats.touched_blocks_4k = INT64_MAX; + else + stats.touched_blocks_4k += blocks_to_add; + } + if (end_blk > last_touched_blk) + last_touched_blk = end_blk; +} + /* write_file does not allow incomplete writes. It loops internally * until len bytes are written or errno is set. Note that use_seek and * offset are only used in sparse processing (see write_sparse()). */ @@ -208,6 +248,8 @@ int write_file(int f, int use_seek, OFF_T offset, const char *buf, int len) { int ret = 0; + if (!use_seek && sparse_files == 0) + track_block_touches(f, offset, len); while (len > 0) { int r1; if (sparse_files > 0) { diff --git a/io.c b/io.c index 451d9b450..44d81ccc3 100644 --- a/io.c +++ b/io.c @@ -1692,6 +1692,18 @@ static void read_a_msg(void) raw_read_buf((char*)&stats.total_read, sizeof stats.total_read); iobuf.in_multiplexed = 1; break; + case MSG_BLOCK_STATS: { + char b[8]; + + if (msg_bytes != 8 || protocol_version < 33 || (!am_generator && !am_sender)) + goto invalid_msg; + raw_read_buf(b, 8); + stats.touched_blocks_4k = IVAL64(b, 0); + iobuf.in_multiplexed = 1; + if (am_server && am_generator) + send_msg(MSG_BLOCK_STATS, b, sizeof b, 0); + break; + } case MSG_REDO: if (msg_bytes != 4 || !am_generator) goto invalid_msg; diff --git a/main.c b/main.c index 8381935cf..604294927 100644 --- a/main.c +++ b/main.c @@ -443,6 +443,9 @@ static void output_summary(void) human_num(stats.total_transferred_size)); rprintf(FINFO,"Literal data: %s bytes\n", human_num(stats.literal_data)); + if (protocol_version >= 33) + rprintf(FINFO,"Number of 4 KiB logical blocks touched: %s\n", + comma_num(stats.touched_blocks_4k)); rprintf(FINFO,"Matched data: %s bytes\n", human_num(stats.matched_data)); rprintf(FINFO,"File list size: %s\n", @@ -1107,6 +1110,11 @@ static int do_recv(int f_in, int f_out, char *local_name) write_int(f_out, NDX_DONE); send_msg(MSG_STATS, (char*)&stats.total_read, sizeof stats.total_read, 0); + if (protocol_version >= 33) { + char b[8]; + SIVAL64(b, 0, stats.touched_blocks_4k); + send_msg(MSG_BLOCK_STATS, b, sizeof b, 0); + } io_flush(FULL_FLUSH); /* Handle any keep-alive packets from the post-processing work diff --git a/receiver.c b/receiver.c index 28d663acc..20d0b21f7 100644 --- a/receiver.c +++ b/receiver.c @@ -472,6 +472,9 @@ static int receive_data(int f_in, char *fname_r, int fd_r, OFF_T size_r, int32 i; char *map = NULL; + /* Reset the block tracker's per-file state in case the OS reuses an fd. */ + reset_block_tracker(); + #ifdef SUPPORT_PREALLOCATION if (preallocate_files && fd != -1 && total_size > 0 && (!inplace_sizing || total_size > size_r)) { /* Try to preallocate enough space for file's eventual length. Can diff --git a/rsync.1.md b/rsync.1.md index ad686b646..d1c46c58d 100644 --- a/rsync.1.md +++ b/rsync.1.md @@ -3537,6 +3537,12 @@ sign) if you want the local shell to expand it. Note that this line is only output if deletions are in effect, and only if the negotiated protocol is at least 31 (the default when both sides are 3.1.0, September 2013, or newer). + - `Number of 4 KiB logical blocks touched` is the number of unique 4 KiB + logical file regions covered by rsync's own write operations. This metric + decouples network payload from the scope of local file modifications. + For example, a small amount of *Literal data* can result in a massive + number of touched blocks if the modifications are highly scattered + across a file, especially when using `--inplace`. - `Number of regular files transferred` is the count of normal files that were updated via rsync's delta-transfer algorithm, which does not include directories, symlinks, etc. diff --git a/rsync.h b/rsync.h index b15aa1af6..e23b188cc 100644 --- a/rsync.h +++ b/rsync.h @@ -111,7 +111,7 @@ /* Update this if you make incompatible changes and ALSO update the * SUBPROTOCOL_VERSION if it is not a final (official) release. */ -#define PROTOCOL_VERSION 32 +#define PROTOCOL_VERSION 33 /* This is used when working on a new protocol version or for any unofficial * protocol tweaks. It should be a non-zero value for each pre-release repo @@ -125,7 +125,7 @@ * All older protocol versions MUST be compatible with the final, official * release of the protocol, so don't tweak the code to change the protocol * behavior for an older protocol version. */ -#define SUBPROTOCOL_VERSION 0 +#define SUBPROTOCOL_VERSION 8392 /* For testing */ /* We refuse to interoperate with versions that are not in this range. * Note that we assume we'll work with later versions: the onus is on @@ -299,6 +299,7 @@ enum msgcode { MSG_LOG=FLOG, MSG_CLIENT=FCLIENT, /* sibling logging */ MSG_REDO=9, /* reprocess indicated flist index */ MSG_STATS=10, /* message has stats data for generator */ + MSG_BLOCK_STATS=11,/* message has block-level stats for sender */ MSG_IO_ERROR=22,/* the sending side had an I/O error */ MSG_IO_TIMEOUT=33,/* tell client about a daemon's timeout value */ MSG_NOOP=42, /* a do-nothing message (legacy protocol-30 only) */ @@ -1080,6 +1081,7 @@ struct stats { int64 total_read; int64 literal_data; int64 matched_data; + int64 touched_blocks_4k; int64 flist_buildtime; int64 flist_xfertime; int64 flist_size; diff --git a/testsuite/skiplist/README.md b/testsuite/skiplist/README.md index 31dd457ed..b7bfbe72f 100644 --- a/testsuite/skiplist/README.md +++ b/testsuite/skiplist/README.md @@ -27,6 +27,7 @@ different tests merge cleanly. | `macos.txt` | macOS-only additions | | `cygwin.txt` | Cygwin-only additions | | `proto29.txt` | additions for a `--protocol=29` run, on any platform | +| `proto30.txt` | additions for a `--protocol=30` run, on any platform | Compose them with commas; the result is the union, so listing a test twice is harmless. Plain test names may be mixed in with `@FILE` entries. diff --git a/testsuite/skiplist/proto29.txt b/testsuite/skiplist/proto29.txt index 2ec82b850..9ce01019d 100644 --- a/testsuite/skiplist/proto29.txt +++ b/testsuite/skiplist/proto29.txt @@ -14,3 +14,4 @@ daemon-copylinks-parent-target-regression # the stdio_daemon client speaks prot partial-protected-regular-retry-linux # one-inplace partial staging needs protocol >= 30 (forced 29) scanner-batch-flag-mismatch # xattrs (-X) need protocol 30+ symlink-exclude-xattr # xattr (-X) transfer requires protocol 30+ (negotiated 29) +write-touched-blocks # logical-block statistics require protocol 33+ (forced 29) diff --git a/testsuite/skiplist/proto30.txt b/testsuite/skiplist/proto30.txt new file mode 100644 index 000000000..f3575cfbe --- /dev/null +++ b/testsuite/skiplist/proto30.txt @@ -0,0 +1,9 @@ +# Tests expected to SKIP. One name per line, '#' starts a comment; the file +# must stay sorted and duplicate-free (runtests.py enforces both). Referenced +# from a workflow as RSYNC_EXPECT_SKIPPED=@testsuite/skiplist/[,@...]. +# See testsuite/skiplist/README.md. +# +# Additions for a --protocol=30 run (make check30), on top of the platform +# files. + +write-touched-blocks # logical-block statistics require protocol 33+ (forced 30) diff --git a/testsuite/write-touched-blocks_test.py b/testsuite/write-touched-blocks_test.py new file mode 100644 index 000000000..0e58571ee --- /dev/null +++ b/testsuite/write-touched-blocks_test.py @@ -0,0 +1,171 @@ +import os +import shlex +import subprocess +import sys +from pathlib import Path + +from rsyncfns import ( + FROMDIR, RSYNC, SCRATCHDIR, + makepath, test_fail, test_skipped +) + +rsync_args = shlex.split(str(RSYNC)) +for arg in rsync_args: + if arg.startswith('--protocol='): + prot_version = int(arg.split('=')[1]) + if prot_version < 33: + test_skipped(f"Skipping write-touched-blocks: feature requires protocol 33, but CI forced {prot_version}") + +src = FROMDIR +makepath(src) + +base_file = src / 'base.bin' + +# Generate 4 MiB of random data in memory +data = os.urandom(4 * 1024 * 1024) + +def setup_test(dest_name): + """Resets the base file and creates a clean destination file.""" + dest_path = SCRATCHDIR / dest_name + base_file.write_bytes(data) + dest_path.write_bytes(data) + return dest_path + +def run_client(src_path, dest_path): + rsync_cmd = shlex.split(str(RSYNC)) + argv = rsync_cmd + ['-a', '--stats', '--inplace', '-I', '--no-whole-file', + str(src_path), str(dest_path)] + return subprocess.run(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True) + +# TEST 1: Contiguous Write (1 Block) +dest_contig = setup_test('dest_contiguous.bin') +with open(base_file, 'r+b') as f: + f.write(b'\x00' * 3000) + +proc = run_client(base_file, dest_contig) +if proc.returncode != 0: + test_fail(f"rsync failed on contiguous test:\n{proc.stdout}") +if "Number of 4 KiB logical blocks touched: 1\n" not in proc.stdout: + test_fail(f"Contiguous check failed! Expected 1 block. Output:\n{proc.stdout}") + +# TEST 2: Scattered Write (10 Blocks) +dest_scatter = setup_test('dest_scattered.bin') +with open(base_file, 'r+b') as f: + for i in range(1, 11): + f.seek(i * 4096) + old_byte = f.read(1)[0] + f.seek(i * 4096) + f.write(bytes([old_byte ^ 0xFF])) + +proc = run_client(base_file, dest_scatter) +if proc.returncode != 0: + test_fail(f"rsync failed on scattered test:\n{proc.stdout}") +if "Number of 4 KiB logical blocks touched: 10\n" not in proc.stdout: + test_fail(f"Scattered check failed! Expected 10 blocks. Output:\n{proc.stdout}") + +# TEST 3: Identical Files (0 Blocks Edge Case) +dest_zero = setup_test('dest_zero.bin') + +proc = run_client(base_file, dest_zero) +if proc.returncode != 0: + test_fail(f"rsync failed on zero-block test:\n{proc.stdout}") +if "Number of 4 KiB logical blocks touched: 0\n" not in proc.stdout: + test_fail(f"Zero check failed! Expected 0 blocks. Output:\n{proc.stdout}") + +# TEST 4: Full File Write (1,024 Blocks) +dest_full = setup_test('dest_full.bin') +# Overwrite the entire 4 MiB base file with brand new random data +# This forces the delta algorithm to find 0 matches and write all 1,024 blocks. +base_file.write_bytes(os.urandom(4 * 1024 * 1024)) + +proc = run_client(base_file, dest_full) +if proc.returncode != 0: + test_fail(f"rsync failed on full write test:\n{proc.stdout}") +if "Number of 4 KiB logical blocks touched: 1,024\n" not in proc.stdout: + test_fail(f"Full write check failed! Expected 1,024 blocks. Output:\n{proc.stdout}") + +# TEST 5: Sparse File Write (Hole skipping) +sparse_src = src / 'sparse.bin' +sparse_dest = SCRATCHDIR / 'sparse_dest.bin' + +# Create a file with written data on the ends, but a massive 4 MiB hole in the middle. +# 1 block data + 1,024 blocks hole + 1 block data = 1,026 blocks total size. +with open(sparse_src, 'wb') as f: + f.write(os.urandom(4096)) # Block 1 (Data) + f.seek(4 * 1024 * 1024, os.SEEK_CUR) # The Hole (4 MiB of nothing) + f.write(os.urandom(4096)) # Block 1026 (Data) + +# We must run this WITH --sparse (-S) and WITHOUT --inplace to force +# the receiver to create a brand new sparse file from scratch using write_sparse(). +rsync_cmd = shlex.split(str(RSYNC)) +argv_sparse = rsync_cmd + ['-a', '--stats', '--sparse', str(sparse_src), str(sparse_dest)] +proc = subprocess.run(argv_sparse, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + +if proc.returncode != 0: + test_fail(f"rsync failed on sparse test:\n{proc.stdout}") + +# Even though the file is over 4 MiB in size, only 2 logical 4K blocks +# should be written because the rest was skipped via lseek. +if "Number of 4 KiB logical blocks touched: 2\n" not in proc.stdout: + test_fail(f"Sparse check failed! Expected 2 logical blocks written. Output:\n{proc.stdout}") + +# TEST 6: Multiple Files +# Creates two separate 4KB files. If the tracker fails to reset between +# files due to FD recycling, it will report 1 block instead of 2. +fd_src_dir = src / 'fd_test' +fd_dest_dir = SCRATCHDIR / 'fd_dest' +makepath(fd_src_dir) +makepath(fd_dest_dir) + +# Create two distinct 1-block files +(fd_src_dir / 'fileA.bin').write_bytes(os.urandom(4096)) +(fd_src_dir / 'fileB.bin').write_bytes(os.urandom(4096)) + +# Sync the whole directory so rsync processes both in one process lifespan +rsync_cmd = shlex.split(str(RSYNC)) +argv_fd = rsync_cmd + ['-a', '--stats', '--inplace', str(fd_src_dir) + '/', str(fd_dest_dir) + '/'] +proc = subprocess.run(argv_fd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + +if proc.returncode != 0: + test_fail(f"rsync failed on multiple files test:\n{proc.stdout}") + +if "Number of 4 KiB logical blocks touched: 2\n" not in proc.stdout: + test_fail(f"FD Reuse bug confirmed! Expected 2 blocks (1 per file). Output:\n{proc.stdout}") + +# TEST 7: Batch Mode State Reset (--read-batch) +batch_src_dir = src / 'batch_src' +batch_dest_dir = SCRATCHDIR / 'batch_dest' +batch_file = SCRATCHDIR / 'test_batch.rsync' +makepath(batch_src_dir) +makepath(batch_dest_dir) + +# Create two 4KB source files with random data +(batch_src_dir / 'fileA.bin').write_bytes(os.urandom(4096)) +(batch_src_dir / 'fileB.bin').write_bytes(os.urandom(4096)) + +# Create two 4KB zeroed destination files (forces the delta algorithm to write exactly 1 block per file) +(batch_dest_dir / 'fileA.bin').write_bytes(b'\x00' * 4096) +(batch_dest_dir / 'fileB.bin').write_bytes(b'\x00' * 4096) + +# Step 1: Generate the batch file (Sender side) +# We MUST use '-I' because the files have identical sizes and timestamps. +rsync_cmd = shlex.split(str(RSYNC)) +argv_write_batch = rsync_cmd + ['-a', '-I', '--only-write-batch=' + str(batch_file), + str(batch_src_dir) + '/', str(batch_dest_dir) + '/'] +subprocess.run(argv_write_batch, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + +# Step 2: Apply the batch file and collect stats (Receiver side) +argv_read_batch = rsync_cmd + ['-a', '-I', '--inplace', '--stats', + '--read-batch=' + str(batch_file), str(batch_dest_dir) + '/'] + +proc = subprocess.run(argv_read_batch, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + +if proc.returncode != 0: + test_fail(f"rsync failed on read-batch test:\n{proc.stdout}") + +# If the state leaked across the batch read, this would output 1 block. +if "Number of 4 KiB logical blocks touched: 2\n" not in proc.stdout: + test_fail(f"Batch Mode tracker check failed! Expected 2 blocks (1 per file). Output:\n{proc.stdout}") + +print("write-touched-blocks: cleanly distinguishes contiguous, scattered, zero, full, sparse, multi-file, and batch writes")