Version 6.4 -

Added ACT storage config item 'no-defrag-reads' to allow simulation of Aerospike Server 7.0+ storage-backed storage-engine memory workloads.
Removed ACT storage config item 'commit-min-size' since it has been removed from Aerospike Server 7.0+.

README.md

@@ -566,6 +566,13 @@ storage write rate, which (for act_storage) is manifest as the large-block read
 and write rates.  For act_index, defragmentation generates an extra internal
 index device read and write load. The default defrag-lwm-pct is 50.
 
+**no-defrag-reads (act_storage ONLY)**
+Flag to model Aerospike 7.0+ storage-engine memory with device/file backing.
+This models defrag for a given write load as usual, but without the large-block
+reads.  To simulate storage-engine memory with device backing, in addition to
+setting this flag, do not specify a read load (read-reqs-per-sec), and do not
+set the tomb-raider flag.  The default no-defrag-reads is no.
+
 **compress-pct (act_storage ONLY)**
 Generate compressible data when writing to devices.  With compress-pct 100, the
 data is fully random (not compressible).  Lower values cause runs of zeros to
@@ -586,14 +593,6 @@ device I/O load with many small, variable-sized writes.  Large block writes (and
 reads) still occur to model defragmentation, but the rate of these is reduced.
 The default commit-to-device is no.
 
-**commit-min-bytes (act_storage ONLY)**
-Minimum size of a write in commit-to-device mode. Must be a power of 2. Each
-write rounds the record size up to a multiple of commit-min-bytes. If
-commit-min-bytes is configured smaller than the minimum I/O size allowed on the
-device, the record size will be rounded up to a multiple of the minimum I/O
-size.  The default commit-min-bytes is 0, meaning writes will round up to a
-multiple of the minimum I/O size.
-
 **tomb-raider (act_storage ONLY)**
 Flag to model the Aerospike tomb raider.  This simply spawns a thread per device
 in which the device is read from beginning to end, one large block at a time.

config/act_storage.conf

@@ -40,12 +40,12 @@ write-reqs-per-sec: 1000
 # replication-factor: 1
 # update-pct: 0
 # defrag-lwm-pct: 50
+# no-defrag-reads: no
 
 # compress-pct: 100
 # disable-odsync: no
 
 # commit-to-device: no
-# commit-min-bytes: 512? # default is detected minimum device IO size
 
 # tomb-raider: no
 # tomb-raider-sleep-usec: 0

src/common/version.h

@@ -1 +1 @@
-#define VERSION "6.3"
+#define VERSION "6.4"

src/storage/act_storage.c

@@ -260,8 +260,9 @@ main(int argc, char* argv[])
 		for (uint32_t n = 0; n < g_scfg.num_devices; n++) {
 			device* dev = &g_devices[n];
 
-			if (pthread_create(&dev->large_block_read_thread, NULL,
-					run_large_block_reads, (void*)dev) != 0) {
+			if (! g_scfg.no_defrag_reads &&
+					pthread_create(&dev->large_block_read_thread, NULL,
+							run_large_block_reads, (void*)dev) != 0) {
 				printf("ERROR: create large op read thread\n");
 				exit(-1);
 			}
@@ -767,37 +768,32 @@ discover_read_pattern(device* dev)
 static void
 discover_write_pattern(device* dev)
 {
-	// Use the larger of min-op bytes and configured commit-min-bytes.
-	dev->min_commit_bytes = dev->min_op_bytes > g_scfg.commit_min_bytes ?
-			dev->min_op_bytes : g_scfg.commit_min_bytes;
-
-	// Total number of "min-commit"-sized blocks on the device. (Excluding
+	// Total number of "min-op"-sized blocks on the device. (Excluding
 	// fractional large block at end of device, if such.)
-	uint64_t n_min_commit_blocks =
+	uint64_t n_min_op_blocks =
 			(dev->n_large_blocks * g_scfg.large_block_ops_bytes) /
-					dev->min_commit_bytes;
+					dev->min_op_bytes;
 
-	// Number of "min-commit"-sized blocks per (smallest) write request.
-	uint32_t write_req_min_commit_blocks =
-			(g_scfg.record_stored_bytes + dev->min_commit_bytes - 1) /
-					dev->min_commit_bytes;
+	// Number of "min-op"-sized blocks per (smallest) write request.
+	uint32_t write_req_min_op_blocks =
+			(g_scfg.record_stored_bytes + dev->min_op_bytes - 1) /
+					dev->min_op_bytes;
 
 	// Size in bytes per (smallest) write request.
-	dev->write_bytes = write_req_min_commit_blocks * dev->min_commit_bytes;
+	dev->write_bytes = write_req_min_op_blocks * dev->min_op_bytes;
 
-	// Number of "min-commit"-sized blocks per (largest) write request.
-	uint32_t write_req_min_commit_blocks_rmx =
-			(g_scfg.record_stored_bytes_rmx + dev->min_commit_bytes - 1) /
-					dev->min_commit_bytes;
+	// Number of "min-op"-sized blocks per (largest) write request.
+	uint32_t write_req_min_op_blocks_rmx =
+			(g_scfg.record_stored_bytes_rmx + dev->min_op_bytes - 1) /
+					dev->min_op_bytes;
 
 	// Number of write request sizes in configured range.
 	dev->n_write_sizes =
-			write_req_min_commit_blocks_rmx - write_req_min_commit_blocks + 1;
+			write_req_min_op_blocks_rmx - write_req_min_op_blocks + 1;
 
 	// Total number of sites on device to write to. (Make sure the last site
 	// has room for largest possible write request.)
-	dev->n_write_offsets =
-			n_min_commit_blocks - write_req_min_commit_blocks_rmx + 1;
+	dev->n_write_offsets = n_min_op_blocks - write_req_min_op_blocks_rmx + 1;
 }
 
 //------------------------------------------------

src/storage/cfg_storage.c

@@ -59,10 +59,10 @@ static const char TAG_LARGE_BLOCK_OP_KBYTES[]   = "large-block-op-kbytes";
 static const char TAG_REPLICATION_FACTOR[]      = "replication-factor";
 static const char TAG_UPDATE_PCT[]              = "update-pct";
 static const char TAG_DEFRAG_LWM_PCT[]          = "defrag-lwm-pct";
+static const char TAG_NO_DEFRAG_READS[]         = "no-defrag-reads";
 static const char TAG_COMPRESS_PCT[]            = "compress-pct";
 static const char TAG_DISABLE_ODSYNC[]          = "disable-odsync";
 static const char TAG_COMMIT_TO_DEVICE[]        = "commit-to-device";
-static const char TAG_COMMIT_MIN_BYTES[]        = "commit-min-bytes";
 static const char TAG_TOMB_RAIDER[]             = "tomb-raider";
 static const char TAG_TOMB_RAIDER_SLEEP_USEC[]  = "tomb-raider-sleep-usec";
 static const char TAG_MAX_LAG_SEC[]             = "max-lag-sec";
@@ -192,6 +192,9 @@ storage_configure(int argc, char* argv[])
 		else if (strcmp(tag, TAG_DEFRAG_LWM_PCT) == 0) {
 			g_scfg.defrag_lwm_pct = parse_uint32();
 		}
+		else if (strcmp(tag, TAG_NO_DEFRAG_READS) == 0) {
+			g_scfg.no_defrag_reads = parse_yes_no();
+		}
 		else if (strcmp(tag, TAG_COMPRESS_PCT) == 0) {
 			g_scfg.compress_pct = parse_uint32();
 		}
@@ -201,9 +204,6 @@ storage_configure(int argc, char* argv[])
 		else if (strcmp(tag, TAG_COMMIT_TO_DEVICE) == 0) {
 			g_scfg.commit_to_device = parse_yes_no();
 		}
-		else if (strcmp(tag, TAG_COMMIT_MIN_BYTES) == 0) {
-			g_scfg.commit_min_bytes = parse_uint32();
-		}
 		else if (strcmp(tag, TAG_TOMB_RAIDER) == 0) {
 			g_scfg.tomb_raider = parse_yes_no();
 		}
@@ -305,13 +305,6 @@ check_configuration()
 		return false;
 	}
 
-	if (g_scfg.commit_min_bytes != 0 &&
-			(g_scfg.commit_min_bytes > g_scfg.large_block_ops_bytes ||
-			! is_power_of_2(g_scfg.commit_min_bytes))) {
-		configuration_error(TAG_COMMIT_MIN_BYTES);
-		return false;
-	}
-
 	return true;
 }
 
@@ -374,6 +367,11 @@ derive_configuration()
 		g_scfg.large_block_writes_per_sec = g_scfg.large_block_reads_per_sec;
 	}
 
+	// To simulate the new storage-engine memory where defrag reads from RAM.
+	if (g_scfg.no_defrag_reads) {
+		g_scfg.large_block_reads_per_sec = 0;
+	}
+
 	// Non-zero load must be enough to calculate service thread rates safely.
 	uint32_t total_reqs_per_sec =
 			g_scfg.internal_read_reqs_per_sec +
@@ -430,14 +428,14 @@ echo_configuration()
 			g_scfg.update_pct);
 	printf("%s: %" PRIu32 "\n", TAG_DEFRAG_LWM_PCT,
 			g_scfg.defrag_lwm_pct);
+	printf("%s: %s\n", TAG_NO_DEFRAG_READS,
+			g_scfg.no_defrag_reads ? "yes" : "no");
 	printf("%s: %" PRIu32 "\n", TAG_COMPRESS_PCT,
 			g_scfg.compress_pct);
 	printf("%s: %s\n", TAG_DISABLE_ODSYNC,
 			g_scfg.disable_odsync ? "yes" : "no");
 	printf("%s: %s\n", TAG_COMMIT_TO_DEVICE,
 			g_scfg.commit_to_device ? "yes" : "no");
-	printf("%s: %" PRIu32 "\n", TAG_COMMIT_MIN_BYTES,
-			g_scfg.commit_min_bytes);
 	printf("%s: %s\n", TAG_TOMB_RAIDER,
 			g_scfg.tomb_raider ? "yes" : "no");
 	printf("%s: %" PRIu32 "\n", TAG_TOMB_RAIDER_SLEEP_USEC,

src/storage/cfg_storage.h

@@ -56,10 +56,10 @@ typedef struct storage_cfg_s {
 	uint32_t replication_factor;
 	uint32_t update_pct;
 	uint32_t defrag_lwm_pct;
+	bool no_defrag_reads;
 	uint32_t compress_pct;
 	bool disable_odsync;
 	bool commit_to_device;
-	uint32_t commit_min_bytes;
 	bool tomb_raider;
 	uint32_t tomb_raider_sleep_us;
 	uint64_t max_lag_usec;          // converted from literal units in seconds