diff --git a/server/src/engine/storage/v2/impls/mdl_journal.rs b/server/src/engine/storage/v2/impls/mdl_journal.rs index 7a0e87eb..fc27f58e 100644 --- a/server/src/engine/storage/v2/impls/mdl_journal.rs +++ b/server/src/engine/storage/v2/impls/mdl_journal.rs @@ -533,8 +533,9 @@ impl BatchAdapterSpec for ModelDataAdapter { f: &mut TrackedReaderContext, batch_info: &Self::BatchMetadata, event_type: Self::EventType, - _: &mut JournalHeuristics, + h: &mut JournalHeuristics, ) -> RuntimeResult<()> { + h.increment_server_event_count(); // get txn id let txn_id = u64::from_le_bytes(f.read_block()?); // get pk diff --git a/server/src/engine/storage/v2/mod.rs b/server/src/engine/storage/v2/mod.rs index 069c132d..a5228e96 100644 --- a/server/src/engine/storage/v2/mod.rs +++ b/server/src/engine/storage/v2/mod.rs @@ -144,6 +144,12 @@ pub fn restore(cfg: &Configuration) -> RuntimeResult { JournalSettings::default(), )?; if mdl_stats.recommended_action().needs_compaction() { + info!( + "{}.{} needs compaction due to {}", + id.space(), + id.entity(), + mdl_stats.recommended_action().reason_str() + ); model_driver = journal::compact_journal::>( &model_data_file_path, model_driver, diff --git a/server/src/engine/storage/v2/raw/journal/mod.rs b/server/src/engine/storage/v2/raw/journal/mod.rs index 5a02192c..8eda0280 100644 --- a/server/src/engine/storage/v2/raw/journal/mod.rs +++ b/server/src/engine/storage/v2/raw/journal/mod.rs @@ -154,6 +154,7 @@ impl RawJournalAdapter for EventLogAdapter { if this_checksum.finish() != expected_checksum { return Err(StorageError::RawJournalDecodeCorruptionInBatchMetadata.into()); } + heuristics.increment_server_event_count(); ::DECODE_DISPATCH [<::EventMeta as TaggedEnum>::dscr_u64(&meta) as usize]( gs, heuristics, pl, diff --git a/server/src/engine/storage/v2/raw/journal/raw/mod.rs b/server/src/engine/storage/v2/raw/journal/raw/mod.rs index eec5892a..2f8f63d1 100644 --- a/server/src/engine/storage/v2/raw/journal/raw/mod.rs +++ b/server/src/engine/storage/v2/raw/journal/raw/mod.rs @@ -965,7 +965,6 @@ impl JournalSettings { #[derive(Debug)] pub struct JournalStats { header: usize, - server_events: usize, driver_events: usize, heuristics: JournalHeuristics, file_size: usize, @@ -973,6 +972,7 @@ pub struct JournalStats { #[derive(Debug)] pub struct JournalHeuristics { + server_events: usize, redundant_records: usize, } @@ -985,6 +985,9 @@ impl JournalHeuristics { pub fn report_new_redundant_record(&mut self) { self.report_additional_redundant_records(1) } + pub fn increment_server_event_count(&mut self) { + self.server_events += 1; + } #[cfg(test)] pub fn get_current_redundant(&self) -> usize { self.redundant_records @@ -1002,6 +1005,13 @@ impl Recommendation { pub const fn needs_compaction(&self) -> bool { matches!(self, Self::CompactDrvHighRatio | Self::CompactRedHighRatio) } + pub const fn reason_str(&self) -> &'static str { + match self { + Self::NoActionNeeded => "no action needed", + Self::CompactDrvHighRatio => "drv_high_ratio", + Self::CompactRedHighRatio => "srv_high_redundancy", + } + } } impl JournalStats { @@ -1012,13 +1022,15 @@ impl JournalStats { } else { 4 * 1024 * 1024 }; - let total_records = self.server_events + self.driver_events; - let server_event_percentage = (self.server_events as f64 / total_records as f64) * 100.0; + let total_records = self.heuristics.server_events + self.driver_events; + let server_event_percentage = + (self.heuristics.server_events as f64 / total_records as f64) * 100.0; let driver_event_percentage = (self.driver_events as f64 / total_records as f64) * 100.0; - let redundant_record_percentage = if self.server_events == 0 { + let redundant_record_percentage = if self.heuristics.server_events == 0 { 0.0 } else { - (self.heuristics.redundant_records as f64 / self.server_events as f64) * 100.00 + (self.heuristics.redundant_records as f64 / self.heuristics.server_events as f64) + * 100.00 }; if self.file_size >= minimum_file_size_compaction_trigger { if driver_event_percentage >= server_event_percentage { @@ -1032,9 +1044,9 @@ impl JournalStats { } fn new() -> Self { Self { - server_events: 0, driver_events: 0, heuristics: JournalHeuristics { + server_events: 0, redundant_records: 0, }, file_size: 0, @@ -1286,7 +1298,6 @@ impl RawJournalReader { Ok(()) => { jtrace_reader!(ServerEventAppliedSuccess); Self::__refresh_known_txn(self); - self.stats.server_events += 1; return Ok(false); } Err(e) => return Err(e), diff --git a/server/src/engine/storage/v2/raw/journal/raw/tests/compaction.rs b/server/src/engine/storage/v2/raw/journal/raw/tests/compaction.rs index 75afd210..e51ec904 100644 --- a/server/src/engine/storage/v2/raw/journal/raw/tests/compaction.rs +++ b/server/src/engine/storage/v2/raw/journal/raw/tests/compaction.rs @@ -153,6 +153,7 @@ impl RawJournalAdapter for CompactDBAdapter { file: &mut TrackedReader, heuristics: &mut JournalHeuristics, ) -> RuntimeResult<()> { + heuristics.increment_server_event_count(); match meta { CompactDBEventKind::Insert => read_kv(file, gs), CompactDBEventKind::Update => {