storage: Fix false positives in compaction triggers

skytable · Apr 24, 2024 · b110071 · b110071
1 parent 548f806
commit b110071
Show file tree

Hide file tree

Showing 5 changed files with 28 additions and 8 deletions.
diff --git a/server/src/engine/storage/v2/impls/mdl_journal.rs b/server/src/engine/storage/v2/impls/mdl_journal.rs
@@ -533,8 +533,9 @@ impl BatchAdapterSpec for ModelDataAdapter {
         f: &mut TrackedReaderContext<Self::Spec>,
         batch_info: &Self::BatchMetadata,
         event_type: Self::EventType,
-        _: &mut JournalHeuristics,
+        h: &mut JournalHeuristics,
     ) -> RuntimeResult<()> {
+        h.increment_server_event_count();
         // get txn id
         let txn_id = u64::from_le_bytes(f.read_block()?);
         // get pk

diff --git a/server/src/engine/storage/v2/mod.rs b/server/src/engine/storage/v2/mod.rs
@@ -144,6 +144,12 @@ pub fn restore(cfg: &Configuration) -> RuntimeResult<SELoaded> {
                 JournalSettings::default(),
             )?;
             if mdl_stats.recommended_action().needs_compaction() {
+                info!(
+                    "{}.{} needs compaction due to {}",
+                    id.space(),
+                    id.entity(),
+                    mdl_stats.recommended_action().reason_str()
+                );
                 model_driver = journal::compact_journal::<true, BatchAdapter<ModelDataAdapter>>(
                     &model_data_file_path,
                     model_driver,

diff --git a/server/src/engine/storage/v2/raw/journal/mod.rs b/server/src/engine/storage/v2/raw/journal/mod.rs
@@ -154,6 +154,7 @@ impl<EL: EventLogSpec> RawJournalAdapter for EventLogAdapter<EL> {
         if this_checksum.finish() != expected_checksum {
             return Err(StorageError::RawJournalDecodeCorruptionInBatchMetadata.into());
         }
+        heuristics.increment_server_event_count();
         <EL as EventLogSpec>::DECODE_DISPATCH
             [<<EL as EventLogSpec>::EventMeta as TaggedEnum>::dscr_u64(&meta) as usize](
             gs, heuristics, pl,

diff --git a/server/src/engine/storage/v2/raw/journal/raw/mod.rs b/server/src/engine/storage/v2/raw/journal/raw/mod.rs
@@ -965,14 +965,14 @@ impl JournalSettings {
 #[derive(Debug)]
 pub struct JournalStats {
     header: usize,
-    server_events: usize,
     driver_events: usize,
     heuristics: JournalHeuristics,
     file_size: usize,
 }
 
 #[derive(Debug)]
 pub struct JournalHeuristics {
+    server_events: usize,
     redundant_records: usize,
 }
 
@@ -985,6 +985,9 @@ impl JournalHeuristics {
     pub fn report_new_redundant_record(&mut self) {
         self.report_additional_redundant_records(1)
     }
+    pub fn increment_server_event_count(&mut self) {
+        self.server_events += 1;
+    }
     #[cfg(test)]
     pub fn get_current_redundant(&self) -> usize {
         self.redundant_records
@@ -1002,6 +1005,13 @@ impl Recommendation {
     pub const fn needs_compaction(&self) -> bool {
         matches!(self, Self::CompactDrvHighRatio | Self::CompactRedHighRatio)
     }
+    pub const fn reason_str(&self) -> &'static str {
+        match self {
+            Self::NoActionNeeded => "no action needed",
+            Self::CompactDrvHighRatio => "drv_high_ratio",
+            Self::CompactRedHighRatio => "srv_high_redundancy",
+        }
+    }
 }
 
 impl JournalStats {
@@ -1012,13 +1022,15 @@ impl JournalStats {
         } else {
             4 * 1024 * 1024
         };
-        let total_records = self.server_events + self.driver_events;
-        let server_event_percentage = (self.server_events as f64 / total_records as f64) * 100.0;
+        let total_records = self.heuristics.server_events + self.driver_events;
+        let server_event_percentage =
+            (self.heuristics.server_events as f64 / total_records as f64) * 100.0;
         let driver_event_percentage = (self.driver_events as f64 / total_records as f64) * 100.0;
-        let redundant_record_percentage = if self.server_events == 0 {
+        let redundant_record_percentage = if self.heuristics.server_events == 0 {
             0.0
         } else {
-            (self.heuristics.redundant_records as f64 / self.server_events as f64) * 100.00
+            (self.heuristics.redundant_records as f64 / self.heuristics.server_events as f64)
+                * 100.00
         };
         if self.file_size >= minimum_file_size_compaction_trigger {
             if driver_event_percentage >= server_event_percentage {
@@ -1032,9 +1044,9 @@ impl JournalStats {
     }
     fn new<J: RawJournalAdapter>() -> Self {
         Self {
-            server_events: 0,
             driver_events: 0,
             heuristics: JournalHeuristics {
+                server_events: 0,
                 redundant_records: 0,
             },
             file_size: 0,
@@ -1286,7 +1298,6 @@ impl<J: RawJournalAdapter> RawJournalReader<J> {
                         Ok(()) => {
                             jtrace_reader!(ServerEventAppliedSuccess);
                             Self::__refresh_known_txn(self);
-                            self.stats.server_events += 1;
                             return Ok(false);
                         }
                         Err(e) => return Err(e),

diff --git a/server/src/engine/storage/v2/raw/journal/raw/tests/compaction.rs b/server/src/engine/storage/v2/raw/journal/raw/tests/compaction.rs
@@ -153,6 +153,7 @@ impl RawJournalAdapter for CompactDBAdapter {
         file: &mut TrackedReader<Self::Spec>,
         heuristics: &mut JournalHeuristics,
     ) -> RuntimeResult<()> {
+        heuristics.increment_server_event_count();
         match meta {
             CompactDBEventKind::Insert => read_kv(file, gs),
             CompactDBEventKind::Update => {