s

MystenLabs · Jun 21, 2024 · 943e55d · 943e55d
1 parent bbfaafc
commit 943e55d
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 4 deletions.
diff --git a/crates/sui-data-ingestion-core/src/progress_store/mod.rs b/crates/sui-data-ingestion-core/src/progress_store/mod.rs
@@ -38,10 +38,12 @@ impl<P: ProgressStore> ProgressStore for ProgressStoreWrapper<P> {
         task_name: String,
         checkpoint_number: CheckpointSequenceNumber,
     ) -> Result<()> {
-        self.progress_store
-            .save(task_name.clone(), checkpoint_number)
-            .await?;
-        self.pending_state.insert(task_name, checkpoint_number);
+        if checkpoint_number > self.load(task_name.clone()).await? {
+            self.progress_store
+                .save(task_name.clone(), checkpoint_number)
+                .await?;
+            self.pending_state.insert(task_name, checkpoint_number);
+        }
         Ok(())
     }
 }

diff --git a/crates/sui-data-ingestion-core/src/reader.rs b/crates/sui-data-ingestion-core/src/reader.rs
@@ -324,6 +324,7 @@ impl CheckpointReader {
         watcher
             .watch(&self.path, RecursiveMode::NonRecursive)
             .expect("Inotify watcher failed");
+        self.gc_processed_files(self.last_pruned_watermark).expect("Failed to clean the directory");
 
         loop {
             tokio::select! {