diff --git a/lazyflow/slot.py b/lazyflow/slot.py
index 1863c7a80..c68a9a9dc 100644
--- a/lazyflow/slot.py
+++ b/lazyflow/slot.py
@@ -932,7 +932,9 @@ def setDirty(self, *args, _mod_time: int = None, **kwargs):
                 try:
                     self.operator.propagateDirty(self.top_level_slot, self.subindex, roi)
                 finally:
-                    self.operator._previous_dirty_mod_time_buffer = self.operator._pending_dirty_mod_time
+                    self.operator._previous_dirty_mod_time_buffer = max(
+                        self.operator._previous_dirty_mod_time_buffer, self.operator._pending_dirty_mod_time
+                    )
                     self.operator._pending_dirty_mod_time = -1
 
     def __iter__(self):
diff --git a/tests/test_lazyflow/test_graph/test_dirty_modtime.py b/tests/test_lazyflow/test_graph/test_dirty_modtime.py
index 415439c54..7d65efbf1 100644
--- a/tests/test_lazyflow/test_graph/test_dirty_modtime.py
+++ b/tests/test_lazyflow/test_graph/test_dirty_modtime.py
@@ -30,6 +30,21 @@ def test_op_mod_time(graph):
         assert op._pending_dirty_mod_time == -1
 
 
+def test_op_lower_mod_time_does_not_modify(graph):
+    """setDirty _mod_time modifies parent op correctly"""
+    op = MockOp(graph=graph)
+
+    assert op._pending_dirty_mod_time == -1
+
+    op.Input.setDirty((), _mod_time=42)
+    assert op._previous_dirty_mod_time_buffer == 42
+    assert op._pending_dirty_mod_time == -1
+
+    op.Input.setDirty((), _mod_time=41)
+    assert op._previous_dirty_mod_time_buffer == 42
+    assert op._pending_dirty_mod_time == -1
+
+
 def test_op_mod_time_chain(graph):
     """mod_time is propagated to all ops in the chain"""
     op1 = MockOp(graph=graph)