diff --git a/mlir/lib/Conversion/XeVMToLLVM/XeVMToLLVM.cpp b/mlir/lib/Conversion/XeVMToLLVM/XeVMToLLVM.cpp
index 75888ba79447a..05485e6257915 100644
--- a/mlir/lib/Conversion/XeVMToLLVM/XeVMToLLVM.cpp
+++ b/mlir/lib/Conversion/XeVMToLLVM/XeVMToLLVM.cpp
@@ -915,7 +915,7 @@ class LLVMLoadStoreToOCLPattern : public OpConversionPattern<OpType> {
     std::optional<ArrayAttr> optCacheControls =
         getCacheControlMetadata(rewriter, op);
     if (!optCacheControls) {
-      op->removeAttr("cache_control");
+      rewriter.modifyOpInPlace(op, [&]() { op->removeAttr("cache_control"); });
       return success();
     }
 
@@ -929,8 +929,10 @@ class LLVMLoadStoreToOCLPattern : public OpConversionPattern<OpType> {
         rewriter, op->getLoc(), ptr, *optCacheControls, moduleOp);
 
     // Replace the pointer operand with the annotated one.
-    op->setOperand(ptrIdx, annotatedPtr);
-    op->removeAttr("cache_control");
+    rewriter.modifyOpInPlace(op, [&]() {
+      op->setOperand(ptrIdx, annotatedPtr);
+      op->removeAttr("cache_control");
+    });
     return success();
   }
 };