From 68c90e305d7ae3d329ea2892d0236b83b33e9f56 Mon Sep 17 00:00:00 2001
From: Guo Yejun <yejun.guo@intel.com>
Date: Tue, 7 Feb 2023 18:24:29 -0800
Subject: [PATCH] deepspeed/runtime/utils.py: reset_peak_memory_stats when
 empty cache

---
 deepspeed/runtime/utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/deepspeed/runtime/utils.py b/deepspeed/runtime/utils.py
index 3e2d454e6e63..2569a44fc4b3 100755
--- a/deepspeed/runtime/utils.py
+++ b/deepspeed/runtime/utils.py
@@ -810,6 +810,7 @@ def get_ma_status():
 
 def empty_cache():
     get_accelerator().empty_cache()
+    get_accelerator().reset_peak_memory_stats()
 
 
 def see_memory_usage(message, force=False):