diff --git a/oneflow/init.py b/oneflow/init.py
index 7c968dbe3ca..92090df08f8 100644
--- a/oneflow/init.py
+++ b/oneflow/init.py
@@ -69,9 +69,13 @@
 del env_util
 
 
-def SyncOnMaster():
-    if oneflow.distributed.get_rank() == 0:
-        oneflow._oneflow_internal.eager.single_client.Sync
+# capture oneflow methods so that they can be still accessed after `del oneflow`
+def _SyncOnMasterFn(get_rank, sync):
+    def SyncOnMaster():
+        if get_rank() == 0:
+            sync()
+
+    return SyncOnMaster
 
 
 atexit.register(oneflow._oneflow_internal.SetShuttingDown)
@@ -80,7 +84,12 @@ def SyncOnMaster():
 # Global<ResourceDesc, ForSession>::Get(), used by vm in background thread,
 # will be set to nullptr by TryCloseDefaultSession,
 # so sync vm in advance to avoid data race
-atexit.register(SyncOnMaster)
+atexit.register(
+    _SyncOnMasterFn(
+        oneflow.python.framework.distribute.get_rank,
+        oneflow._oneflow_internal.eager.single_client.Sync,
+    )
+)
 del atexit
 
 import sys