diff --git a/ngraph_bridge/ngraph_executor.cc b/ngraph_bridge/ngraph_executor.cc
index 7d4fe2c2a..590ab70bb 100644
--- a/ngraph_bridge/ngraph_executor.cc
+++ b/ngraph_bridge/ngraph_executor.cc
@@ -294,13 +294,6 @@ NGraphExecutor::CreateCallback(const std::string signature,
 
   // Serialize to nGraph if needed
   if (std::getenv("NGRAPH_ENABLE_SERIALIZE") != nullptr) {
-    std::string file_name = "tf_function_" + m_node_name + ".json";
-    auto status_ser = StringToFile("tf_function_" + m_node_name + ".json",
-                                   serialized_ng_func);
-    if (status_ser != Status::OK()) {
-      return std::make_pair(status_ser,
-                            std::make_tuple(ng_exec, serialized_ng_func, pts));
-    }
 #if defined NGRAPH_DISTRIBUTED
     int rank_id;
     rank_id = ng::get_distributed_interface()->get_rank();
@@ -311,6 +304,13 @@ NGraphExecutor::CreateCallback(const std::string signature,
       return std::make_pair(status,
                             std::make_tuple(ng_exec, serialized_ng_func, pts));
     }
+#else
+    auto status_ser = StringToFile("tf_function_" + m_node_name + ".json",
+                                   serialized_ng_func);
+    if (status_ser != Status::OK()) {
+      return std::make_pair(status_ser,
+                            std::make_tuple(ng_exec, serialized_ng_func, pts));
+    }
 #endif
   }
   // Get NgExecutable