diff --git a/cmd/koord-runtime-proxy/main.go b/cmd/koord-runtime-proxy/main.go
index 4b00fcc2b..264940e66 100644
--- a/cmd/koord-runtime-proxy/main.go
+++ b/cmd/koord-runtime-proxy/main.go
@@ -48,6 +48,7 @@ func main() {
 	if err != nil {
 		klog.Fatalf("Failed to create socket dir, err: %v", err)
 	}
+	defer os.Remove(options.RuntimeProxyEndpoint)
 
 	switch options.BackendRuntimeMode {
 	case options.BackendRuntimeModeContainerd:
@@ -62,6 +63,5 @@ func main() {
 
 	stopCh := genericapiserver.SetupSignalHandler()
 	<-stopCh
-	os.Remove(options.RuntimeProxyEndpoint)
 	klog.Info("RuntimeManager shutting down")
 }