diff --git a/pkg/agent/run.go b/pkg/agent/run.go
index dacdf08f5c87..4a3dd0691b9a 100644
--- a/pkg/agent/run.go
+++ b/pkg/agent/run.go
@@ -344,13 +344,13 @@ func configureNode(ctx context.Context, nodeConfig *daemonconfig.Node, nodes typ
 		}
 
 		// inject node config
-		if changed, err := nodeconfig.SetNodeConfigAnnotations(node); err != nil {
+		if changed, err := nodeconfig.SetNodeConfigAnnotations(nodeConfig, node); err != nil {
 			return false, err
 		} else if changed {
 			updateNode = true
 		}
 
-		if changed, err := nodeconfig.SetNodeConfigLabels(node); err != nil {
+		if changed, err := nodeconfig.SetNodeConfigLabels(nodeConfig, node); err != nil {
 			return false, err
 		} else if changed {
 			updateNode = true
diff --git a/pkg/nodeconfig/nodeconfig.go b/pkg/nodeconfig/nodeconfig.go
index ee6dc46ac7c6..237a77617b67 100644
--- a/pkg/nodeconfig/nodeconfig.go
+++ b/pkg/nodeconfig/nodeconfig.go
@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/k3s-io/k3s/pkg/configfilearg"
+	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
@@ -73,7 +74,7 @@ func getNodeEnv() (string, error) {
 // environment variables as annotations on the node object. It also stores a
 // hash of the combined args + variables. These are used by other components
 // to determine if the node configuration has been changed.
-func SetNodeConfigAnnotations(node *corev1.Node) (bool, error) {
+func SetNodeConfigAnnotations(nodeConfig *config.Node, node *corev1.Node) (bool, error) {
 	nodeArgs, err := getNodeArgs()
 	if err != nil {
 		return false, err
@@ -106,13 +107,22 @@ func SetNodeConfigAnnotations(node *corev1.Node) (bool, error) {
 // that may not be present on down-level or up-level nodes.
 // These labels are used by other components to determine whether
 // or not a node supports particular functionality.
-func SetNodeConfigLabels(node *corev1.Node) (bool, error) {
+func SetNodeConfigLabels(nodeConfig *config.Node, node *corev1.Node) (bool, error) {
 	if node.Labels == nil {
 		node.Labels = make(map[string]string)
 	}
-	if _, ok := node.Labels[ClusterEgressLabel]; !ok {
-		node.Labels[ClusterEgressLabel] = "true"
-		return true, nil
+	_, hasLabel := node.Labels[ClusterEgressLabel]
+	switch nodeConfig.EgressSelectorMode {
+	case config.EgressSelectorModeCluster, config.EgressSelectorModePod:
+		if !hasLabel {
+			node.Labels[ClusterEgressLabel] = "true"
+			return true, nil
+		}
+	default:
+		if hasLabel {
+			delete(node.Labels, ClusterEgressLabel)
+			return true, nil
+		}
 	}
 	return false, nil
 }
diff --git a/pkg/nodeconfig/nodeconfig_test.go b/pkg/nodeconfig/nodeconfig_test.go
index ef1c2dca60d2..08355a453558 100644
--- a/pkg/nodeconfig/nodeconfig_test.go
+++ b/pkg/nodeconfig/nodeconfig_test.go
@@ -4,6 +4,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/version"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -20,6 +21,7 @@ var FakeNodeWithNoAnnotation = &corev1.Node{
 }
 
 var TestEnvName = version.ProgramUpper + "_NODE_NAME"
+var FakeNodeConfig = &config.Node{}
 var FakeNodeWithAnnotation = &corev1.Node{
 	TypeMeta: metav1.TypeMeta{
 		Kind:       "Node",
@@ -39,7 +41,7 @@ func Test_UnitSetExistingNodeConfigAnnotations(t *testing.T) {
 	// adding same config
 	os.Args = []string{version.Program, "server", "--no-flannel"}
 	os.Setenv(version.ProgramUpper+"_NODE_NAME", "fakeNode-with-annotation")
-	nodeUpdated, err := SetNodeConfigAnnotations(FakeNodeWithAnnotation)
+	nodeUpdated, err := SetNodeConfigAnnotations(FakeNodeConfig, FakeNodeWithAnnotation)
 	if err != nil {
 		t.Fatalf("Failed to set node config annotation: %v", err)
 	}
@@ -50,6 +52,7 @@ func Test_UnitSetExistingNodeConfigAnnotations(t *testing.T) {
 
 func Test_UnitSetNodeConfigAnnotations(t *testing.T) {
 	type args struct {
+		config *config.Node
 		node   *corev1.Node
 		osArgs []string
 	}
@@ -72,6 +75,7 @@ func Test_UnitSetNodeConfigAnnotations(t *testing.T) {
 		{
 			name: "Set empty NodeConfigAnnotations",
 			args: args{
+				config: FakeNodeConfig,
 				node:   FakeNodeWithAnnotation,
 				osArgs: []string{version.Program, "server", "--no-flannel"},
 			},
@@ -83,6 +87,7 @@ func Test_UnitSetNodeConfigAnnotations(t *testing.T) {
 		{
 			name: "Set args with equal",
 			args: args{
+				config: FakeNodeConfig,
 				node:   FakeNodeWithNoAnnotation,
 				osArgs: []string{version.Program, "server", "--no-flannel", "--write-kubeconfig-mode=777"},
 			},
@@ -98,7 +103,7 @@ func Test_UnitSetNodeConfigAnnotations(t *testing.T) {
 				t.Errorf("Setup for SetNodeConfigAnnotations() failed = %v", err)
 				return
 			}
-			got, err := SetNodeConfigAnnotations(tt.args.node)
+			got, err := SetNodeConfigAnnotations(tt.args.config, tt.args.node)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("SetNodeConfigAnnotations() error = %v, wantErr %v", err, tt.wantErr)
 				return