route: Clean up ip rules with "proto unspec"

This commit adds "removeStaleProxyRulesIPvX()" which removes any ip
rules with "proto unspec" to ensure upgrade/downgrade goes smoothly.

Scenario 1: upgrade from v1.15-old to v1.15-tip
  v1.15-old cilium installs ip rules with "proto unspec", then
  v1.15-tip will install "duplicate" ip rules with "proto kernel".
  This is the moment when "removeStaleProxyRulesIPvX()" plays a role,
  it cleans those "proto unspec" stale rules without breaking
  connectivity.

Scenario 2: downgrade from v1.15-tip to v1.15-old
  v1.15-tip has rules with "proto kernel". When v1.15-old tries to
  "ReplaceRule()" with "proto unspec", thanks to
  [this](https://github.com/cilium/cilium/blob/v1.15.3/pkg/datapath/linux/route/route_linux.go#L402),
  "ReplaceRule()" won't replace the rules because they already exist
  (with a different proto). This ensures connectivity can survive
  the downgrade too.

Scenario 3: upgrade from v1.15-tip to v1.16
  Since v1.15-tip installs correct rules with "proto kernel", v1.16
  will do nothing after confirming existance by "lookupRule()". It
  should be painless as well.

This is a v1.15-only commit because:
1. v1.14 is still using bpf/init.sh which sets rules with "proto kernel"
   properly;
2. v1.16 has been fixed to set "proto kernel";
3. v1.15-tip -> v1.16 upgrade has been discussed above without any
   issue;

Also please note that we don't have to clean up leftover ip routes with
"proto unspec", because we replace them via "route.Upsert()" which
replaces the old ones unconditionally, leaving no stale routes.

Signed-off-by: Zhichuan Liang <gray.liang@isovalent.com>

pkg/proxy/proxy.go

@@ -442,6 +442,9 @@ func (p *Proxy) ReinstallRoutingRules() error {
 			return err
 		}
 	}
+	if err := removeStaleProxyRulesIPv4(); err != nil {
+		return err
+	}
 
 	if option.Config.EnableIPv6 {
 		if err := installToProxyRoutesIPv6(); err != nil {
@@ -469,6 +472,9 @@ func (p *Proxy) ReinstallRoutingRules() error {
 			return err
 		}
 	}
+	if err := removeStaleProxyRulesIPv6(); err != nil {
+		return err
+	}
 
 	return nil
 }

pkg/proxy/routes.go

@@ -10,6 +10,8 @@ import (
 	"syscall"
 
 	"github.com/vishvananda/netlink"
+	"golang.org/x/exp/slices"
+	"golang.org/x/sys/unix"
 
 	"github.com/cilium/cilium/pkg/datapath/linux/linux_defaults"
 	"github.com/cilium/cilium/pkg/datapath/linux/route"
@@ -155,6 +157,11 @@ func removeFromProxyRoutesIPv4() error {
 	return nil
 }
 
+// removeStaleProxyRulesIPv4 removes stale proxy rules. This is a v1.15 only function.
+func removeStaleProxyRulesIPv4() error {
+	return removeProtoUnspecRules(netlink.FAMILY_V4)
+}
+
 // installFromProxyRoutesIPv6 configures routes and rules needed to redirect ingress
 // packets from the proxy.
 func installFromProxyRoutesIPv6(ipv6 net.IP, device string) error {
@@ -201,3 +208,30 @@ func removeFromProxyRoutesIPv6() error {
 
 	return nil
 }
+
+// removeStaleProxyRulesIPv6 removes stale proxy rules. This is a v1.15 only function.
+func removeStaleProxyRulesIPv6() error {
+	return removeProtoUnspecRules(netlink.FAMILY_V6)
+}
+
+// removeProtoUnspecRules removes all routing rules with protocol RTPROT_UNSPEC. This is a v1.15 only function.
+func removeProtoUnspecRules(family int) error {
+	rules, err := route.ListRules(family, nil)
+	if err != nil {
+		return fmt.Errorf("listing routing rules: %w", err)
+	}
+	ciliumRouteTables := []int{
+		linux_defaults.RouteTableToProxy,
+		linux_defaults.RouteTableFromProxy,
+	}
+	for _, r := range rules {
+		if r.Protocol == unix.RTPROT_UNSPEC && slices.Contains(ciliumRouteTables, r.Table) {
+			if err := netlink.RuleDel(&r); err != nil {
+				if !errors.Is(err, syscall.ENOENT) && !errors.Is(err, syscall.EAFNOSUPPORT) {
+					return fmt.Errorf("removing proto unspec routing rule: %w", err)
+				}
+			}
+		}
+	}
+	return nil
+}