Skip to content

Commit

Permalink
Re-introduce 2005 route table
Browse files Browse the repository at this point in the history 
This commit re-introduced the 2005 routes that were removed by
cilium@9dd6cfc (datapath: remove 2005 route table for ipv6 only)
and cilium@c1a0dba (datapath: remove 2005 route table for ipv4 only).

Signed-off-by: Robin Gögge <r.goegge@gmail.com>
Signed-off-by: Zhichuan Liang <gray.liang@isovalent.com>
  • Loading branch information
@rgo3 @julianwiedmann
rgo3 authored and julianwiedmann committed Dec 1, 2023
1 parent ac63856 commit 217ae4f
Show file tree
Hide file tree
Showing 4 changed files with 371 additions and 105 deletions.
19 changes: 12 additions & 7 deletions pkg/datapath/linux/linux_defaults/linux_defaults.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,11 @@ const (
// RouteTableVtep is the default table ID to use for VTEP routing rules
RouteTableVtep = 202

// RouteTableProxy is the default table ID to use for proxy routing rules.
RouteTableProxy = 2004
// RouteTableToProxy is the default table ID to use routing rules to the proxy.
RouteTableToProxy = 2004

// RouteTableFromProxy is the default table ID to use routing rules from the proxy.
RouteTableFromProxy = 2005

// RouteTableInterfacesOffset is the offset for the per-ENI routing tables.
// Each ENI interface will have its own table starting with this offset. It
Expand Down Expand Up @@ -70,11 +73,13 @@ const (
// RulePriorityWireguard is the priority of the rule used for routing packets to WireGuard device for encryption
RulePriorityWireguard = 1

// RulePriorityProxyIngress is the priority of the routing rule installed by
// the proxy package for redirecting inbound packets to the proxy. Priority 10
// used to be for outgoing packets from the proxy (see PROXY_RT_TABLE in older
// versions), but is no longer used.
RulePriorityProxyIngress = 9
// RulePriorityToProxyIngress is the priority of the routing rule installed by
// the proxy package for redirecting inbound packets to the proxy.
RulePriorityToProxyIngress = 9

// RulePriorityFromProxyIngress is the priority of the routing rule installed by
// the proxy package for redirecting inbound packets from the proxy.
RulePriorityFromProxyIngress = 10

// RulePriorityIngress is the priority of the rule used for ingress routing
// of endpoints. This priority is after encryption and proxy rules, and
Expand Down
59 changes: 54 additions & 5 deletions pkg/proxy/proxy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,14 @@ package proxy
import (
"context"
"fmt"
"net"

"github.com/sirupsen/logrus"
"github.com/vishvananda/netlink"

"github.com/cilium/cilium/api/v1/models"
"github.com/cilium/cilium/pkg/completion"
"github.com/cilium/cilium/pkg/defaults"
"github.com/cilium/cilium/pkg/envoy"
"github.com/cilium/cilium/pkg/ipcache"
"github.com/cilium/cilium/pkg/lock"
Expand Down Expand Up @@ -415,31 +418,77 @@ func (p *Proxy) SetProxyPort(name string, proxyType types.ProxyType, port uint16
}

// ReinstallRoutingRules ensures the presence of routing rules and tables needed
// to route packets to the L7 proxy.
// to route packets to and from the L7 proxy.
func (p *Proxy) ReinstallRoutingRules() error {
if option.Config.EnableIPv4 {
if err := installRoutesIPv4(); err != nil {
if err := installToProxyRoutesIPv4(); err != nil {
return err
}

if !option.Config.EnableIPSec {
if err := removeFromProxyRoutesIPv4(); err != nil {
return err
}
} else {
if err := installFromProxyRoutesIPv4(node.GetInternalIPv4Router(), defaults.HostDevice); err != nil {
return err
}
}
} else {
if err := removeRoutesIPv4(); err != nil {
if err := removeToProxyRoutesIPv4(); err != nil {
return err
}
if err := removeFromProxyRoutesIPv4(); err != nil {
return err
}
}

if option.Config.EnableIPv6 {
if err := installRoutesIPv6(); err != nil {
if err := installToProxyRoutesIPv6(); err != nil {
return err
}

if !option.Config.EnableIPSec {
if err := removeFromProxyRoutesIPv6(); err != nil {
return err
}
} else {
ipv6, err := getCiliumNetIPv6()
if err != nil {
return err
}
if err := installFromProxyRoutesIPv6(ipv6, defaults.HostDevice); err != nil {
return err
}
}
} else {
if err := removeRoutesIPv6(); err != nil {
if err := removeToProxyRoutesIPv6(); err != nil {
return err
}
if err := removeFromProxyRoutesIPv6(); err != nil {
return err
}
}

return nil
}

// getCiliumNetIPv6 retrieves the first IPv6 address from the cilium_net device.
func getCiliumNetIPv6() (net.IP, error) {
link, err := netlink.LinkByName(defaults.SecondHostDevice)
if err != nil {
return nil, fmt.Errorf("cannot find link '%s': %w", defaults.SecondHostDevice, err)
}

addrList, err := netlink.AddrList(link, netlink.FAMILY_V6)
if err == nil && len(addrList) > 0 {
return addrList[0].IP, nil
}

return nil, fmt.Errorf("failed to find valid IPv6 address for cilium_net")

}

// ReinstallIPTablesRules is called by daemon reconfiguration to reinstall
// proxy ports rules that were removed during the removal of all Cilium rules.
func (p *Proxy) ReinstallIPTablesRules(ctx context.Context) error {
Expand Down
140 changes: 119 additions & 21 deletions pkg/proxy/routes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,80 +17,178 @@ import (

var (
// Routing rule for traffic to proxy.
tproxyRule = route.Rule{
toProxyRule = route.Rule{
// Cilium bumps the default catch-all pref 0 routing rule that points at
// table 255 to pref 100 during startup, to create space to insert its own
// rules between 0-99.
Priority: linux_defaults.RulePriorityProxyIngress,
Priority: linux_defaults.RulePriorityToProxyIngress,
Mark: int(linux_defaults.MagicMarkIsToProxy),
Mask: linux_defaults.MagicMarkHostMask,
Table: linux_defaults.RouteTableProxy,
Table: linux_defaults.RouteTableToProxy,
}

// Default IPv4 route for local delivery.
route4 = route.Route{
Table: linux_defaults.RouteTableProxy,
Table: linux_defaults.RouteTableToProxy,
Type: route.RTN_LOCAL,
Local: net.IPv4zero,
Device: "lo",
Proto: linux_defaults.RTProto}

// Default IPv6 route for local delivery.
route6 = route.Route{
Table: linux_defaults.RouteTableProxy,
Table: linux_defaults.RouteTableToProxy,
Type: route.RTN_LOCAL,
Local: net.IPv6zero,
Device: "lo",
Proto: linux_defaults.RTProto,
}
)

// installRoutesIPv4 configures routes and rules needed to redirect ingress
// installToProxyRoutesIPv4 configures routes and rules needed to redirect ingress
// packets to the proxy.
func installRoutesIPv4() error {
func installToProxyRoutesIPv4() error {
if err := route.Upsert(route4); err != nil {
return fmt.Errorf("inserting ipv4 proxy route %v: %w", route4, err)
}
if err := route.ReplaceRule(tproxyRule); err != nil {
return fmt.Errorf("inserting ipv4 proxy routing rule %v: %w", tproxyRule, err)
if err := route.ReplaceRule(toProxyRule); err != nil {
return fmt.Errorf("inserting ipv4 proxy routing rule %v: %w", toProxyRule, err)
}

return nil
}

// removeRoutesIPv4 ensures routes and rules for proxy traffic are removed.
func removeRoutesIPv4() error {
if err := route.DeleteRule(netlink.FAMILY_V4, tproxyRule); err != nil && !errors.Is(err, syscall.ENOENT) {
// removeToProxyRoutesIPv4 ensures routes and rules for proxy traffic are removed.
func removeToProxyRoutesIPv4() error {
if err := route.DeleteRule(netlink.FAMILY_V4, toProxyRule); err != nil && !errors.Is(err, syscall.ENOENT) {
return fmt.Errorf("removing ipv4 proxy routing rule: %w", err)
}
if err := route.DeleteRouteTable(linux_defaults.RouteTableProxy, netlink.FAMILY_V4); err != nil {
if err := route.DeleteRouteTable(linux_defaults.RouteTableToProxy, netlink.FAMILY_V4); err != nil {
return fmt.Errorf("removing ipv4 proxy route table: %w", err)
}

return nil
}

// installRoutesIPv6 configures routes and rules needed to redirect ingress
// installToProxyRoutesIPv6 configures routes and rules needed to redirect ingress
// packets to the proxy.
func installRoutesIPv6() error {
func installToProxyRoutesIPv6() error {
if err := route.Upsert(route6); err != nil {
return fmt.Errorf("inserting ipv6 proxy route %v: %w", route6, err)
}
if err := route.ReplaceRuleIPv6(tproxyRule); err != nil {
return fmt.Errorf("inserting ipv6 proxy routing rule %v: %w", tproxyRule, err)
if err := route.ReplaceRuleIPv6(toProxyRule); err != nil {
return fmt.Errorf("inserting ipv6 proxy routing rule %v: %w", toProxyRule, err)
}

return nil
}

// removeRoutesIPv6 ensures routes and rules for proxy traffic are removed.
func removeRoutesIPv6() error {
if err := route.DeleteRule(netlink.FAMILY_V6, tproxyRule); err != nil && !errors.Is(err, syscall.ENOENT) {
// removeToProxyRoutesIPv6 ensures routes and rules for proxy traffic are removed.
func removeToProxyRoutesIPv6() error {
if err := route.DeleteRule(netlink.FAMILY_V6, toProxyRule); err != nil && !errors.Is(err, syscall.ENOENT) {
return fmt.Errorf("removing ipv6 proxy routing rule: %w", err)
}
if err := route.DeleteRouteTable(linux_defaults.RouteTableProxy, netlink.FAMILY_V6); err != nil {
if err := route.DeleteRouteTable(linux_defaults.RouteTableToProxy, netlink.FAMILY_V6); err != nil {
return fmt.Errorf("removing ipv6 proxy route table: %w", err)
}

return nil
}

var (
// Routing rule for traffic from proxy.
fromProxyRule = route.Rule{
Priority: linux_defaults.RulePriorityFromProxyIngress,
Mark: linux_defaults.MagicMarkIsProxy,
Mask: linux_defaults.MagicMarkHostMask,
Table: linux_defaults.RouteTableFromProxy,
}
)

// installFromProxyRoutesIPv4 configures routes and rules needed to redirect ingress
// packets from the proxy.
func installFromProxyRoutesIPv4(ipv4 net.IP, device string) error {
fromProxyToCiliumHostRoute4 := route.Route{
Table: linux_defaults.RouteTableFromProxy,
Prefix: net.IPNet{
IP: ipv4,
Mask: net.CIDRMask(32, 32),
},
Device: device,
Type: route.RTN_LOCAL,
Proto: linux_defaults.RTProto,
}
fromProxyDefaultRoute4 := route.Route{
Table: linux_defaults.RouteTableFromProxy,
Nexthop: &ipv4,
Device: device,
}

if err := route.ReplaceRule(fromProxyRule); err != nil {
return fmt.Errorf("inserting ipv4 from proxy routing rule %v: %w", fromProxyRule, err)
}
if err := route.Upsert(fromProxyToCiliumHostRoute4); err != nil {
return fmt.Errorf("inserting ipv4 from proxy to cilium_host route %v: %w", fromProxyToCiliumHostRoute4, err)
}
if err := route.Upsert(fromProxyDefaultRoute4); err != nil {
return fmt.Errorf("inserting ipv4 from proxy default route %v: %w", fromProxyDefaultRoute4, err)
}

return nil
}

// removeFromProxyRoutesIPv4 ensures routes and rules for traffic from the proxy are removed.
func removeFromProxyRoutesIPv4() error {
if err := route.DeleteRule(netlink.FAMILY_V4, fromProxyRule); err != nil && !errors.Is(err, syscall.ENOENT) {
return fmt.Errorf("removing ipv4 from proxy routing rule: %w", err)
}
if err := route.DeleteRouteTable(linux_defaults.RouteTableFromProxy, netlink.FAMILY_V4); err != nil {
return fmt.Errorf("removing ipv4 from proxy route table: %w", err)
}

return nil
}

// installFromProxyRoutesIPv6 configures routes and rules needed to redirect ingress
// packets from the proxy.
func installFromProxyRoutesIPv6(ipv6 net.IP, device string) error {
fromProxyToCiliumHostRoute6 := route.Route{
Table: linux_defaults.RouteTableFromProxy,
Prefix: net.IPNet{
IP: ipv6,
Mask: net.CIDRMask(128, 128),
},
Device: device,
Proto: linux_defaults.RTProto,
}

fromProxyDefaultRoute6 := route.Route{
Table: linux_defaults.RouteTableFromProxy,
Nexthop: &ipv6,
Device: device,
}

if err := route.ReplaceRuleIPv6(fromProxyRule); err != nil {
return fmt.Errorf("inserting ipv6 from proxy routing rule %v: %w", fromProxyRule, err)
}
if err := route.Upsert(fromProxyToCiliumHostRoute6); err != nil {
return fmt.Errorf("inserting ipv6 from proxy to cilium_host route %v: %w", fromProxyToCiliumHostRoute6, err)
}
if err := route.Upsert(fromProxyDefaultRoute6); err != nil {
return fmt.Errorf("inserting ipv6 from proxy default route %v: %w", fromProxyDefaultRoute6, err)
}

return nil
}

// removeFromProxyRoutesIPv6 ensures routes and rules for traffic from the proxy are removed.
func removeFromProxyRoutesIPv6() error {
if err := route.DeleteRule(netlink.FAMILY_V6, fromProxyRule); err != nil && !errors.Is(err, syscall.ENOENT) {
return fmt.Errorf("removing ipv6 from proxy routing rule: %w", err)
}
if err := route.DeleteRouteTable(linux_defaults.RouteTableFromProxy, netlink.FAMILY_V6); err != nil {
return fmt.Errorf("removing ipv6 from proxy route table: %w", err)
}

return nil
}

0 comments on commit 217ae4f

Please sign in to comment.