diff --git a/.github/workflows/cni-plugin-integration.yml b/.github/workflows/cni-plugin-integration.yml index 4e80b8e4..d08c7826 100644 --- a/.github/workflows/cni-plugin-integration.yml +++ b/.github/workflows/cni-plugin-integration.yml @@ -12,33 +12,20 @@ on: - justfile* jobs: - cni-flannel-test: - continue-on-error: true - timeout-minutes: 15 - runs-on: ubuntu-latest - steps: - - uses: linkerd/dev/actions/setup-tools@v43 - - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 - - name: Run CNI integration tests - run: just cni-plugin-test-integration-flannel - cni-calico-test: - continue-on-error: true - timeout-minutes: 15 - runs-on: ubuntu-latest - steps: - - uses: linkerd/dev/actions/setup-tools@v43 - - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 - - name: Run CNI integration tests - run: just cni-plugin-test-integration-calico - cni-cilium-test: - continue-on-error: true + cni-test: + strategy: + matrix: + cni: [flannel, calico, cilium] + iptables-mode: [legacy, nft] timeout-minutes: 15 runs-on: ubuntu-latest steps: - uses: linkerd/dev/actions/setup-tools@v43 - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 - name: Run CNI integration tests - run: just cni-plugin-test-integration-cilium + env: + IPTABLES_MODE: ${{ matrix.iptables-mode }} + run: just cni-plugin-test-integration-${{ matrix.cni }} ordering-test: continue-on-error: true timeout-minutes: 15 diff --git a/cni-plugin/integration/manifests/calico/linkerd-cni.yaml b/cni-plugin/integration/manifests/calico/linkerd-cni.yaml index 05f110bd..ec009421 100644 --- a/cni-plugin/integration/manifests/calico/linkerd-cni.yaml +++ b/cni-plugin/integration/manifests/calico/linkerd-cni.yaml @@ -80,7 +80,9 @@ data: "ports-to-redirect": [], "inbound-ports-to-ignore": ["4191","4190"], "simulate": false, - "use-wait-flag": false + "use-wait-flag": false, + "iptables-mode": "$IPTABLES_MODE", + "ipv6": true } } --- diff --git a/cni-plugin/integration/manifests/cilium/linkerd-cni.yaml b/cni-plugin/integration/manifests/cilium/linkerd-cni.yaml index 05f110bd..ec009421 100644 --- a/cni-plugin/integration/manifests/cilium/linkerd-cni.yaml +++ b/cni-plugin/integration/manifests/cilium/linkerd-cni.yaml @@ -80,7 +80,9 @@ data: "ports-to-redirect": [], "inbound-ports-to-ignore": ["4191","4190"], "simulate": false, - "use-wait-flag": false + "use-wait-flag": false, + "iptables-mode": "$IPTABLES_MODE", + "ipv6": true } } --- diff --git a/cni-plugin/integration/manifests/flannel/linkerd-cni.yaml b/cni-plugin/integration/manifests/flannel/linkerd-cni.yaml index 5b9166ad..3774ca7a 100644 --- a/cni-plugin/integration/manifests/flannel/linkerd-cni.yaml +++ b/cni-plugin/integration/manifests/flannel/linkerd-cni.yaml @@ -84,7 +84,9 @@ data: "ports-to-redirect": [], "inbound-ports-to-ignore": ["4191","4190"], "simulate": false, - "use-wait-flag": false + "use-wait-flag": false, + "iptables-mode": "$IPTABLES_MODE", + "ipv6": true } } --- diff --git a/cni-plugin/integration/run.sh b/cni-plugin/integration/run.sh index 29414462..3b51339a 100755 --- a/cni-plugin/integration/run.sh +++ b/cni-plugin/integration/run.sh @@ -4,9 +4,8 @@ set -euxo pipefail cd "${BASH_SOURCE[0]%/*}" -# Integration tests to run. Scenario is passed in as an environment variable. -# Default is 'flannel' SCENARIO=${CNI_TEST_SCENARIO:-flannel} +IPTABLES_MODE=${IPTABLES_MODE:-legacy} # Run kubectl with the correct context. function k() { @@ -25,7 +24,10 @@ function create_test_lab() { # can enable a testing matrix? # Apply all files in scenario directory. For non-flannel CNIs, this will # include the CNI manifest itself. - k apply -f "manifests/$SCENARIO/" + for f in ./manifests/"$SCENARIO"/*.yaml + do + envsubst < "$f" | k apply -f - + done } function cleanup() { diff --git a/cni-plugin/integration/testutil/test_util.go b/cni-plugin/integration/testutil/test_util.go index 3d2429e9..1691e11d 100644 --- a/cni-plugin/integration/testutil/test_util.go +++ b/cni-plugin/integration/testutil/test_util.go @@ -49,8 +49,11 @@ type ProxyInit struct { PortsToRedirect []int `json:"ports-to-redirect"` InboundPortsToIgnore []string `json:"inbound-ports-to-ignore"` OutboundPortsToIgnore []string `json:"outbound-ports-to-ignore"` + SubnetsToIgnore []string `json:"subnets-to-ignore"` Simulate bool `json:"simulate"` UseWaitFlag bool `json:"use-wait-flag"` + IPTablesMode string `json:"iptables-mode"` + IPv6 bool `json:"ipv6"` } // LinkerdPlugin is what we use for CNI configuration in the plugins section diff --git a/cni-plugin/main.go b/cni-plugin/main.go index 9eb68980..ab441e1d 100644 --- a/cni-plugin/main.go +++ b/cni-plugin/main.go @@ -52,6 +52,8 @@ type ProxyInit struct { SubnetsToIgnore []string `json:"subnets-to-ignore"` Simulate bool `json:"simulate"` UseWaitFlag bool `json:"use-wait-flag"` + IPTablesMode string `json:"iptables-mode"` + IPv6 bool `json:"ipv6"` } // Kubernetes a K8s specific struct to hold config @@ -219,8 +221,8 @@ func cmdAdd(args *skel.CmdArgs) error { SimulateOnly: conf.ProxyInit.Simulate, NetNs: args.Netns, UseWaitFlag: conf.ProxyInit.UseWaitFlag, - FirewallBinPath: "iptables-legacy", - FirewallSaveBinPath: "iptables-legacy-save", + IPTablesMode: conf.ProxyInit.IPTablesMode, + IPv6: conf.ProxyInit.IPv6, } // Check if there are any overridden ports to be skipped @@ -292,17 +294,24 @@ func cmdAdd(args *skel.CmdArgs) error { options.OutboundPortsToIgnore = append(options.OutboundPortsToIgnore, skippedPorts...) } - firewallConfiguration, err := cmd.BuildFirewallConfiguration(&options) - if err != nil { - logEntry.Errorf("linkerd-cni: could not create a Firewall Configuration from the options: %v", options) - return err + // This ensures BC against linkerd2-cni older versions not yet passing this flag + if options.IPTablesMode == "" { + options.IPTablesMode = cmd.IPTablesModeLegacy } - err = iptables.ConfigureFirewall(*firewallConfiguration) - if err != nil { - logEntry.Errorf("linkerd-cni: could not configure firewall: %s", err) + // always trigger the IPv4 rules + optIPv4 := options + optIPv4.IPv6 = false + if err := buildAndConfigure(logEntry, &optIPv4); err != nil { return err } + + // trigger the IPv6 rules + if options.IPv6 { + if err := buildAndConfigure(logEntry, &options); err != nil { + return err + } + } } else { if containsInitContainer { logEntry.Debug("linkerd-cni: linkerd-init initContainer is present, skipping.") @@ -353,6 +362,24 @@ func getAPIServerPorts(ctx context.Context, api *kubernetes.Clientset) ([]string return ports, nil } +func buildAndConfigure(logEntry *logrus.Entry, options *cmd.RootOptions) error { + firewallConfiguration, err := cmd.BuildFirewallConfiguration(options) + if err != nil { + logEntry.Errorf("linkerd-cni: could not create a Firewall Configuration from the options: %v", options) + return err + } + + err = iptables.ConfigureFirewall(*firewallConfiguration) + // We couldn't find a robust way of checking IPv6 support besides trying to just call ip6tables-save. + // If IPv4 rules worked but not IPv6, let's not fail the container (the actual problem will get logged). + if !options.IPv6 && err != nil { + logEntry.Errorf("linkerd-cni: could not configure firewall: %s", err) + return err + } + + return nil +} + func getAnnotationOverride(ctx context.Context, api *kubernetes.Clientset, pod *v1.Pod, key string) (string, error) { // Check if the annotation is present on the pod if override := pod.GetObjectMeta().GetAnnotations()[key]; override != "" { diff --git a/justfile b/justfile index ba3cb506..b279fd34 100644 --- a/justfile +++ b/justfile @@ -224,6 +224,7 @@ _cni-plugin-setup-cilium: echo "Mounted /sys/fs/bpf to cilium-test-server cluster" helm repo add cilium https://helm.cilium.io/ helm install cilium cilium/cilium --version 1.13.0 \ + --kube-context k3d-l5d-cilium-test \ --namespace kube-system \ --set kubeProxyReplacement=partial \ --set hostServices.enabled=false \ diff --git a/proxy-init/cmd/root.go b/proxy-init/cmd/root.go index 3c8fa8f7..6df5852b 100644 --- a/proxy-init/cmd/root.go +++ b/proxy-init/cmd/root.go @@ -13,6 +13,22 @@ import ( "github.com/linkerd/linkerd2-proxy-init/internal/util" ) +const ( + // IPTablesModeLegacy signals the usage of the iptables-legacy commands + IPTablesModeLegacy = "legacy" + // IPTablesModeNFT signals the usage of the iptables-nft commands + IPTablesModeNFT = "nft" + + cmdLegacy = "iptables-legacy" + cmdLegacySave = "iptables-legacy-save" + cmdLegacyIPv6 = "ip6tables-legacy" + cmdLegacyIPv6Save = "ip6tables-legacy-save" + cmdNFT = "iptables-nft" + cmdNFTSave = "iptables-nft-save" + cmdNFTIPv6 = "ip6tables-nft" + cmdNFTIPv6Save = "ip6tables-nft-save" +) + // RootOptions provides the information that will be used to build a firewall configuration. type RootOptions struct { IncomingProxyPort int @@ -30,6 +46,8 @@ type RootOptions struct { LogLevel string FirewallBinPath string FirewallSaveBinPath string + IPTablesMode string + IPv6 bool } func newRootOptions() *RootOptions { @@ -47,8 +65,10 @@ func newRootOptions() *RootOptions { TimeoutCloseWaitSecs: 0, LogFormat: "plain", LogLevel: "info", - FirewallBinPath: "iptables-legacy", - FirewallSaveBinPath: "iptables-legacy-save", + FirewallBinPath: "", + FirewallSaveBinPath: "", + IPTablesMode: "", + IPv6: true, } } @@ -61,7 +81,7 @@ func NewRootCmd() *cobra.Command { Use: "proxy-init", Short: "proxy-init adds a Kubernetes pod to the Linkerd service mesh", Long: "proxy-init adds a Kubernetes pod to the Linkerd service mesh.", - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, _ []string) error { if options.TimeoutCloseWaitSecs != 0 { sysctl := exec.Command("sysctl", "-w", @@ -75,16 +95,39 @@ func NewRootCmd() *cobra.Command { log.Info(string(out)) } - config, err := BuildFirewallConfiguration(options) + log.SetFormatter(getFormatter(options.LogFormat)) + err := setLogLevel(options.LogLevel) if err != nil { return err } - log.SetFormatter(getFormatter(options.LogFormat)) - err = setLogLevel(options.LogLevel) + + // always trigger the IPv4 rules + optIPv4 := *options + optIPv4.IPv6 = false + config, err := BuildFirewallConfiguration(&optIPv4) if err != nil { return err } - return iptables.ConfigureFirewall(*config) + + if err = iptables.ConfigureFirewall(*config); err != nil { + return err + } + + if !options.IPv6 { + return nil + } + + // trigger the IPv6 rules + config, err = BuildFirewallConfiguration(options) + if err != nil { + return err + } + + // We couldn't find a robust way of checking IPv6 support besides trying to just call ip6tables-save. + // If IPv4 rules worked but not IPv6, let's not fail the container (the actual problem will get logged). + _ = iptables.ConfigureFirewall(*config) + + return nil }, } @@ -101,6 +144,10 @@ func NewRootCmd() *cobra.Command { cmd.PersistentFlags().IntVar(&options.TimeoutCloseWaitSecs, "timeout-close-wait-secs", options.TimeoutCloseWaitSecs, "Sets nf_conntrack_tcp_timeout_close_wait") cmd.PersistentFlags().StringVar(&options.LogFormat, "log-format", options.LogFormat, "Configure log format ('plain' or 'json')") cmd.PersistentFlags().StringVar(&options.LogLevel, "log-level", options.LogLevel, "Configure log level") + cmd.PersistentFlags().StringVar(&options.IPTablesMode, "iptables-mode", options.IPTablesMode, "Variant of iptables command to use (\"legacy\" or \"nft\"); overrides --firewall-bin-path and --firewall-save-bin-path") + cmd.PersistentFlags().BoolVar(&options.IPv6, "ipv6", options.IPv6, "Set rules both via iptables and ip6tables to support dual-stack networking") + + // these two flags are kept for backwards-compatibility, but --iptables-mode is preferred cmd.PersistentFlags().StringVar(&options.FirewallBinPath, "firewall-bin-path", options.FirewallBinPath, "Path to iptables binary") cmd.PersistentFlags().StringVar(&options.FirewallSaveBinPath, "firewall-save-bin-path", options.FirewallSaveBinPath, "Path to iptables-save binary") return cmd @@ -108,6 +155,21 @@ func NewRootCmd() *cobra.Command { // BuildFirewallConfiguration returns an iptables FirewallConfiguration suitable to use to configure iptables. func BuildFirewallConfiguration(options *RootOptions) (*iptables.FirewallConfiguration, error) { + if options.IPTablesMode != "" && options.IPTablesMode != IPTablesModeLegacy && options.IPTablesMode != IPTablesModeNFT { + return nil, fmt.Errorf("--iptables-mode valid values are only \"%s\" and \"%s\"", IPTablesModeLegacy, IPTablesModeNFT) + } + + if options.IPTablesMode == "" { + switch options.FirewallBinPath { + case "", cmdLegacy: + options.IPTablesMode = IPTablesModeLegacy + case cmdNFT: + options.IPTablesMode = IPTablesModeNFT + default: + return nil, fmt.Errorf("--firewall-bin-path valid values are only \"%s\" and \"%s\"", cmdLegacy, cmdNFT) + } + } + if !util.IsValidPort(options.IncomingProxyPort) { return nil, fmt.Errorf("--incoming-proxy-port must be a valid TCP port number") } @@ -116,6 +178,8 @@ func BuildFirewallConfiguration(options *RootOptions) (*iptables.FirewallConfigu return nil, fmt.Errorf("--outgoing-proxy-port must be a valid TCP port number") } + cmd, cmdSave := getCommands(options) + sanitizedSubnets := []string{} for _, subnet := range options.SubnetsToIgnore { subnet := strings.TrimSpace(subnet) @@ -138,8 +202,8 @@ func BuildFirewallConfiguration(options *RootOptions) (*iptables.FirewallConfigu SimulateOnly: options.SimulateOnly, NetNs: options.NetNs, UseWaitFlag: options.UseWaitFlag, - BinPath: options.FirewallBinPath, - SaveBinPath: options.FirewallSaveBinPath, + BinPath: cmd, + SaveBinPath: cmdSave, } if len(options.PortsToRedirect) > 0 { @@ -160,6 +224,21 @@ func getFormatter(format string) log.Formatter { } } +func getCommands(options *RootOptions) (string, string) { + if options.IPTablesMode == IPTablesModeLegacy { + if options.IPv6 { + return cmdLegacyIPv6, cmdLegacyIPv6Save + } + return cmdLegacy, cmdLegacySave + } + + if options.IPv6 { + return cmdNFTIPv6, cmdNFTIPv6Save + } + + return cmdNFT, cmdNFTSave +} + func setLogLevel(logLevel string) error { level, err := log.ParseLevel(logLevel) if err != nil { diff --git a/proxy-init/cmd/root_test.go b/proxy-init/cmd/root_test.go index d1b31b28..5a2ca97b 100644 --- a/proxy-init/cmd/root_test.go +++ b/proxy-init/cmd/root_test.go @@ -31,6 +31,7 @@ func TestBuildFirewallConfiguration(t *testing.T) { options.IncomingProxyPort = expectedIncomingProxyPort options.OutgoingProxyPort = expectedOutgoingProxyPort options.ProxyUserID = expectedProxyUserID + options.IPv6 = false config, err := BuildFirewallConfiguration(options) if err != nil { @@ -51,6 +52,7 @@ func TestBuildFirewallConfiguration(t *testing.T) { options: &RootOptions{ IncomingProxyPort: -1, OutgoingProxyPort: 1234, + IPTablesMode: IPTablesModeLegacy, }, errorMessage: "--incoming-proxy-port must be a valid TCP port number", }, @@ -58,6 +60,7 @@ func TestBuildFirewallConfiguration(t *testing.T) { options: &RootOptions{ IncomingProxyPort: 100000, OutgoingProxyPort: 1234, + IPTablesMode: IPTablesModeLegacy, }, errorMessage: "--incoming-proxy-port must be a valid TCP port number", }, @@ -65,6 +68,7 @@ func TestBuildFirewallConfiguration(t *testing.T) { options: &RootOptions{ IncomingProxyPort: 1234, OutgoingProxyPort: -1, + IPTablesMode: IPTablesModeLegacy, }, errorMessage: "--outgoing-proxy-port must be a valid TCP port number", }, @@ -72,12 +76,14 @@ func TestBuildFirewallConfiguration(t *testing.T) { options: &RootOptions{ IncomingProxyPort: 1234, OutgoingProxyPort: 100000, + IPTablesMode: IPTablesModeLegacy, }, errorMessage: "--outgoing-proxy-port must be a valid TCP port number", }, { options: &RootOptions{ SubnetsToIgnore: []string{"1.1.1.1/24", "0.0.0.0"}, + IPTablesMode: IPTablesModeLegacy, }, errorMessage: "0.0.0.0 is not a valid CIDR address", }, @@ -102,6 +108,7 @@ func TestBuildFirewallConfiguration(t *testing.T) { // Tests that subnets are parsed properly and trimmed of excess whitespace options: &RootOptions{ SubnetsToIgnore: []string{"1.1.1.1/24 "}, + IPTablesMode: IPTablesModeLegacy, }, errorMessage: "", },