Skip to content

use 'nf_conntrack' instead of 'nf_conntrack_ipv4' for linux kernel >= 4.19 #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
May 29, 2020
Merged

use 'nf_conntrack' instead of 'nf_conntrack_ipv4' for linux kernel >= 4.19 #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
74e40cf
use 'nf_conntrack' instead of 'nf_conntrack_ipv4' for linux kernel >=…
islinwb Nov 2, 2018
7720365
update bazel
islinwb Nov 2, 2018
24262f5
Import k8s.io/apimachinery/pkg/util/version by hand
EricMountain May 29, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 5 additions & 13 deletions pkg/proxy/ipvs/proxier.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,14 +151,6 @@ var ipsetWithIptablesChain = []struct {
{kubeNodePortLocalSetTCP, string(KubeNodePortChain), "RETURN", "dst"},
}

var ipvsModules = []string{
"ip_vs",
"ip_vs_rr",
"ip_vs_wrr",
"ip_vs_sh",
"nf_conntrack_ipv4",
}

// In IPVS proxy mode, the following flags need to be set
const sysctlRouteLocalnet = "net/ipv4/conf/all/route_localnet"
const sysctlBridgeCallIPTables = "net/bridge/bridge-nf-call-iptables"
Expand Down Expand Up @@ -449,14 +441,12 @@ func NewLinuxKernelHandler() *LinuxKernelHandler {
// GetModules returns all installed kernel modules.
func (handle *LinuxKernelHandler) GetModules() ([]string, error) {
// Check whether IPVS required kernel modules are built-in
kernelVersionFile := "/proc/sys/kernel/osrelease"
b, err := ioutil.ReadFile(kernelVersionFile)
kernelVersion, ipvsModules, err := utilipvs.GetKernelVersionAndIPVSMods(handle.executor)
if err != nil {
glog.Errorf("Failed to read file %s with error %v", kernelVersionFile, err)
return nil, err
}
kernelVersion := strings.TrimSpace(string(b))
builtinModsFilePath := fmt.Sprintf("/lib/modules/%s/modules.builtin", kernelVersion)
b, err = ioutil.ReadFile(builtinModsFilePath)
b, err := ioutil.ReadFile(builtinModsFilePath)
if err != nil {
glog.Errorf("Failed to read file %s with error %v", builtinModsFilePath, err)
}
Expand Down Expand Up @@ -497,6 +487,8 @@ func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner) (bool, err
}
wantModules := sets.NewString()
loadModules := sets.NewString()
linuxKernelHandler := NewLinuxKernelHandler()
_, ipvsModules, _ := utilipvs.GetKernelVersionAndIPVSMods(linuxKernelHandler.executor)
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Check err ?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Interestingly upstream chose to explicitly ignore errors.

Basically we're only interested in the 2nd return value, and in error paths it is nil, so ipvsModules will be nil. This leads to wantModules being empty, and then potentially the len(modules) bit fails lower down. So there's something handling it it seems. Haven't looked at the details of mods, err := handle.GetModules(), however I not particularly tempted to deviate from upstream just for this.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok

wantModules.Insert(ipvsModules...)
loadModules.Insert(mods...)
modules := wantModules.Difference(loadModules).UnsortedList()
Expand Down
36 changes: 4 additions & 32 deletions pkg/util/ipvs/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,42 +80,14 @@ go_library(
"//conditions:default": [],
}),
importpath = "k8s.io/kubernetes/pkg/util/ipvs",
deps = select({
"@io_bazel_rules_go//go/platform:android": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:darwin": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:dragonfly": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:freebsd": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
deps = [
"//vendor/k8s.io/apimachinery/pkg/util/version:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
] + select({
"@io_bazel_rules_go//go/platform:linux": [
"//vendor/github.com/docker/libnetwork/ipvs:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:nacl": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:netbsd": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:openbsd": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:plan9": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:solaris": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:windows": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"//conditions:default": [],
}),
Expand Down
54 changes: 46 additions & 8 deletions pkg/util/ipvs/ipvs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,12 @@ package ipvs
import (
"net"
"strconv"
"strings"
"time"

"fmt"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/utils/exec"
)

// Interface is an injectable interface for running ipvs commands. Implementations must be goroutine-safe.
Expand Down Expand Up @@ -68,14 +73,21 @@ const (
IPVSProxyMode = "ipvs"
)

// Sets of IPVS required kernel modules.
var ipvsModules = []string{
"ip_vs",
"ip_vs_rr",
"ip_vs_wrr",
"ip_vs_sh",
"nf_conntrack_ipv4",
}
// IPVS required kernel modules.
const (
// ModIPVS is the kernel module "ip_vs"
ModIPVS string = "ip_vs"
// ModIPVSRR is the kernel module "ip_vs_rr"
ModIPVSRR string = "ip_vs_rr"
// ModIPVSWRR is the kernel module "ip_vs_wrr"
ModIPVSWRR string = "ip_vs_wrr"
// ModIPVSSH is the kernel module "ip_vs_sh"
ModIPVSSH string = "ip_vs_sh"
// ModNfConntrackIPV4 is the module "nf_conntrack_ipv4"
ModNfConntrackIPV4 string = "nf_conntrack_ipv4"
// ModNfConntrack is the kernel module "nf_conntrack"
ModNfConntrack string = "nf_conntrack"
)

// Equal check the equality of virtual server.
// We don't use struct == since it doesn't work because of slice.
Expand Down Expand Up @@ -110,3 +122,29 @@ func (rs *RealServer) Equal(other *RealServer) bool {
rs.Port == other.Port &&
rs.Weight == other.Weight
}

// GetKernelVersionAndIPVSMods returns the linux kernel version and the required ipvs modules
func GetKernelVersionAndIPVSMods(Executor exec.Interface) (kernelVersion string, ipvsModules []string, err error) {
kernelVersionFile := "/proc/sys/kernel/osrelease"
out, err := Executor.Command("cut", "-f1", "-d", " ", kernelVersionFile).CombinedOutput()
if err != nil {
return "", nil, fmt.Errorf("error getting os release kernel version: %v(%s)", err, out)
}
kernelVersion = strings.TrimSpace(string(out))
// parse kernel version
ver1, err := version.ParseGeneric(kernelVersion)
if err != nil {
return kernelVersion, nil, fmt.Errorf("error parsing kernel version: %v(%s)", err, kernelVersion)
}
// "nf_conntrack_ipv4" has been removed since v4.19
// see https://github.com/torvalds/linux/commit/a0ae2562c6c4b2721d9fddba63b7286c13517d9f
ver2, _ := version.ParseGeneric("4.19")
// get required ipvs modules
if ver1.LessThan(ver2) {
ipvsModules = append(ipvsModules, ModIPVS, ModIPVSRR, ModIPVSWRR, ModIPVSSH, ModNfConntrackIPV4)
} else {
ipvsModules = append(ipvsModules, ModIPVS, ModIPVSRR, ModIPVSWRR, ModIPVSSH, ModNfConntrack)
}

return kernelVersion, ipvsModules, nil
}
13 changes: 5 additions & 8 deletions pkg/util/ipvs/kernelcheck_linux.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,11 @@ func (r RequiredIPVSKernelModulesAvailableCheck) Name() string {
func (r RequiredIPVSKernelModulesAvailableCheck) Check() (warnings, errors []error) {
glog.V(1).Infoln("validating the kernel module IPVS required exists in machine or not")

kernelVersion, ipvsModules, err := GetKernelVersionAndIPVSMods(r.Executor)
if err != nil {
errors = append(errors, err)
}

// Find out loaded kernel modules
out, err := r.Executor.Command("cut", "-f1", "-d", " ", "/proc/modules").CombinedOutput()
if err != nil {
Expand All @@ -60,14 +65,6 @@ func (r RequiredIPVSKernelModulesAvailableCheck) Check() (warnings, errors []err

// Check builtin modules exist or not
if len(modules) != 0 {
kernelVersionFile := "/proc/sys/kernel/osrelease"
b, err := r.Executor.Command("cut", "-f1", "-d", " ", kernelVersionFile).CombinedOutput()
if err != nil {
errors = append(errors, fmt.Errorf("error getting os release kernel version: %v(%s)", err, out))
return nil, errors
}

kernelVersion := strings.TrimSpace(string(b))
builtinModsFilePath := fmt.Sprintf("/lib/modules/%s/modules.builtin", kernelVersion)
out, err := r.Executor.Command("cut", "-f1", "-d", " ", builtinModsFilePath).CombinedOutput()
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion pkg/util/ipvs/kernelcheck_linux_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,8 @@ func TestRequiredIPVSKernelModulesAvailableCheck(t *testing.T) {
for i, tc := range cases {
fcmd := fakeexec.FakeCmd{
CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
func() ([]byte, error) { return []byte(cases[i].loadedKernel), nil },
func() ([]byte, error) { return []byte(cases[i].kernelVersion), nil },
func() ([]byte, error) { return []byte(cases[i].loadedKernel), nil },
func() ([]byte, error) { return []byte(cases[i].builtinKernel), nil },
},
}
Expand Down
36 changes: 36 additions & 0 deletions staging/src/k8s.io/apimachinery/pkg/util/version/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package(default_visibility = ["//visibility:public"])

load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)

go_library(
name = "go_default_library",
srcs = [
"doc.go",
"version.go",
],
importmap = "k8s.io/kubernetes/vendor/k8s.io/apimachinery/pkg/util/version",
importpath = "k8s.io/apimachinery/pkg/util/version",
)

go_test(
name = "go_default_test",
srcs = ["version_test.go"],
embed = [":go_default_library"],
)

filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)

filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)
18 changes: 18 additions & 0 deletions staging/src/k8s.io/apimachinery/pkg/util/version/doc.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package version provides utilities for version number comparisons
package version // import "k8s.io/apimachinery/pkg/util/version"
Loading