Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

ipvs: ensure selected scheduler kernel modules are loaded #93040

Merged
merged 3 commits into from Oct 26, 2020
Merged

ipvs: ensure selected scheduler kernel modules are loaded #93040

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e2a0edd
ipvs: ensure selected scheduler kernel modules are loaded
Jul 13, 2020
65ff4e8
ipvs: log error if scheduler does not exist and fallback to rr
Jul 23, 2020
51ed242
ipvs: check for existence of scheduler module and fail if not found
Oct 5, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/kube-proxy/app/server_others.go
View file
Open in desktop
Expand Up @@ -122,7 +122,7 @@ func newProxyServer(
iptInterface = utiliptables.New(execer, protocol)
kernelHandler = ipvs.NewLinuxKernelHandler()
ipsetInterface = utilipset.New(execer)
canUseIPVS, err := ipvs.CanUseIPVSProxier(kernelHandler, ipsetInterface)
canUseIPVS, err := ipvs.CanUseIPVSProxier(kernelHandler, ipsetInterface, config.IPVS.Scheduler)
if string(config.Mode) == proxyModeIPVS && err != nil {
klog.Errorf("Can't use the IPVS proxier: %v", err)
}
Expand Down
22 changes: 21 additions & 1 deletion cmd/kube-proxy/app/server_others_test.go
View file
Open in desktop
Expand Up @@ -79,6 +79,7 @@ func Test_getProxyMode(t *testing.T) {
kernelCompat bool
ipsetError error
expected string
scheduler string
}{
{ // flag says userspace
flag: "userspace",
Expand Down Expand Up @@ -110,13 +111,15 @@ func Test_getProxyMode(t *testing.T) {
kernelVersion: "4.18",
ipsetVersion: ipvs.MinIPSetCheckVersion,
expected: proxyModeIPVS,
scheduler: "rr",
},
{ // flag says ipvs, ipset version ok, kernel modules installed for linux kernel 4.19
flag: "ipvs",
kmods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"},
kernelVersion: "4.19",
ipsetVersion: ipvs.MinIPSetCheckVersion,
expected: proxyModeIPVS,
scheduler: "rr",
},
{ // flag says ipvs, ipset version too low, fallback on iptables mode
flag: "ipvs",
Expand All @@ -142,6 +145,23 @@ func Test_getProxyMode(t *testing.T) {
kernelCompat: true,
expected: proxyModeIPTables,
},
{ // flag says ipvs, ipset version ok, kernel modules installed for sed scheduler
cmluciano marked this conversation as resolved.
Show resolved Hide resolved
flag: "ipvs",
kmods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack", "ip_vs_sed"},
kernelVersion: "4.19",
ipsetVersion: ipvs.MinIPSetCheckVersion,
expected: proxyModeIPVS,
scheduler: "sed",
},
{ // flag says ipvs, kernel modules not installed for sed scheduler, fallback to iptables
flag: "ipvs",
kmods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"},
kernelVersion: "4.19",
ipsetVersion: ipvs.MinIPSetCheckVersion,
expected: proxyModeIPTables,
kernelCompat: true,
scheduler: "sed",
},
}
for i, c := range cases {
kcompater := &fakeKernelCompatTester{c.kernelCompat}
Expand All @@ -150,7 +170,7 @@ func Test_getProxyMode(t *testing.T) {
modules: c.kmods,
kernelVersion: c.kernelVersion,
}
canUseIPVS, _ := ipvs.CanUseIPVSProxier(khandler, ipsetver)
canUseIPVS, _ := ipvs.CanUseIPVSProxier(khandler, ipsetver, cases[i].scheduler)
r := getProxyMode(c.flag, canUseIPVS, kcompater)
if r != c.expected {
t.Errorf("Case[%d] Expected %q, got %q", i, c.expected, r)
Expand Down
8 changes: 7 additions & 1 deletion pkg/proxy/ipvs/proxier.go
View file
Open in desktop
Expand Up @@ -693,7 +693,7 @@ func (handle *LinuxKernelHandler) GetKernelVersion() (string, error) {
// This is determined by checking if all the required kernel modules can be loaded. It may
// return an error if it fails to get the kernel modules information without error, in which
// case it will also return false.
func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner) (bool, error) {
func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner, scheduler string) (bool, error) {
mods, err := handle.GetModules()
if err != nil {
return false, fmt.Errorf("error getting installed ipvs required kernel modules: %v", err)
Expand All @@ -711,6 +711,12 @@ func CanUseIPVSProxier(handle KernelHandler, ipsetver IPSetVersioner) (bool, err
}
mods = utilipvs.GetRequiredIPVSModules(kernelVersion)
wantModules := sets.NewString()
// We check for the existence of the scheduler mod and will trigger a missingMods error if not found
if scheduler == "" {
scheduler = DefaultScheduler
}
schedulerMod := "ip_vs_" + scheduler
mods = append(mods, schedulerMod)
wantModules.Insert(mods...)

modules := wantModules.Difference(loadModules).UnsortedList()
Expand Down
35 changes: 34 additions & 1 deletion pkg/proxy/ipvs/proxier_test.go
View file
Open in desktop
Expand Up @@ -259,6 +259,7 @@ func TestCleanupLeftovers(t *testing.T) {
func TestCanUseIPVSProxier(t *testing.T) {
testCases := []struct {
mods []string
scheduler string
kernelVersion string
kernelErr error
ipsetVersion string
Expand All @@ -268,6 +269,7 @@ func TestCanUseIPVSProxier(t *testing.T) {
// case 0, kernel error
{
mods: []string{"foo", "bar", "baz"},
scheduler: "",
kernelVersion: "4.19",
kernelErr: fmt.Errorf("oops"),
ipsetVersion: "0.0",
Expand All @@ -276,6 +278,7 @@ func TestCanUseIPVSProxier(t *testing.T) {
// case 1, ipset error
{
mods: []string{"foo", "bar", "baz"},
scheduler: "",
kernelVersion: "4.19",
ipsetVersion: MinIPSetCheckVersion,
ipsetErr: fmt.Errorf("oops"),
Expand All @@ -284,51 +287,81 @@ func TestCanUseIPVSProxier(t *testing.T) {
// case 2, missing required kernel modules and ipset version too low
{
mods: []string{"foo", "bar", "baz"},
scheduler: "rr",
kernelVersion: "4.19",
ipsetVersion: "1.1",
ok: false,
},
// case 3, missing required ip_vs_* kernel modules
{
mods: []string{"ip_vs", "a", "bc", "def"},
scheduler: "sed",
kernelVersion: "4.19",
ipsetVersion: MinIPSetCheckVersion,
ok: false,
},
// case 4, ipset version too low
{
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"},
scheduler: "rr",
kernelVersion: "4.19",
ipsetVersion: "4.3.0",
ok: false,
},
// case 5, ok for linux kernel 4.19
{
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"},
scheduler: "rr",
kernelVersion: "4.19",
ipsetVersion: MinIPSetCheckVersion,
ok: true,
},
// case 6, ok for linux kernel 4.18
{
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
scheduler: "rr",
kernelVersion: "4.18",
ipsetVersion: MinIPSetCheckVersion,
ok: true,
},
// case 7. ok when module list has extra modules
{
mods: []string{"foo", "ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack", "bar"},
scheduler: "rr",
kernelVersion: "4.19",
ipsetVersion: "6.19",
ok: true,
},
// case 8, not ok for sed based IPVS scheduling
{
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack"},
scheduler: "sed",
kernelVersion: "4.19",
ipsetVersion: MinIPSetCheckVersion,
ok: false,
},
// case 9, ok for dh based IPVS scheduling
{
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack", "ip_vs_dh"},
scheduler: "dh",
kernelVersion: "4.19",
ipsetVersion: MinIPSetCheckVersion,
ok: true,
},
// case 10, non-existent scheduler, error due to modules not existing
{
mods: []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack", "ip_vs_dh"},
scheduler: "foobar",
kernelVersion: "4.19",
ipsetVersion: MinIPSetCheckVersion,
ok: false,
},
}

for i := range testCases {
handle := &fakeKernelHandler{modules: testCases[i].mods, kernelVersion: testCases[i].kernelVersion}
versioner := &fakeIPSetVersioner{version: testCases[i].ipsetVersion, err: testCases[i].ipsetErr}
ok, err := CanUseIPVSProxier(handle, versioner)
ok, err := CanUseIPVSProxier(handle, versioner, testCases[i].scheduler)
if ok != testCases[i].ok {
t.Errorf("Case [%d], expect %v, got %v: err: %v", i, testCases[i].ok, ok, err)
}
Expand Down