Permalink
Browse files

Merge pull request #853 from Mirantis/jell/cpu

Add new tests for cpusets
  • Loading branch information...
ivan4th committed Feb 6, 2019
2 parents 4ea5dff + ab9ab88 commit a653f48d1878557ea7dbf58dd34f88117b49d45c
@@ -2,7 +2,7 @@ version: "2"
checks:
argument-count:
config:
threshold: 9
threshold: 10
complex-logic:
config:
threshold: 8
@@ -73,8 +73,9 @@ func main() {

// FIXME: move the pid of qemu instance out of /kubepods/podxxxxxxx
// for some cases it will be killed by kubelet after the virtlet pod is deleted/recreated
if _, err := cgroups.GetProcessController(os.Getpid(), "hugetlb"); err == nil {
err = cgroups.MoveProcess(os.Getpid(), "hugetlb", "/")
cm := cgroups.NewManager(os.Getpid(), nil)
if _, err := cm.GetProcessController("hugetlb"); err == nil {
err = cm.MoveProcess("hugetlb", "/")
if err != nil {
glog.Warningf("failed to move pid into hugetlb path /: %v", err)
}
@@ -149,7 +150,7 @@ func main() {
glog.Fatalf("Error reexecuting vmwrapper: %v", err)
}
} else {
if err := setupCPUSets(); err != nil {
if err := setupCPUSets(cm); err != nil {
glog.Errorf("Can't set cpusets for emulator: %v", err)
os.Exit(1)
}
@@ -163,13 +164,13 @@ func main() {
}
}

func setupCPUSets() error {
func setupCPUSets(cm cgroups.Manager) error {
cpusets := os.Getenv(config.CpusetsEnvVarName)
if cpusets == "" {
return nil
}

controller, err := cgroups.GetProcessController("self", "cpuset")
controller, err := cm.GetProcessController("cpuset")
if err != nil {
return err
}
@@ -0,0 +1,163 @@
- name: GetImagePathDigestAndVirtualSize
value: fake/image1
- name: 'storage: CreateStoragePool'
value: |-
<pool type="dir">
<name>volumes</name>
<target>
<path>/var/lib/virtlet/volumes</path>
</target>
</pool>
- name: 'storage: volumes: CreateStorageVol'
value: |-
<volume type="file">
<name>virtlet_root_231700d5-c9a6-5a49-738d-99a954c51550</name>
<allocation unit="b">0</allocation>
<capacity unit="b">424242</capacity>
<target>
<format type="qcow2"></format>
</target>
<backingStore>
<path>/fake/volume/path</path>
<format type="qcow2"></format>
</backingStore>
</volume>
- name: 'domain conn: DefineDomain'
value: |-
<domain type="kvm">
<name>virtlet-231700d5-c9a6-container1</name>
<uuid>231700d5-c9a6-5a49-738d-99a954c51550</uuid>
<memory unit="MiB">1024</memory>
<vcpu>1</vcpu>
<cputune>
<shares>0</shares>
<period>0</period>
<quota>0</quota>
</cputune>
<os>
<type>hvm</type>
<boot dev="hd"></boot>
</os>
<features>
<acpi></acpi>
</features>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>restart</on_crash>
<devices>
<emulator>/vmwrapper</emulator>
<disk type="file" device="disk">
<driver name="qemu" type="qcow2"></driver>
<source file="/var/lib/virtlet/volumes/virtlet_root_231700d5-c9a6-5a49-738d-99a954c51550"></source>
<target dev="sda" bus="scsi"></target>
<address type="drive" controller="0" bus="0" target="0" unit="0"></address>
</disk>
<disk type="file" device="cdrom">
<driver name="qemu" type="raw"></driver>
<source file="/var/lib/virtlet/config/config-231700d5-c9a6-5a49-738d-99a954c51550.iso"></source>
<target dev="sdb" bus="scsi"></target>
<readonly></readonly>
<address type="drive" controller="0" bus="0" target="0" unit="1"></address>
</disk>
<controller type="scsi" index="0" model="virtio-scsi">
<address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x0"></address>
</controller>
<controller type="pci" model="pci-root"></controller>
<serial type="unix">
<source mode="connect" path="/var/lib/libvirt/streamer.sock">
<reconnect enabled="yes" timeout="1"></reconnect>
</source>
<target port="0"></target>
</serial>
<input type="tablet" bus="usb"></input>
<graphics type="vnc" port="-1"></graphics>
<video>
<model type="cirrus"></model>
</video>
</devices>
<commandline xmlns="http://libvirt.org/schemas/domain/qemu/1.0">
<env name="VIRTLET_EMULATOR" value="/usr/bin/kvm"></env>
<env name="VIRTLET_NET_KEY" value="/tmp/fakenetns"></env>
<env name="VIRTLET_CONTAINER_ID" value="231700d5-c9a6-5a49-738d-99a954c51550"></env>
<env name="VIRTLET_CONTAINER_LOG_PATH" value="/var/log/pods/69eec606-0493-5825-73a4-c5e0c0236155/container1_42.log"></env>
</commandline>
</domain>
- name: Calling setting cpuset for emulator proces
- name: FileReader
value: /run/libvirt/qemu/virtlet-231700d5-c9a6-testName_0.pid
- name: 'ReadString: 4242'
value: {}
- name: FileReader
value: /proc/4242/cgroup
- name: |
ReadString: 3:cpuset:/somepath/in/cgroups/emulator
value: {}
- name: 'WriteFile: /sys/fs/cgroup/cpuset/somepath/in/cgroups/emulator/cpuset.cpus'
value: "42"
- name: Calling setting cpuset for domain definition
- name: 'domain conn: virtlet-231700d5-c9a6-container1: Undefine'
- name: 'domain conn: DefineDomain'
value: |-
<domain type="kvm">
<name>virtlet-231700d5-c9a6-container1</name>
<uuid>231700d5-c9a6-5a49-738d-99a954c51550</uuid>
<memory unit="MiB">1024</memory>
<vcpu>1</vcpu>
<cputune>
<shares>0</shares>
<period>0</period>
<quota>0</quota>
</cputune>
<os>
<type>hvm</type>
<boot dev="hd"></boot>
</os>
<features>
<acpi></acpi>
</features>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>restart</on_crash>
<devices>
<emulator>/vmwrapper</emulator>
<disk type="file" device="disk">
<driver name="qemu" type="qcow2"></driver>
<source file="/var/lib/virtlet/volumes/virtlet_root_231700d5-c9a6-5a49-738d-99a954c51550"></source>
<target dev="sda" bus="scsi"></target>
<address type="drive" controller="0" bus="0" target="0" unit="0"></address>
</disk>
<disk type="file" device="cdrom">
<driver name="qemu" type="raw"></driver>
<source file="/var/lib/virtlet/config/config-231700d5-c9a6-5a49-738d-99a954c51550.iso"></source>
<target dev="sdb" bus="scsi"></target>
<readonly></readonly>
<address type="drive" controller="0" bus="0" target="0" unit="1"></address>
</disk>
<controller type="scsi" index="0" model="virtio-scsi">
<address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x0"></address>
</controller>
<controller type="pci" model="pci-root"></controller>
<serial type="unix">
<source mode="connect" path="/var/lib/libvirt/streamer.sock">
<reconnect enabled="yes" timeout="1"></reconnect>
</source>
<target port="0"></target>
</serial>
<input type="tablet" bus="usb"></input>
<graphics type="vnc" port="-1"></graphics>
<video>
<model type="cirrus"></model>
</video>
</devices>
<commandline xmlns="http://libvirt.org/schemas/domain/qemu/1.0">
<env name="VIRTLET_EMULATOR" value="/usr/bin/kvm"></env>
<env name="VIRTLET_NET_KEY" value="/tmp/fakenetns"></env>
<env name="VIRTLET_CONTAINER_ID" value="231700d5-c9a6-5a49-738d-99a954c51550"></env>
<env name="VIRTLET_CONTAINER_LOG_PATH" value="/var/log/pods/69eec606-0493-5825-73a4-c5e0c0236155/container1_42.log"></env>
<env name="VIRTLET_CPUSETS" value="42"></env>
</commandline>
</domain>
- name: Invoking RemoveContainer()
- name: 'domain conn: virtlet-231700d5-c9a6-container1: Undefine'
- name: 'storage: volumes: RemoveVolumeByName'
value: virtlet_root_231700d5-c9a6-5a49-738d-99a954c51550
@@ -17,12 +17,9 @@ limitations under the License.
package libvirttools

import (
"bytes"
"io/ioutil"
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"strings"

libvirtxml "github.com/libvirt/libvirt-go-xml"

@@ -74,65 +71,49 @@ func (v *VirtualizationTool) UpdateCpusetsInContainerDefinition(containerID, cpu
// UpdateCpusetsForEmulatorProcess looks through /proc for emulator process
// to find its cgroup manager for cpusets then uses it to adjust the setting
func (v *VirtualizationTool) UpdateCpusetsForEmulatorProcess(containerID, cpusets string) (bool, error) {
// TODO: replace iterating over the procfs with reading pid from
// /run/libvirt/qemu/virtlet-CONTAINER_ID[:12]-DOMAIN_NAME.pid
d, err := os.Open(procfsLocation)
pidFilePath, err := v.getEmulatorPidFileLocation(containerID)
if err != nil {
return false, err
}
defer d.Close()

entries, err := d.Readdirnames(-1)
f, err := v.filesManipulator.FileReader(pidFilePath)
if err != nil {
// File not found - so there is no emulator yet
if _, ok := err.(*os.PathError); ok {
return false, nil
}
return false, err
}
defer f.Close()

for _, name := range entries {
_, err := strconv.ParseInt(name, 10, 32)
if err != nil {
// skip non numeric names
continue
}

isContainerPid, err := isEmulatorPid(name, containerID)
if err != nil {
// there should be only a single line without eol, but use eol as
// a marker to read data to EOF.
pid, err := f.ReadString('\n')
if err != nil {
if err != io.EOF {
return false, err
}

if isContainerPid {
controller, err := cgroups.GetProcessController(name, "cpuset")
if err != nil {
return false, err
}

if err := controller.Set("cpus", cpusets); err != nil {
return false, err
}
return true, nil
}
}

return false, nil
}

func isEmulatorPid(pid, containerID string) (bool, error) {
data, err := ioutil.ReadFile(filepath.Join(procfsLocation, pid, "cmdline"))
cm := cgroups.NewManager(pid, v.filesManipulator)
controller, err := cm.GetProcessController("cpuset")
if err != nil {
return false, err
}

cmdline := bytes.Split(data, []byte{0})

if string(cmdline[0]) != emulatorProcessName {
return false, nil
if err := controller.Set("cpus", cpusets); err != nil {
return false, err
}
return true, nil
}

searchTerm := "virtlet-" + containerID[:12]
for _, param := range cmdline {
if strings.Contains(string(param), searchTerm) {
return true, nil
}
func (v *VirtualizationTool) getEmulatorPidFileLocation(containerID string) (string, error) {
container, err := v.metadataStore.Container(containerID).Retrieve()
if err != nil {
return "", err
}

return false, nil
return fmt.Sprintf(
"/run/libvirt/qemu/virtlet-%s-%s.pid",
containerID[:13], container.Config.PodName,
), nil
}
@@ -0,0 +1,50 @@
/*
Copyright 2019 Mirantis
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package libvirttools

import (
"fmt"
"testing"

fakemeta "github.com/Mirantis/virtlet/pkg/metadata/fake"
testutils "github.com/Mirantis/virtlet/pkg/utils/testing"
"github.com/Mirantis/virtlet/tests/gm"
)

func TestUpdateCpusets(t *testing.T) {
files := map[string]string{
"/proc/4242/cgroup": "3:cpuset:/somepath/in/cgroups/emulator\n",
}
ct := newContainerTester(t, testutils.NewToplevelRecorder(), nil, files)
defer ct.teardown()

sandbox := fakemeta.GetSandboxes(1)[0]
ct.setPodSandbox(sandbox)
containerID := ct.createContainer(sandbox, nil, nil)
pidFilePath := fmt.Sprintf("/run/libvirt/qemu/virtlet-%s-%s.pid", containerID[:13], sandbox.Name)
files[pidFilePath] = "4242"

ct.rec.Rec("Calling setting cpuset for emulator proces", nil)
ct.virtTool.UpdateCpusetsForEmulatorProcess(containerID, "42")

ct.rec.Rec("Calling setting cpuset for domain definition", nil)
ct.virtTool.UpdateCpusetsInContainerDefinition(containerID, "42")

ct.rec.Rec("Invoking RemoveContainer()", nil)
ct.removeContainer(containerID)
gm.Verify(t, gm.NewYamlVerifier(ct.rec.Content()))
}
@@ -25,7 +25,7 @@ import (
)

func TestDump(t *testing.T) {
ct := newContainerTester(t, testutils.NewToplevelRecorder(), nil)
ct := newContainerTester(t, testutils.NewToplevelRecorder(), nil, nil)
defer ct.teardown()

sandbox := fakemeta.GetSandboxes(1)[0]
Oops, something went wrong.

0 comments on commit a653f48

Please sign in to comment.