View
@@ -0,0 +1,86 @@
package health
import (
"encoding/json"
"fmt"
"net/http"
"time"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/onsi/gomega/ghttp"
"k8s.io/api/core/v1"
v12 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/onsi/ginkgo/extensions/table"
"k8s.io/apimachinery/pkg/util/clock"
v13 "kubevirt.io/kubevirt/pkg/api/v1"
"kubevirt.io/kubevirt/pkg/kubecli"
"kubevirt.io/kubevirt/pkg/log"
"kubevirt.io/kubevirt/pkg/virt-handler/devices"
"kubevirt.io/kubevirt/pkg/virt-handler/isolation"
)
var _ = Describe("Health", func() {
var server *ghttp.Server
var client kubecli.KubevirtClient
log.Log.SetIOWriter(GinkgoWriter)
BeforeEach(func() {
var err error
server = ghttp.NewServer()
client, err = kubecli.GetKubevirtClientFromFlags(server.URL(), "")
Expect(err).ToNot(HaveOccurred())
})
table.DescribeTable("should mark the nodes", func(schedulable bool, device devices.Device) {
now := v12.Now()
t, err := json.Marshal(now)
Expect(err).ToNot(HaveOccurred())
patch := fmt.Sprintf(`{"metadata": { "labels": {"kubevirt.io/schedulable": "%t"}, "annotations": {"kubevirt.io/heartbeat": %s}}}`, schedulable, string(t))
server.AppendHandlers(ghttp.CombineHandlers(
ghttp.VerifyRequest(http.MethodPatch, "/api/v1/nodes/testhost"),
ghttp.VerifyBody([]byte(patch)),
ghttp.RespondWithJSONEncoded(http.StatusOK, &v1.Node{}),
))
stop := make(chan struct{})
defer close(stop)
go func() {
GinkgoRecover()
checker := &ReadinessChecker{
clientset: client,
host: "testhost",
plugins: map[string]devices.Device{"test": device},
clock: clock.NewFakeClock(now.Time),
}
checker.HeartBeat(1*time.Second, 100, stop)
}()
time.Sleep(500 * time.Millisecond)
Expect(server.ReceivedRequests()).To(HaveLen(1))
},
table.Entry("should mark the node as non-schedulable because of a failing device check", false, &testDevice{fail: true}),
table.Entry("should mark the node as schedulable", true, &testDevice{fail: false}),
)
AfterEach(func() {
server.Close()
})
})
type testDevice struct {
fail bool
}
func (*testDevice) Setup(vmi *v13.VirtualMachineInstance, hostNamespaces *isolation.IsolationResult, podNamespaces *isolation.IsolationResult) error {
return nil
}
func (t *testDevice) Available() error {
if t.fail {
return fmt.Errorf("failing")
}
return nil
}
View
@@ -0,0 +1,52 @@
// Automatically generated by MockGen. DO NOT EDIT!
// Source: isolation.go
package isolation
import (
gomock "github.com/golang/mock/gomock"
v1 "kubevirt.io/kubevirt/pkg/api/v1"
)
// Mock of PodIsolationDetector interface
type MockPodIsolationDetector struct {
ctrl *gomock.Controller
recorder *_MockPodIsolationDetectorRecorder
}
// Recorder for MockPodIsolationDetector (not exported)
type _MockPodIsolationDetectorRecorder struct {
mock *MockPodIsolationDetector
}
func NewMockPodIsolationDetector(ctrl *gomock.Controller) *MockPodIsolationDetector {
mock := &MockPodIsolationDetector{ctrl: ctrl}
mock.recorder = &_MockPodIsolationDetectorRecorder{mock}
return mock
}
func (_m *MockPodIsolationDetector) EXPECT() *_MockPodIsolationDetectorRecorder {
return _m.recorder
}
func (_m *MockPodIsolationDetector) Detect(vm *v1.VirtualMachineInstance) (*IsolationResult, error) {
ret := _m.ctrl.Call(_m, "Detect", vm)
ret0, _ := ret[0].(*IsolationResult)
ret1, _ := ret[1].(error)
return ret0, ret1
}
func (_mr *_MockPodIsolationDetectorRecorder) Detect(arg0 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCall(_mr.mock, "Detect", arg0)
}
func (_m *MockPodIsolationDetector) Whitelist(controller []string) PodIsolationDetector {
ret := _m.ctrl.Call(_m, "Whitelist", controller)
ret0, _ := ret[0].(PodIsolationDetector)
return ret0
}
func (_mr *_MockPodIsolationDetectorRecorder) Whitelist(arg0 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCall(_mr.mock, "Whitelist", arg0)
}
View
@@ -0,0 +1,206 @@
/*
* This file is part of the kubevirt project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* Copyright 2017 Red Hat, Inc.
*
*/
package isolation
//go:generate mockgen -source $GOFILE -package=$GOPACKAGE -destination=generated_mock_$GOFILE
/*
ATTENTION: Rerun code generators when interface signatures are modified.
*/
import (
"bufio"
"fmt"
"net"
"os"
"strings"
"syscall"
"kubevirt.io/kubevirt/pkg/api/v1"
"kubevirt.io/kubevirt/pkg/log"
"kubevirt.io/kubevirt/pkg/virt-handler/cmd-client"
)
// PodIsolationDetector helps detecting cgroups, namespaces and PIDs of Pods from outside of them.
// Different strategies may be applied to do that.
type PodIsolationDetector interface {
// Detect takes a vm, looks up a socket based the VM and detects pid, cgroups and namespaces of the owner of that socket.
// It returns an IsolationResult containing all isolation information
Detect(vm *v1.VirtualMachineInstance) (*IsolationResult, error)
// Whitelist allows specifying cgroup controller which should be considered to detect the cgroup slice
// It returns a PodIsolationDetector to allow configuring the PodIsolationDetector via the builder pattern.
Whitelist(controller []string) PodIsolationDetector
}
type socketBasedIsolationDetector struct {
socketDir string
controller []string
}
// NewSocketBasedIsolationDetector takes socketDir and creates a socket based IsolationDetector
// It returns a PodIsolationDetector which detects pid, cgroups and namespaces of the socket owner.
func NewSocketBasedIsolationDetector(socketDir string) PodIsolationDetector {
return &socketBasedIsolationDetector{socketDir: socketDir, controller: []string{"devices"}}
}
func (s *socketBasedIsolationDetector) Whitelist(controller []string) PodIsolationDetector {
s.controller = controller
return s
}
func (s *socketBasedIsolationDetector) Detect(vm *v1.VirtualMachineInstance) (*IsolationResult, error) {
var pid int
var slice string
var err error
var controller []string
// Look up the socket of the virt-launcher Pod which was created for that VM, and extract the PID from it
socket := cmdclient.SocketFromUID(s.socketDir, string(vm.UID))
if pid, err = s.getPid(socket); err != nil {
log.Log.Object(vm).Reason(err).Errorf("Could not get owner Pid of socket %s", socket)
return nil, err
}
// Look up the cgroup slice based on the whitelisted controller
if controller, slice, err = s.getSlice(pid); err != nil {
log.Log.Object(vm).Reason(err).Errorf("Could not get cgroup slice for Pid %d", pid)
return nil, err
}
return NewIsolationResult(pid, slice, controller), nil
}
func NewIsolationResult(pid int, slice string, controller []string) *IsolationResult {
return &IsolationResult{pid: pid, slice: slice, controller: controller}
}
type IsolationResult struct {
pid int
slice string
controller []string
}
func (r *IsolationResult) Slice() string {
return r.slice
}
func (r *IsolationResult) PIDNamespace() string {
return fmt.Sprintf("/proc/%d/ns/pid", r.pid)
}
func (r *IsolationResult) NetNamespace() string {
return fmt.Sprintf("/proc/%d/ns/net", r.pid)
}
func (r *IsolationResult) MountRoot() string {
return fmt.Sprintf("/proc/%d/root", r.pid)
}
func (r *IsolationResult) Pid() int {
return r.pid
}
func (r *IsolationResult) Controller() []string {
return r.controller
}
func (s *socketBasedIsolationDetector) getPid(socket string) (int, error) {
sock, err := net.Dial("unix", socket)
if err != nil {
return -1, err
}
defer sock.Close()
ufile, err := sock.(*net.UnixConn).File()
if err != nil {
return -1, err
}
// This is the tricky part, which will give us the PID of the owning socket
ucreds, err := syscall.GetsockoptUcred(int(ufile.Fd()), syscall.SOL_SOCKET, syscall.SO_PEERCRED)
if err != nil {
return -1, err
}
if int(ucreds.Pid) == 0 {
return -1, fmt.Errorf("The detected PID is 0. Is the isolation detector running in the host PID namespace?")
}
return int(ucreds.Pid), nil
}
func (s *socketBasedIsolationDetector) getSlice(pid int) (controller []string, slice string, err error) {
cgroups, err := os.Open(fmt.Sprintf("/proc/%d/cgroup", pid))
if err != nil {
return
}
defer cgroups.Close()
scanner := bufio.NewScanner(cgroups)
for scanner.Scan() {
cgEntry := strings.Split(scanner.Text(), ":")
// Check if we have a sane cgroup line
if len(cgEntry) != 3 {
err = fmt.Errorf("Could not extract slice from cgroup line: %s", scanner.Text())
return
}
// Skip not supported cgroup controller
if !sliceContains(s.controller, cgEntry[1]) {
continue
}
// Set and check cgroup slice
if slice == "" {
slice = cgEntry[2]
} else if slice != cgEntry[2] {
err = fmt.Errorf("Process is part of more than one slice. Expected %s, found %s", slice, cgEntry[2])
return
}
// Add controller
controller = append(controller, cgEntry[1])
}
// Check if we encountered a read error
if scanner.Err() != nil {
err = scanner.Err()
return
}
if slice == "" {
err = fmt.Errorf("Could not detect slice of whitelisted controller: %v", s.controller)
return
}
return
}
func sliceContains(controllers []string, value string) bool {
for _, c := range controllers {
if c == value {
return true
}
}
return false
}
func NodeIsolationResult() *IsolationResult {
return &IsolationResult{
pid: 1,
}
}
View
@@ -0,0 +1,16 @@
package isolation
import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"testing"
"kubevirt.io/kubevirt/pkg/log"
)
func TestIsolation(t *testing.T) {
RegisterFailHandler(Fail)
log.Log.SetIOWriter(GinkgoWriter)
RunSpecs(t, "Isolation Suite")
}
View
@@ -0,0 +1,75 @@
package isolation
import (
"fmt"
"io/ioutil"
"net"
"os"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"kubevirt.io/kubevirt/pkg/api/v1"
"kubevirt.io/kubevirt/pkg/virt-handler/cmd-client"
)
var _ = Describe("Isolation", func() {
Context("With an existing socket", func() {
var socket net.Listener
var tmpDir string
vm := v1.NewMinimalVMIWithNS("default", "testvm")
BeforeEach(func() {
var err error
tmpDir, err = ioutil.TempDir("", "kubevirt")
os.MkdirAll(tmpDir+"/sockets/", os.ModePerm)
socket, err = net.Listen("unix", cmdclient.SocketFromUID(
tmpDir,
string(vm.UID)),
)
Expect(err).ToNot(HaveOccurred())
})
It("Should detect the PID of the test suite", func() {
result, err := NewSocketBasedIsolationDetector(tmpDir).Whitelist([]string{"devices"}).Detect(vm)
Expect(err).ToNot(HaveOccurred())
Expect(result.Pid()).To(Equal(os.Getpid()))
})
It("Should not detect any slice if there is no matching controller", func() {
_, err := NewSocketBasedIsolationDetector(tmpDir).Whitelist([]string{"not_existing_slice"}).Detect(vm)
Expect(err).To(HaveOccurred())
})
It("Should detect the 'devices' controller slice of the test suite", func() {
result, err := NewSocketBasedIsolationDetector(tmpDir).Whitelist([]string{"devices"}).Detect(vm)
Expect(err).ToNot(HaveOccurred())
Expect(result.Slice()).To(HavePrefix("/"))
})
It("Should detect the PID namespace of the test suite", func() {
result, err := NewSocketBasedIsolationDetector(tmpDir).Whitelist([]string{"devices"}).Detect(vm)
Expect(err).ToNot(HaveOccurred())
Expect(result.PIDNamespace()).To(Equal(fmt.Sprintf("/proc/%d/ns/pid", os.Getpid())))
})
It("Should detect the Mount root of the test suite", func() {
result, err := NewSocketBasedIsolationDetector(tmpDir).Whitelist([]string{"devices"}).Detect(vm)
Expect(err).ToNot(HaveOccurred())
Expect(result.MountRoot()).To(Equal(fmt.Sprintf("/proc/%d/root", os.Getpid())))
})
It("Should detect the Network namespace of the test suite", func() {
result, err := NewSocketBasedIsolationDetector(tmpDir).Whitelist([]string{"devices"}).Detect(vm)
Expect(err).ToNot(HaveOccurred())
Expect(result.NetNamespace()).To(Equal(fmt.Sprintf("/proc/%d/ns/net", os.Getpid())))
})
AfterEach(func() {
socket.Close()
os.RemoveAll(tmpDir)
})
})
})
View
@@ -0,0 +1,224 @@
// Copyright 2015 CNI authors
// Copyright 2017 rmohr@redhat.com
// Copyright 2018 KubeVirt authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// GO version 1.10 or greater is required. Before that, switching namespaces in
// long running processes in go did not work in a reliable way.
// +build go1.10
package ns
import (
"fmt"
"os"
"regexp"
"runtime"
"sync"
"syscall"
"golang.org/x/sys/unix"
)
type Namespace interface {
// Executes the passed closure in this object's namespace,
// attempting to restore the original namespace before returning.
// No code called from Do() should create go-routines, or the risk
// of executing code in an incorrect namespace exists.
Do(toRun func(Namespace) error) error
// Sets the current namespace to this object's namespace.
Set() error
// Returns the filesystem path representing this object's namespace
Path() string
// Returns a file descriptor representing this object's namespace
Fd() uintptr
// Cleans up this instance of the namespace; if this instance
// is the last user the namespace will be destroyed
Close() error
}
type namespace struct {
file *os.File
closed bool
nsType int
nsTypeStr string
}
// Returns an object representing the namespace referred to by @path
func GetNS(nspath string) (Namespace, error) {
err := IsNSorErr(nspath)
if err != nil {
return nil, err
}
//cgroup ipc mnt net pid user uts
res := regexp.MustCompile(`^/proc/.+/ns/([^/]+)`).FindStringSubmatch(nspath)
if len(res) != 2 {
return nil, fmt.Errorf("Error detecting namespace type from path: %s", nspath)
}
var nsType int
switch res[1] {
case "pid":
nsType = unix.CLONE_NEWPID
case "net":
nsType = unix.CLONE_NEWNET
default:
return nil, fmt.Errorf("Error unsupported namespace requested: %s", res[1])
}
fd, err := os.Open(nspath)
if err != nil {
return nil, err
}
return &namespace{file: fd, nsType: nsType, nsTypeStr: res[1]}, nil
}
func (ns *namespace) Set() error {
if err := ns.errorIfClosed(); err != nil {
return err
}
if _, _, err := unix.Syscall(unix.SYS_SETNS, ns.Fd(), uintptr(ns.nsType), 0); err != 0 {
return fmt.Errorf("Error switching to ns %v: %v", ns.file.Name(), err)
}
return nil
}
func (ns *namespace) Close() error {
if err := ns.errorIfClosed(); err != nil {
return err
}
if err := ns.file.Close(); err != nil {
return fmt.Errorf("Failed to close %q: %v", ns.file.Name(), err)
}
ns.closed = true
return nil
}
func (ns *namespace) Path() string {
return ns.file.Name()
}
func (ns *namespace) Fd() uintptr {
return ns.file.Fd()
}
func (ns *namespace) errorIfClosed() error {
if ns.closed {
return fmt.Errorf("%q has already been closed", ns.file.Name())
}
return nil
}
func (ns *namespace) Do(toRun func(Namespace) error) error {
if err := ns.errorIfClosed(); err != nil {
return err
}
containedCall := func(hostNS Namespace) error {
threadNS, err := GetNS(ns.getCurrentThreadNSPath())
if err != nil {
return fmt.Errorf("failed to open current netns: %v", err)
}
defer threadNS.Close()
// switch to target namespace
if err = ns.Set(); err != nil {
return fmt.Errorf("error switching to ns %v: %v", ns.file.Name(), err)
}
defer threadNS.Set() // switch back
return toRun(hostNS)
}
// save a handle to current network namespace
hostNS, err := GetNS(ns.getCurrentThreadNSPath())
if err != nil {
return fmt.Errorf("Failed to open current namespace: %v", err)
}
defer hostNS.Close()
var wg sync.WaitGroup
wg.Add(1)
var innerError error
go func() {
defer wg.Done()
runtime.LockOSThread()
innerError = containedCall(hostNS)
}()
wg.Wait()
return innerError
}
func (ns *namespace) getCurrentThreadNSPath() string {
// /proc/self/ns/net returns the namespace of the main thread, not
// of whatever thread this goroutine is running on. Make sure we
// use the thread's net namespace since the thread is switching around
return fmt.Sprintf("/proc/%d/task/%d/ns/%s", os.Getpid(), unix.Gettid(), ns.nsTypeStr)
}
// WithNSPath executes the passed closure under the given
// namespace, restoring the original namespace afterwards.
func WithNSPath(nspath string, toRun func(Namespace) error) error {
ns, err := GetNS(nspath)
if err != nil {
return err
}
defer ns.Close()
return ns.Do(toRun)
}
const (
// https://github.com/torvalds/linux/blob/master/include/uapi/linux/magic.h
NSFS_MAGIC = 0x6e736673
PROCFS_MAGIC = 0x9fa0
)
type NSPathNotExistErr struct{ msg string }
func (e NSPathNotExistErr) Error() string { return e.msg }
type NSPathNotNSErr struct{ msg string }
func (e NSPathNotNSErr) Error() string { return e.msg }
func IsNSorErr(nspath string) error {
stat := syscall.Statfs_t{}
if err := syscall.Statfs(nspath, &stat); err != nil {
if os.IsNotExist(err) {
err = NSPathNotExistErr{msg: fmt.Sprintf("failed to Statfs %q: %v", nspath, err)}
} else {
err = fmt.Errorf("failed to Statfs %q: %v", nspath, err)
}
return err
}
switch stat.Type {
case PROCFS_MAGIC, NSFS_MAGIC:
return nil
default:
return NSPathNotNSErr{msg: fmt.Sprintf("unknown FS magic on %q: %x", nspath, stat.Type)}
}
}
View
@@ -34,20 +34,17 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
"encoding/json"
v12 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"kubevirt.io/kubevirt/pkg/api/v1"
"kubevirt.io/kubevirt/pkg/cloud-init"
"kubevirt.io/kubevirt/pkg/config"
"kubevirt.io/kubevirt/pkg/controller"
"kubevirt.io/kubevirt/pkg/host-disk"
"kubevirt.io/kubevirt/pkg/kubecli"
"kubevirt.io/kubevirt/pkg/log"
"kubevirt.io/kubevirt/pkg/precond"
"kubevirt.io/kubevirt/pkg/virt-handler/cmd-client"
"kubevirt.io/kubevirt/pkg/virt-handler/device-manager"
"kubevirt.io/kubevirt/pkg/virt-handler/devices"
"kubevirt.io/kubevirt/pkg/virt-handler/isolation"
"kubevirt.io/kubevirt/pkg/virt-launcher"
"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api"
"kubevirt.io/kubevirt/pkg/watchdog"
@@ -62,7 +59,8 @@ func NewController(
domainInformer cache.SharedInformer,
gracefulShutdownInformer cache.SharedIndexInformer,
watchdogTimeoutSeconds int,
maxDevices int,
isolationDetector isolation.PodIsolationDetector,
clusterConfig *config.ClusterConfig,
) *VirtualMachineController {
queue := workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter())
@@ -76,7 +74,6 @@ func NewController(
vmiInformer: vmiInformer,
domainInformer: domainInformer,
gracefulShutdownInformer: gracefulShutdownInformer,
heartBeatInterval: 1 * time.Minute,
watchdogTimeoutSeconds: watchdogTimeoutSeconds,
}
@@ -100,7 +97,8 @@ func NewController(
c.launcherClients = make(map[string]cmdclient.LauncherClient)
c.kvmController = device_manager.NewDeviceController(c.host, maxDevices)
c.DevicePlugins = []devices.Device{&devices.KVM{ClusterConfig: clusterConfig}, &devices.TUN{}}
c.isolationDetector = isolationDetector
return c
}
@@ -116,9 +114,9 @@ type VirtualMachineController struct {
gracefulShutdownInformer cache.SharedIndexInformer
launcherClients map[string]cmdclient.LauncherClient
launcherClientLock sync.Mutex
heartBeatInterval time.Duration
watchdogTimeoutSeconds int
kvmController *device_manager.DeviceController
DevicePlugins []devices.Device
isolationDetector isolation.PodIsolationDetector
}
// Determines if a domain's grace period has expired during shutdown.
@@ -183,7 +181,15 @@ func (d *VirtualMachineController) updateVMIStatus(vmi *v1.VirtualMachineInstanc
return err
}
controller.NewVirtualMachineInstanceConditionManager().CheckFailure(vmi, syncError, "Synchronizing with the Domain failed.")
// In case we have no syncError from direct communication,
// let's check if we got asynchronous error reports from virt-launcher
if syncError == nil {
cond := d.checkDomainForSyncErrors(domain)
if cond != nil {
syncError = fmt.Errorf(cond.Message)
}
}
controller.NewVirtualMachineInstanceConditionManager().CheckFailure(vmi, syncError, "DomainSync")
if !reflect.DeepEqual(oldStatus, vmi.Status) {
_, err = d.clientset.VirtualMachineInstance(vmi.ObjectMeta.Namespace).Update(vmi)
@@ -214,8 +220,6 @@ func (c *VirtualMachineController) Run(threadiness int, stopCh chan struct{}) {
go c.domainInformer.Run(stopCh)
cache.WaitForCacheSync(stopCh, c.domainInformer.HasSynced)
go c.kvmController.Run(stopCh)
// Poplulate the VirtualMachineInstance store with known Domains on the host, to get deletes since the last run
for _, domain := range c.domainInformer.GetStore().List() {
d := domain.(*api.Domain)
@@ -232,8 +236,6 @@ func (c *VirtualMachineController) Run(threadiness int, stopCh chan struct{}) {
go c.gracefulShutdownInformer.Run(stopCh)
cache.WaitForCacheSync(stopCh, c.domainInformer.HasSynced, c.vmiInformer.HasSynced, c.gracefulShutdownInformer.HasSynced)
go c.heartBeat(c.heartBeatInterval, stopCh)
// Start the actual work
for i := 0; i < threadiness; i++ {
go wait.Until(c.runWorker, time.Second, stopCh)
@@ -425,7 +427,7 @@ func (d *VirtualMachineController) execute(key string) error {
syncErr = d.processVmCleanup(vmi)
} else if shouldUpdate {
log.Log.Object(vmi).V(3).Info("Processing vmi update")
syncErr = d.processVmUpdate(vmi)
syncErr = d.processVmUpdate(vmi, domain)
} else {
log.Log.Object(vmi).V(3).Info("No update processing required")
}
@@ -644,7 +646,7 @@ func (d *VirtualMachineController) getVerifiedLauncherClient(vmi *v1.VirtualMach
return
}
func (d *VirtualMachineController) processVmUpdate(origVMI *v1.VirtualMachineInstance) error {
func (d *VirtualMachineController) processVmUpdate(origVMI *v1.VirtualMachineInstance, domain *api.Domain) error {
vmi := origVMI.DeepCopy()
@@ -670,6 +672,23 @@ func (d *VirtualMachineController) processVmUpdate(origVMI *v1.VirtualMachineIns
if err != nil {
return err
}
// No need to do the setup again if we obviously did it at least once succeed with the setup
if domain == nil {
podEnv, err := d.isolationDetector.Detect(vmi)
if err != nil {
return err
}
nodeEnv := isolation.NodeIsolationResult()
for _, device := range d.DevicePlugins {
if err := device.Setup(vmi, nodeEnv, podEnv); err != nil {
return err
}
}
}
err = client.SyncVirtualMachine(vmi)
if err != nil {
return err
@@ -687,6 +706,23 @@ func (d *VirtualMachineController) setVmPhaseForStatusReason(domain *api.Domain,
vmi.Status.Phase = phase
return nil
}
// checkDomainForSyncErrors returns a synchronization error if present
func (d *VirtualMachineController) checkDomainForSyncErrors(domain *api.Domain) *api.DomainCondition {
if domain == nil {
return nil
}
for _, cond := range domain.Status.Conditions {
if cond.Type == api.DomainConditionSynchronized {
if cond.Status == k8sv1.ConditionFalse {
return &cond
}
return nil
}
}
return nil
}
func (d *VirtualMachineController) calculateVmPhaseForStatusReason(domain *api.Domain, vmi *v1.VirtualMachineInstance) (v1.VirtualMachineInstancePhase, error) {
if domain == nil {
@@ -730,6 +766,11 @@ func (d *VirtualMachineController) calculateVmPhaseForStatusReason(domain *api.D
}
case api.Running, api.Paused, api.Blocked, api.PMSuspended:
return v1.Running, nil
case api.Error:
switch domain.Status.Reason {
case api.ReasonLibvirtUnreachable:
return v1.Failed, nil
}
}
}
return vmi.Status.Phase, nil
@@ -795,25 +836,6 @@ func (d *VirtualMachineController) updateDomainFunc(old, new interface{}) {
}
}
func (d *VirtualMachineController) heartBeat(interval time.Duration, stopCh chan struct{}) {
for {
wait.JitterUntil(func() {
now, err := json.Marshal(v12.Now())
if err != nil {
log.DefaultLogger().Reason(err).Errorf("Can't determine date")
return
}
data := []byte(fmt.Sprintf(`{"metadata": { "labels": {"%s": "true"}, "annotations": {"%s": %s}}}`, v1.NodeSchedulable, v1.VirtHandlerHeartbeat, string(now)))
_, err = d.clientset.CoreV1().Nodes().Patch(d.host, types.StrategicMergePatchType, data)
if err != nil {
log.DefaultLogger().Reason(err).Errorf("Can't patch node %s", d.host)
return
}
log.DefaultLogger().V(4).Infof("Heartbeat sent")
}, interval, 1.2, true, stopCh)
}
}
func isACPIEnabled(vmi *v1.VirtualMachineInstance, domain *api.Domain) bool {
zero := int64(0)
return vmi.Spec.TerminationGracePeriodSeconds != &zero &&
View
@@ -39,11 +39,14 @@ import (
"k8s.io/client-go/kubernetes/fake"
"kubevirt.io/kubevirt/pkg/api/v1"
"kubevirt.io/kubevirt/pkg/config"
"kubevirt.io/kubevirt/pkg/kubecli"
"kubevirt.io/kubevirt/pkg/log"
"kubevirt.io/kubevirt/pkg/precond"
"kubevirt.io/kubevirt/pkg/testutils"
"kubevirt.io/kubevirt/pkg/virt-handler/cmd-client"
"kubevirt.io/kubevirt/pkg/virt-handler/devices"
"kubevirt.io/kubevirt/pkg/virt-handler/isolation"
"kubevirt.io/kubevirt/pkg/virt-launcher"
"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api"
"kubevirt.io/kubevirt/pkg/watchdog"
@@ -98,6 +101,8 @@ var _ = Describe("VirtualMachineInstance", func() {
mockWatchdog = &MockWatchdog{shareDir}
mockGracefulShutdown = &MockGracefulShutdown{shareDir}
mockIsolationDetector := isolation.NewMockPodIsolationDetector(ctrl)
mockIsolationDetector.EXPECT().Detect(gomock.Any()).AnyTimes()
controller = NewController(recorder,
virtClient,
@@ -107,8 +112,10 @@ var _ = Describe("VirtualMachineInstance", func() {
domainInformer,
gracefulShutdownInformer,
1,
10,
mockIsolationDetector,
config.NewClusterConfig(nil),
)
controller.DevicePlugins = nil
testUUID = uuid.NewUUID()
client = cmdclient.NewMockLauncherClient(ctrl)
@@ -265,6 +272,43 @@ var _ = Describe("VirtualMachineInstance", func() {
controller.Execute()
})
It("should invoke device extensions if a Domain does not yet exist", func() {
vmi := v1.NewMinimalVMI("testvmi")
vmi.ObjectMeta.ResourceVersion = "1"
vmi.UID = testUUID
vmi.Status.Phase = v1.Scheduled
fakeExtension := devices.NewMockDevice(ctrl)
controller.DevicePlugins = []devices.Device{fakeExtension}
mockWatchdog.CreateFile(vmi)
vmiFeeder.Add(vmi)
fakeExtension.EXPECT().Setup(gomock.Any(), gomock.Any(), gomock.Any())
client.EXPECT().SyncVirtualMachine(vmi)
controller.Execute()
})
It("should not invoke device extensions if a Domain exist", func() {
vmi := v1.NewMinimalVMI("testvmi")
vmi.ObjectMeta.ResourceVersion = "1"
vmi.UID = testUUID
vmi.Status.Phase = v1.Running
fakeExtension := devices.NewMockDevice(ctrl)
controller.DevicePlugins = []devices.Device{fakeExtension}
domain := api.NewMinimalDomainWithUUID("testvmi", testUUID)
domain.Status.Status = api.Running
mockWatchdog.CreateFile(vmi)
domainFeeder.Add(domain)
vmiFeeder.Add(vmi)
client.EXPECT().SyncVirtualMachine(vmi)
fakeExtension.EXPECT().Setup(gomock.Any(), gomock.Any(), gomock.Any()).MaxTimes(0)
controller.Execute()
})
It("should update from Scheduled to Running, if it sees a running Domain", func() {
vmi := v1.NewMinimalVMI("testvmi")
vmi.UID = testUUID
@@ -298,6 +342,54 @@ var _ = Describe("VirtualMachineInstance", func() {
controller.Execute()
})
It("should update from Running to Failed if an unrecoverable error occurs in the launcher", func() {
vmi := v1.NewMinimalVMI("testvmi")
vmi.ObjectMeta.ResourceVersion = "1"
vmi.Status.Phase = v1.Running
updatedVMI := vmi.DeepCopy()
updatedVMI.Status.Phase = v1.Failed
mockWatchdog.CreateFile(vmi)
domain := api.NewMinimalDomain("testvmi")
domain.Status.Status = api.Error
domain.Status.Reason = api.ReasonLibvirtUnreachable
vmiFeeder.Add(vmi)
domainFeeder.Add(domain)
vmiInterface.EXPECT().Update(updatedVMI)
controller.Execute()
})
It("should propagate asynchronous sync errors from the Domain to the VMI", func() {
vmi := v1.NewMinimalVMI("testvmi")
vmi.ObjectMeta.ResourceVersion = "1"
vmi.Status.Phase = v1.Scheduled
mockWatchdog.CreateFile(vmi)
domain := api.NewMinimalDomain("testvmi")
domain.Status.Status = api.Running
domain.Status.Reason = api.ReasonUnknown
domain.Status.Conditions = []api.DomainCondition{{
Type: api.DomainConditionSynchronized,
Status: k8sv1.ConditionFalse,
Reason: "blub",
Message: "bla",
}}
vmiFeeder.Add(vmi)
domainFeeder.Add(domain)
vmiInterface.EXPECT().Update(gomock.Any()).Do(func(vmi *v1.VirtualMachineInstance) {
Expect(vmi.Status.Conditions).To(HaveLen(1))
Expect(vmi.Status.Conditions[0].Type).To(Equal(v1.VirtualMachineInstanceSynchronized))
Expect(vmi.Status.Conditions[0].Reason).To(Equal("DomainSync"))
Expect(vmi.Status.Conditions[0].Message).To(Equal("bla"))
Expect(vmi.Status.Conditions[0].Status).To(Equal(k8sv1.ConditionFalse))
})
controller.Execute()
})
It("should move VirtualMachineInstance from Scheduled to Failed if watchdog file is missing", func() {
vmi := v1.NewMinimalVMI("testvmi")
vmi.ObjectMeta.ResourceVersion = "1"
View
@@ -203,7 +203,7 @@ func (mon *monitor) refresh() {
if mon.pid == 0 {
var err error
mon.pid, err = findPid(mon.cmdlineMatchStr)
mon.pid, err = FindPid(mon.cmdlineMatchStr)
if err != nil {
log.Log.Infof("Still missing PID for %s, %v", mon.cmdlineMatchStr, err)
@@ -315,7 +315,7 @@ func pidExists(pid int) (bool, error) {
return true, nil
}
func findPid(commandNamePrefix string) (int, error) {
func FindPid(commandNamePrefix string) (int, error) {
entries, err := filepath.Glob("/proc/*/cmdline")
if err != nil {
return 0, err
View
@@ -10,6 +10,10 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/apimachinery/pkg/types"
"k8s.io/api/core/v1"
"kubevirt.io/kubevirt/pkg/log"
notifyserver "kubevirt.io/kubevirt/pkg/virt-handler/notify-server"
"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api"
@@ -71,6 +75,21 @@ func (c *DomainEventClient) SendDomainEvent(event watch.Event) error {
return nil
}
func (c *DomainEventClient) SendErrorDomainEvent(name string, namespace string, uid types.UID, reason api.StateChangeReason, err error) error {
domain := api.NewDomainReferenceFromName(namespace, name)
domain.GetObjectMeta().SetUID(uid)
domain.Spec.Metadata.KubeVirt.UID = uid
domain.Status.Status = api.Error
domain.Status.Reason = reason
domain.Status.Conditions = []api.DomainCondition{{
Type: api.DomainConditionSynchronized,
Status: v1.ConditionFalse,
Reason: string(reason),
Message: err.Error(),
}}
return c.SendDomainEvent(watch.Event{Type: watch.Modified, Object: domain})
}
func newWatchEventError(err error) watch.Event {
return watch.Event{Type: watch.Error, Object: &metav1.Status{Status: metav1.StatusFailure, Message: err.Error()}}
}
View
@@ -37,7 +37,7 @@ import (
"kubevirt.io/kubevirt/pkg/api/v1"
"kubevirt.io/kubevirt/pkg/cloud-init"
"kubevirt.io/kubevirt/pkg/config"
"kubevirt.io/kubevirt/pkg/config-disk"
"kubevirt.io/kubevirt/pkg/emptydisk"
"kubevirt.io/kubevirt/pkg/ephemeral-disk"
"kubevirt.io/kubevirt/pkg/log"
@@ -163,24 +163,24 @@ func Convert_v1_Volume_To_api_Disk(source *v1.Volume, disk *Disk, c *ConverterCo
return Convert_v1_EmptyDiskSource_To_api_Disk(source.Name, source.EmptyDisk, disk, c)
}
if source.ConfigMap != nil {
return Convert_v1_Config_To_api_Disk(source.Name, disk, config.ConfigMap)
return Convert_v1_Config_To_api_Disk(source.Name, disk, config_disk.ConfigMap)
}
if source.Secret != nil {
return Convert_v1_Config_To_api_Disk(source.Name, disk, config.Secret)
return Convert_v1_Config_To_api_Disk(source.Name, disk, config_disk.Secret)
}
return fmt.Errorf("disk %s references an unsupported source", disk.Alias.Name)
}
func Convert_v1_Config_To_api_Disk(volumeName string, disk *Disk, configType config.Type) error {
func Convert_v1_Config_To_api_Disk(volumeName string, disk *Disk, configType config_disk.Type) error {
disk.Type = "file"
disk.Driver.Type = "raw"
switch configType {
case config.ConfigMap:
disk.Source.File = config.GetConfigMapDiskPath(volumeName)
case config_disk.ConfigMap:
disk.Source.File = config_disk.GetConfigMapDiskPath(volumeName)
break
case config.Secret:
disk.Source.File = config.GetSecretDiskPath(volumeName)
case config_disk.Secret:
disk.Source.File = config_disk.GetSecretDiskPath(volumeName)
break
default:
return fmt.Errorf("Cannot convert config '%s' to disk, unrecognized type", configType)
View
@@ -804,7 +804,7 @@ func (in *Domain) DeepCopyInto(out *Domain) {
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
in.Spec.DeepCopyInto(&out.Spec)
out.Status = in.Status
in.Status.DeepCopyInto(&out.Status)
return
}
@@ -827,6 +827,24 @@ func (in *Domain) DeepCopyObject() runtime.Object {
}
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *DomainCondition) DeepCopyInto(out *DomainCondition) {
*out = *in
in.LastProbeTime.DeepCopyInto(&out.LastProbeTime)
in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime)
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DomainCondition.
func (in *DomainCondition) DeepCopy() *DomainCondition {
if in == nil {
return nil
}
out := new(DomainCondition)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *DomainList) DeepCopyInto(out *DomainList) {
*out = *in
@@ -949,6 +967,13 @@ func (in *DomainSpec) DeepCopy() *DomainSpec {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *DomainStatus) DeepCopyInto(out *DomainStatus) {
*out = *in
if in.Conditions != nil {
in, out := &in.Conditions, &out.Conditions
*out = make([]DomainCondition, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
return
}
View
@@ -42,6 +42,9 @@ type LifeCycle string
type StateChangeReason string
const (
// TODO migrate most of the lifecycle stuff to conditions
// this is better extensible and will help us to stay compatible with
// different virt-handler and virt-launcher versions
NoState LifeCycle = "NoState"
Running LifeCycle = "Running"
Blocked LifeCycle = "Blocked"
@@ -50,6 +53,11 @@ const (
Shutoff LifeCycle = "Shutoff"
Crashed LifeCycle = "Crashed"
PMSuspended LifeCycle = "PMSuspended"
Error LifeCycle = "Error"
// Error reasons
ReasonLibvirtUnreachable StateChangeReason = "LibvirtUnreachable"
ReasonDomainSyncError StateChangeReason = "SyncError"
// Common reasons
ReasonUnknown StateChangeReason = "Unknown"
@@ -98,8 +106,24 @@ type Domain struct {
}
type DomainStatus struct {
Status LifeCycle
Reason StateChangeReason
Status LifeCycle
Reason StateChangeReason
Conditions []DomainCondition
}
type DomainConditionType string
const (
DomainConditionSynchronized = "Synchronized"
)
type DomainCondition struct {
Type DomainConditionType
Status kubev1.ConditionStatus
LastProbeTime metav1.Time
LastTransitionTime metav1.Time
Reason string
Message string
}
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
View
@@ -63,7 +63,7 @@ func (s *Launcher) Sync(args *cmdclient.Args, reply *cmdclient.Reply) error {
return nil
}
_, err = s.domainManager.SyncVMI(vmi, s.useEmulation)
err = s.domainManager.SyncVMI(vmi, s.useEmulation)
if err != nil {
log.Log.Object(vmi).Reason(err).Errorf("Failed to sync vmi")
reply.Success = false
View
@@ -77,8 +77,7 @@ var _ = Describe("Virt remote commands", func() {
Context("server", func() {
It("should start a vmi", func() {
vmi := v1.NewVMIReferenceFromName("testvmi")
domain := api.NewMinimalDomain("testvmi")
domainManager.EXPECT().SyncVMI(vmi, useEmulation).Return(&domain.Spec, nil)
domainManager.EXPECT().SyncVMI(vmi, useEmulation).Return(nil)
err := client.SyncVirtualMachine(vmi)
Expect(err).ToNot(HaveOccurred())
View
@@ -31,19 +31,18 @@ func (_m *MockDomainManager) EXPECT() *_MockDomainManagerRecorder {
return _m.recorder
}
func (_m *MockDomainManager) SyncVMI(_param0 *v1.VirtualMachineInstance, _param1 bool) (*api.DomainSpec, error) {
ret := _m.ctrl.Call(_m, "SyncVMI", _param0, _param1)
ret0, _ := ret[0].(*api.DomainSpec)
ret1, _ := ret[1].(error)
return ret0, ret1
func (_m *MockDomainManager) SyncVMI(vmi *v1.VirtualMachineInstance, useEmulation bool) error {
ret := _m.ctrl.Call(_m, "SyncVMI", vmi, useEmulation)
ret0, _ := ret[0].(error)
return ret0
}
func (_mr *_MockDomainManagerRecorder) SyncVMI(arg0, arg1 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCall(_mr.mock, "SyncVMI", arg0, arg1)
}
func (_m *MockDomainManager) KillVMI(_param0 *v1.VirtualMachineInstance) error {
ret := _m.ctrl.Call(_m, "KillVMI", _param0)
func (_m *MockDomainManager) KillVMI(vmi *v1.VirtualMachineInstance) error {
ret := _m.ctrl.Call(_m, "KillVMI", vmi)
ret0, _ := ret[0].(error)
return ret0
}
@@ -52,8 +51,8 @@ func (_mr *_MockDomainManagerRecorder) KillVMI(arg0 interface{}) *gomock.Call {
return _mr.mock.ctrl.RecordCall(_mr.mock, "KillVMI", arg0)
}
func (_m *MockDomainManager) DeleteVMI(_param0 *v1.VirtualMachineInstance) error {
ret := _m.ctrl.Call(_m, "DeleteVMI", _param0)
func (_m *MockDomainManager) DeleteVMI(vmi *v1.VirtualMachineInstance) error {
ret := _m.ctrl.Call(_m, "DeleteVMI", vmi)
ret0, _ := ret[0].(error)
return ret0
}
@@ -62,8 +61,8 @@ func (_mr *_MockDomainManagerRecorder) DeleteVMI(arg0 interface{}) *gomock.Call
return _mr.mock.ctrl.RecordCall(_mr.mock, "DeleteVMI", arg0)
}
func (_m *MockDomainManager) SignalShutdownVMI(_param0 *v1.VirtualMachineInstance) error {
ret := _m.ctrl.Call(_m, "SignalShutdownVMI", _param0)
func (_m *MockDomainManager) SignalShutdownVMI(vmi *v1.VirtualMachineInstance) error {
ret := _m.ctrl.Call(_m, "SignalShutdownVMI", vmi)
ret0, _ := ret[0].(error)
return ret0
}
View
@@ -33,16 +33,24 @@ import (
k8sv1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sync"
"time"
"k8s.io/apimachinery/pkg/watch"
"os"
"kubevirt.io/kubevirt/pkg/api/v1"
cloudinit "kubevirt.io/kubevirt/pkg/cloud-init"
"kubevirt.io/kubevirt/pkg/config"
"kubevirt.io/kubevirt/pkg/cloud-init"
"kubevirt.io/kubevirt/pkg/config-disk"
"kubevirt.io/kubevirt/pkg/emptydisk"
"kubevirt.io/kubevirt/pkg/ephemeral-disk"
"kubevirt.io/kubevirt/pkg/hooks"
"kubevirt.io/kubevirt/pkg/host-disk"
"kubevirt.io/kubevirt/pkg/log"
"kubevirt.io/kubevirt/pkg/registry-disk"
"kubevirt.io/kubevirt/pkg/util/net/dns"
"kubevirt.io/kubevirt/pkg/virt-launcher/notify-client"
"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api"
"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/cli"
domainerrors "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/errors"
@@ -51,23 +59,145 @@ import (
)
type DomainManager interface {
SyncVMI(*v1.VirtualMachineInstance, bool) (*api.DomainSpec, error)
KillVMI(*v1.VirtualMachineInstance) error
DeleteVMI(*v1.VirtualMachineInstance) error
SignalShutdownVMI(*v1.VirtualMachineInstance) error
SyncVMI(vmi *v1.VirtualMachineInstance, useEmulation bool) error
KillVMI(vmi *v1.VirtualMachineInstance) error
DeleteVMI(vmi *v1.VirtualMachineInstance) error
SignalShutdownVMI(vmi *v1.VirtualMachineInstance) error
ListAllDomains() ([]*api.Domain, error)
}
type LazyLibvirtDomainManager struct {
pointerLock *sync.Mutex
_manager DomainManager
StopChan chan struct{}
events chan watch.Event
virtShareDir string
initOnce *sync.Once
}
func NewLazyLibvirtDomainManager(virtShareDir string, stopChan chan struct{}) (manager DomainManager, events chan watch.Event) {
events = make(chan watch.Event, 10)
return &LazyLibvirtDomainManager{
StopChan: stopChan,
pointerLock: &sync.Mutex{},
events: events,
virtShareDir: virtShareDir,
initOnce: &sync.Once{},
}, events
}
func (l *LazyLibvirtDomainManager) SyncVMI(vmi *v1.VirtualMachineInstance, useEmulation bool) error {
pending, err := l.initialSync(vmi, useEmulation)
if err != nil {
return err
}
if pending {
return nil
}
return l.get().SyncVMI(vmi, useEmulation)
}
func (l *LazyLibvirtDomainManager) KillVMI(vmi *v1.VirtualMachineInstance) error {
if l.get() == nil {
return nil
}
return l.get().KillVMI(vmi)
}
func (l *LazyLibvirtDomainManager) DeleteVMI(vmi *v1.VirtualMachineInstance) error {
if l.get() == nil {
return nil
}
return l.get().DeleteVMI(vmi)
}
func (l *LazyLibvirtDomainManager) SignalShutdownVMI(vmi *v1.VirtualMachineInstance) error {
if l.get() == nil {
return nil
}
return l.get().SignalShutdownVMI(vmi)
}
func (l *LazyLibvirtDomainManager) get() DomainManager {
l.pointerLock.Lock()
defer l.pointerLock.Unlock()
return l._manager
}
func (l *LazyLibvirtDomainManager) initialSync(vmi *v1.VirtualMachineInstance, useEmulation bool) (pending bool, err error) {
l.pointerLock.Lock()
defer l.pointerLock.Unlock()
var domainCon cli.Connection
if l._manager == nil {
l.initOnce.Do(func() {
util.StartLibvirt(l.StopChan)
go func() {
domainCon, err = cli.NewConnection("qemu:///system", "", "", 10*time.Second)
if err != nil {
log.Log.Object(vmi).Reason(err).Error("Could not establish initial connection with libvirt, exiting.")
l.fatalError(vmi, err)
}
util.StartDomainEventMonitoring()
err = eventsclient.StartNotifier(l.virtShareDir, domainCon, l.events)
if err != nil {
log.Log.Object(vmi).Reason(err).Error("Could not establish initial connection with libvirt, exiting.")
l.fatalError(vmi, err)
}
manager := NewLibvirtDomainManager(domainCon)
err := manager.SyncVMI(vmi, useEmulation)
if err != nil {
log.Log.Object(vmi).Reason(err).Error("Initial synchronization with libvirt failed.")
l.syncError(vmi, api.ReasonDomainSyncError, err)
}
l.pointerLock.Lock()
defer l.pointerLock.Unlock()
l._manager = manager
}()
})
return true, nil
}
return false, nil
}
func (l LazyLibvirtDomainManager) fatalError(vmi *v1.VirtualMachineInstance, syncError error) {
l.syncError(vmi, api.ReasonLibvirtUnreachable, syncError)
os.Exit(1)
}
func (l LazyLibvirtDomainManager) syncError(vmi *v1.VirtualMachineInstance, reason api.StateChangeReason, syncError error) {
cli, err := eventsclient.NewDomainEventClient(l.virtShareDir)
if err != nil {
log.Log.Object(vmi).Reason(err).Error("Can't inform virt-handler about a sync error.")
return
}
err = cli.SendErrorDomainEvent(vmi.Name, vmi.Namespace, vmi.UID, reason, syncError)
if err != nil {
log.Log.Object(vmi).Reason(err).Error("Can't inform virt-handler about a sync error.")
return
}
}
func (l LazyLibvirtDomainManager) ListAllDomains() ([]*api.Domain, error) {
if l.get() == nil {
return nil, nil
}
return l.get().ListAllDomains()
}
type LibvirtDomainManager struct {
virConn cli.Connection
}
func NewLibvirtDomainManager(connection cli.Connection) (DomainManager, error) {
func NewLibvirtDomainManager(connection cli.Connection) DomainManager {
manager := LibvirtDomainManager{
virConn: connection,
}
return &manager, nil
return &manager
}
// All local environment setup that needs to occur before VirtualMachineInstance starts
@@ -121,11 +251,11 @@ func (l *LibvirtDomainManager) preStartHook(vmi *v1.VirtualMachineInstance, doma
return domain, fmt.Errorf("creating empty disks failed: %v", err)
}
// create ConfigMap disks if they exists
if err := config.CreateConfigMapDisks(vmi); err != nil {
if err := config_disk.CreateConfigMapDisks(vmi); err != nil {
return domain, fmt.Errorf("creating config map disks failed: %v", err)
}
// create Secret disks if they exists
if err := config.CreateSecretDisks(vmi); err != nil {
if err := config_disk.CreateSecretDisks(vmi); err != nil {
return domain, fmt.Errorf("creating secret disks failed: %v", err)
}
@@ -139,7 +269,7 @@ func (l *LibvirtDomainManager) preStartHook(vmi *v1.VirtualMachineInstance, doma
return domain, err
}
func (l *LibvirtDomainManager) SyncVMI(vmi *v1.VirtualMachineInstance, useEmulation bool) (*api.DomainSpec, error) {
func (l *LibvirtDomainManager) SyncVMI(vmi *v1.VirtualMachineInstance, useEmulation bool) error {
logger := log.Log.Object(vmi)
domain := &api.Domain{}
@@ -151,7 +281,7 @@ func (l *LibvirtDomainManager) SyncVMI(vmi *v1.VirtualMachineInstance, useEmulat
}
if err := api.Convert_v1_VirtualMachine_To_api_Domain(vmi, domain, c); err != nil {
logger.Error("Conversion failed.")
return nil, err
return err
}
// Set defaults which are not coming from the cluster
@@ -166,31 +296,31 @@ func (l *LibvirtDomainManager) SyncVMI(vmi *v1.VirtualMachineInstance, useEmulat
domain, err = l.preStartHook(vmi, domain)
if err != nil {
logger.Reason(err).Error("pre start setup for VirtualMachineInstance failed.")
return nil, err
return err
}
dom, err = util.SetDomainSpec(l.virConn, vmi, domain.Spec)
if err != nil {
return nil, err
return err
}
logger.Info("Domain defined.")
} else {
logger.Reason(err).Error("Getting the domain failed.")
return nil, err
return err
}
}
defer dom.Free()
domState, _, err := dom.GetState()
if err != nil {
logger.Reason(err).Error("Getting the domain state failed.")
return nil, err
return err
}
// To make sure, that we set the right qemu wrapper arguments,
// we update the domain XML whenever a VirtualMachineInstance was already defined but not running
if !newDomain && cli.IsDown(domState) {
dom, err = util.SetDomainSpec(l.virConn, vmi, domain.Spec)
if err != nil {
return nil, err
return err
}
}
@@ -201,15 +331,15 @@ func (l *LibvirtDomainManager) SyncVMI(vmi *v1.VirtualMachineInstance, useEmulat
err = dom.Create()
if err != nil {
logger.Reason(err).Error("Starting the VirtualMachineInstance failed.")
return nil, err
return err
}
logger.Info("Domain started.")
} else if cli.IsPaused(domState) {
// TODO: if state change reason indicates a system error, we could try something smarter
err := dom.Resume()
if err != nil {
logger.Reason(err).Error("Resuming the VirtualMachineInstance failed.")
return nil, err
return err
}
logger.Info("Domain resumed.")
} else {
@@ -218,18 +348,18 @@ func (l *LibvirtDomainManager) SyncVMI(vmi *v1.VirtualMachineInstance, useEmulat
xmlstr, err := dom.GetXMLDesc(0)
if err != nil {
return nil, err
return err
}
var newSpec api.DomainSpec
err = xml.Unmarshal([]byte(xmlstr), &newSpec)
if err != nil {
logger.Reason(err).Error("Parsing domain XML failed.")
return nil, err
return err
}
// TODO: check if VirtualMachineInstance Spec and Domain Spec are equal or if we have to sync
return &newSpec, nil
return nil
}
func (l *LibvirtDomainManager) getDomainSpec(dom cli.VirDomain) (*api.DomainSpec, error) {
View
@@ -81,9 +81,8 @@ var _ = Describe("Manager", func() {
mockDomain.EXPECT().GetState().Return(libvirt.DOMAIN_SHUTDOWN, 1, nil)
mockDomain.EXPECT().Create().Return(nil)
mockDomain.EXPECT().GetXMLDesc(libvirt.DomainXMLFlags(0)).Return(string(xml), nil)
manager, _ := NewLibvirtDomainManager(mockConn)
newspec, err := manager.SyncVMI(vmi, true)
Expect(newspec).ToNot(BeNil())
manager := NewLibvirtDomainManager(mockConn)
err = manager.SyncVMI(vmi, true)
Expect(err).To(BeNil())
})
It("should leave a defined and started VirtualMachineInstance alone", func() {
@@ -94,9 +93,8 @@ var _ = Describe("Manager", func() {
mockConn.EXPECT().LookupDomainByName(testDomainName).Return(mockDomain, nil)
mockDomain.EXPECT().GetState().Return(libvirt.DOMAIN_RUNNING, 1, nil)
mockDomain.EXPECT().GetXMLDesc(libvirt.DomainXMLFlags(0)).Return(string(xml), nil)
manager, _ := NewLibvirtDomainManager(mockConn)
newspec, err := manager.SyncVMI(vmi, true)
Expect(newspec).ToNot(BeNil())
manager := NewLibvirtDomainManager(mockConn)
err = manager.SyncVMI(vmi, true)
Expect(err).To(BeNil())
})
table.DescribeTable("should try to start a VirtualMachineInstance in state",
@@ -110,9 +108,8 @@ var _ = Describe("Manager", func() {
mockConn.EXPECT().DomainDefineXML(string(xml)).Return(mockDomain, nil)
mockDomain.EXPECT().Create().Return(nil)
mockDomain.EXPECT().GetXMLDesc(libvirt.DomainXMLFlags(0)).Return(string(xml), nil)
manager, _ := NewLibvirtDomainManager(mockConn)
newspec, err := manager.SyncVMI(vmi, true)
Expect(newspec).ToNot(BeNil())
manager := NewLibvirtDomainManager(mockConn)
err = manager.SyncVMI(vmi, true)
Expect(err).To(BeNil())
},
table.Entry("crashed", libvirt.DOMAIN_CRASHED),
@@ -129,9 +126,8 @@ var _ = Describe("Manager", func() {
mockDomain.EXPECT().GetState().Return(libvirt.DOMAIN_PAUSED, 1, nil)
mockDomain.EXPECT().Resume().Return(nil)
mockDomain.EXPECT().GetXMLDesc(libvirt.DomainXMLFlags(0)).Return(string(xml), nil)
manager, _ := NewLibvirtDomainManager(mockConn)
newspec, err := manager.SyncVMI(vmi, true)
Expect(newspec).ToNot(BeNil())
manager := NewLibvirtDomainManager(mockConn)
err = manager.SyncVMI(vmi, true)
Expect(err).To(BeNil())
})
})
@@ -140,7 +136,7 @@ var _ = Describe("Manager", func() {
func(state libvirt.DomainState) {
mockConn.EXPECT().LookupDomainByName(testDomainName).Return(mockDomain, nil)
mockDomain.EXPECT().Undefine().Return(nil)
manager, _ := NewLibvirtDomainManager(mockConn)
manager := NewLibvirtDomainManager(mockConn)
err := manager.DeleteVMI(newVMI(testNamespace, testVmName))
Expect(err).To(BeNil())
},
@@ -152,7 +148,7 @@ var _ = Describe("Manager", func() {
mockConn.EXPECT().LookupDomainByName(testDomainName).Return(mockDomain, nil)
mockDomain.EXPECT().GetState().Return(state, 1, nil)
mockDomain.EXPECT().DestroyFlags(libvirt.DOMAIN_DESTROY_GRACEFUL).Return(nil)
manager, _ := NewLibvirtDomainManager(mockConn)
manager := NewLibvirtDomainManager(mockConn)
err := manager.KillVMI(newVMI(testNamespace, testVmName))
Expect(err).To(BeNil())
},
@@ -173,7 +169,7 @@ var _ = Describe("Manager", func() {
mockDomain.EXPECT().GetXMLDesc(gomock.Eq(libvirt.DOMAIN_XML_INACTIVE)).Return(string(x), nil)
mockConn.EXPECT().ListAllDomains(gomock.Eq(libvirt.CONNECT_LIST_DOMAINS_ACTIVE|libvirt.CONNECT_LIST_DOMAINS_INACTIVE)).Return([]cli.VirDomain{mockDomain}, nil)
manager, _ := NewLibvirtDomainManager(mockConn)
manager := NewLibvirtDomainManager(mockConn)
doms, err := manager.ListAllDomains()
Expect(len(doms)).To(Equal(1))
View
@@ -141,13 +141,11 @@ func StartLibvirt(stopChan chan struct{}) {
// doesn't exit until virt-launcher is ready for it to. Virt-launcher traps signals
// to perform special shutdown logic. These processes need to live in the same
// container.
go func() {
for {
exitChan := make(chan struct{})
cmd := exec.Command("/usr/share/kubevirt/virt-launcher/libvirtd.sh")
cmd := exec.Command("/usr/sbin/libvirtd")
// libvirtd logs to stderr (see configuration in libvirtd.sh)
// connect libvirt's stderr to our own stdout in order to see the logs in the container logs
cmd.Stderr = os.Stdout
@@ -242,3 +240,50 @@ func NewDomain(dom cli.VirDomain) (*api.Domain, error) {
domain.GetObjectMeta().SetUID(domain.Spec.Metadata.KubeVirt.UID)
return domain, nil
}
func SetupLibvirt() error {
qemuConf, err := os.OpenFile("/etc/libvirt/qemu.conf", os.O_APPEND|os.O_WRONLY, 0644)
if err != nil {
return err
}
defer qemuConf.Close()
// We are in a container, don't try to stuff qemu inside special cgroups
_, err = qemuConf.WriteString("cgroup_controllers = [ ]\n")
if err != nil {
return err
}
// If hugepages exist, tell libvirt about them
_, err = os.Stat("/dev/hugepages")
if err == nil {
_, err = qemuConf.WriteString("hugetlbfs_mount = \"/dev/hugepages\"\n")
} else if !os.IsNotExist(err) {
return err
}
// Let libvirt log to stderr
libvirtConf, err := os.OpenFile("/etc/libvirt/libvirtd.conf", os.O_APPEND|os.O_WRONLY, 0644)
if err != nil {
return err
}
defer libvirtConf.Close()
_, err = libvirtConf.WriteString("log_outputs = \"1:stderr\"\n")
if err != nil {
return err
}
return nil
}
func StartDomainEventMonitoring() {
go func() {
for {
if res := libvirt.EventRunDefaultImpl(); res != nil {
log.Log.Reason(res).Error("Listening to libvirt events failed, retrying.")
time.Sleep(time.Second)
}
}
}()
}
View
@@ -28,7 +28,7 @@ import (
"github.com/google/goexpect"
"github.com/pborman/uuid"
"kubevirt.io/kubevirt/pkg/config"
"kubevirt.io/kubevirt/pkg/config-disk"
"kubevirt.io/kubevirt/pkg/kubecli"
"kubevirt.io/kubevirt/tests"
)
@@ -54,7 +54,7 @@ var _ = Describe("Config", func() {
BeforeEach(func() {
configMapName = "configmap-" + uuid.NewRandom().String()
configMapPath = config.GetConfigMapSourcePath(configMapName + "-vol")
configMapPath = config_disk.GetConfigMapSourcePath(configMapName + "-vol")
data := map[string]string{
"option1": "value1",
@@ -151,7 +151,7 @@ var _ = Describe("Config", func() {
BeforeEach(func() {
secretName = "secret-" + uuid.NewRandom().String()
secretPath = config.GetSecretSourcePath(secretName + "-vol")
secretPath = config_disk.GetSecretSourcePath(secretName + "-vol")
data := map[string]string{
"user": "admin",
View
@@ -65,7 +65,6 @@ import (
"kubevirt.io/kubevirt/pkg/kubecli"
"kubevirt.io/kubevirt/pkg/log"
"kubevirt.io/kubevirt/pkg/util/net/dns"
"kubevirt.io/kubevirt/pkg/virt-controller/services"
"kubevirt.io/kubevirt/pkg/virtctl"
cdiv1 "kubevirt.io/containerized-data-importer/pkg/apis/datavolumecontroller/v1alpha1"
@@ -346,46 +345,6 @@ func BeforeTestSuitSetup() {
CreatePVC(osWindows, defaultWindowsDiskSize)
EnsureKVMPresent()
}
func EnsureKVMPresent() {
useEmulation := false
virtClient, err := kubecli.GetKubevirtClient()
PanicOnError(err)
options := metav1.GetOptions{}
cfgMap, err := virtClient.CoreV1().ConfigMaps("kube-system").Get("kubevirt-config", options)
if err == nil {
val, ok := cfgMap.Data["debug.useEmulation"]
useEmulation = ok && (val == "true")
} else {
// If the cfgMap is missing, default to useEmulation=false
// no other error is expected
if !errors.IsNotFound(err) {
ExpectWithOffset(1, err).ToNot(HaveOccurred())
}
}
if !useEmulation {
listOptions := metav1.ListOptions{LabelSelector: v1.AppLabel + "=virt-handler"}
virtHandlerPods, err := virtClient.CoreV1().Pods(metav1.NamespaceSystem).List(listOptions)
ExpectWithOffset(1, err).ToNot(HaveOccurred())
EventuallyWithOffset(1, func() bool {
ready := true
// cluster is not ready until all nodes are ready.
for _, pod := range virtHandlerPods.Items {
virtHandlerNode, err := virtClient.CoreV1().Nodes().Get(pod.Spec.NodeName, metav1.GetOptions{})
ExpectWithOffset(1, err).ToNot(HaveOccurred())
allocatable, ok := virtHandlerNode.Status.Allocatable[services.KvmDevice]
ready = ready && ok
ready = ready && (allocatable.Value() > 0)
}
return ready
}, 120*time.Second, 1*time.Second).Should(BeTrue(),
"KVM devices are required for testing, but are not present on cluster nodes")
}
}
func CreateConfigMap(name string, data map[string]string) {
View
@@ -41,7 +41,6 @@ import (
"kubevirt.io/kubevirt/pkg/api/v1"
"kubevirt.io/kubevirt/pkg/kubecli"
"kubevirt.io/kubevirt/pkg/util/net/dns"
"kubevirt.io/kubevirt/pkg/virt-controller/services"
"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api"
"kubevirt.io/kubevirt/tests"
)
@@ -85,7 +84,6 @@ var _ = Describe("VMIlifecycle", func() {
AfterEach(func() {
// Not every test causes virt-handler to restart, but a few different contexts do.
// This check is fast and non-intrusive if virt-handler is already running.
tests.EnsureKVMPresent()
})
Describe("Creating a VirtualMachineInstance", func() {
@@ -741,41 +739,6 @@ var _ = Describe("VMIlifecycle", func() {
Expect(domain.Spec.Type).To(Equal(expectedType))
})
It("should request a TUN device but not KVM", func() {
err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Error()
Expect(err).To(BeNil())
listOptions := metav1.ListOptions{}
var pod k8sv1.Pod
Eventually(func() error {
podList, err := virtClient.CoreV1().Pods(tests.NamespaceTestDefault).List(listOptions)
Expect(err).ToNot(HaveOccurred())
for _, item := range podList.Items {
if strings.HasPrefix(item.Name, vmi.ObjectMeta.GenerateName) {
pod = item
return nil
}
}
return fmt.Errorf("Associated pod for VM '%s' not found", vmi.Name)
}, 75, 0.5).Should(Succeed())
computeContainerFound := false
for _, container := range pod.Spec.Containers {
if container.Name == "compute" {
computeContainerFound = true
_, ok := container.Resources.Limits[services.KvmDevice]
Expect(ok).To(BeFalse(), "Container should not have requested KVM device")
_, ok = container.Resources.Limits[services.TunDevice]
Expect(ok).To(BeTrue(), "Container should have requested TUN device")
}
}
Expect(computeContainerFound).To(BeTrue(), "Compute container was not found in pod")
})
})
Context("VM Accelerated Mode", func() {
@@ -790,41 +753,6 @@ var _ = Describe("VMIlifecycle", func() {
}
})
It("should request a KVM and TUN device", func() {
err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Error()
Expect(err).To(BeNil())
listOptions := metav1.ListOptions{}
var pod k8sv1.Pod
Eventually(func() error {
podList, err := virtClient.CoreV1().Pods(tests.NamespaceTestDefault).List(listOptions)
Expect(err).ToNot(HaveOccurred())
for _, item := range podList.Items {
if strings.HasPrefix(item.Name, vmi.ObjectMeta.GenerateName) {
pod = item
return nil
}
}
return fmt.Errorf("Associated pod for VM '%s' not found", vmi.Name)
}, 75, 0.5).Should(Succeed())
computeContainerFound := false
for _, container := range pod.Spec.Containers {
if container.Name == "compute" {
computeContainerFound = true
_, ok := container.Resources.Limits[services.KvmDevice]
Expect(ok).To(BeTrue(), "Container should have requested KVM device")
_, ok = container.Resources.Limits[services.TunDevice]
Expect(ok).To(BeTrue(), "Container should have requested TUN device")
}
}
Expect(computeContainerFound).To(BeTrue(), "Compute container was not found in pod")
})
It("should not enable emulation in virt-launcher", func() {
err := virtClient.RestClient().Post().Resource("virtualmachineinstances").Namespace(tests.NamespaceTestDefault).Body(vmi).Do().Error()
Expect(err).To(BeNil())
@@ -861,23 +789,6 @@ var _ = Describe("VMIlifecycle", func() {
Expect(computeContainerFound).To(BeTrue(), "Compute container was not found in pod")
Expect(emulationFlagFound).To(BeFalse(), "Expected VM pod not to have '--use-emulation' flag")
})
It("Should provide KVM via plugin framework", func() {
listOptions := metav1.ListOptions{}
nodeList, err := virtClient.CoreV1().Nodes().List(listOptions)
Expect(err).ToNot(HaveOccurred())
if len(nodeList.Items) == 0 {
Skip("Unable to inspect nodes in cluster")
}
node := nodeList.Items[0]
_, ok := node.Status.Allocatable[services.KvmDevice]
Expect(ok).To(BeTrue(), "KVM devices not allocatable on node: %s", node.Name)
_, ok = node.Status.Capacity[services.KvmDevice]
Expect(ok).To(BeTrue(), "No Capacity for KVM devices on node: %s", node.Name)
})
})
})
View
@@ -42,7 +42,6 @@ import (
"kubevirt.io/kubevirt/pkg/api/v1"
"kubevirt.io/kubevirt/pkg/kubecli"
"kubevirt.io/kubevirt/pkg/log"
"kubevirt.io/kubevirt/pkg/virt-controller/services"
"kubevirt.io/kubevirt/tests"
)
@@ -479,7 +478,7 @@ var _ = Describe("Networking", func() {
Expect(err).ToNot(HaveOccurred())
})
It("should not request a tun device", func() {
It("should not request NET_ADMIN", func() {
By("Creating random VirtualMachineInstance")
autoAttach := false
vmi := tests.NewRandomVMIWithEphemeralDisk(tests.RegistryDiskFor(tests.RegistryDiskAlpine))
@@ -503,13 +502,6 @@ var _ = Describe("Networking", func() {
foundContainer := false
for _, container := range pod.Spec.Containers {
if container.Name == "compute" {
foundContainer = true
_, ok := container.Resources.Requests[services.TunDevice]
Expect(ok).To(BeFalse())
_, ok = container.Resources.Limits[services.TunDevice]
Expect(ok).To(BeFalse())
netAdminCap := false
caps := container.SecurityContext.Capabilities
for _, cap := range caps.Add {
View
@@ -24,7 +24,7 @@ import (
"strings"
"time"
expect "github.com/google/goexpect"
"github.com/google/goexpect"
. "github.com/onsi/ginkgo"
"github.com/onsi/ginkgo/extensions/table"
. "github.com/onsi/gomega"
@@ -82,11 +82,12 @@ var _ = Describe("Slirp", func() {
virtClient,
vmiPod,
vmiPod.Spec.Containers[1].Name,
[]string{"netstat", "-tnlp"},
[]string{"cat", "/proc/net/tcp"},
)
log.Log.Infof("%v", output)
Expect(err).ToNot(HaveOccurred())
Expect(strings.Contains(output, "0.0.0.0:80")).To(BeTrue())
// :0050 is port 80, 0A is listening
Expect(strings.Contains(output, "0: 00000000:0050 00000000:0000 0A")).To(BeTrue())
By("return \"Hello World!\" when connecting to localhost on port 80")
output, err = tests.ExecuteCommandOnPod(