-
Notifications
You must be signed in to change notification settings - Fork 43
hypervisor: add hot plugging support for Qemu Q35 #456
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,104 @@ | ||
// | ||
// Copyright (c) 2017 Intel Corporation | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
// | ||
|
||
package virtcontainers | ||
|
||
import "fmt" | ||
|
||
type bridgeType string | ||
|
||
const ( | ||
pciBridge bridgeType = "pci" | ||
pcieBridge = "pcie" | ||
) | ||
|
||
const pciBridgeMaxCapacity = 30 | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Qemu's documentation is not very good, I found this [1], but I'm not able to hot plug 32 devices,
I don't think so [1] https://github.com/qemu/qemu/blob/master/docs/pcie.txt#L76 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Having a peek at: https://en.wikipedia.org/wiki/PCI_configuration_space |
||
|
||
// Bridge is a bridge where devices can be hot plugged | ||
type Bridge struct { | ||
// Address contains information about devices plugged and its address in the bridge | ||
Address map[uint32]string | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Correct me if I am wrong but no need for a very specific There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. pcie has more capacity, for this reason I'd like to keep |
||
|
||
// Type is the type of the bridge (pci, pcie, etc) | ||
Type bridgeType | ||
|
||
//ID is used to identify the bridge in the hypervisor | ||
ID string | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can we also add the chassis information here. |
||
} | ||
|
||
// NewBridges creates n new pci(e) bridges depending of the machine type | ||
func NewBridges(n uint32, machine string) []Bridge { | ||
var bridges []Bridge | ||
var bt bridgeType | ||
|
||
switch machine { | ||
case QemuQ35: | ||
// currently only pci bridges are supported | ||
// qemu-2.10 will introduce pcie bridges | ||
fallthrough | ||
case QemuPC: | ||
bt = pciBridge | ||
default: | ||
return nil | ||
} | ||
|
||
for i := uint32(0); i < n; i++ { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Once again, if we really want to limit the memory, then let's use |
||
bridges = append(bridges, Bridge{ | ||
Type: bt, | ||
ID: fmt.Sprintf("%s-bridge-%d", bt, i), | ||
Address: make(map[uint32]string), | ||
}) | ||
} | ||
|
||
return bridges | ||
} | ||
|
||
// addDevice on success adds the device ID to the bridge and return the address | ||
// where the device was added, otherwise an error is returned | ||
func (b *Bridge) addDevice(ID string) (uint32, error) { | ||
var addr uint32 | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It's more a key than an address. |
||
|
||
// looking for the first available address | ||
for i := uint32(1); i <= pciBridgeMaxCapacity; i++ { | ||
if _, ok := b.Address[i]; !ok { | ||
addr = i | ||
break | ||
} | ||
} | ||
|
||
if addr == 0 { | ||
return 0, fmt.Errorf("Unable to hot plug device on bridge: there are not empty slots") | ||
} | ||
|
||
// save address and device | ||
b.Address[addr] = ID | ||
return addr, nil | ||
} | ||
|
||
// removeDevice on success removes the device ID from the bridge and return nil, | ||
// otherwise an error is returned | ||
func (b *Bridge) removeDevice(ID string) error { | ||
// check if the device was hot plugged in the bridge | ||
for addr, devID := range b.Address { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It's more a key than an address. Let's call it |
||
if devID == ID { | ||
// free address to re-use the same slot with other devices | ||
delete(b.Address, addr) | ||
return nil | ||
} | ||
} | ||
|
||
return fmt.Errorf("Unable to hot unplug device %s: not present on bridge", ID) | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,65 @@ | ||
// | ||
// Copyright (c) 2017 Intel Corporation | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
// | ||
|
||
package virtcontainers | ||
|
||
import ( | ||
"testing" | ||
|
||
"github.com/stretchr/testify/assert" | ||
) | ||
|
||
func TestNewBridges(t *testing.T) { | ||
assert := assert.New(t) | ||
var countBridges uint32 = 1 | ||
|
||
bridges := NewBridges(countBridges, "") | ||
assert.Nil(bridges) | ||
|
||
bridges = NewBridges(countBridges, QemuQ35) | ||
assert.Len(bridges, int(countBridges)) | ||
|
||
b := bridges[0] | ||
assert.NotEmpty(b.ID) | ||
assert.NotNil(b.Address) | ||
} | ||
|
||
func TestAddRemoveDevice(t *testing.T) { | ||
assert := assert.New(t) | ||
var countBridges uint32 = 1 | ||
|
||
// create a bridge | ||
bridges := NewBridges(countBridges, "") | ||
assert.Nil(bridges) | ||
bridges = NewBridges(countBridges, QemuQ35) | ||
assert.Len(bridges, int(countBridges)) | ||
|
||
// add device | ||
devID := "abc123" | ||
b := bridges[0] | ||
addr, err := b.addDevice(devID) | ||
assert.NoError(err) | ||
if addr < 1 { | ||
assert.Fail("address cannot be less then 1") | ||
} | ||
|
||
// remove device | ||
err = b.removeDevice("") | ||
assert.Error(err) | ||
|
||
err = b.removeDevice(devID) | ||
assert.NoError(err) | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -42,6 +42,9 @@ const ( | |
// networkFileType represents a network file type (pod only) | ||
networkFileType | ||
|
||
// hypervisorFileType represents a hypervisor file type (pod only) | ||
hypervisorFileType | ||
|
||
// processFileType represents a process file type | ||
processFileType | ||
|
||
|
@@ -64,6 +67,9 @@ const stateFile = "state.json" | |
// networkFile is the file name storing a pod network. | ||
const networkFile = "network.json" | ||
|
||
// hypervisorFile is the file name storing a hypervisor's state. | ||
const hypervisorFile = "hypervisor.json" | ||
|
||
// processFile is the file name storing a container process. | ||
const processFile = "process.json" | ||
|
||
|
@@ -109,6 +115,10 @@ type resourceStorage interface { | |
fetchPodNetwork(podID string) (NetworkNamespace, error) | ||
storePodNetwork(podID string, networkNS NetworkNamespace) error | ||
|
||
// Hypervisor resources | ||
fetchHypervisorState(podID string, state interface{}) error | ||
storeHypervisorState(podID string, state interface{}) error | ||
|
||
// Container resources | ||
storeContainerResource(podID, containerID string, resource podResource, data interface{}) error | ||
deleteContainerResources(podID, containerID string, resources []podResource) error | ||
|
@@ -319,7 +329,7 @@ func (fs *filesystem) fetchDeviceFile(fileData []byte, devices *[]Device) error | |
func resourceNeedsContainerID(podSpecific bool, resource podResource) bool { | ||
|
||
switch resource { | ||
case lockFileType, networkFileType: | ||
case lockFileType, networkFileType, hypervisorFileType: | ||
// pod-specific resources | ||
return false | ||
default: | ||
|
@@ -342,7 +352,7 @@ func resourceDir(podSpecific bool, podID, containerID string, resource podResour | |
case configFileType: | ||
path = configStoragePath | ||
break | ||
case stateFileType, networkFileType, processFileType, lockFileType, mountsFileType, devicesFileType: | ||
case stateFileType, networkFileType, processFileType, lockFileType, mountsFileType, devicesFileType, hypervisorFileType: | ||
path = runStoragePath | ||
break | ||
default: | ||
|
@@ -378,6 +388,8 @@ func (fs *filesystem) resourceURI(podSpecific bool, podID, containerID string, r | |
filename = stateFile | ||
case networkFileType: | ||
filename = networkFile | ||
case hypervisorFileType: | ||
filename = hypervisorFile | ||
case processFileType: | ||
filename = processFile | ||
case lockFileType: | ||
|
@@ -429,6 +441,7 @@ func (fs *filesystem) commonResourceChecks(podSpecific bool, podID, containerID | |
case configFileType: | ||
case stateFileType: | ||
case networkFileType: | ||
case hypervisorFileType: | ||
case processFileType: | ||
case mountsFileType: | ||
case devicesFileType: | ||
|
@@ -598,10 +611,23 @@ func (fs *filesystem) fetchPodNetwork(podID string) (NetworkNamespace, error) { | |
return networkNS, nil | ||
} | ||
|
||
func (fs *filesystem) fetchHypervisorState(podID string, state interface{}) error { | ||
return fs.fetchResource(true, podID, "", hypervisorFileType, state) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is perfect, but look at There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. change applied, thanks |
||
} | ||
|
||
func (fs *filesystem) storePodNetwork(podID string, networkNS NetworkNamespace) error { | ||
return fs.storePodResource(podID, networkFileType, networkNS) | ||
} | ||
|
||
func (fs *filesystem) storeHypervisorState(podID string, state interface{}) error { | ||
hypervisorFile, _, err := fs.resourceURI(true, podID, "", hypervisorFileType) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
return fs.storeFile(hypervisorFile, state) | ||
} | ||
|
||
func (fs *filesystem) deletePodResources(podID string, resources []podResource) error { | ||
if resources == nil { | ||
resources = []podResource{configFileType, stateFileType} | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Please fix errors in your commit message:
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
fixed, thanks
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The title of the commit message is still wrong.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
ahh ok s/Implements/Implement/
done, thanks