Skip to content

Commit

Permalink
Remove patches of PCI Interrupt affinity for PT, SRIOV, crypto
Browse files Browse the repository at this point in the history 
    Create an agent which runs on each worker node to do pci interrupt
    affinity work instead of current implementation of this feature in
    nova

    Below test done and pass, see detailed test spec in story link
    1) deployment test
    2) Periodic audit pci irq affinity
    3) Remove VM without sriov pci port
    4) Remove VM with sriov pci port
    5) Add VM without sriov pci port
    6) Add VM with sriov pci port
    7) Add VM without pci_irq_affinity_mask
    8) Add VM without cpu policy set
    9) VM resize test
    10) Remove one pci port for VM

    Story: 2004600
    Task: 28851

Signed-off-by: zhipengl <zhipengs.liu@intel.com>
  • Loading branch information
@hustlzp1981
hustlzp1981 committed Mar 1, 2019
1 parent 3155137 commit c52432b
Show file tree
Hide file tree
Showing 6 changed files with 0 additions and 342 deletions.
22 changes: 0 additions & 22 deletions nova/compute/manager.py
View file
Expand Up @@ -6678,28 +6678,6 @@ def _poll_rescued_instances(self, context):
for instance in to_unrescue:
self.compute_api.unrescue(context, instance)

@periodic_task.periodic_task(spacing=CONF.pci_affine_interval)
def _affine_pci_dev_instances(self, context):
"""Periodically reaffine pci device irqs. This will correct the
affinity setting of dynamically created irqs.
"""
filters = {'vm_state': vm_states.ACTIVE,
'task_state': None,
'deleted': False,
'host': self.host}
instances = objects.InstanceList.get_by_filters(
context, filters, expected_attrs=[], use_slave=True)
for instance in instances:
if len(instance.pci_devices.objects) == 0:
continue
try:
self.driver.affine_pci_dev_irqs(instance, wait_for_irqs=False)
except NotImplementedError:
return
except Exception as e:
LOG.info("Error affining pci device irqs: %s.",
e, instance=instance)

@periodic_task.periodic_task
def _poll_unconfirmed_resizes(self, context):
if CONF.resize_confirm_window == 0:
Expand Down
17 changes: 0 additions & 17 deletions nova/conf/compute.py
View file
Expand Up @@ -950,23 +950,6 @@ def host_extend_options(config_opts_file='/etc/nova/compute_extend.conf'):
Possible values:
* 0: Will run at the default periodic interval.
* Any value < 0: Disables the option.
* Any positive integer in seconds.
"""),
cfg.IntOpt('pci_affine_interval',
default=60,
help="""
Number of seconds between pci affinity updates
This option specifies how often the pci_affine_interval
periodic task should run. A number less than 0 means to disable the
task completely. Leaving this at the default of 0 will cause this to
run at the default periodic interval. Setting it to any positive
value will cause it to run at approximately that number of seconds.
Possible values:
* 0: Will run at the default periodic interval.
* Any value < 0: Disables the option.
* Any positive integer in seconds.
Expand Down
14 changes: 0 additions & 14 deletions nova/conf/libvirt.py
View file
Expand Up @@ -1044,19 +1044,6 @@
),
]

# WRS: add options for PCI IRQ affinity, msi irq detection parameters
libvirt_pci_irq_opts = [
cfg.IntOpt('msi_irq_timeout',
default=45,
help='Number of seconds to wait for msi irq configuration'),
cfg.IntOpt('msi_irq_since',
default=6,
help='Number of seconds to wait for msi irqs to stabilize.'),
cfg.IntOpt('msi_irq_check_interval',
default=2,
help='Check interval in seconds for msi irqs to stabilize.'),
]

ALL_OPTS = list(itertools.chain(
libvirt_general_opts,
libvirt_imagebackend_opts,
Expand All @@ -1074,7 +1061,6 @@
libvirt_volume_smbfs_opts,
libvirt_remotefs_opts,
libvirt_volume_vzstorage_opts,
libvirt_pci_irq_opts,
))


Expand Down
170 changes: 0 additions & 170 deletions nova/pci/utils.py
View file
Expand Up @@ -18,7 +18,6 @@
#


import errno
import glob
import os
import re
Expand All @@ -28,8 +27,6 @@

from nova import exception
from nova.network import model as network_model
from nova import utils
from nova.virt import hardware

LOG = logging.getLogger(__name__)

Expand Down Expand Up @@ -133,173 +130,6 @@ def is_physical_function(domain, bus, slot, function):
return False


def get_irqs_by_pci_address(pci_addr):
"""Get list of PCI IRQs based on a VF's pci address
Raises PciDeviceNotFoundById in case the pci device is not found,
or when there is an underlying problem getting associated irqs.
:param pci_addr: PCI address
:return: irqs, msi_irqs
"""
irqs = set()
msi_irqs = set()

dev_path = "/sys/bus/pci/devices/%s" % (pci_addr)
if not os.path.isdir(dev_path):
raise exception.PciDeviceNotFoundById(id=pci_addr)

_irqs = set()
irq_path = "/sys/bus/pci/devices/%s/irq" % (pci_addr)
try:
with open(irq_path) as f:
_irqs.update([int(x) for x in f.readline().split() if int(x) > 0])
except Exception as e:
LOG.error('get_irqs_by_pci_address: '
'pci_addr=%(A)s: irq_path=%(P)s; error=%(E)s',
{'A': pci_addr, 'P': irq_path, 'E': e})
raise exception.PciDeviceNotFoundById(id=pci_addr)

_msi_irqs = set()
msi_path = "/sys/bus/pci/devices/%s/msi_irqs" % (pci_addr)
try:
_msi_irqs.update([int(x) for x in os.listdir(msi_path) if int(x) > 0])
except OSError as e:
# msi_path disappears during configuration; do not treat
# non-existance as fatal
if e.errno == errno.ENOENT:
return (irqs, msi_irqs)
else:
LOG.error('get_irqs_by_pci_address: '
'pci_addr=%(A)s: msi_path=%(P)s; error=%(E)s',
{'A': pci_addr, 'P': msi_path, 'E': e})
raise exception.PciDeviceNotFoundById(id=pci_addr)
except Exception as e:
LOG.error('get_irqs_by_pci_address: '
'pci_addr=%(A)s: msi_path=%(P)s; error=%(E)s',
{'A': pci_addr, 'P': msi_path, 'E': e})
raise exception.PciDeviceNotFoundById(id=pci_addr)

# Return only configured irqs, ignore any that are missing.
for irq in _irqs:
irq_path = "/proc/irq/%s" % (irq)
if os.path.isdir(irq_path):
irqs.update([irq])
for irq in _msi_irqs:
irq_path = "/proc/irq/%s" % (irq)
if os.path.isdir(irq_path):
msi_irqs.update([irq])
return (irqs, msi_irqs)


def get_pci_irqs_pinned_cpuset(flavor=None, numa_topology=None,
pci_numa_node=None):
"""Get pinned cpuset where pci irq are affined.
:param flavor: flavor
:param pci_numa_node: numa node of a specific PCI device
:param numa_topology: instance numa topology
:return: cpuset, cpulist
"""
cpuset = set()
cpulist = ''

if numa_topology is None or pci_numa_node is None or pci_numa_node < 0:
return (cpuset, cpulist)

# Determine full affinity cpuset, but restrict to pci's numa node
for cell in numa_topology.cells:
if cell.id == pci_numa_node:
if cell.cpu_pinning is not None:
cpuset.update(set(cell.cpu_pinning.values()))

# Use extra-spec hw:pci_irq_affinity_mask only when the instance is pinned.
if cpuset:
pci_cpuset = hardware._get_pci_affinity_mask(flavor)
if pci_cpuset:
cpuset = set()
for cell in numa_topology.cells:
if cell.cpu_pinning is not None:
for vcpu in cell.cpuset:
if vcpu in pci_cpuset:
vcpu_cell, pcpu = numa_topology.vcpu_to_pcpu(vcpu)
cpuset.update(set([pcpu]))

cpulist = utils.list_to_range(input_list=list(cpuset))
return (cpuset, cpulist)


def set_irqs_affinity_by_pci_address(pci_addr, flavor=None,
numa_topology=None):
"""Set cpu affinity for list of PCI IRQs with a VF's pci address,
but restrict cpuset to the numa node of the PCI.
Return list
Raises PciDeviceNotFoundById in case the pci device is not found,
or when there is an underlying problem getting associated irqs.
:param pci_addr: PCI address
:param flavor: flavor
:param numa_topology: instance numa topology
:return: irqs, msi_irqs, numa_node, cpulist
"""
irqs = set()
msi_irqs = set()
numa_node = None
cpulist = ''

if numa_topology is None:
return (irqs, msi_irqs, numa_node, cpulist)

# Get the irqs associated with pci addr
_irqs, _msi_irqs = get_irqs_by_pci_address(pci_addr)

# Obtain physical numa_node for this pci addr
numa_path = "/sys/bus/pci/devices/%s/numa_node" % (pci_addr)
try:
with open(numa_path) as f:
numa_node = [int(x) for x in f.readline().split()][0]
except Exception as e:
LOG.error('set_irqs_affinity_by_pci_address: '
'pci_addr=%(A)s: numa_path=%(P)s; error=%(E)s',
{'A': pci_addr, 'P': numa_path, 'E': e})
raise exception.PciDeviceNotFoundById(id=pci_addr)

# Skip irq configuration if there is no associated numa node
if numa_node is None or numa_node < 0:
return (irqs, msi_irqs, numa_node, cpulist)

# Determine the pinned cpuset where irqs are to be affined
cpuset, cpulist = get_pci_irqs_pinned_cpuset(flavor=flavor,
numa_topology=numa_topology,
pci_numa_node=numa_node)

# Skip irq configuration if there are no pinned cpus
if not cpuset:
return (irqs, msi_irqs, numa_node, cpulist)

# Set IRQ affinity, but do not treat errors as fatal.
for irq in _irqs:
irq_aff_path = "/proc/irq/%s/smp_affinity_list" % (irq)
try:
with open(irq_aff_path, 'w') as f:
f.write(cpulist)
irqs.update([irq])
except Exception as e:
LOG.warning("Could not affine pci_addr:%(A)s, irq:%(I)s, "
"error=%(E)s",
{"A": pci_addr, "I": irq, "E": e})
for irq in _msi_irqs:
irq_aff_path = "/proc/irq/%s/smp_affinity_list" % (irq)
try:
with open(irq_aff_path, 'w') as f:
f.write(cpulist)
msi_irqs.update([irq])
except Exception as e:
LOG.warning("Could not affine pci_addr:%(A)s, irq:%(I)s, "
"error=%(E)s",
{"A": pci_addr, "I": irq, "E": e})
return (irqs, msi_irqs, numa_node, cpulist)


def format_instance_pci_devices(pci_devices=None,
delim='\n'):
"""Returns formated pci devices list.
Expand Down
6 changes: 0 additions & 6 deletions nova/virt/driver.py
View file
Expand Up @@ -1667,12 +1667,6 @@ def get_l3_closids(self):
def get_l3_closids_used(self):
return 0

# WRS: extension
def affine_pci_dev_irqs(self, instance, wait_for_irqs=True):
"""Affine PCI device irqs to VM's pcpus."""
raise NotImplementedError()


def load_compute_driver(virtapi, compute_driver=None):
"""Load a compute driver module.
Expand Down

0 comments on commit c52432b

Please sign in to comment.