Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
hv: introduce SRIOV interception
VF_ENABLE is one field of SRIOV capability that is used to create
or remove VF physical devices. If VF_ENABLE is set, hv can detect
if the VF physical devices are ready after waiting 100 ms.

v2: Add sanity check for writing NumVFs register, add precondition
    and application constraints when VF_ENABLE is set and refine
    code style.

Tracked-On: #4433

Signed-off-by: Yuan Liu <yuan1.liu@intel.com>
Acked-by: Eddie Dong <eddie.dong@intel.com>
  • Loading branch information
yliu80 authored and wenlingz committed Feb 28, 2020
1 parent 14931d1 commit 2f74830
Show file tree
Hide file tree
Showing 2 changed files with 180 additions and 2 deletions.
176 changes: 174 additions & 2 deletions hypervisor/dm/vpci/vsriov.c
Expand Up @@ -27,8 +27,143 @@
* $FreeBSD$
*/

#include <vm.h>
#include <ptdev.h>
#include <vpci.h>
#include <logmsg.h>

#include "vpci_priv.h"

/**
* @pre pf_vdev != NULL
*/
static inline uint8_t get_vf_devfun(const struct pci_vdev *pf_vdev, uint16_t fst_off, uint16_t stride, uint16_t id)
{
return ((uint8_t)((pf_vdev->bdf.fields.devfun + fst_off + (stride * id)) & 0xFFU));
}

/**
* @pre pf_vdev != NULL
*/
static inline uint8_t get_vf_bus(const struct pci_vdev *pf_vdev, uint16_t fst_off, uint16_t stride, uint16_t id)
{
return ((uint8_t)(pf_vdev->bdf.fields.bus + ((pf_vdev->bdf.fields.devfun + fst_off + (stride * id)) >> 8U)));
}

/**
* @pre pf_vdev != NULL
*/
static inline uint16_t read_sriov_reg(const struct pci_vdev *pf_vdev, uint16_t reg)
{
return ((uint16_t)(pci_pdev_read_cfg(pf_vdev->bdf, pf_vdev->sriov.capoff + reg, 2U)));
}

/**
* @pre pf_vdev != NULL
*/
static bool is_vf_enabled(const struct pci_vdev *pf_vdev)
{
uint16_t control;

control = read_sriov_reg(pf_vdev, PCIR_SRIOV_CONTROL);
return ((control & PCIM_SRIOV_VF_ENABLE) != 0U);
}

/**
* @pre pf_vdev != NULL
*/
static void init_sriov_vf_bar(struct pci_vdev *pf_vdev)
{
/* Implementation in next patch */
(void)pf_vdev;
}

/**
* @pre pf_vdev != NULL
*/
static void create_vf(struct pci_vdev *pf_vdev, union pci_bdf vf_bdf)
{
/* Implementation in next patch */
(void)pf_vdev;
(void)vf_bdf;
}

/**
* @pre pf_vdev != NULL
* @pre is_vf_enabled(pf_dev) == true
* @Application constraints: PCIR_SRIOV_NUMVFS register value cannot be 0 if VF_ENABLE is set.
*/
static void enable_vf(struct pci_vdev *pf_vdev)
{
union pci_bdf vf_bdf;
uint16_t idx;
uint16_t sub_vid = 0U;

/* Confirm that the physical VF_ENABLE register has been set successfully */
ASSERT(is_vf_enabled(pf_vdev), "VF_ENABLE was not set successfully on the hardware");

/*
* All VFs bars information are located at PF VF_BAR fields of SRIOV capability.
* Initialize the PF's VF_BAR registers before initialize each VF device bar.
*/
init_sriov_vf_bar(pf_vdev);

/*
* Per PCIE base spec 9.3.3.3.1, VF Enable bit from cleared to set, the
* system is not perrmitted to issue requests to the VFs until one of
* the following is true:
* 1. at least 100ms has passed.
* 2. An FRS message has been received from the PF with a reason code
* of VF Enabled.
* 3. At least VF Enable Time has passed since VF Enable was Set.
* VF Enable Time is either the Reset Time value in the Readiness Time
* Reporting capability associated with the VF or a value determined
* by system software/firmware.
*
* Curerntly, we use the first way to wait for VF physical devices to be ready.
*/
udelay (100U * 1000U);

/*
* Due to VF's DEVICE ID and VENDOR ID are 0xFFFF, so check if VF physical
* device has been created by the value of SUBSYSTEM VENDOR ID.
* To check if all enabled VFs are ready, just check the first VF already exists,
* do not need to check all.
*/
sub_vid = (uint16_t) pci_pdev_read_cfg(vf_bdf, PCIV_SUB_VENDOR_ID, 2U);
if ((sub_vid != 0xFFFFU) && (sub_vid != 0U)) {
uint16_t num_vfs, stride, fst_off;

num_vfs = read_sriov_reg(pf_vdev, PCIR_SRIOV_NUMVFS);
fst_off = read_sriov_reg(pf_vdev, PCIR_SRIOV_FST_VF_OFF);
stride = read_sriov_reg(pf_vdev, PCIR_SRIOV_VF_STRIDE);
for (idx = 0U; idx < num_vfs; idx++) {
vf_bdf.fields.bus = get_vf_bus(pf_vdev, fst_off, stride, idx);
vf_bdf.fields.devfun = get_vf_devfun(pf_vdev, fst_off, stride, idx);

/* if one VF has never been created then create new pdev/vdev for this VF */
if (pci_find_vdev(&pf_vdev->vpci->vm->vpci, vf_bdf) == NULL) {
create_vf(pf_vdev, vf_bdf);
}
}
} else {
/*
* If the VF physical device was not created successfully, the pdev/vdev
* will also not be created so that SOS can aware of VF creation failure,
*/
pr_err("PF %x:%x.%x can't create VFs after 100 ms",
pf_vdev->bdf.bits.b, pf_vdev->bdf.bits.d, pf_vdev->bdf.bits.f);
}
}

/**
* @pre pf_vdev != NULL
*/
static void disable_vf(struct pci_vdev *pf_vdev)
{
/* Implementation in next patch */
(void)pf_vdev;
}

/**
* @pre vdev != NULL
Expand Down Expand Up @@ -58,7 +193,44 @@ void read_sriov_cap_reg(const struct pci_vdev *vdev, uint32_t offset, uint32_t b
*/
void write_sriov_cap_reg(struct pci_vdev *vdev, uint32_t offset, uint32_t bytes, uint32_t val)
{
/* Needs to intercept VF_ENABLE and add it in next patch. */
pci_pdev_write_cfg(vdev->pdev->bdf, offset, bytes, val);

uint32_t reg;

reg = offset - vdev->sriov.capoff;
if (reg == PCIR_SRIOV_CONTROL) {
bool enable;

enable = (((val & PCIM_SRIOV_VF_ENABLE) != 0U) ? true : false);
if (enable != is_vf_enabled(vdev)) {
if (enable) {
/*
* set VF_ENABLE to PF physical device before enable_vf
* since need to ask hardware to create VF physical
* devices firstly
*/
pci_pdev_write_cfg(vdev->pdev->bdf, offset, bytes, val);
enable_vf(vdev);
} else {
disable_vf(vdev);
pci_pdev_write_cfg(vdev->pdev->bdf, offset, bytes, val);
}
} else {
pci_pdev_write_cfg(vdev->pdev->bdf, offset, bytes, val);
}
} else if (reg == PCIR_SRIOV_NUMVFS) {
uint16_t total;

total = read_sriov_reg(vdev, PCIR_SRIOV_TOTAL_VFS);
/*
* sanity check for NumVFs register based on PCE Express Base 4.0 9.3.3.7 chapter
* The results are undefined if NumVFs is set to a value greater than TotalVFs
* NumVFs may only be written while VF Enable is Clear
* If NumVFs is written when VF Enable is Set, the results are undefined
*/
if ((((uint16_t)(val & 0xFFU)) <= total) && (!is_vf_enabled(vdev))) {
pci_pdev_write_cfg(vdev->pdev->bdf, offset, bytes, val);
}
} else {
pci_pdev_write_cfg(vdev->pdev->bdf, offset, bytes, val);
}
}
6 changes: 6 additions & 0 deletions hypervisor/include/hw/pci.h
Expand Up @@ -86,6 +86,7 @@
#define PCIM_BAR_MEM_1MB 0x02U
#define PCIM_BAR_MEM_64 0x04U
#define PCIM_BAR_MEM_BASE 0xFFFFFFF0U
#define PCIV_SUB_VENDOR_ID 0x2CU
#define PCIR_CAP_PTR 0x34U
#define PCIR_CAP_PTR_CARDBUS 0x14U
#define PCI_BASE_ADDRESS_MEM_MASK (~0x0fUL)
Expand Down Expand Up @@ -114,7 +115,12 @@

/* SRIOV Definitions */
#define PCI_SRIOV_CAP_LEN 0x40U
#define PCIR_SRIOV_CONTROL 0x8U
#define PCIR_SRIOV_TOTAL_VFS 0xEU
#define PCIR_SRIOV_NUMVFS 0x10U
#define PCIR_SRIOV_FST_VF_OFF 0x14U
#define PCIR_SRIOV_VF_STRIDE 0x16U
#define PCIM_SRIOV_VF_ENABLE 0x1U

/* PCI Message Signalled Interrupts (MSI) */
#define PCIR_MSI_CTRL 0x02U
Expand Down

0 comments on commit 2f74830

Please sign in to comment.