Skip to content

Commit

Permalink
vpci/header: program p2m with guest BAR view
Browse files Browse the repository at this point in the history 
Take into account guest's BAR view and program its p2m accordingly:
gfn is guest's view of the BAR and mfn is the physical BAR value.
This way hardware domain sees physical BAR values and guest sees
emulated ones.

Hardware domain continues getting the BARs identity mapped, while for
domUs the BARs are mapped at the requested guest address without
modifying the BAR address in the device PCI config space.

Signed-off-by: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
Signed-off-by: Volodymyr Babchuk <volodymyr_babchuk@epam.com>
Signed-off-by: Stewart Hildebrand <stewart.hildebrand@amd.com>
Reviewed-by: Roger Pau Monné <roger.pau@citrix.com>
  • Loading branch information
@jbeulich
Oleksandr Andrushchenko authored and jbeulich committed Feb 27, 2024
1 parent 622bdd9 commit 52ebde3
Show file tree
Hide file tree
Showing 2 changed files with 66 additions and 20 deletions.
83 changes: 64 additions & 19 deletions xen/drivers/vpci/header.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,33 +34,45 @@

struct map_data {
struct domain *d;
const struct vpci_bar *bar;
bool map;
};

static int cf_check map_range(
unsigned long s, unsigned long e, void *data, unsigned long *c)
{
const struct map_data *map = data;
/* Start address of the BAR as seen by the guest. */
unsigned long start_gfn = PFN_DOWN(map->bar->guest_addr);
/* Physical start address of the BAR. */
unsigned long start_mfn = PFN_DOWN(map->bar->addr);
int rc;

for ( ; ; )
{
unsigned long size = e - s + 1;
/*
* Ranges to be mapped don't always start at the BAR start address, as
* there can be holes or partially consumed ranges. Account for the
* offset of the current address from the BAR start.
*/
unsigned long map_mfn = start_mfn + s - start_gfn;
unsigned long m_end = map_mfn + size - 1;

if ( !iomem_access_permitted(map->d, s, e) )
if ( !iomem_access_permitted(map->d, map_mfn, m_end) )
{
printk(XENLOG_G_WARNING
"%pd denied access to MMIO range [%#lx, %#lx]\n",
map->d, s, e);
map->d, map_mfn, m_end);
return -EPERM;
}

rc = xsm_iomem_mapping(XSM_HOOK, map->d, s, e, map->map);
rc = xsm_iomem_mapping(XSM_HOOK, map->d, map_mfn, m_end, map->map);
if ( rc )
{
printk(XENLOG_G_WARNING
"%pd XSM denied access to MMIO range [%#lx, %#lx]: %d\n",
map->d, s, e, rc);
map->d, map_mfn, m_end, rc);
return rc;
}

Expand All @@ -73,8 +85,8 @@ static int cf_check map_range(
* - {un}map_mmio_regions doesn't support preemption.
*/

rc = map->map ? map_mmio_regions(map->d, _gfn(s), size, _mfn(s))
: unmap_mmio_regions(map->d, _gfn(s), size, _mfn(s));
rc = map->map ? map_mmio_regions(map->d, _gfn(s), size, _mfn(map_mfn))
: unmap_mmio_regions(map->d, _gfn(s), size, _mfn(map_mfn));
if ( rc == 0 )
{
*c += size;
Expand All @@ -83,8 +95,9 @@ static int cf_check map_range(
if ( rc < 0 )
{
printk(XENLOG_G_WARNING
"Failed to identity %smap [%lx, %lx] for d%d: %d\n",
map->map ? "" : "un", s, e, map->d->domain_id, rc);
"Failed to %smap [%lx %lx] -> [%lx %lx] for %pd: %d\n",
map->map ? "" : "un", s, e, map_mfn,
map_mfn + size, map->d, rc);
break;
}
ASSERT(rc < size);
Expand Down Expand Up @@ -163,10 +176,6 @@ static void modify_decoding(const struct pci_dev *pdev, uint16_t cmd,
bool vpci_process_pending(struct vcpu *v)
{
struct pci_dev *pdev = v->vpci.pdev;
struct map_data data = {
.d = v->domain,
.map = v->vpci.cmd & PCI_COMMAND_MEMORY,
};
struct vpci_header *header = NULL;
unsigned int i;

Expand All @@ -186,6 +195,11 @@ bool vpci_process_pending(struct vcpu *v)
for ( i = 0; i < ARRAY_SIZE(header->bars); i++ )
{
struct vpci_bar *bar = &header->bars[i];
struct map_data data = {
.d = v->domain,
.map = v->vpci.cmd & PCI_COMMAND_MEMORY,
.bar = bar,
};
int rc;

if ( rangeset_is_empty(bar->mem) )
Expand Down Expand Up @@ -236,7 +250,6 @@ bool vpci_process_pending(struct vcpu *v)
static int __init apply_map(struct domain *d, const struct pci_dev *pdev,
uint16_t cmd)
{
struct map_data data = { .d = d, .map = true };
struct vpci_header *header = &pdev->vpci->header;
int rc = 0;
unsigned int i;
Expand All @@ -246,6 +259,7 @@ static int __init apply_map(struct domain *d, const struct pci_dev *pdev,
for ( i = 0; i < ARRAY_SIZE(header->bars); i++ )
{
struct vpci_bar *bar = &header->bars[i];
struct map_data data = { .d = d, .map = true, .bar = bar };

if ( rangeset_is_empty(bar->mem) )
continue;
Expand Down Expand Up @@ -311,12 +325,16 @@ static int modify_bars(const struct pci_dev *pdev, uint16_t cmd, bool rom_only)
* First fill the rangesets with the BAR of this device or with the ROM
* BAR only, depending on whether the guest is toggling the memory decode
* bit of the command register, or the enable bit of the ROM BAR register.
*
* For non-hardware domain we use guest physical addresses.
*/
for ( i = 0; i < ARRAY_SIZE(header->bars); i++ )
{
struct vpci_bar *bar = &header->bars[i];
unsigned long start = PFN_DOWN(bar->addr);
unsigned long end = PFN_DOWN(bar->addr + bar->size - 1);
unsigned long start_guest = PFN_DOWN(bar->guest_addr);
unsigned long end_guest = PFN_DOWN(bar->guest_addr + bar->size - 1);

if ( !bar->mem )
continue;
Expand All @@ -336,11 +354,26 @@ static int modify_bars(const struct pci_dev *pdev, uint16_t cmd, bool rom_only)
continue;
}

rc = rangeset_add_range(bar->mem, start, end);
ASSERT(rangeset_is_empty(bar->mem));

/*
* Make sure that the guest set address has the same page offset
* as the physical address on the host or otherwise things won't work as
* expected.
*/
if ( PAGE_OFFSET(bar->guest_addr) != PAGE_OFFSET(bar->addr) )
{
gprintk(XENLOG_G_WARNING,
"%pp: can't map BAR%u - offset mismatch: %#lx vs %#lx\n",
&pdev->sbdf, i, bar->guest_addr, bar->addr);
return -EINVAL;
}

rc = rangeset_add_range(bar->mem, start_guest, end_guest);
if ( rc )
{
printk(XENLOG_G_WARNING "Failed to add [%lx, %lx]: %d\n",
start, end, rc);
start_guest, end_guest, rc);
return rc;
}

Expand All @@ -352,12 +385,12 @@ static int modify_bars(const struct pci_dev *pdev, uint16_t cmd, bool rom_only)
if ( rangeset_is_empty(prev_bar->mem) )
continue;

rc = rangeset_remove_range(prev_bar->mem, start, end);
rc = rangeset_remove_range(prev_bar->mem, start_guest, end_guest);
if ( rc )
{
gprintk(XENLOG_WARNING,
"%pp: failed to remove overlapping range [%lx, %lx]: %d\n",
&pdev->sbdf, start, end, rc);
&pdev->sbdf, start_guest, end_guest, rc);
return rc;
}
}
Expand Down Expand Up @@ -425,8 +458,8 @@ static int modify_bars(const struct pci_dev *pdev, uint16_t cmd, bool rom_only)
for ( i = 0; i < ARRAY_SIZE(tmp->vpci->header.bars); i++ )
{
const struct vpci_bar *remote_bar = &tmp->vpci->header.bars[i];
unsigned long start = PFN_DOWN(remote_bar->addr);
unsigned long end = PFN_DOWN(remote_bar->addr +
unsigned long start = PFN_DOWN(remote_bar->guest_addr);
unsigned long end = PFN_DOWN(remote_bar->guest_addr +
remote_bar->size - 1);

if ( !remote_bar->enabled )
Expand Down Expand Up @@ -513,6 +546,8 @@ static void cf_check bar_write(
struct vpci_bar *bar = data;
bool hi = false;

ASSERT(is_hardware_domain(pdev->domain));

if ( bar->type == VPCI_BAR_MEM64_HI )
{
ASSERT(reg > PCI_BASE_ADDRESS_0);
Expand Down Expand Up @@ -543,6 +578,8 @@ static void cf_check bar_write(
*/
bar->addr &= ~(0xffffffffULL << (hi ? 32 : 0));
bar->addr |= (uint64_t)val << (hi ? 32 : 0);
/* Update guest address, so hardware domain BAR is identity mapped. */
bar->guest_addr = bar->addr;

/* Make sure Xen writes back the same value for the BAR RO bits. */
if ( !hi )
Expand Down Expand Up @@ -639,11 +676,14 @@ static void cf_check rom_write(
}

if ( !rom->enabled )
{
/*
* If the ROM BAR is not mapped update the address field so the
* correct address is mapped into the p2m.
*/
rom->addr = val & PCI_ROM_ADDRESS_MASK;
rom->guest_addr = rom->addr;
}

if ( !header->bars_mapped || rom->enabled == new_enabled )
{
Expand All @@ -667,7 +707,10 @@ static void cf_check rom_write(
return;

if ( !new_enabled )
{
rom->addr = val & PCI_ROM_ADDRESS_MASK;
rom->guest_addr = rom->addr;
}
}

static int bar_add_rangeset(const struct pci_dev *pdev, struct vpci_bar *bar,
Expand Down Expand Up @@ -862,6 +905,7 @@ static int cf_check init_header(struct pci_dev *pdev)
}

bars[i].addr = addr;
bars[i].guest_addr = addr;
bars[i].size = size;
bars[i].prefetchable = val & PCI_BASE_ADDRESS_MEM_PREFETCH;

Expand All @@ -884,6 +928,7 @@ static int cf_check init_header(struct pci_dev *pdev)
rom->type = VPCI_BAR_ROM;
rom->size = size;
rom->addr = addr;
rom->guest_addr = addr;
header->rom_enabled = pci_conf_read32(pdev->sbdf, rom_reg) &
PCI_ROM_ADDRESS_ENABLE;

Expand Down
3 changes: 2 additions & 1 deletion xen/include/xen/vpci.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,8 @@ int vpci_msix_arch_print(const struct vpci_msix *msix);
*/
static inline paddr_t vmsix_table_base(const struct vpci *vpci, unsigned int nr)
{
return vpci->header.bars[vpci->msix->tables[nr] & PCI_MSIX_BIRMASK].addr;
return vpci->header.bars[vpci->msix->tables[nr] &
PCI_MSIX_BIRMASK].guest_addr;
}

static inline paddr_t vmsix_table_addr(const struct vpci *vpci, unsigned int nr)
Expand Down

0 comments on commit 52ebde3

Please sign in to comment.