HV: add support for 64-bit bar emulation

Enable 64-bit bar emulation, if pbar is of type PCIBAR_MEM64, vbar will also be
of type PCIBAR_MEM64 instead of PCIBAR_MEM32

With 64-bit bar emulation code in place, we can remove enum pci_bar_type type
from struct pci_bar as bar type can be derived from struct pci_bar's reg member
by using the pci_get_bar_type function

Rename functions:
  pci_base_from_size_mask --> git_size_masked_bar_base

Remove unused functions

Tracked-On: #3241
Signed-off-by: dongshen <dongsheng.x.zhang@intel.com>
Reviewed-by: Eddie Dong <eddie.dong@intel.com>

Author: donsheng

hypervisor/dm/vpci/pci_pt.c

@@ -346,7 +346,6 @@ static void set_vbar_base(struct pci_bar *vbar, uint32_t base)
 
 /**
  * @pre vdev != NULL
- * @pre (vdev->bar[idx].type == PCIBAR_NONE) || (vdev->bar[idx].type == PCIBAR_MEM32)
  */
 static void vdev_pt_write_vbar(struct pci_vdev *vdev, uint32_t offset, uint32_t val)
 {
@@ -361,22 +360,41 @@ static void vdev_pt_write_vbar(struct pci_vdev *vdev, uint32_t offset, uint32_t
 
 	vbar = &vdev->bar[idx];
 
-	switch (vdev->bar[idx].type) {
-	case PCIBAR_NONE:
-		break;
+	if (vbar->is_64bit_high) {
+		if (idx > 0U) {
+			uint32_t prev_idx = idx - 1U;
 
-	case PCIBAR_MEM32:
-		base = pci_base_from_size_mask(vbar->size, (uint64_t)val);
-		set_vbar_base(vbar, (uint32_t)base);
+			base = git_size_masked_bar_base(vdev->bar[prev_idx].size, ((uint64_t)val) << 32U) >> 32U;
+			set_vbar_base(vbar, (uint32_t)base);
 
-		if (bar_update_normal) {
-			vdev_pt_remap_mem_vbar(vdev, idx);
+			if (bar_update_normal) {
+				vdev_pt_remap_mem_vbar(vdev, prev_idx);
+			}
+		} else {
+			ASSERT(false, "idx for upper 32-bit of the 64-bit bar should be greater than 0!");
 		}
-		break;
+	} else {
+		enum pci_bar_type type = pci_get_bar_type(vbar->reg.value);
+
+		switch (type) {
+		case PCIBAR_MEM32:
+			base = git_size_masked_bar_base(vbar->size, (uint64_t)val);
+			set_vbar_base(vbar, (uint32_t)base);
 
-	default:
-		/* Should never reach here, init_vdev_pt() only sets vbar type to PCIBAR_NONE and PCIBAR_MEM32 */
-		break;
+			if (bar_update_normal) {
+				vdev_pt_remap_mem_vbar(vdev, idx);
+			}
+			break;
+
+		case PCIBAR_MEM64:
+			base = git_size_masked_bar_base(vbar->size, (uint64_t)val);
+			set_vbar_base(vbar, (uint32_t)base);
+			break;
+
+		default:
+			/* Nothing to do */
+			break;
+		}
 	}
 
 	/* Write the vbar value to corresponding virtualized vbar reg */
@@ -403,15 +421,6 @@ int32_t vdev_pt_write_cfg(struct pci_vdev *vdev, uint32_t offset, uint32_t bytes
 	return ret;
 }
 
-/**
- * For bar emulation, currently only MMIO is supported and bar size cannot be greater than 4GB
- * @pre bar != NULL
- */
-static inline bool is_bar_supported(const struct pci_bar *bar)
-{
-	return (is_mmio_bar(bar) && is_valid_bar_size(bar));
-}
-
 /**
  * PCI base address register (bar) virtualization:
  *
@@ -453,40 +462,47 @@ void init_vdev_pt(struct pci_vdev *vdev)
 			pbar = &vdev->pdev->bar[idx];
 			vbar = &vdev->bar[idx];
 
-			if (is_bar_supported(pbar)) {
-				vbar->reg.value = pbar->reg.value;
-				vbar->reg.bits.mem.base = 0x0U; /* clear vbar base */
-				if (vbar->reg.bits.mem.type == 0x2U) {
-					/* Clear vbar 64-bit flag and set it to 32-bit */
-					vbar->reg.bits.mem.type = 0x0U;
-				}
-
-				/**
-				 * If vbar->base is 0 (unassigned), Linux kernel will reprogram the vbar on
-				 * its bar size boundary, so in order to ensure the MMIO vbar allocated by guest
-				 * is 4k aligned, set its size to be 4K aligned.
-				 */
-				vbar->size = round_page_up(pbar->size);
-
-				/**
-				 * Only 32-bit bar is supported for now so both PCIBAR_MEM32 and PCIBAR_MEM64
-				 * are reported to guest as PCIBAR_MEM32
-				 */
-				vbar->type = PCIBAR_MEM32;
+			vbar->size = 0UL;
+			vbar->reg.value = pbar->reg.value;
+			vbar->is_64bit_high = pbar->is_64bit_high;
 
-				/* For pre-launched VMs: vbar base is predefined in vm_config */
-				vbar_base = vdev->ptdev_config->vbar_base[idx];
+			if (pbar->is_64bit_high) {
+				ASSERT(idx > 0U, "idx for upper 32-bit of the 64-bit bar should be greater than 0!");
 
-				/* Set the new vbar base */
-				vdev_pt_write_vbar(vdev, pci_bar_offset(idx), (uint32_t)vbar_base);
+				if (idx > 0U) {
+					/* For pre-launched VMs: vbar base is predefined in vm_config */
+					vbar_base = vdev->ptdev_config->vbar_base[idx - 1U];
+					/* Write the upper 32-bit of a 64-bit bar */
+					vdev_pt_write_vbar(vdev, pci_bar_offset(idx), (uint32_t)(vbar_base >> 32U));
+				}
 			} else {
-				vbar->reg.value = 0x0U;
-				vbar->size = 0UL;
-				vbar->type = PCIBAR_NONE;
+				enum pci_bar_type type = pci_get_bar_type(pbar->reg.value);
+
+				switch (type) {
+				case PCIBAR_MEM32:
+				case PCIBAR_MEM64:
+					/**
+					 * If vbar->base is 0 (unassigned), Linux kernel will reprogram the vbar on
+					 * its bar size boundary, so in order to ensure the MMIO vbar allocated by guest
+					 * is 4k aligned, set its size to be 4K aligned.
+					 */
+					vbar->size = round_page_up(pbar->size);
+
+					/* For pre-launched VMs: vbar base is predefined in vm_config */
+					vbar_base = vdev->ptdev_config->vbar_base[idx];
+					vdev_pt_write_vbar(vdev, pci_bar_offset(idx), (uint32_t)vbar_base);
+					break;
+
+				default:
+					vbar->reg.value = 0x0U;
+					vbar->size = 0UL;
+					break;
+				}
 			}
 		}
 
 		pci_command = (uint16_t)pci_pdev_read_cfg(vdev->pdev->bdf, PCIR_COMMAND, 2U);
+
 		/* Disable INTX */
 		pci_command |= 0x400U;
 		pci_pdev_write_cfg(vdev->pdev->bdf, PCIR_COMMAND, 2U, pci_command);

hypervisor/hw/pci.c

@@ -287,7 +287,6 @@ static uint32_t pci_pdev_read_bar(union pci_bdf bdf, uint32_t idx, struct pci_ba
 	}
 
 	bar->size = size;
-	bar->type = type;
 
 	return (type == PCIBAR_MEM64)?2U:1U;
 }

hypervisor/include/hw/pci.h

@@ -183,7 +183,6 @@ struct pci_bar {
 	/* Base Address Register */
 	union pci_bar_reg reg;
 	uint64_t size;
-	enum pci_bar_type type;
 	bool is_64bit_high; /* true if this is the upper 32-bit of a 64-bit bar */
 };
 
@@ -270,7 +269,7 @@ static inline enum pci_bar_type pci_get_bar_type(uint32_t val)
  * Given bar size and raw bar value, return bar base address by masking off its lower flag bits
  * size/val: all in 64-bit values to accommodate 64-bit MMIO bar size masking
  */
-static inline uint64_t pci_base_from_size_mask(uint64_t size, uint64_t val)
+static inline uint64_t git_size_masked_bar_base(uint64_t size, uint64_t val)
 {
 	uint64_t mask;
 
@@ -308,22 +307,6 @@ static inline bool bdf_is_equal(const union pci_bdf *a, const union pci_bdf *b)
 	return (a->value == b->value);
 }
 
-/**
- * @pre bar != NULL
- */
-static inline bool is_mmio_bar(const struct pci_bar *bar)
-{
-	return (bar->type == PCIBAR_MEM32) || (bar->type == PCIBAR_MEM64);
-}
-
-/**
- * @pre bar != NULL
- */
-static inline bool is_valid_bar_size(const struct pci_bar *bar)
-{
-	return (bar->size > 0UL) && (bar->size <= 0xffffffffU);
-}
-
 uint32_t pci_pdev_read_cfg(union pci_bdf bdf, uint32_t offset, uint32_t bytes);
 void pci_pdev_write_cfg(union pci_bdf bdf, uint32_t offset, uint32_t bytes, uint32_t val);
 void enable_disable_pci_intx(union pci_bdf bdf, bool enable);