hv:Replace dynamic memory with static for ptdev

-- Config MAX_PT_IRQ_ENTRIES 64 in Kconfig
-- Remove ptdev list
-- Add alloc_ptdev_entry_id api

v3-->v4:
  -- move is_entry_active from assign.c to ptdev.h
  -- Add clear active flag in release_entry
v2-->v3:
  -- Remove redundancy active check for ptdev entry
     in release_all_entries and get_ptdev_info
v1-->v2:
  -- split to small patches

Tracked-On: #861
Signed-off-by: Mingqiang Chi <mingqiang.chi@intel.com>
Reviewed-by: Jason Chen CJ <jason.cj.chen@intel.com>
Reviewed-by: Binbin Wu <binbin.wu@intel.com>
Reviewed-by: Eddie Dong <eddie.dong@intel.com>

Author: mingqiangchi

hypervisor/arch/x86/Kconfig

@@ -113,6 +113,11 @@ config MAX_EMULATED_MMIO_REGIONS
 	range 0 128
 	default 16
 
+config MAX_PT_IRQ_ENTRIES
+	int "Maximum number of interrupt source for PT devices"
+	range 0 128
+	default 64
+
 config MAX_IOMMU_NUM
 	int "Maximum number of IOMMU devices"
 	range 1 2

hypervisor/arch/x86/assign.c

@@ -19,12 +19,14 @@ static inline struct ptdev_remapping_info *
 ptdev_lookup_entry_by_sid(uint32_t intr_type,
 		const union source_id *sid,const struct acrn_vm *vm)
 {
+	uint16_t idx;
 	struct ptdev_remapping_info *entry;
-	struct list_head *pos;
 
-	list_for_each(pos, &ptdev_list) {
-		entry = list_entry(pos, struct ptdev_remapping_info,
-				entry_node);
+	for (idx = 0U; idx < CONFIG_MAX_PT_IRQ_ENTRIES; idx++) {
+		entry = &ptdev_irq_entries[idx];
+		if (!is_entry_active(entry)) {
+			continue;
+		}
 		if ((intr_type == entry->intr_type) &&
 			((vm == NULL) ?
 			(sid->value == entry->phys_sid.value) :
@@ -37,12 +39,6 @@ ptdev_lookup_entry_by_sid(uint32_t intr_type,
 	return NULL;
 }
 
-static inline bool
-is_entry_active(const struct ptdev_remapping_info *entry)
-{
-	return atomic_load32(&entry->active) == ACTIVE_FLAG;
-}
-
 #ifdef HV_DEBUG
 static bool ptdev_hv_owned_intx(const struct acrn_vm *vm, const union source_id *virt_sid)
 {
@@ -209,6 +205,9 @@ static struct ptdev_remapping_info *add_msix_remapping(struct acrn_vm *vm,
 		}
 
 		entry = alloc_entry(vm, PTDEV_INTR_MSI);
+		if (entry == NULL) {
+			return NULL;
+		}
 		entry->phys_sid.value = phys_sid.value;
 		entry->virt_sid.value = virt_sid.value;
 
@@ -299,6 +298,9 @@ static struct ptdev_remapping_info *add_intx_remapping(struct acrn_vm *vm, uint8
 			return NULL;
 		}
 		entry = alloc_entry(vm, PTDEV_INTR_INTX);
+		if (entry == NULL) {
+			return NULL;
+		}
 		entry->phys_sid.value = phys_sid.value;
 		entry->virt_sid.value = virt_sid.value;
 
@@ -830,14 +832,14 @@ void get_ptdev_info(char *str_arg, size_t str_max)
 {
 	char *str = str_arg;
 	struct ptdev_remapping_info *entry;
+	uint16_t idx;
 	size_t len, size = str_max;
 	uint32_t irq, vector;
 	char type[16];
 	uint64_t dest;
 	bool lvl_tm;
 	uint8_t pin, vpin;
 	uint32_t bdf, vbdf;
-	struct list_head *pos;
 
 	len = snprintf(str, size, "\r\nVM\tTYPE\tIRQ\tVEC\tDEST\tTM\tPIN\tVPIN\tBDF\tVBDF");
 	if (len >= size) {
@@ -847,9 +849,8 @@ void get_ptdev_info(char *str_arg, size_t str_max)
 	str += len;
 
 	spinlock_obtain(&ptdev_lock);
-	list_for_each(pos, &ptdev_list) {
-		entry = list_entry(pos, struct ptdev_remapping_info,
-				entry_node);
+	for (idx = 0U; idx < CONFIG_MAX_PT_IRQ_ENTRIES; idx++) {
+		entry = &ptdev_irq_entries[idx];
 		if (is_entry_active(entry)) {
 			get_entry_info(entry, type, &irq, &vector,
 					&dest, &lvl_tm, &pin, &vpin,

hypervisor/common/ptdev.c

@@ -8,17 +8,30 @@
 #include <softirq.h>
 #include <ptdev.h>
 
-/* passthrough device link */
-struct list_head ptdev_list;
+#define PTDEV_BITMAP_ARRAY_SIZE	INT_DIV_ROUNDUP(CONFIG_MAX_PT_IRQ_ENTRIES, 64U)
+struct ptdev_remapping_info ptdev_irq_entries[CONFIG_MAX_PT_IRQ_ENTRIES];
+static uint64_t ptdev_entry_bitmaps[PTDEV_BITMAP_ARRAY_SIZE];
+
 spinlock_t ptdev_lock;
 
-/*
- * entry could both be in ptdev_list and vm->softirq_dev_entry_list.
- * When release entry, we need make sure entry deleted from both
- * lists. We have to require two locks and the lock sequence is:
- *   ptdev_lock
- *     vm->softirq_dev_lock
- */
+bool is_entry_active(const struct ptdev_remapping_info *entry)
+{
+	return atomic_load32(&entry->active) == ACTIVE_FLAG;
+}
+
+static inline uint16_t alloc_ptdev_entry_id(void)
+{
+	uint16_t id = (uint16_t)ffz64_ex(ptdev_entry_bitmaps, CONFIG_MAX_PT_IRQ_ENTRIES);
+
+	while (id < CONFIG_MAX_PT_IRQ_ENTRIES) {
+		if (!bitmap_test_and_set_lock((id & 0x3FU), &ptdev_entry_bitmaps[id >> 6U])) {
+			return id;
+		}
+		id = (uint16_t)ffz64_ex(ptdev_entry_bitmaps, CONFIG_MAX_PT_IRQ_ENTRIES);
+	}
+
+	return INVALID_PTDEV_ENTRY_ID;
+}
 
 static void ptdev_enqueue_softirq(struct ptdev_remapping_info *entry)
 {
@@ -78,22 +91,26 @@ struct ptdev_remapping_info *
 alloc_entry(struct acrn_vm *vm, uint32_t intr_type)
 {
 	struct ptdev_remapping_info *entry;
+	uint16_t ptdev_id = alloc_ptdev_entry_id();
 
-	/* allocate */
-	entry = calloc(1U, sizeof(*entry));
-	ASSERT(entry != NULL, "alloc memory failed");
+	if (ptdev_id >= CONFIG_MAX_PT_IRQ_ENTRIES) {
+		pr_err("Alloc ptdev irq entry failed");
+		return NULL;
+	}
+
+	entry = &ptdev_irq_entries[ptdev_id];
+	(void)memset((void *)entry, 0U, sizeof(struct ptdev_remapping_info));
+	entry->ptdev_entry_id = ptdev_id;
 	entry->intr_type = intr_type;
 	entry->vm = vm;
+	entry->intr_count = 0UL;
 
 	INIT_LIST_HEAD(&entry->softirq_node);
-	INIT_LIST_HEAD(&entry->entry_node);
 
-	entry->intr_count = 0UL;
 	initialize_timer(&entry->intr_delay_timer, ptdev_intr_delay_callback,
 		entry, 0UL, 0, 0UL);
 
 	atomic_clear32(&entry->active, ACTIVE_FLAG);
-	list_add(&entry->entry_node, &ptdev_list);
 
 	return entry;
 }
@@ -104,30 +121,26 @@ release_entry(struct ptdev_remapping_info *entry)
 {
 	uint64_t rflags;
 
-	/* remove entry from ptdev_list */
-	list_del_init(&entry->entry_node);
-
 	/*
 	 * remove entry from softirq list.the ptdev_lock
 	 * is required before calling release_entry.
 	 */
 	spinlock_irqsave_obtain(&entry->vm->softirq_dev_lock, &rflags);
 	list_del_init(&entry->softirq_node);
 	spinlock_irqrestore_release(&entry->vm->softirq_dev_lock, rflags);
-
-	free(entry);
+	atomic_clear32(&entry->active, ACTIVE_FLAG);
+	bitmap_clear_nolock((entry->ptdev_entry_id) & 0x3FU, &ptdev_entry_bitmaps[(entry->ptdev_entry_id) >> 6U]);
 }
 
 /* require ptdev_lock protect */
 static void
 release_all_entries(const struct acrn_vm *vm)
 {
 	struct ptdev_remapping_info *entry;
-	struct list_head *pos, *tmp;
+	uint16_t idx;
 
-	list_for_each_safe(pos, tmp, &ptdev_list) {
-		entry = list_entry(pos, struct ptdev_remapping_info,
-				entry_node);
+	for (idx = 0U; idx < CONFIG_MAX_PT_IRQ_ENTRIES; idx++) {
+		entry = &ptdev_irq_entries[idx];
 		if (entry->vm == vm) {
 			release_entry(entry);
 		}
@@ -200,7 +213,6 @@ void ptdev_init(void)
 		return;
 	}
 
-	INIT_LIST_HEAD(&ptdev_list);
 	spinlock_init(&ptdev_lock);
 
 	register_softirq(SOFTIRQ_PTDEV, ptdev_softirq);
@@ -218,12 +230,14 @@ uint32_t get_vm_ptdev_intr_data(const struct acrn_vm *target_vm, uint64_t *buffe
 	uint32_t buffer_cnt)
 {
 	uint32_t index = 0U;
+	uint16_t i;
 	struct ptdev_remapping_info *entry;
-	struct list_head *pos, *tmp;
 
-	list_for_each_safe(pos, tmp, &ptdev_list) {
-		entry = list_entry(pos, struct ptdev_remapping_info,
-				entry_node);
+	for (i = 0U; i < CONFIG_MAX_PT_IRQ_ENTRIES; i++) {
+		entry = &ptdev_irq_entries[i];
+		if (!is_entry_active(entry)) {
+			continue;
+		}
 		if (entry->vm == target_vm) {
 			buffer[index] = entry->allocated_pirq;
 			buffer[index + 1U] = entry->intr_count;

hypervisor/include/common/ptdev.h

@@ -12,6 +12,8 @@
 #define PTDEV_INTR_MSI		(1U << 0U)
 #define PTDEV_INTR_INTX		(1U << 1U)
 
+#define INVALID_PTDEV_ENTRY_ID 0xffffU
+
 enum ptdev_vpin_source {
 	PTDEV_VPIN_IOAPIC,
 	PTDEV_VPIN_PIC,
@@ -51,6 +53,7 @@ struct ptdev_msi_info {
  * with interrupt handler and softirq.
  */
 struct ptdev_remapping_info {
+	uint16_t ptdev_entry_id;
 	uint32_t intr_type;
 	union source_id phys_sid;
 	union source_id virt_sid;
@@ -59,16 +62,16 @@ struct ptdev_remapping_info {
 	uint32_t allocated_pirq;
 	uint32_t polarity; /* 0=active high, 1=active low*/
 	struct list_head softirq_node;
-	struct list_head entry_node;
 	struct ptdev_msi_info msi;
 
 	uint64_t intr_count;
 	struct hv_timer intr_delay_timer; /* used for delay intr injection */
 };
 
-extern struct list_head ptdev_list;
+extern struct ptdev_remapping_info ptdev_irq_entries[];
 extern spinlock_t ptdev_lock;
 
+bool is_entry_active(const struct ptdev_remapping_info *entry);
 void ptdev_softirq(uint16_t pcpu_id);
 void ptdev_init(void);
 void ptdev_release_all_entries(const struct acrn_vm *vm);