diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h index d465ece58151..a09c20c606cb 100644 --- a/arch/x86/include/asm/hw_irq.h +++ b/arch/x86/include/asm/hw_irq.h @@ -85,6 +85,7 @@ struct irq_alloc_info { struct ioapic_alloc_info ioapic; struct uv_alloc_info uv; }; + unsigned int vector_align; }; struct irq_cfg { diff --git a/arch/x86/kernel/apic/msi.c b/arch/x86/kernel/apic/msi.c index 7517eb05bdc1..d7b6e94b1ceb 100644 --- a/arch/x86/kernel/apic/msi.c +++ b/arch/x86/kernel/apic/msi.c @@ -168,6 +168,8 @@ int pci_msi_prepare(struct irq_domain *domain, struct device *dev, int nvec, arg->flags |= X86_IRQ_ALLOC_CONTIGUOUS_VECTORS; } + arg->vector_align = to_pci_dev(dev)->align_msi_vector; + return 0; } EXPORT_SYMBOL_GPL(pci_msi_prepare); diff --git a/arch/x86/kernel/apic/vector.c b/arch/x86/kernel/apic/vector.c index 3e6f6b448f6a..2e984516df12 100644 --- a/arch/x86/kernel/apic/vector.c +++ b/arch/x86/kernel/apic/vector.c @@ -30,6 +30,7 @@ struct apic_chip_data { unsigned int cpu; unsigned int prev_cpu; unsigned int irq; + unsigned int vector_align; struct hlist_node clist; unsigned int move_in_progress : 1, is_managed : 1, @@ -190,7 +191,8 @@ static int reserve_managed_vector(struct irq_data *irqd) raw_spin_lock_irqsave(&vector_lock, flags); apicd->is_managed = true; - ret = irq_matrix_reserve_managed(vector_matrix, affmsk); + ret = irq_matrix_reserve_managed(vector_matrix, affmsk, + apicd->vector_align); raw_spin_unlock_irqrestore(&vector_lock, flags); trace_vector_reserve_managed(irqd->irq, ret); return ret; @@ -245,7 +247,8 @@ assign_vector_locked(struct irq_data *irqd, const struct cpumask *dest) if (apicd->move_in_progress || !hlist_unhashed(&apicd->clist)) return -EBUSY; - vector = irq_matrix_alloc(vector_matrix, dest, resvd, &cpu); + vector = irq_matrix_alloc(vector_matrix, dest, resvd, &cpu, + apicd->vector_align); trace_vector_alloc(irqd->irq, vector, resvd, vector); if (vector < 0) return vector; @@ -322,7 +325,7 @@ assign_managed_vector(struct irq_data *irqd, const struct cpumask *dest) if (apicd->vector && cpumask_test_cpu(apicd->cpu, vector_searchmask)) return 0; vector = irq_matrix_alloc_managed(vector_matrix, vector_searchmask, - &cpu); + &cpu, apicd->vector_align); trace_vector_alloc_managed(irqd->irq, vector, vector); if (vector < 0) return vector; @@ -562,6 +565,10 @@ static int x86_vector_alloc_irqs(struct irq_domain *domain, unsigned int virq, goto error; } + if (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI + || info->type == X86_IRQ_ALLOC_TYPE_PCI_MSIX) + apicd->vector_align = info->vector_align; + apicd->irq = virq + i; irqd->chip = &lapic_controller; irqd->chip_data = apicd; diff --git a/include/linux/irq.h b/include/linux/irq.h index c3eb89606c2b..19cb159292d6 100644 --- a/include/linux/irq.h +++ b/include/linux/irq.h @@ -1242,14 +1242,14 @@ struct irq_matrix *irq_alloc_matrix(unsigned int matrix_bits, void irq_matrix_online(struct irq_matrix *m); void irq_matrix_offline(struct irq_matrix *m); void irq_matrix_assign_system(struct irq_matrix *m, unsigned int bit, bool replace); -int irq_matrix_reserve_managed(struct irq_matrix *m, const struct cpumask *msk); +int irq_matrix_reserve_managed(struct irq_matrix *m, const struct cpumask *msk, unsigned int align); void irq_matrix_remove_managed(struct irq_matrix *m, const struct cpumask *msk); int irq_matrix_alloc_managed(struct irq_matrix *m, const struct cpumask *msk, - unsigned int *mapped_cpu); + unsigned int *mapped_cpu, unsigned int align); void irq_matrix_reserve(struct irq_matrix *m); void irq_matrix_remove_reserved(struct irq_matrix *m); int irq_matrix_alloc(struct irq_matrix *m, const struct cpumask *msk, - bool reserved, unsigned int *mapped_cpu); + bool reserved, unsigned int *mapped_cpu, unsigned int align); void irq_matrix_free(struct irq_matrix *m, unsigned int cpu, unsigned int bit, bool managed); void irq_matrix_assign(struct irq_matrix *m, unsigned int bit); diff --git a/include/linux/pci.h b/include/linux/pci.h index b8c354e62937..213103612691 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -485,6 +485,7 @@ struct pci_dev { #ifdef CONFIG_PCI_MSI void __iomem *msix_base; raw_spinlock_t msi_lock; + unsigned int align_msi_vector; #endif struct pci_vpd vpd; #ifdef CONFIG_PCIE_DPC diff --git a/kernel/irq/matrix.c b/kernel/irq/matrix.c index 1698e77645ac..4db1b022e904 100644 --- a/kernel/irq/matrix.c +++ b/kernel/irq/matrix.c @@ -108,14 +108,17 @@ void irq_matrix_offline(struct irq_matrix *m) } static unsigned int matrix_alloc_area(struct irq_matrix *m, struct cpumap *cm, - unsigned int num, bool managed) + unsigned int num, bool managed, unsigned int align) { unsigned int area, start = m->alloc_start; unsigned int end = m->alloc_end; + if (align > 0) + align--; + bitmap_or(m->scratch_map, cm->managed_map, m->system_map, end); bitmap_or(m->scratch_map, m->scratch_map, cm->alloc_map, end); - area = bitmap_find_next_zero_area(m->scratch_map, end, start, num, 0); + area = bitmap_find_next_zero_area(m->scratch_map, end, start, num, align); if (area >= end) return area; if (managed) @@ -207,7 +210,7 @@ void irq_matrix_assign_system(struct irq_matrix *m, unsigned int bit, * on all CPUs in @msk, but it's not guaranteed that the bits are at the * same offset on all CPUs */ -int irq_matrix_reserve_managed(struct irq_matrix *m, const struct cpumask *msk) +int irq_matrix_reserve_managed(struct irq_matrix *m, const struct cpumask *msk, unsigned int align) { unsigned int cpu, failed_cpu; @@ -215,7 +218,7 @@ int irq_matrix_reserve_managed(struct irq_matrix *m, const struct cpumask *msk) struct cpumap *cm = per_cpu_ptr(m->maps, cpu); unsigned int bit; - bit = matrix_alloc_area(m, cm, 1, true); + bit = matrix_alloc_area(m, cm, 1, true, align); if (bit >= m->alloc_end) goto cleanup; cm->managed++; @@ -284,7 +287,7 @@ void irq_matrix_remove_managed(struct irq_matrix *m, const struct cpumask *msk) * @mapped_cpu: Pointer to store the CPU for which the irq was allocated */ int irq_matrix_alloc_managed(struct irq_matrix *m, const struct cpumask *msk, - unsigned int *mapped_cpu) + unsigned int *mapped_cpu, unsigned int align) { unsigned int bit, cpu, end; struct cpumap *cm; @@ -298,9 +301,14 @@ int irq_matrix_alloc_managed(struct irq_matrix *m, const struct cpumask *msk, cm = per_cpu_ptr(m->maps, cpu); end = m->alloc_end; + + if (align > 0) + align--; + /* Get managed bit which are not allocated */ bitmap_andnot(m->scratch_map, cm->managed_map, cm->alloc_map, end); - bit = find_first_bit(m->scratch_map, end); + bitmap_complement(m->scratch_map, m->scratch_map, end); + bit = bitmap_find_next_zero_area(m->scratch_map, end, 0, 1, align); if (bit >= end) return -ENOSPC; set_bit(bit, cm->alloc_map); @@ -375,7 +383,7 @@ void irq_matrix_remove_reserved(struct irq_matrix *m) * @mapped_cpu: Pointer to store the CPU for which the irq was allocated */ int irq_matrix_alloc(struct irq_matrix *m, const struct cpumask *msk, - bool reserved, unsigned int *mapped_cpu) + bool reserved, unsigned int *mapped_cpu, unsigned int align) { unsigned int cpu, bit; struct cpumap *cm; @@ -392,7 +400,7 @@ int irq_matrix_alloc(struct irq_matrix *m, const struct cpumask *msk, return -ENOSPC; cm = per_cpu_ptr(m->maps, cpu); - bit = matrix_alloc_area(m, cm, 1, false); + bit = matrix_alloc_area(m, cm, 1, false, align); if (bit >= m->alloc_end) return -ENOSPC; cm->allocated++; @@ -404,7 +412,6 @@ int irq_matrix_alloc(struct irq_matrix *m, const struct cpumask *msk, *mapped_cpu = cpu; trace_irq_matrix_alloc(bit, cpu, m, cm); return bit; - } /**