From e5648b75349246a55ad2c3d36807cc10fca698ff Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Wed, 6 Jul 2022 17:07:51 +0530 Subject: [PATCH 01/34] iommu/amd: Update struct iommu_dev_data definition commit d02674d71c5a625e5b2e4323edbf05ebedaf4273 upstream. struct iommu_dev_data contains member "pdev" to point to pci_dev. This is valid for only PCI devices and for other devices this will be NULL. This causes unnecessary "pdev != NULL" check at various places. Replace "struct pci_dev" member with "struct device" and use to_pci_dev() to get pci device reference as needed. Also adjust setup_aliases() and clone_aliases() function. No functional change intended. Co-developed-by: Suravee Suthikulpanit Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20220706113825.25582-2-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: Malathi Signed-off-by: PvsNarasimha --- drivers/iommu/amd/amd_iommu_types.h | 2 +- drivers/iommu/amd/iommu.c | 32 +++++++++++++++++------------ 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 111c4cad5d9b5..c0f7c8f4f75f6 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -687,7 +687,7 @@ struct iommu_dev_data { struct list_head list; /* For domain->dev_list */ struct llist_node dev_data_list; /* For global dev_data_list */ struct protection_domain *domain; /* Domain the device is bound to */ - struct pci_dev *pdev; + struct device *dev; u16 devid; /* PCI Device ID */ bool iommu_v2; /* Device can make use of IOMMUv2 */ struct { diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index ea44e8c2412a6..504ed97f44fe6 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -187,10 +187,13 @@ static int clone_alias(struct pci_dev *pdev, u16 alias, void *data) return 0; } -static void clone_aliases(struct pci_dev *pdev) +static void clone_aliases(struct device *dev) { - if (!pdev) + struct pci_dev *pdev; + + if (!dev_is_pci(dev)) return; + pdev = to_pci_dev(dev); /* * The IVRS alias stored in the alias table may not be @@ -202,14 +205,14 @@ static void clone_aliases(struct pci_dev *pdev) pci_for_each_dma_alias(pdev, clone_alias, NULL); } -static struct pci_dev *setup_aliases(struct device *dev) +static void setup_aliases(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); u16 ivrs_alias; /* For ACPI HID devices, there are no aliases */ if (!dev_is_pci(dev)) - return NULL; + return; /* * Add the IVRS alias to the pci aliases if it is on the same @@ -220,9 +223,7 @@ static struct pci_dev *setup_aliases(struct device *dev) PCI_BUS_NUM(ivrs_alias) == pdev->bus->number) pci_add_dma_alias(pdev, ivrs_alias & 0xff, 1); - clone_aliases(pdev); - - return pdev; + clone_aliases(dev); } static struct iommu_dev_data *find_dev_data(u16 devid) @@ -330,7 +331,8 @@ static int iommu_init_device(struct device *dev) if (!dev_data) return -ENOMEM; - dev_data->pdev = setup_aliases(dev); + dev_data->dev = dev; + setup_aliases(dev); /* * By default we use passthrough mode for IOMMUv2 capable device. @@ -1247,13 +1249,17 @@ static int device_flush_dte_alias(struct pci_dev *pdev, u16 alias, void *data) static int device_flush_dte(struct iommu_dev_data *dev_data) { struct amd_iommu *iommu; + struct pci_dev *pdev = NULL; u16 alias; int ret; iommu = amd_iommu_rlookup_table[dev_data->devid]; - if (dev_data->pdev) - ret = pci_for_each_dma_alias(dev_data->pdev, + if (dev_is_pci(dev_data->dev)) + pdev = to_pci_dev(dev_data->dev); + + if (pdev) + ret = pci_for_each_dma_alias(pdev, device_flush_dte_alias, iommu); else ret = iommu_flush_dte(iommu, dev_data->devid); @@ -1576,7 +1582,7 @@ static void do_attach(struct iommu_dev_data *dev_data, /* Update device table */ set_dte_entry(dev_data->devid, domain, ats, dev_data->iommu_v2); - clone_aliases(dev_data->pdev); + clone_aliases(dev_data->dev); device_flush_dte(dev_data); } @@ -1592,7 +1598,7 @@ static void do_detach(struct iommu_dev_data *dev_data) dev_data->domain = NULL; list_del(&dev_data->list); clear_dte_entry(dev_data->devid); - clone_aliases(dev_data->pdev); + clone_aliases(dev_data->dev); /* Flush the DTE entry */ device_flush_dte(dev_data); @@ -1835,7 +1841,7 @@ static void update_device_table(struct protection_domain *domain) list_for_each_entry(dev_data, &domain->dev_list, list) { set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled, dev_data->iommu_v2); - clone_aliases(dev_data->pdev); + clone_aliases(dev_data->dev); } } From f81329258c888accecfc62e5bc45975c2650ecca Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Wed, 6 Jul 2022 17:07:52 +0530 Subject: [PATCH 02/34] iommu/amd: Introduce pci segment structure commit 404ec4e4c169fb64da6b2a38b471c13ac0897c76 upstream. Newer AMD systems can support multiple PCI segments, where each segment contains one or more IOMMU instances. However, an IOMMU instance can only support a single PCI segment. Current code assumes that system contains only one pci segment (segment 0) and creates global data structures such as device table, rlookup table, etc. Introducing per PCI segment data structure, which contains segment specific data structures. This will eventually replace the global data structures. Also update `amd_iommu->pci_seg` variable to point to PCI segment structure instead of PCI segment ID. Co-developed-by: Suravee Suthikulpanit Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20220706113825.25582-3-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: Malathi Signed-off-by: PvsNarasimha --- drivers/iommu/amd/amd_iommu_types.h | 24 ++++++++++++++- drivers/iommu/amd/init.c | 46 ++++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index c0f7c8f4f75f6..bee4d351d2207 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -451,6 +451,11 @@ extern bool amd_iommu_irq_remap; /* kmem_cache to get tables with 128 byte alignement */ extern struct kmem_cache *amd_iommu_irq_cache; +/* Make iterating over all pci segment easier */ +#define for_each_pci_segment(pci_seg) \ + list_for_each_entry((pci_seg), &amd_iommu_pci_seg_list, list) +#define for_each_pci_segment_safe(pci_seg, next) \ + list_for_each_entry_safe((pci_seg), (next), &amd_iommu_pci_seg_list, list) /* * Make iterating over all IOMMUs easier */ @@ -525,6 +530,17 @@ struct protection_domain { unsigned dev_iommu[MAX_IOMMUS]; /* per-IOMMU reference count */ }; +/* + * This structure contains information about one PCI segment in the system. + */ +struct amd_iommu_pci_seg { + /* List with all PCI segments in the system */ + struct list_head list; + + /* PCI segment number */ + u16 id; +}; + /* * Structure where we save information about one hardware AMD IOMMU in the * system. @@ -576,7 +592,7 @@ struct amd_iommu { u16 cap_ptr; /* pci domain of this IOMMU */ - u16 pci_seg; + struct amd_iommu_pci_seg *pci_seg; /* start of exclusion range of that IOMMU */ u64 exclusion_start; @@ -707,6 +723,12 @@ extern struct list_head ioapic_map; extern struct list_head hpet_map; extern struct list_head acpihid_map; +/* + * List with all PCI segments in the system. This list is not locked because + * it is only written at driver initialization time + */ +extern struct list_head amd_iommu_pci_seg_list; + /* * List with all IOMMUs in the system. This list is not locked because it is * only written and read at driver initialization or suspend time diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index fafb7bd798394..4387a0b7976d3 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -168,6 +168,7 @@ u16 amd_iommu_last_bdf; /* largest PCI device id we have LIST_HEAD(amd_iommu_unity_map); /* a list of required unity mappings we find in ACPI */ +LIST_HEAD(amd_iommu_pci_seg_list); /* list of all PCI segments */ LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the system */ @@ -1483,6 +1484,43 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, return 0; } +/* Allocate PCI segment data structure */ +static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id) +{ + struct amd_iommu_pci_seg *pci_seg; + + pci_seg = kzalloc(sizeof(struct amd_iommu_pci_seg), GFP_KERNEL); + if (pci_seg == NULL) + return NULL; + + pci_seg->id = id; + list_add_tail(&pci_seg->list, &amd_iommu_pci_seg_list); + + return pci_seg; +} + +static struct amd_iommu_pci_seg *__init get_pci_segment(u16 id) +{ + struct amd_iommu_pci_seg *pci_seg; + + for_each_pci_segment(pci_seg) { + if (pci_seg->id == id) + return pci_seg; + } + + return alloc_pci_segment(id); +} + +static void __init free_pci_segments(void) +{ + struct amd_iommu_pci_seg *pci_seg, *next; + + for_each_pci_segment_safe(pci_seg, next) { + list_del(&pci_seg->list); + kfree(pci_seg); + } +} + static void __init free_iommu_one(struct amd_iommu *iommu) { free_cwwb_sem(iommu); @@ -1569,8 +1607,14 @@ static void amd_iommu_ats_write_check_workaround(struct amd_iommu *iommu) */ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h) { + struct amd_iommu_pci_seg *pci_seg; int ret; + pci_seg = get_pci_segment(h->pci_seg); + if (pci_seg == NULL) + return -ENOMEM; + iommu->pci_seg = pci_seg; + raw_spin_lock_init(&iommu->lock); atomic64_set(&iommu->cmd_sem_val, 0); @@ -1591,7 +1635,6 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h) */ iommu->devid = h->devid; iommu->cap_ptr = h->cap_ptr; - iommu->pci_seg = h->pci_seg; iommu->mmio_phys = h->mmio_phys; switch (h->type) { @@ -2659,6 +2702,7 @@ static void __init free_iommu_resources(void) amd_iommu_dev_table = NULL; free_iommu_all(); + free_pci_segments(); } /* SB IOAPIC is always on this device in AMD systems */ From 2375f1f92a42f16ff5b2bd45d7fb30cd9ab06c3d Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 6 Jul 2022 17:07:53 +0530 Subject: [PATCH 03/34] iommu/amd: Introduce per PCI segment device table commit 04230c119930299f2a30de783c0a643481a66eed upstream. Introduce per PCI segment device table. All IOMMUs within the segment will share this device table. This will replace global device table i.e. amd_iommu_dev_table. Also introduce helper function to get the device table for the given IOMMU. Co-developed-by: Vasant Hegde Signed-off-by: Vasant Hegde Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20220706113825.25582-4-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: Malathi Signed-off-by: PvsNarasimha --- drivers/iommu/amd/amd_iommu.h | 1 + drivers/iommu/amd/amd_iommu_types.h | 10 ++++++++++ drivers/iommu/amd/init.c | 26 ++++++++++++++++++++++++-- drivers/iommu/amd/iommu.c | 12 ++++++++++++ 4 files changed, 47 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 5a3a2ae52b5df..af915e970f56f 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -131,4 +131,5 @@ static inline void amd_iommu_apply_ivrs_quirks(void) { } extern void amd_iommu_domain_set_pgtable(struct protection_domain *domain, u64 *root, int mode); +extern struct dev_table_entry *get_dev_table(struct amd_iommu *iommu); #endif diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index bee4d351d2207..f779ff0bc7ab6 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -539,6 +539,16 @@ struct amd_iommu_pci_seg { /* PCI segment number */ u16 id; + + /* + * device table virtual address + * + * Pointer to the per PCI segment device table. + * It is indexed by the PCI device id or the HT unit id and contains + * information about the domain the device belongs to as well as the + * page table root pointer. + */ + struct dev_table_entry *dev_table; }; /* diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 4387a0b7976d3..ef74f00f8ca78 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -646,11 +646,29 @@ static int __init find_last_devid_acpi(struct acpi_table_header *table) * * The following functions belong to the code path which parses the ACPI table * the second time. In this ACPI parsing iteration we allocate IOMMU specific - * data structures, initialize the device/alias/rlookup table and also - * basically initialize the hardware. + * data structures, initialize the per PCI segment device/alias/rlookup table + * and also basically initialize the hardware. * ****************************************************************************/ +/* Allocate per PCI segment device table */ +static inline int __init alloc_dev_table(struct amd_iommu_pci_seg *pci_seg) +{ + pci_seg->dev_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO | GFP_DMA32, + get_order(dev_table_size)); + if (!pci_seg->dev_table) + return -ENOMEM; + + return 0; +} + +static inline void free_dev_table(struct amd_iommu_pci_seg *pci_seg) +{ + free_pages((unsigned long)pci_seg->dev_table, + get_order(dev_table_size)); + pci_seg->dev_table = NULL; +} + /* * Allocates the command buffer. This buffer is per AMD IOMMU. We can * write commands to that buffer later and the IOMMU will execute them @@ -1496,6 +1514,9 @@ static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id) pci_seg->id = id; list_add_tail(&pci_seg->list, &amd_iommu_pci_seg_list); + if (alloc_dev_table(pci_seg)) + return NULL; + return pci_seg; } @@ -1517,6 +1538,7 @@ static void __init free_pci_segments(void) for_each_pci_segment_safe(pci_seg, next) { list_del(&pci_seg->list); + free_dev_table(pci_seg); kfree(pci_seg); } } diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 504ed97f44fe6..344677dbe4e37 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -133,6 +133,18 @@ static inline int get_device_id(struct device *dev) return devid; } +struct dev_table_entry *get_dev_table(struct amd_iommu *iommu) +{ + struct dev_table_entry *dev_table; + struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; + + BUG_ON(pci_seg == NULL); + dev_table = pci_seg->dev_table; + BUG_ON(dev_table == NULL); + + return dev_table; +} + static struct protection_domain *to_pdomain(struct iommu_domain *dom) { return container_of(dom, struct protection_domain, domain); From 0f09f474085b04ca83bc0a34b2ce56c043448423 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 6 Jul 2022 17:07:54 +0530 Subject: [PATCH 04/34] iommu/amd: Introduce per PCI segment rlookup table commit eda797a2779509280c84c557a9caf568b23db4e6 upstream. This will replace global rlookup table (amd_iommu_rlookup_table). Add helper functions to set/get rlookup table for the given device. Also add macros to get seg/devid from sbdf. Co-developed-by: Vasant Hegde Signed-off-by: Vasant Hegde Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20220706113825.25582-5-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: Malathi Signed-off-by: PvsNarasimha --- drivers/iommu/amd/amd_iommu.h | 1 + drivers/iommu/amd/amd_iommu_types.h | 11 ++++++++ drivers/iommu/amd/init.c | 23 +++++++++++++++ drivers/iommu/amd/iommu.c | 44 +++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index af915e970f56f..53f554a11b774 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -21,6 +21,7 @@ extern int amd_iommu_init_devices(void); extern void amd_iommu_uninit_devices(void); extern void amd_iommu_init_notifier(void); extern int amd_iommu_init_api(void); +extern void amd_iommu_set_rlookup_table(struct amd_iommu *iommu, u16 devid); #ifdef CONFIG_AMD_IOMMU_DEBUGFS void amd_iommu_debugfs_setup(struct amd_iommu *iommu); diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index f779ff0bc7ab6..f87d657081941 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -451,6 +451,9 @@ extern bool amd_iommu_irq_remap; /* kmem_cache to get tables with 128 byte alignement */ extern struct kmem_cache *amd_iommu_irq_cache; +#define PCI_SBDF_TO_SEGID(sbdf) (((sbdf) >> 16) & 0xffff) +#define PCI_SBDF_TO_DEVID(sbdf) ((sbdf) & 0xffff) + /* Make iterating over all pci segment easier */ #define for_each_pci_segment(pci_seg) \ list_for_each_entry((pci_seg), &amd_iommu_pci_seg_list, list) @@ -485,6 +488,7 @@ struct amd_iommu_fault { }; +struct amd_iommu; struct iommu_domain; struct irq_domain; struct amd_irte_ops; @@ -549,6 +553,13 @@ struct amd_iommu_pci_seg { * page table root pointer. */ struct dev_table_entry *dev_table; + + /* + * The rlookup iommu table is used to find the IOMMU which is + * responsible for a specific device. It is indexed by the PCI + * device id. + */ + struct amd_iommu **rlookup_table; }; /* diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index ef74f00f8ca78..7a97ea1cbff1d 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -669,6 +669,26 @@ static inline void free_dev_table(struct amd_iommu_pci_seg *pci_seg) pci_seg->dev_table = NULL; } +/* Allocate per PCI segment IOMMU rlookup table. */ +static inline int __init alloc_rlookup_table(struct amd_iommu_pci_seg *pci_seg) +{ + pci_seg->rlookup_table = (void *)__get_free_pages( + GFP_KERNEL | __GFP_ZERO, + get_order(rlookup_table_size)); + if (pci_seg->rlookup_table == NULL) + return -ENOMEM; + + return 0; +} + +static inline void free_rlookup_table(struct amd_iommu_pci_seg *pci_seg) +{ + free_pages((unsigned long)pci_seg->rlookup_table, + get_order(rlookup_table_size)); + pci_seg->rlookup_table = NULL; +} + + /* * Allocates the command buffer. This buffer is per AMD IOMMU. We can * write commands to that buffer later and the IOMMU will execute them @@ -1516,6 +1536,8 @@ static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id) if (alloc_dev_table(pci_seg)) return NULL; + if (alloc_rlookup_table(pci_seg)) + return NULL; return pci_seg; } @@ -1538,6 +1560,7 @@ static void __init free_pci_segments(void) for_each_pci_segment_safe(pci_seg, next) { list_del(&pci_seg->list); + free_rlookup_table(pci_seg); free_dev_table(pci_seg); kfree(pci_seg); } diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 344677dbe4e37..617b1760fed20 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -145,6 +145,50 @@ struct dev_table_entry *get_dev_table(struct amd_iommu *iommu) return dev_table; } +static inline u16 get_device_segment(struct device *dev) +{ + u16 seg; + + if (dev_is_pci(dev)) { + struct pci_dev *pdev = to_pci_dev(dev); + + seg = pci_domain_nr(pdev->bus); + } else { + u32 devid = get_acpihid_device_id(dev, NULL); + + seg = PCI_SBDF_TO_SEGID(devid); + } + + return seg; +} + +/* Writes the specific IOMMU for a device into the PCI segment rlookup table */ +void amd_iommu_set_rlookup_table(struct amd_iommu *iommu, u16 devid) +{ + struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; + + pci_seg->rlookup_table[devid] = iommu; +} + +static struct amd_iommu *__rlookup_amd_iommu(u16 seg, u16 devid) +{ + struct amd_iommu_pci_seg *pci_seg; + + for_each_pci_segment(pci_seg) { + if (pci_seg->id == seg) + return pci_seg->rlookup_table[devid]; + } + return NULL; +} + +static struct amd_iommu *rlookup_amd_iommu(struct device *dev) +{ + u16 seg = get_device_segment(dev); + u16 devid = get_device_id(dev); + + return __rlookup_amd_iommu(seg, devid); +} + static struct protection_domain *to_pdomain(struct iommu_domain *dom) { return container_of(dom, struct protection_domain, domain); From e82b87efd0d98aaea1dec6ec7cfbaa891f40d9d3 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Wed, 6 Jul 2022 17:07:55 +0530 Subject: [PATCH 05/34] iommu/amd: Introduce per PCI segment irq_lookup_table commit 333e581bcdffc09e6c6c95a0c24bfc66b3b4dcfd upstream. This will replace global irq lookup table (irq_lookup_table). Co-developed-by: Suravee Suthikulpanit Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20220706113825.25582-6-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: Malathi Signed-off-by: PvsNarasimha --- drivers/iommu/amd/amd_iommu_types.h | 6 ++++++ drivers/iommu/amd/init.c | 27 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index f87d657081941..cce18f7afe83e 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -560,6 +560,12 @@ struct amd_iommu_pci_seg { * device id. */ struct amd_iommu **rlookup_table; + + /* + * This table is used to find the irq remapping table for a given + * device id quickly. + */ + struct irq_remap_table **irq_lookup_table; }; /* diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 7a97ea1cbff1d..de2803486029f 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -688,6 +688,26 @@ static inline void free_rlookup_table(struct amd_iommu_pci_seg *pci_seg) pci_seg->rlookup_table = NULL; } +static inline int __init alloc_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg) +{ + pci_seg->irq_lookup_table = (void *)__get_free_pages( + GFP_KERNEL | __GFP_ZERO, + get_order(rlookup_table_size)); + kmemleak_alloc(pci_seg->irq_lookup_table, + rlookup_table_size, 1, GFP_KERNEL); + if (pci_seg->irq_lookup_table == NULL) + return -ENOMEM; + + return 0; +} + +static inline void free_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg) +{ + kmemleak_free(pci_seg->irq_lookup_table); + free_pages((unsigned long)pci_seg->irq_lookup_table, + get_order(rlookup_table_size)); + pci_seg->irq_lookup_table = NULL; +} /* * Allocates the command buffer. This buffer is per AMD IOMMU. We can @@ -1560,6 +1580,7 @@ static void __init free_pci_segments(void) for_each_pci_segment_safe(pci_seg, next) { list_del(&pci_seg->list); + free_irq_lookup_table(pci_seg); free_rlookup_table(pci_seg); free_dev_table(pci_seg); kfree(pci_seg); @@ -2945,6 +2966,7 @@ static int __init early_amd_iommu_init(void) amd_iommu_irq_remap = check_ioapic_information(); if (amd_iommu_irq_remap) { + struct amd_iommu_pci_seg *pci_seg; /* * Interrupt remapping enabled, create kmem_cache for the * remapping tables. @@ -2961,6 +2983,11 @@ static int __init early_amd_iommu_init(void) if (!amd_iommu_irq_cache) goto out; + for_each_pci_segment(pci_seg) { + if (alloc_irq_lookup_table(pci_seg)) + goto out; + } + irq_lookup_table = (void *)__get_free_pages( GFP_KERNEL | __GFP_ZERO, get_order(rlookup_table_size)); From 85a8e734dac9131d3eef9b16878b9f8ade3dff06 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Wed, 6 Jul 2022 17:07:56 +0530 Subject: [PATCH 06/34] iommu/amd: Introduce per PCI segment dev_data_list commit 39a303ba4a57f263aa2ed4908161bca3583f35f9 upstream. This will replace global dev_data_list. Co-developed-by: Suravee Suthikulpanit Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20220706113825.25582-7-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: PvsNarasimha --- drivers/iommu/amd/amd_iommu_types.h | 3 +++ drivers/iommu/amd/init.c | 1 + drivers/iommu/amd/iommu.c | 21 ++++++++++----------- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index cce18f7afe83e..2264087dcad28 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -541,6 +541,9 @@ struct amd_iommu_pci_seg { /* List with all PCI segments in the system */ struct list_head list; + /* List of all available dev_data structures */ + struct llist_head dev_data_list; + /* PCI segment number */ u16 id; diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index de2803486029f..16380fb2d17c5 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1552,6 +1552,7 @@ static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id) return NULL; pci_seg->id = id; + init_llist_head(&pci_seg->dev_data_list); list_add_tail(&pci_seg->list, &amd_iommu_pci_seg_list); if (alloc_dev_table(pci_seg)) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 617b1760fed20..84c2fb9b86179 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -61,9 +61,6 @@ static DEFINE_SPINLOCK(pd_bitmap_lock); -/* List of all available dev_data structures */ -static LLIST_HEAD(dev_data_list); - LIST_HEAD(ioapic_map); LIST_HEAD(hpet_map); LIST_HEAD(acpihid_map); @@ -194,9 +191,10 @@ static struct protection_domain *to_pdomain(struct iommu_domain *dom) return container_of(dom, struct protection_domain, domain); } -static struct iommu_dev_data *alloc_dev_data(u16 devid) +static struct iommu_dev_data *alloc_dev_data(struct amd_iommu *iommu, u16 devid) { struct iommu_dev_data *dev_data; + struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL); if (!dev_data) @@ -206,19 +204,20 @@ static struct iommu_dev_data *alloc_dev_data(u16 devid) dev_data->devid = devid; ratelimit_default_init(&dev_data->rs); - llist_add(&dev_data->dev_data_list, &dev_data_list); + llist_add(&dev_data->dev_data_list, &pci_seg->dev_data_list); return dev_data; } -static struct iommu_dev_data *search_dev_data(u16 devid) +static struct iommu_dev_data *search_dev_data(struct amd_iommu *iommu, u16 devid) { struct iommu_dev_data *dev_data; struct llist_node *node; + struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; - if (llist_empty(&dev_data_list)) + if (llist_empty(&pci_seg->dev_data_list)) return NULL; - node = dev_data_list.first; + node = pci_seg->dev_data_list.first; llist_for_each_entry(dev_data, node, dev_data_list) { if (dev_data->devid == devid) return dev_data; @@ -287,10 +286,10 @@ static struct iommu_dev_data *find_dev_data(u16 devid) struct iommu_dev_data *dev_data; struct amd_iommu *iommu = amd_iommu_rlookup_table[devid]; - dev_data = search_dev_data(devid); + dev_data = search_dev_data(iommu, devid); if (dev_data == NULL) { - dev_data = alloc_dev_data(devid); + dev_data = alloc_dev_data(iommu, devid); if (!dev_data) return NULL; @@ -3481,7 +3480,7 @@ static int amd_ir_set_vcpu_affinity(struct irq_data *data, void *vcpu_info) struct vcpu_data *vcpu_pi_info = pi_data->vcpu_data; struct amd_ir_data *ir_data = data->chip_data; struct irq_2_irte *irte_info = &ir_data->irq_2_irte; - struct iommu_dev_data *dev_data = search_dev_data(irte_info->devid); + struct iommu_dev_data *dev_data = search_dev_data(NULL, irte_info->devid); /* Note: * This device has never been set up for guest mode. From d63b654bf6aec641e64c604efbe16df74d326f79 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 6 Jul 2022 17:07:57 +0530 Subject: [PATCH 07/34] iommu/amd: Introduce per PCI segment old_dev_tbl_cpy commit eb21ef0227915ba61c8927fddca53571a0a35de8 upstream. It will remove global old_dev_tbl_cpy. Also update copy_device_table() copy device table for all PCI segments. Co-developed-by: Vasant Hegde Signed-off-by: Vasant Hegde Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20220706113825.25582-8-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: PvsNarasimha --- drivers/iommu/amd/amd_iommu_types.h | 6 ++ drivers/iommu/amd/init.c | 109 ++++++++++++++++------------ 2 files changed, 70 insertions(+), 45 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 2264087dcad28..bcf1eb7169985 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -569,6 +569,12 @@ struct amd_iommu_pci_seg { * device id quickly. */ struct irq_remap_table **irq_lookup_table; + + /* + * Pointer to a device table which the content of old device table + * will be copied to. It's only be used in kdump kernel. + */ + struct dev_table_entry *old_dev_tbl_cpy; }; /* diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 16380fb2d17c5..a9266d8f6b6a3 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -196,11 +196,6 @@ bool amd_iommu_force_isolation __read_mostly; * page table root pointer. */ struct dev_table_entry *amd_iommu_dev_table; -/* - * Pointer to a device table which the content of old device table - * will be copied to. It's only be used in kdump kernel. - */ -static struct dev_table_entry *old_dev_tbl_cpy; /* * The alias table is a driver specific data structure which contains the @@ -1020,39 +1015,27 @@ static int get_dev_entry_bit(u16 devid, u8 bit) } -static bool copy_device_table(void) +static bool __copy_device_table(struct amd_iommu *iommu) { - u64 int_ctl, int_tab_len, entry = 0, last_entry = 0; + u64 int_ctl, int_tab_len, entry = 0; + struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; struct dev_table_entry *old_devtb = NULL; u32 lo, hi, devid, old_devtb_size; phys_addr_t old_devtb_phys; - struct amd_iommu *iommu; u16 dom_id, dte_v, irq_v; gfp_t gfp_flag; u64 tmp; - if (!amd_iommu_pre_enabled) - return false; - - pr_warn("Translation is already enabled - trying to copy translation structures\n"); - for_each_iommu(iommu) { - /* All IOMMUs should use the same device table with the same size */ - lo = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET); - hi = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET + 4); - entry = (((u64) hi) << 32) + lo; - if (last_entry && last_entry != entry) { - pr_err("IOMMU:%d should use the same dev table as others!\n", - iommu->index); - return false; - } - last_entry = entry; + /* Each IOMMU use separate device table with the same size */ + lo = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET); + hi = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET + 4); + entry = (((u64) hi) << 32) + lo; - old_devtb_size = ((entry & ~PAGE_MASK) + 1) << 12; - if (old_devtb_size != dev_table_size) { - pr_err("The device table size of IOMMU:%d is not expected!\n", - iommu->index); - return false; - } + old_devtb_size = ((entry & ~PAGE_MASK) + 1) << 12; + if (old_devtb_size != dev_table_size) { + pr_err("The device table size of IOMMU:%d is not expected!\n", + iommu->index); + return false; } /* @@ -1075,30 +1058,30 @@ static bool copy_device_table(void) return false; gfp_flag = GFP_KERNEL | __GFP_ZERO | GFP_DMA32; - old_dev_tbl_cpy = (void *)__get_free_pages(gfp_flag, - get_order(dev_table_size)); - if (old_dev_tbl_cpy == NULL) { + pci_seg->old_dev_tbl_cpy = (void *)__get_free_pages(gfp_flag, + get_order(dev_table_size)); + if (pci_seg->old_dev_tbl_cpy == NULL) { pr_err("Failed to allocate memory for copying old device table!\n"); return false; } for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) { - old_dev_tbl_cpy[devid] = old_devtb[devid]; + pci_seg->old_dev_tbl_cpy[devid] = old_devtb[devid]; dom_id = old_devtb[devid].data[1] & DEV_DOMID_MASK; dte_v = old_devtb[devid].data[0] & DTE_FLAG_V; if (dte_v && dom_id) { - old_dev_tbl_cpy[devid].data[0] = old_devtb[devid].data[0]; - old_dev_tbl_cpy[devid].data[1] = old_devtb[devid].data[1]; + pci_seg->old_dev_tbl_cpy[devid].data[0] = old_devtb[devid].data[0]; + pci_seg->old_dev_tbl_cpy[devid].data[1] = old_devtb[devid].data[1]; __set_bit(dom_id, amd_iommu_pd_alloc_bitmap); /* If gcr3 table existed, mask it out */ if (old_devtb[devid].data[0] & DTE_FLAG_GV) { tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B; tmp |= DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C; - old_dev_tbl_cpy[devid].data[1] &= ~tmp; + pci_seg->old_dev_tbl_cpy[devid].data[1] &= ~tmp; tmp = DTE_GCR3_VAL_A(~0ULL) << DTE_GCR3_SHIFT_A; tmp |= DTE_FLAG_GV; - old_dev_tbl_cpy[devid].data[0] &= ~tmp; + pci_seg->old_dev_tbl_cpy[devid].data[0] &= ~tmp; } } @@ -1112,7 +1095,7 @@ static bool copy_device_table(void) return false; } - old_dev_tbl_cpy[devid].data[2] = old_devtb[devid].data[2]; + pci_seg->old_dev_tbl_cpy[devid].data[2] = old_devtb[devid].data[2]; } } memunmap(old_devtb); @@ -1120,6 +1103,33 @@ static bool copy_device_table(void) return true; } +static bool copy_device_table(void) +{ + struct amd_iommu *iommu; + struct amd_iommu_pci_seg *pci_seg; + + if (!amd_iommu_pre_enabled) + return false; + + pr_warn("Translation is already enabled - trying to copy translation structures\n"); + + /* + * All IOMMUs within PCI segment shares common device table. + * Hence copy device table only once per PCI segment. + */ + for_each_pci_segment(pci_seg) { + for_each_iommu(iommu) { + if (pci_seg->id != iommu->pci_seg->id) + continue; + if (!__copy_device_table(iommu)) + return false; + break; + } + } + + return true; +} + void amd_iommu_apply_erratum_63(u16 devid) { int sysmgt; @@ -2634,7 +2644,7 @@ static void early_enable_iommu(struct amd_iommu *iommu) static void early_enable_iommus(void) { struct amd_iommu *iommu; - + struct amd_iommu_pci_seg *pci_seg; if (!copy_device_table()) { /* @@ -2644,9 +2654,14 @@ static void early_enable_iommus(void) */ if (amd_iommu_pre_enabled) pr_err("Failed to copy DEV table from previous kernel.\n"); - if (old_dev_tbl_cpy != NULL) - free_pages((unsigned long)old_dev_tbl_cpy, - get_order(dev_table_size)); + + for_each_pci_segment(pci_seg) { + if (pci_seg->old_dev_tbl_cpy != NULL) { + free_pages((unsigned long)pci_seg->old_dev_tbl_cpy, + get_order(dev_table_size)); + pci_seg->old_dev_tbl_cpy = NULL; + } + } for_each_iommu(iommu) { clear_translation_pre_enabled(iommu); @@ -2654,9 +2669,13 @@ static void early_enable_iommus(void) } } else { pr_info("Copied DEV table from previous kernel.\n"); - free_pages((unsigned long)amd_iommu_dev_table, - get_order(dev_table_size)); - amd_iommu_dev_table = old_dev_tbl_cpy; + + for_each_pci_segment(pci_seg) { + free_pages((unsigned long)pci_seg->dev_table, + get_order(dev_table_size)); + pci_seg->dev_table = pci_seg->old_dev_tbl_cpy; + } + for_each_iommu(iommu) { iommu_disable_command_buffer(iommu); iommu_disable_event_buffer(iommu); From e5c1a9b631ed2c86925ccacd6c32bf58dc12f347 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 6 Jul 2022 17:07:58 +0530 Subject: [PATCH 08/34] iommu/amd: Introduce per PCI segment alias_table commit 99fc4ac3d2978cec1ae80dbc452fac1bb2cc976f upstream. This will replace global alias table (amd_iommu_alias_table). Co-developed-by: Vasant Hegde Signed-off-by: Vasant Hegde Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20220706113825.25582-9-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: PvsNarasimha --- drivers/iommu/amd/amd_iommu_types.h | 7 +++++ drivers/iommu/amd/init.c | 41 ++++++++++++++++++++++------- drivers/iommu/amd/iommu.c | 41 ++++++++++++++++++----------- 3 files changed, 64 insertions(+), 25 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index bcf1eb7169985..274eb7022a0ef 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -575,6 +575,13 @@ struct amd_iommu_pci_seg { * will be copied to. It's only be used in kdump kernel. */ struct dev_table_entry *old_dev_tbl_cpy; + + /* + * The alias table is a driver specific data structure which contains the + * mappings of the PCI device ids to the actual requestor ids on the IOMMU. + * More than one device can share the same requestor id. + */ + u16 *alias_table; }; /* diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index a9266d8f6b6a3..780c393afd42a 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -704,6 +704,31 @@ static inline void free_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg) pci_seg->irq_lookup_table = NULL; } +static int __init alloc_alias_table(struct amd_iommu_pci_seg *pci_seg) +{ + int i; + + pci_seg->alias_table = (void *)__get_free_pages(GFP_KERNEL, + get_order(alias_table_size)); + if (!pci_seg->alias_table) + return -ENOMEM; + + /* + * let all alias entries point to itself + */ + for (i = 0; i <= amd_iommu_last_bdf; ++i) + pci_seg->alias_table[i] = i; + + return 0; +} + +static void __init free_alias_table(struct amd_iommu_pci_seg *pci_seg) +{ + free_pages((unsigned long)pci_seg->alias_table, + get_order(alias_table_size)); + pci_seg->alias_table = NULL; +} + /* * Allocates the command buffer. This buffer is per AMD IOMMU. We can * write commands to that buffer later and the IOMMU will execute them @@ -1294,6 +1319,7 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, u32 dev_i, ext_flags = 0; bool alias = false; struct ivhd_entry *e; + struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; u32 ivhd_size; int ret; @@ -1375,7 +1401,7 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, devid_to = e->ext >> 8; set_dev_entry_from_acpi(iommu, devid , e->flags, 0); set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0); - amd_iommu_alias_table[devid] = devid_to; + pci_seg->alias_table[devid] = devid_to; break; case IVHD_DEV_ALIAS_RANGE: @@ -1433,7 +1459,7 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, devid = e->devid; for (dev_i = devid_start; dev_i <= devid; ++dev_i) { if (alias) { - amd_iommu_alias_table[dev_i] = devid_to; + pci_seg->alias_table[dev_i] = devid_to; set_dev_entry_from_acpi(iommu, devid_to, flags, ext_flags); } @@ -1567,6 +1593,8 @@ static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id) if (alloc_dev_table(pci_seg)) return NULL; + if (alloc_alias_table(pci_seg)) + return NULL; if (alloc_rlookup_table(pci_seg)) return NULL; @@ -1593,6 +1621,7 @@ static void __init free_pci_segments(void) list_del(&pci_seg->list); free_irq_lookup_table(pci_seg); free_rlookup_table(pci_seg); + free_alias_table(pci_seg); free_dev_table(pci_seg); kfree(pci_seg); } @@ -2887,7 +2916,7 @@ static void __init ivinfo_init(void *ivrs) static int __init early_amd_iommu_init(void) { struct acpi_table_header *ivrs_base; - int i, remap_cache_sz, ret; + int remap_cache_sz, ret; acpi_status status; if (!amd_iommu_detected) @@ -2958,12 +2987,6 @@ static int __init early_amd_iommu_init(void) if (amd_iommu_pd_alloc_bitmap == NULL) goto out; - /* - * let all alias entries point to itself - */ - for (i = 0; i <= amd_iommu_last_bdf; ++i) - amd_iommu_alias_table[i] = i; - /* * never allocate domain 0 because its used as the non-allocated and * error value placeholder diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 84c2fb9b86179..4991799c71831 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -242,7 +242,7 @@ static int clone_alias(struct pci_dev *pdev, u16 alias, void *data) return 0; } -static void clone_aliases(struct device *dev) +static void clone_aliases(struct amd_iommu *iommu, struct device *dev) { struct pci_dev *pdev; @@ -255,14 +255,15 @@ static void clone_aliases(struct device *dev) * part of the PCI DMA aliases if it's bus differs * from the original device. */ - clone_alias(pdev, amd_iommu_alias_table[pci_dev_id(pdev)], NULL); + clone_alias(pdev, iommu->pci_seg->alias_table[pci_dev_id(pdev)], NULL); pci_for_each_dma_alias(pdev, clone_alias, NULL); } -static void setup_aliases(struct device *dev) +static void setup_aliases(struct amd_iommu *iommu, struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); + struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; u16 ivrs_alias; /* For ACPI HID devices, there are no aliases */ @@ -273,12 +274,12 @@ static void setup_aliases(struct device *dev) * Add the IVRS alias to the pci aliases if it is on the same * bus. The IVRS table may know about a quirk that we don't. */ - ivrs_alias = amd_iommu_alias_table[pci_dev_id(pdev)]; + ivrs_alias = pci_seg->alias_table[pci_dev_id(pdev)]; if (ivrs_alias != pci_dev_id(pdev) && PCI_BUS_NUM(ivrs_alias) == pdev->bus->number) pci_add_dma_alias(pdev, ivrs_alias & 0xff, 1); - clone_aliases(dev); + clone_aliases(iommu, dev); } static struct iommu_dev_data *find_dev_data(u16 devid) @@ -370,7 +371,7 @@ static bool check_device(struct device *dev) return true; } -static int iommu_init_device(struct device *dev) +static int iommu_init_device(struct amd_iommu *iommu, struct device *dev) { struct iommu_dev_data *dev_data; int devid; @@ -387,7 +388,7 @@ static int iommu_init_device(struct device *dev) return -ENOMEM; dev_data->dev = dev; - setup_aliases(dev); + setup_aliases(iommu, dev); /* * By default we use passthrough mode for IOMMUv2 capable device. @@ -408,7 +409,7 @@ static int iommu_init_device(struct device *dev) return 0; } -static void iommu_ignore_device(struct device *dev) +static void iommu_ignore_device(struct amd_iommu *iommu, struct device *dev) { int devid; @@ -419,7 +420,7 @@ static void iommu_ignore_device(struct device *dev) amd_iommu_rlookup_table[devid] = NULL; memset(&amd_iommu_dev_table[devid], 0, sizeof(struct dev_table_entry)); - setup_aliases(dev); + setup_aliases(iommu, dev); } static void amd_iommu_uninit_device(struct device *dev) @@ -1305,6 +1306,7 @@ static int device_flush_dte(struct iommu_dev_data *dev_data) { struct amd_iommu *iommu; struct pci_dev *pdev = NULL; + struct amd_iommu_pci_seg *pci_seg; u16 alias; int ret; @@ -1321,7 +1323,8 @@ static int device_flush_dte(struct iommu_dev_data *dev_data) if (ret) return ret; - alias = amd_iommu_alias_table[dev_data->devid]; + pci_seg = iommu->pci_seg; + alias = pci_seg->alias_table[dev_data->devid]; if (alias != dev_data->devid) { ret = iommu_flush_dte(iommu, alias); if (ret) @@ -1637,7 +1640,7 @@ static void do_attach(struct iommu_dev_data *dev_data, /* Update device table */ set_dte_entry(dev_data->devid, domain, ats, dev_data->iommu_v2); - clone_aliases(dev_data->dev); + clone_aliases(iommu, dev_data->dev); device_flush_dte(dev_data); } @@ -1653,7 +1656,7 @@ static void do_detach(struct iommu_dev_data *dev_data) dev_data->domain = NULL; list_del(&dev_data->list); clear_dte_entry(dev_data->devid); - clone_aliases(dev_data->dev); + clone_aliases(iommu, dev_data->dev); /* Flush the DTE entry */ device_flush_dte(dev_data); @@ -1838,12 +1841,12 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) if (dev_iommu_priv_get(dev)) return &iommu->iommu; - ret = iommu_init_device(dev); + ret = iommu_init_device(iommu, dev); if (ret) { if (ret != -ENOTSUPP) dev_err(dev, "Failed to initialize - trying to proceed anyway\n"); iommu_dev = ERR_PTR(ret); - iommu_ignore_device(dev); + iommu_ignore_device(iommu, dev); } else { amd_iommu_set_pci_msi_domain(dev, iommu); iommu_dev = &iommu->iommu; @@ -1894,9 +1897,13 @@ static void update_device_table(struct protection_domain *domain) struct iommu_dev_data *dev_data; list_for_each_entry(dev_data, &domain->dev_list, list) { + struct amd_iommu *iommu = rlookup_amd_iommu(dev_data->dev); + + if (!iommu) + continue; set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled, dev_data->iommu_v2); - clone_aliases(dev_data->dev); + clone_aliases(iommu, dev_data->dev); } } @@ -2790,6 +2797,7 @@ static struct irq_remap_table *alloc_irq_table(u16 devid, struct pci_dev *pdev) { struct irq_remap_table *table = NULL; struct irq_remap_table *new_table = NULL; + struct amd_iommu_pci_seg *pci_seg; struct amd_iommu *iommu; unsigned long flags; u16 alias; @@ -2800,11 +2808,12 @@ static struct irq_remap_table *alloc_irq_table(u16 devid, struct pci_dev *pdev) if (!iommu) goto out_unlock; + pci_seg = iommu->pci_seg; table = irq_lookup_table[devid]; if (table) goto out_unlock; - alias = amd_iommu_alias_table[devid]; + alias = pci_seg->alias_table[devid]; table = irq_lookup_table[alias]; if (table) { set_remap_table_entry(iommu, devid, table); From 420d42d7e5f8b4090ff6db6ae2ffc34ba8692d3c Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Wed, 6 Jul 2022 17:07:59 +0530 Subject: [PATCH 09/34] iommu/amd: Introduce per PCI segment unity map list commit b618ae6247bbafe0844355bafd948e59ebd77098 upstream. Newer AMD systems can support multiple PCI segments. In order to support multiple PCI segments IVMD table in IVRS structure is enhanced to include pci segment id. Update ivmd_header structure to include "pci_seg". Also introduce per PCI segment unity map list. It will replace global amd_iommu_unity_map list. Note that we have used "reserved" field in IVMD table to include "pci_seg id" which was set to zero. It will take care of backward compatibility (new kernel will work fine on older systems). Co-developed-by: Suravee Suthikulpanit Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20220706113825.25582-10-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: PvsNarasimha --- drivers/iommu/amd/amd_iommu_types.h | 13 +++++++------ drivers/iommu/amd/init.c | 30 +++++++++++++++++++---------- drivers/iommu/amd/iommu.c | 8 +++++++- 3 files changed, 34 insertions(+), 17 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 274eb7022a0ef..f4bc6bbe2b087 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -582,6 +582,13 @@ struct amd_iommu_pci_seg { * More than one device can share the same requestor id. */ u16 *alias_table; + + /* + * A list of required unity mappings we find in ACPI. It is not locked + * because as runtime it is only read. It is created at ACPI table + * parsing time. + */ + struct list_head unity_map; }; /* @@ -811,12 +818,6 @@ struct unity_map_entry { int prot; }; -/* - * List of all unity mappings. It is not locked because as runtime it is only - * read. It is created at ACPI table parsing time. - */ -extern struct list_head amd_iommu_unity_map; - /* * Data structures for device handling */ diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 780c393afd42a..b961b435a5ea7 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -144,7 +144,8 @@ struct ivmd_header { u16 length; u16 devid; u16 aux; - u64 resv; + u16 pci_seg; + u8 resv[6]; u64 range_start; u64 range_length; } __attribute__((packed)); @@ -165,8 +166,6 @@ static int amd_iommu_target_ivhd_type; u16 amd_iommu_last_bdf; /* largest PCI device id we have to handle */ -LIST_HEAD(amd_iommu_unity_map); /* a list of required unity mappings - we find in ACPI */ LIST_HEAD(amd_iommu_pci_seg_list); /* list of all PCI segments */ LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the @@ -1589,6 +1588,7 @@ static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id) pci_seg->id = id; init_llist_head(&pci_seg->dev_data_list); + INIT_LIST_HEAD(&pci_seg->unity_map); list_add_tail(&pci_seg->list, &amd_iommu_pci_seg_list); if (alloc_dev_table(pci_seg)) @@ -2416,10 +2416,13 @@ static int iommu_init_irq(struct amd_iommu *iommu) static void __init free_unity_maps(void) { struct unity_map_entry *entry, *next; + struct amd_iommu_pci_seg *p, *pci_seg; - list_for_each_entry_safe(entry, next, &amd_iommu_unity_map, list) { - list_del(&entry->list); - kfree(entry); + for_each_pci_segment_safe(pci_seg, p) { + list_for_each_entry_safe(entry, next, &pci_seg->unity_map, list) { + list_del(&entry->list); + kfree(entry); + } } } @@ -2427,8 +2430,13 @@ static void __init free_unity_maps(void) static int __init init_unity_map_range(struct ivmd_header *m) { struct unity_map_entry *e = NULL; + struct amd_iommu_pci_seg *pci_seg; char *s; + pci_seg = get_pci_segment(m->pci_seg); + if (pci_seg == NULL) + return -ENOMEM; + e = kzalloc(sizeof(*e), GFP_KERNEL); if (e == NULL) return -ENOMEM; @@ -2466,14 +2474,16 @@ static int __init init_unity_map_range(struct ivmd_header *m) if (m->flags & IVMD_FLAG_EXCL_RANGE) e->prot = (IVMD_FLAG_IW | IVMD_FLAG_IR) >> 1; - DUMP_printk("%s devid_start: %02x:%02x.%x devid_end: %02x:%02x.%x" - " range_start: %016llx range_end: %016llx flags: %x\n", s, + DUMP_printk("%s devid_start: %04x:%02x:%02x.%x devid_end: " + "%04x:%02x:%02x.%x range_start: %016llx range_end: %016llx" + " flags: %x\n", s, m->pci_seg, PCI_BUS_NUM(e->devid_start), PCI_SLOT(e->devid_start), - PCI_FUNC(e->devid_start), PCI_BUS_NUM(e->devid_end), + PCI_FUNC(e->devid_start), m->pci_seg, + PCI_BUS_NUM(e->devid_end), PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end), e->address_start, e->address_end, m->flags); - list_add_tail(&e->list, &amd_iommu_unity_map); + list_add_tail(&e->list, &pci_seg->unity_map); return 0; } diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 4991799c71831..5cbb4d241480d 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2252,13 +2252,19 @@ static void amd_iommu_get_resv_regions(struct device *dev, { struct iommu_resv_region *region; struct unity_map_entry *entry; + struct amd_iommu *iommu; + struct amd_iommu_pci_seg *pci_seg; int devid; devid = get_device_id(dev); if (devid < 0) return; + iommu = rlookup_amd_iommu(dev); + if (!iommu) + return; + pci_seg = iommu->pci_seg; - list_for_each_entry(entry, &amd_iommu_unity_map, list) { + list_for_each_entry(entry, &pci_seg->unity_map, list) { int type, prot = 0; size_t length; From ffee3804df76aee3b04ea3446b5cb3851c0c92f9 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Wed, 6 Jul 2022 17:08:00 +0530 Subject: [PATCH 10/34] iommu/amd: Introduce per PCI segment last_bdf commit 307959008d80cdfd67fb19a146932582bb68a399 upstream. Current code uses global "amd_iommu_last_bdf" to track the last bdf supported by the system. This value is used for various memory allocation, device data flushing, etc. Introduce per PCI segment last_bdf which will be used to track last bdf supported by the given PCI segment and use this value for all per segment memory allocations. Eventually it will replace global "amd_iommu_last_bdf". Co-developed-by: Suravee Suthikulpanit Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20220706113825.25582-11-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: PvsNarasimha --- drivers/iommu/amd/amd_iommu_types.h | 3 ++ drivers/iommu/amd/init.c | 69 ++++++++++++++++++----------- 2 files changed, 45 insertions(+), 27 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index f4bc6bbe2b087..f6cfe3e8ca8fa 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -547,6 +547,9 @@ struct amd_iommu_pci_seg { /* PCI segment number */ u16 id; + /* Largest PCI device id we expect translation requests for */ + u16 last_bdf; + /* * device table virtual address * diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index b961b435a5ea7..d289d4068667e 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -556,6 +556,7 @@ static int __init find_last_devid_from_ivhd(struct ivhd_header *h) { u8 *p = (void *)h, *end = (void *)h; struct ivhd_entry *dev; + int last_devid = -EINVAL; u32 ivhd_size = get_ivhd_header_size(h); @@ -573,13 +574,15 @@ static int __init find_last_devid_from_ivhd(struct ivhd_header *h) case IVHD_DEV_ALL: /* Use maximum BDF value for DEV_ALL */ update_last_devid(0xffff); - break; + return 0xffff; case IVHD_DEV_SELECT: case IVHD_DEV_RANGE_END: case IVHD_DEV_ALIAS: case IVHD_DEV_EXT_SELECT: /* all the above subfield types refer to device ids */ update_last_devid(dev->devid); + if (dev->devid > last_devid) + last_devid = dev->devid; break; default: break; @@ -589,7 +592,7 @@ static int __init find_last_devid_from_ivhd(struct ivhd_header *h) WARN_ON(p != end); - return 0; + return last_devid; } static int __init check_ivrs_checksum(struct acpi_table_header *table) @@ -613,27 +616,31 @@ static int __init check_ivrs_checksum(struct acpi_table_header *table) * id which we need to handle. This is the first of three functions which parse * the ACPI table. So we check the checksum here. */ -static int __init find_last_devid_acpi(struct acpi_table_header *table) +static int __init find_last_devid_acpi(struct acpi_table_header *table, u16 pci_seg) { u8 *p = (u8 *)table, *end = (u8 *)table; struct ivhd_header *h; + int last_devid, last_bdf = 0; p += IVRS_HEADER_LENGTH; end += table->length; while (p < end) { h = (struct ivhd_header *)p; - if (h->type == amd_iommu_target_ivhd_type) { - int ret = find_last_devid_from_ivhd(h); - - if (ret) - return ret; + if (h->pci_seg == pci_seg && + h->type == amd_iommu_target_ivhd_type) { + last_devid = find_last_devid_from_ivhd(h); + + if (last_devid < 0) + return -EINVAL; + if (last_devid > last_bdf) + last_bdf = last_devid; } p += h->length; } WARN_ON(p != end); - return 0; + return last_bdf; } /**************************************************************************** @@ -1578,14 +1585,28 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, } /* Allocate PCI segment data structure */ -static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id) +static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id, + struct acpi_table_header *ivrs_base) { struct amd_iommu_pci_seg *pci_seg; + int last_bdf; + + /* + * First parse ACPI tables to find the largest Bus/Dev/Func we need to + * handle in this PCI segment. Upon this information the shared data + * structures for the PCI segments in the system will be allocated. + */ + last_bdf = find_last_devid_acpi(ivrs_base, id); + if (last_bdf < 0) + return NULL; pci_seg = kzalloc(sizeof(struct amd_iommu_pci_seg), GFP_KERNEL); if (pci_seg == NULL) return NULL; + pci_seg->last_bdf = last_bdf; + DUMP_printk("PCI segment : 0x%0x, last bdf : 0x%04x\n", id, last_bdf); + pci_seg->id = id; init_llist_head(&pci_seg->dev_data_list); INIT_LIST_HEAD(&pci_seg->unity_map); @@ -1601,7 +1622,8 @@ static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id) return pci_seg; } -static struct amd_iommu_pci_seg *__init get_pci_segment(u16 id) +static struct amd_iommu_pci_seg *__init get_pci_segment(u16 id, + struct acpi_table_header *ivrs_base) { struct amd_iommu_pci_seg *pci_seg; @@ -1610,7 +1632,7 @@ static struct amd_iommu_pci_seg *__init get_pci_segment(u16 id) return pci_seg; } - return alloc_pci_segment(id); + return alloc_pci_segment(id, ivrs_base); } static void __init free_pci_segments(void) @@ -1711,12 +1733,13 @@ static void amd_iommu_ats_write_check_workaround(struct amd_iommu *iommu) * together and also allocates the command buffer and programs the * hardware. It does NOT enable the IOMMU. This is done afterwards. */ -static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h) +static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, + struct acpi_table_header *ivrs_base) { struct amd_iommu_pci_seg *pci_seg; int ret; - pci_seg = get_pci_segment(h->pci_seg); + pci_seg = get_pci_segment(h->pci_seg, ivrs_base); if (pci_seg == NULL) return -ENOMEM; iommu->pci_seg = pci_seg; @@ -1891,7 +1914,7 @@ static int __init init_iommu_all(struct acpi_table_header *table) if (iommu == NULL) return -ENOMEM; - ret = init_iommu_one(iommu, h); + ret = init_iommu_one(iommu, h, table); if (ret) return ret; } @@ -2427,13 +2450,14 @@ static void __init free_unity_maps(void) } /* called for unity map ACPI definition */ -static int __init init_unity_map_range(struct ivmd_header *m) +static int __init init_unity_map_range(struct ivmd_header *m, + struct acpi_table_header *ivrs_base) { struct unity_map_entry *e = NULL; struct amd_iommu_pci_seg *pci_seg; char *s; - pci_seg = get_pci_segment(m->pci_seg); + pci_seg = get_pci_segment(m->pci_seg, ivrs_base); if (pci_seg == NULL) return -ENOMEM; @@ -2500,7 +2524,7 @@ static int __init init_memory_definitions(struct acpi_table_header *table) while (p < end) { m = (struct ivmd_header *)p; if (m->flags & (IVMD_FLAG_UNITY_MAP | IVMD_FLAG_EXCL_RANGE)) - init_unity_map_range(m); + init_unity_map_range(m, table); p += m->length; } @@ -2954,15 +2978,6 @@ static int __init early_amd_iommu_init(void) amd_iommu_target_ivhd_type = get_highest_supported_ivhd_type(ivrs_base); DUMP_printk("Using IVHD type %#x\n", amd_iommu_target_ivhd_type); - /* - * First parse ACPI tables to find the largest Bus/Dev/Func - * we need to handle. Upon this information the shared data - * structures for the IOMMUs in the system will be allocated - */ - ret = find_last_devid_acpi(ivrs_base); - if (ret) - goto out; - dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE); alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE); rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE); From 377791aec206d5fae6237d443bc90b957707b853 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Wed, 6 Jul 2022 17:08:01 +0530 Subject: [PATCH 11/34] iommu/amd: Introduce per PCI segment device table size commit b5c852907e0e8a52f8660235390b831ce0a2b50c upstream With multiple pci segment support, number of BDF supported by each segment may differ. Hence introduce per segment device table size which depends on last_bdf. This will replace global "device_table_size" variable. Co-developed-by: Suravee Suthikulpanit Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20220706113825.25582-12-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: chaithanyaLagisetty Signed-off-by: PvsNarasimha --- drivers/iommu/amd/amd_iommu_types.h | 3 +++ drivers/iommu/amd/init.c | 18 ++++++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index f6cfe3e8ca8fa..9be10bf8e38e1 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -550,6 +550,9 @@ struct amd_iommu_pci_seg { /* Largest PCI device id we expect translation requests for */ u16 last_bdf; + /* Size of the device table */ + u32 dev_table_size; + /* * device table virtual address * diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index d289d4068667e..663ff35f8b981 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -417,6 +417,7 @@ static void iommu_set_cwwb_range(struct amd_iommu *iommu) static void iommu_set_device_table(struct amd_iommu *iommu) { u64 entry; + u32 dev_table_size = iommu->pci_seg->dev_table_size; BUG_ON(iommu->mmio_base == NULL); @@ -656,7 +657,7 @@ static int __init find_last_devid_acpi(struct acpi_table_header *table, u16 pci_ static inline int __init alloc_dev_table(struct amd_iommu_pci_seg *pci_seg) { pci_seg->dev_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO | GFP_DMA32, - get_order(dev_table_size)); + get_order(pci_seg->dev_table_size)); if (!pci_seg->dev_table) return -ENOMEM; @@ -666,7 +667,7 @@ static inline int __init alloc_dev_table(struct amd_iommu_pci_seg *pci_seg) static inline void free_dev_table(struct amd_iommu_pci_seg *pci_seg) { free_pages((unsigned long)pci_seg->dev_table, - get_order(dev_table_size)); + get_order(pci_seg->dev_table_size)); pci_seg->dev_table = NULL; } @@ -1063,7 +1064,7 @@ static bool __copy_device_table(struct amd_iommu *iommu) entry = (((u64) hi) << 32) + lo; old_devtb_size = ((entry & ~PAGE_MASK) + 1) << 12; - if (old_devtb_size != dev_table_size) { + if (old_devtb_size != pci_seg->dev_table_size) { pr_err("The device table size of IOMMU:%d is not expected!\n", iommu->index); return false; @@ -1082,15 +1083,15 @@ static bool __copy_device_table(struct amd_iommu *iommu) } old_devtb = (sme_active() && is_kdump_kernel()) ? (__force void *)ioremap_encrypted(old_devtb_phys, - dev_table_size) - : memremap(old_devtb_phys, dev_table_size, MEMREMAP_WB); + pci_seg->dev_table_size) + : memremap(old_devtb_phys, pci_seg->dev_table_size, MEMREMAP_WB); if (!old_devtb) return false; gfp_flag = GFP_KERNEL | __GFP_ZERO | GFP_DMA32; pci_seg->old_dev_tbl_cpy = (void *)__get_free_pages(gfp_flag, - get_order(dev_table_size)); + get_order(pci_seg->dev_table_size)); if (pci_seg->old_dev_tbl_cpy == NULL) { pr_err("Failed to allocate memory for copying old device table!\n"); return false; @@ -1606,6 +1607,7 @@ static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id, pci_seg->last_bdf = last_bdf; DUMP_printk("PCI segment : 0x%0x, last bdf : 0x%04x\n", id, last_bdf); + pci_seg->dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE); pci_seg->id = id; init_llist_head(&pci_seg->dev_data_list); @@ -2721,7 +2723,7 @@ static void early_enable_iommus(void) for_each_pci_segment(pci_seg) { if (pci_seg->old_dev_tbl_cpy != NULL) { free_pages((unsigned long)pci_seg->old_dev_tbl_cpy, - get_order(dev_table_size)); + get_order(pci_seg->dev_table_size)); pci_seg->old_dev_tbl_cpy = NULL; } } @@ -2735,7 +2737,7 @@ static void early_enable_iommus(void) for_each_pci_segment(pci_seg) { free_pages((unsigned long)pci_seg->dev_table, - get_order(dev_table_size)); + get_order(pci_seg->dev_table_size)); pci_seg->dev_table = pci_seg->old_dev_tbl_cpy; } From 014b6a0457efa2a76e51c278f93fdca3d6ee6737 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Wed, 6 Jul 2022 17:08:02 +0530 Subject: [PATCH 12/34] iommu/amd: Introduce per PCI segment alias table size commit 74ce42a9ab7a245c293c58a4cb964ad285e8fb2e upstream. It will replace global "alias_table_size" variable. Co-developed-by: Suravee Suthikulpanit Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20220706113825.25582-13-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: chaithanyaLagisetty Signed-off-by: PvsNarasimha --- drivers/iommu/amd/amd_iommu_types.h | 3 +++ drivers/iommu/amd/init.c | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 9be10bf8e38e1..b12d13a9e8013 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -553,6 +553,9 @@ struct amd_iommu_pci_seg { /* Size of the device table */ u32 dev_table_size; + /* Size of the alias table */ + u32 alias_table_size; + /* * device table virtual address * diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 663ff35f8b981..dd70c6759b55b 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -716,7 +716,7 @@ static int __init alloc_alias_table(struct amd_iommu_pci_seg *pci_seg) int i; pci_seg->alias_table = (void *)__get_free_pages(GFP_KERNEL, - get_order(alias_table_size)); + get_order(pci_seg->alias_table_size)); if (!pci_seg->alias_table) return -ENOMEM; @@ -732,7 +732,7 @@ static int __init alloc_alias_table(struct amd_iommu_pci_seg *pci_seg) static void __init free_alias_table(struct amd_iommu_pci_seg *pci_seg) { free_pages((unsigned long)pci_seg->alias_table, - get_order(alias_table_size)); + get_order(pci_seg->alias_table_size)); pci_seg->alias_table = NULL; } @@ -1608,6 +1608,7 @@ static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id, pci_seg->last_bdf = last_bdf; DUMP_printk("PCI segment : 0x%0x, last bdf : 0x%04x\n", id, last_bdf); pci_seg->dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE); + pci_seg->alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE); pci_seg->id = id; init_llist_head(&pci_seg->dev_data_list); From 3885fd88cab11c0a028b3aaeea5b1eaeebd920b7 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Wed, 6 Jul 2022 17:08:03 +0530 Subject: [PATCH 13/34] iommu/amd: Introduce per PCI segment rlookup table size commit ec12dd139252cede736b8f98c5ced9644121de30 upstream. It will replace global "rlookup_table_size" variable. Co-developed-by: Suravee Suthikulpanit Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20220706113825.25582-14-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: chaithanyaLagisetty Signed-off-by: PvsNarasimha --- drivers/iommu/amd/amd_iommu_types.h | 3 +++ drivers/iommu/amd/init.c | 11 ++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index b12d13a9e8013..278537296abda 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -556,6 +556,9 @@ struct amd_iommu_pci_seg { /* Size of the alias table */ u32 alias_table_size; + /* Size of the rlookup table */ + u32 rlookup_table_size; + /* * device table virtual address * diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index dd70c6759b55b..a612a12a8183d 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -676,7 +676,7 @@ static inline int __init alloc_rlookup_table(struct amd_iommu_pci_seg *pci_seg) { pci_seg->rlookup_table = (void *)__get_free_pages( GFP_KERNEL | __GFP_ZERO, - get_order(rlookup_table_size)); + get_order(pci_seg->rlookup_table_size)); if (pci_seg->rlookup_table == NULL) return -ENOMEM; @@ -686,7 +686,7 @@ static inline int __init alloc_rlookup_table(struct amd_iommu_pci_seg *pci_seg) static inline void free_rlookup_table(struct amd_iommu_pci_seg *pci_seg) { free_pages((unsigned long)pci_seg->rlookup_table, - get_order(rlookup_table_size)); + get_order(pci_seg->rlookup_table_size)); pci_seg->rlookup_table = NULL; } @@ -694,9 +694,9 @@ static inline int __init alloc_irq_lookup_table(struct amd_iommu_pci_seg *pci_se { pci_seg->irq_lookup_table = (void *)__get_free_pages( GFP_KERNEL | __GFP_ZERO, - get_order(rlookup_table_size)); + get_order(pci_seg->rlookup_table_size)); kmemleak_alloc(pci_seg->irq_lookup_table, - rlookup_table_size, 1, GFP_KERNEL); + pci_seg->rlookup_table_size, 1, GFP_KERNEL); if (pci_seg->irq_lookup_table == NULL) return -ENOMEM; @@ -707,7 +707,7 @@ static inline void free_irq_lookup_table(struct amd_iommu_pci_seg *pci_seg) { kmemleak_free(pci_seg->irq_lookup_table); free_pages((unsigned long)pci_seg->irq_lookup_table, - get_order(rlookup_table_size)); + get_order(pci_seg->rlookup_table_size)); pci_seg->irq_lookup_table = NULL; } @@ -1609,6 +1609,7 @@ static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id, DUMP_printk("PCI segment : 0x%0x, last bdf : 0x%04x\n", id, last_bdf); pci_seg->dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE); pci_seg->alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE); + pci_seg->rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE); pci_seg->id = id; init_llist_head(&pci_seg->dev_data_list); From fca9ab619ed0d69ef759add2b499e6b0e92657c2 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Wed, 6 Jul 2022 17:08:04 +0530 Subject: [PATCH 14/34] iommu/amd: Convert to use per PCI segment irq_lookup_table commit 0217ed5a946b61cca3a608d1ba9a422f8f85baf6 upstream. Then, remove the global irq_lookup_table. [Backport change] In file `drivers/iommu/amd/iommu.c`, in function `amd_iommu_update_ga()`, the call get_irq_table(iommu, devid) was already removed by commit 0a27c660d4d3, so this change is not required. Co-developed-by: Suravee Suthikulpanit Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20220706113825.25582-15-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: chaithanyaLagisetty Signed-off-by: PvsNarasimha --- drivers/iommu/amd/amd_iommu_types.h | 2 -- drivers/iommu/amd/init.c | 19 ---------------- drivers/iommu/amd/iommu.c | 34 +++++++++++++++++++---------- 3 files changed, 22 insertions(+), 33 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 278537296abda..bb33244e14b7d 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -443,8 +443,6 @@ struct irq_remap_table { u32 *table; }; -extern struct irq_remap_table **irq_lookup_table; - /* Interrupt remapping feature used? */ extern bool amd_iommu_irq_remap; diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index a612a12a8183d..472354b21e78a 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -209,12 +209,6 @@ u16 *amd_iommu_alias_table; */ struct amd_iommu **amd_iommu_rlookup_table; -/* - * This table is used to find the irq remapping table for a given device id - * quickly. - */ -struct irq_remap_table **irq_lookup_table; - /* * AMD IOMMU allows up to 2^16 different protection domains. This is a bitmap * to know which ones are already in use. @@ -2834,11 +2828,6 @@ static struct syscore_ops amd_iommu_syscore_ops = { static void __init free_iommu_resources(void) { - kmemleak_free(irq_lookup_table); - free_pages((unsigned long)irq_lookup_table, - get_order(rlookup_table_size)); - irq_lookup_table = NULL; - kmem_cache_destroy(amd_iommu_irq_cache); amd_iommu_irq_cache = NULL; @@ -3059,14 +3048,6 @@ static int __init early_amd_iommu_init(void) if (alloc_irq_lookup_table(pci_seg)) goto out; } - - irq_lookup_table = (void *)__get_free_pages( - GFP_KERNEL | __GFP_ZERO, - get_order(rlookup_table_size)); - kmemleak_alloc(irq_lookup_table, rlookup_table_size, - 1, GFP_KERNEL); - if (!irq_lookup_table) - goto out; } ret = init_memory_definitions(ivrs_base); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 5cbb4d241480d..3a697f11113e6 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2739,16 +2739,18 @@ static void set_dte_irq_entry(u16 devid, struct irq_remap_table *table) amd_iommu_dev_table[devid].data[2] = dte; } -static struct irq_remap_table *get_irq_table(u16 devid) +static struct irq_remap_table *get_irq_table(struct amd_iommu *iommu, u16 devid) { struct irq_remap_table *table; + struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; if (WARN_ONCE(!amd_iommu_rlookup_table[devid], "%s: no iommu for devid %x\n", __func__, devid)) return NULL; - table = irq_lookup_table[devid]; - if (WARN_ONCE(!table, "%s: no table for devid %x\n", __func__, devid)) + table = pci_seg->irq_lookup_table[devid]; + if (WARN_ONCE(!table, "%s: no table for devid %x:%x\n", + __func__, pci_seg->id, devid)) return NULL; return table; @@ -2781,7 +2783,9 @@ static struct irq_remap_table *__alloc_irq_table(void) static void set_remap_table_entry(struct amd_iommu *iommu, u16 devid, struct irq_remap_table *table) { - irq_lookup_table[devid] = table; + struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; + + pci_seg->irq_lookup_table[devid] = table; set_dte_irq_entry(devid, table); iommu_flush_dte(iommu, devid); } @@ -2790,8 +2794,14 @@ static int set_remap_table_entry_alias(struct pci_dev *pdev, u16 alias, void *data) { struct irq_remap_table *table = data; + struct amd_iommu_pci_seg *pci_seg; + struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev); - irq_lookup_table[alias] = table; + if (!iommu) + return -EINVAL; + + pci_seg = iommu->pci_seg; + pci_seg->irq_lookup_table[alias] = table; set_dte_irq_entry(alias, table); iommu_flush_dte(amd_iommu_rlookup_table[alias], alias); @@ -2815,12 +2825,12 @@ static struct irq_remap_table *alloc_irq_table(u16 devid, struct pci_dev *pdev) goto out_unlock; pci_seg = iommu->pci_seg; - table = irq_lookup_table[devid]; + table = pci_seg->irq_lookup_table[devid]; if (table) goto out_unlock; alias = pci_seg->alias_table[devid]; - table = irq_lookup_table[alias]; + table = pci_seg->irq_lookup_table[alias]; if (table) { set_remap_table_entry(iommu, devid, table); goto out_wait; @@ -2834,11 +2844,11 @@ static struct irq_remap_table *alloc_irq_table(u16 devid, struct pci_dev *pdev) spin_lock_irqsave(&iommu_table_lock, flags); - table = irq_lookup_table[devid]; + table = pci_seg->irq_lookup_table[devid]; if (table) goto out_unlock; - table = irq_lookup_table[alias]; + table = pci_seg->irq_lookup_table[alias]; if (table) { set_remap_table_entry(iommu, devid, table); goto out_wait; @@ -2931,7 +2941,7 @@ static int __modify_irte_ga(u16 devid, int index, struct irte_ga *irte) if (iommu == NULL) return -EINVAL; - table = get_irq_table(devid); + table = get_irq_table(iommu, devid); if (!table) return -ENOMEM; @@ -2984,7 +2994,7 @@ static int modify_irte(u16 devid, int index, union irte *irte) if (iommu == NULL) return -EINVAL; - table = get_irq_table(devid); + table = get_irq_table(iommu, devid); if (!table) return -ENOMEM; @@ -3007,7 +3017,7 @@ static void free_irte(u16 devid, int index) if (iommu == NULL) return; - table = get_irq_table(devid); + table = get_irq_table(iommu, devid); if (!table) return; From 3cb1e0009bc83829a919d920bc982a684d0a8f1b Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 6 Jul 2022 17:08:05 +0530 Subject: [PATCH 15/34] iommu/amd: Convert to use rlookup_amd_iommu helper function commit 8b71c9bf4df8294f775b81babfd384d1b3c00aa8 upstream. Use rlookup_amd_iommu() helper function which will give per PCI segment rlookup_table. Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20220706113825.25582-16-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: chaithanyaLagisetty Signed-off-by: PvsNarasimha --- drivers/iommu/amd/iommu.c | 64 +++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 3a697f11113e6..85bd6a94461de 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -228,13 +228,17 @@ static struct iommu_dev_data *search_dev_data(struct amd_iommu *iommu, u16 devid static int clone_alias(struct pci_dev *pdev, u16 alias, void *data) { + struct amd_iommu *iommu; u16 devid = pci_dev_id(pdev); if (devid == alias) return 0; - amd_iommu_rlookup_table[alias] = - amd_iommu_rlookup_table[devid]; + iommu = rlookup_amd_iommu(&pdev->dev); + if (!iommu) + return 0; + + amd_iommu_set_rlookup_table(iommu, alias); memcpy(amd_iommu_dev_table[alias].data, amd_iommu_dev_table[devid].data, sizeof(amd_iommu_dev_table[alias].data)); @@ -365,7 +369,7 @@ static bool check_device(struct device *dev) if (devid > amd_iommu_last_bdf) return false; - if (amd_iommu_rlookup_table[devid] == NULL) + if (rlookup_amd_iommu(dev) == NULL) return false; return true; @@ -1285,7 +1289,9 @@ static int device_flush_iotlb(struct iommu_dev_data *dev_data, int qdep; qdep = dev_data->ats.qdep; - iommu = amd_iommu_rlookup_table[dev_data->devid]; + iommu = rlookup_amd_iommu(dev_data->dev); + if (!iommu) + return -EINVAL; build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size); @@ -1310,7 +1316,9 @@ static int device_flush_dte(struct iommu_dev_data *dev_data) u16 alias; int ret; - iommu = amd_iommu_rlookup_table[dev_data->devid]; + iommu = rlookup_amd_iommu(dev_data->dev); + if (!iommu) + return -EINVAL; if (dev_is_pci(dev_data->dev)) pdev = to_pci_dev(dev_data->dev); @@ -1540,8 +1548,8 @@ static void free_gcr3_table(struct protection_domain *domain) free_page((unsigned long)domain->gcr3_tbl); } -static void set_dte_entry(u16 devid, struct protection_domain *domain, - bool ats, bool ppr) +static void set_dte_entry(struct amd_iommu *iommu, u16 devid, + struct protection_domain *domain, bool ats, bool ppr) { u64 pte_root = 0; u64 flags = 0; @@ -1560,8 +1568,6 @@ static void set_dte_entry(u16 devid, struct protection_domain *domain, flags |= DTE_FLAG_IOTLB; if (ppr) { - struct amd_iommu *iommu = amd_iommu_rlookup_table[devid]; - if (iommu_feature(iommu, FEATURE_EPHSUP)) pte_root |= 1ULL << DEV_ENTRY_PPR; } @@ -1605,8 +1611,6 @@ static void set_dte_entry(u16 devid, struct protection_domain *domain, * entries for the old domain ID that is being overwritten */ if (old_domid) { - struct amd_iommu *iommu = amd_iommu_rlookup_table[devid]; - amd_iommu_flush_tlb_domid(iommu, old_domid); } } @@ -1626,7 +1630,9 @@ static void do_attach(struct iommu_dev_data *dev_data, struct amd_iommu *iommu; bool ats; - iommu = amd_iommu_rlookup_table[dev_data->devid]; + iommu = rlookup_amd_iommu(dev_data->dev); + if (!iommu) + return; ats = dev_data->ats.enabled; /* Update data structures */ @@ -1638,7 +1644,7 @@ static void do_attach(struct iommu_dev_data *dev_data, domain->dev_cnt += 1; /* Update device table */ - set_dte_entry(dev_data->devid, domain, + set_dte_entry(iommu, dev_data->devid, domain, ats, dev_data->iommu_v2); clone_aliases(iommu, dev_data->dev); @@ -1650,7 +1656,9 @@ static void do_detach(struct iommu_dev_data *dev_data) struct protection_domain *domain = dev_data->domain; struct amd_iommu *iommu; - iommu = amd_iommu_rlookup_table[dev_data->devid]; + iommu = rlookup_amd_iommu(dev_data->dev); + if (!iommu) + return; /* Update data structures */ dev_data->domain = NULL; @@ -1830,13 +1838,14 @@ static struct iommu_device *amd_iommu_probe_device(struct device *dev) { struct iommu_device *iommu_dev; struct amd_iommu *iommu; - int ret, devid; + int ret; if (!check_device(dev)) return ERR_PTR(-ENODEV); - devid = get_device_id(dev); - iommu = amd_iommu_rlookup_table[devid]; + iommu = rlookup_amd_iommu(dev); + if (!iommu) + return ERR_PTR(-ENODEV); if (dev_iommu_priv_get(dev)) return &iommu->iommu; @@ -1866,13 +1875,14 @@ static void amd_iommu_probe_finalize(struct device *dev) static void amd_iommu_release_device(struct device *dev) { - int devid = get_device_id(dev); struct amd_iommu *iommu; if (!check_device(dev)) return; - iommu = amd_iommu_rlookup_table[devid]; + iommu = rlookup_amd_iommu(dev); + if (!iommu) + return; amd_iommu_uninit_device(dev); iommu_completion_wait(iommu); @@ -1901,7 +1911,7 @@ static void update_device_table(struct protection_domain *domain) if (!iommu) continue; - set_dte_entry(dev_data->devid, domain, + set_dte_entry(iommu, dev_data->devid, domain, dev_data->ats.enabled, dev_data->iommu_v2); clone_aliases(iommu, dev_data->dev); } @@ -2089,7 +2099,6 @@ static void amd_iommu_detach_device(struct iommu_domain *dom, struct device *dev) { struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); - int devid = get_device_id(dev); struct amd_iommu *iommu; if (!check_device(dev)) @@ -2098,7 +2107,7 @@ static void amd_iommu_detach_device(struct iommu_domain *dom, if (dev_data->domain != NULL) detach_device(dev); - iommu = amd_iommu_rlookup_table[devid]; + iommu = rlookup_amd_iommu(dev); if (!iommu) return; @@ -2125,7 +2134,7 @@ static int amd_iommu_attach_device(struct iommu_domain *dom, dev_data = dev_iommu_priv_get(dev); dev_data->defer_attach = false; - iommu = amd_iommu_rlookup_table[dev_data->devid]; + iommu = rlookup_amd_iommu(dev); if (!iommu) return -EINVAL; @@ -2500,8 +2509,9 @@ static int __flush_pasid(struct protection_domain *domain, u32 pasid, continue; qdep = dev_data->ats.qdep; - iommu = amd_iommu_rlookup_table[dev_data->devid]; - + iommu = rlookup_amd_iommu(dev_data->dev); + if (!iommu) + continue; build_inv_iotlb_pasid(&cmd, dev_data->devid, pasid, qdep, address, size); @@ -2663,7 +2673,9 @@ int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid, struct iommu_cmd cmd; dev_data = dev_iommu_priv_get(&pdev->dev); - iommu = amd_iommu_rlookup_table[dev_data->devid]; + iommu = rlookup_amd_iommu(&pdev->dev); + if (!iommu) + return -ENODEV; build_complete_ppr(&cmd, dev_data->devid, pasid, status, tag, dev_data->pri_tlp); From b3eeaa1c5185f7d54f2bfeceeb2ea415c0d05bf6 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 6 Jul 2022 17:08:06 +0530 Subject: [PATCH 16/34] iommu/amd: Update irq_remapping_alloc to use IOMMU lookup helper function commit 9873ae6e944d2e6f939ab81b25b2be7f0ee92d18 upstream. To allow IOMMU rlookup using both PCI segment and device ID. Co-developed-by: Vasant Hegde Signed-off-by: Vasant Hegde Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20220706113825.25582-17-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: Arukonda Rahul Signed-off-by: PvsNarasimha --- drivers/iommu/amd/iommu.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 85bd6a94461de..9f80beafbc961 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -3262,8 +3262,9 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, struct irq_alloc_info *info = arg; struct irq_data *irq_data; struct amd_ir_data *data = NULL; + struct amd_iommu *iommu; struct irq_cfg *cfg; - int i, ret, devid; + int i, ret, devid, seg, sbdf; int index; if (!info) @@ -3279,8 +3280,14 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, if (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI) info->flags &= ~X86_IRQ_ALLOC_CONTIGUOUS_VECTORS; - devid = get_devid(info); - if (devid < 0) + sbdf = get_devid(info); + if (sbdf < 0) + return -EINVAL; + + seg = PCI_SBDF_TO_SEGID(sbdf); + devid = PCI_SBDF_TO_DEVID(sbdf); + iommu = __rlookup_amd_iommu(seg, devid); + if (!iommu) return -EINVAL; ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, arg); @@ -3289,7 +3296,6 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC) { struct irq_remap_table *table; - struct amd_iommu *iommu; table = alloc_irq_table(devid, NULL); if (table) { @@ -3299,7 +3305,6 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, * interrupts. */ table->min_index = 32; - iommu = amd_iommu_rlookup_table[devid]; for (i = 0; i < 32; ++i) iommu->irte_ops->set_allocated(table, i); } From b88a0235e014574aa66c353b1175e570c532bf70 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 6 Jul 2022 17:08:07 +0530 Subject: [PATCH 17/34] iommu/amd: Introduce struct amd_ir_data.iommu commit 9457d75c056ad52b6c33693c87500e38fdd41c6f upstream. Add a pointer to struct amd_iommu to amd_ir_data structure, which can be used to correlate interrupt remapping data to a per-PCI-segment interrupt remapping table. [Backport change] This patch triggers a compilation warning due to the unused variable int devid = ir_data->irq_2_irte.devid;. Although commit 0a27c660d4d3 (already backported to the kernel) was supposed to remove it, the line wasn't removed. To avoid compilation warning, it has now been removed. Co-developed-by: Vasant Hegde Signed-off-by: Vasant Hegde Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20220706113825.25582-18-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: Arukonda Rahul Signed-off-by: PvsNarasimha --- drivers/iommu/amd/amd_iommu_types.h | 1 + drivers/iommu/amd/iommu.c | 35 +++++++++++++---------------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index bb33244e14b7d..7a68244f32597 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -987,6 +987,7 @@ struct irq_2_irte { struct amd_ir_data { u32 cached_ga_tag; + struct amd_iommu *iommu; struct irq_2_irte irq_2_irte; struct msi_msg msi_entry; void *entry; /* Pointer to union irte or struct irte_ga */ diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 9f80beafbc961..0c1ef57017447 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -3019,16 +3019,11 @@ static int modify_irte(u16 devid, int index, union irte *irte) return 0; } -static void free_irte(u16 devid, int index) +static void free_irte(struct amd_iommu *iommu, u16 devid, int index) { struct irq_remap_table *table; - struct amd_iommu *iommu; unsigned long flags; - iommu = amd_iommu_rlookup_table[devid]; - if (iommu == NULL) - return; - table = get_irq_table(iommu, devid); if (!table) return; @@ -3211,7 +3206,7 @@ static void irq_remapping_prepare_irte(struct amd_ir_data *data, int devid, int index, int sub_handle) { struct irq_2_irte *irte_info = &data->irq_2_irte; - struct amd_iommu *iommu = amd_iommu_rlookup_table[devid]; + struct amd_iommu *iommu = data->iommu; if (!iommu) return; @@ -3352,6 +3347,7 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, goto out_free_data; } + data->iommu = iommu; irq_data->hwirq = (devid << 16) + i; irq_data->chip_data = data; irq_data->chip = &amd_ir_chip; @@ -3368,7 +3364,7 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, kfree(irq_data->chip_data); } for (i = 0; i < nr_irqs; i++) - free_irte(devid, index + i); + free_irte(iommu, devid, index + i); out_free_parent: irq_domain_free_irqs_common(domain, virq, nr_irqs); return ret; @@ -3387,7 +3383,7 @@ static void irq_remapping_free(struct irq_domain *domain, unsigned int virq, if (irq_data && irq_data->chip_data) { data = irq_data->chip_data; irte_info = &data->irq_2_irte; - free_irte(irte_info->devid, irte_info->index); + free_irte(data->iommu, irte_info->devid, irte_info->index); kfree(data->entry); kfree(data); } @@ -3405,7 +3401,7 @@ static int irq_remapping_activate(struct irq_domain *domain, { struct amd_ir_data *data = irq_data->chip_data; struct irq_2_irte *irte_info = &data->irq_2_irte; - struct amd_iommu *iommu = amd_iommu_rlookup_table[irte_info->devid]; + struct amd_iommu *iommu = data->iommu; struct irq_cfg *cfg = irqd_cfg(irq_data); if (!iommu) @@ -3422,7 +3418,7 @@ static void irq_remapping_deactivate(struct irq_domain *domain, { struct amd_ir_data *data = irq_data->chip_data; struct irq_2_irte *irte_info = &data->irq_2_irte; - struct amd_iommu *iommu = amd_iommu_rlookup_table[irte_info->devid]; + struct amd_iommu *iommu = data->iommu; if (iommu) iommu->irte_ops->deactivate(data->entry, irte_info->devid, @@ -3517,12 +3513,16 @@ EXPORT_SYMBOL(amd_iommu_deactivate_guest_mode); static int amd_ir_set_vcpu_affinity(struct irq_data *data, void *vcpu_info) { int ret; - struct amd_iommu *iommu; struct amd_iommu_pi_data *pi_data = vcpu_info; struct vcpu_data *vcpu_pi_info = pi_data->vcpu_data; struct amd_ir_data *ir_data = data->chip_data; struct irq_2_irte *irte_info = &ir_data->irq_2_irte; - struct iommu_dev_data *dev_data = search_dev_data(NULL, irte_info->devid); + struct iommu_dev_data *dev_data; + + if (ir_data->iommu == NULL) + return -EINVAL; + + dev_data = search_dev_data(ir_data->iommu, irte_info->devid); /* Note: * This device has never been set up for guest mode. @@ -3544,10 +3544,6 @@ static int amd_ir_set_vcpu_affinity(struct irq_data *data, void *vcpu_info) pi_data->is_guest_mode = false; } - iommu = amd_iommu_rlookup_table[irte_info->devid]; - if (iommu == NULL) - return -EINVAL; - pi_data->prev_ga_tag = ir_data->cached_ga_tag; if (pi_data->is_guest_mode) { ir_data->ga_root_ptr = (pi_data->base >> 12); @@ -3593,7 +3589,7 @@ static int amd_ir_set_affinity(struct irq_data *data, struct irq_2_irte *irte_info = &ir_data->irq_2_irte; struct irq_cfg *cfg = irqd_cfg(data); struct irq_data *parent = data->parent_data; - struct amd_iommu *iommu = amd_iommu_rlookup_table[irte_info->devid]; + struct amd_iommu *iommu = ir_data->iommu; int ret; if (!iommu) @@ -3653,14 +3649,13 @@ int amd_iommu_update_ga(int cpu, bool is_run, void *data) { struct amd_iommu *iommu; struct amd_ir_data *ir_data = (struct amd_ir_data *)data; - int devid = ir_data->irq_2_irte.devid; struct irte_ga *entry = (struct irte_ga *) ir_data->entry; if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) || !entry || !entry->lo.fields_vapic.guest_mode) return 0; - iommu = amd_iommu_rlookup_table[devid]; + iommu = ir_data->iommu; if (!iommu) return -ENODEV; From 408205ef1d33146537c87a4eba58c084302868b7 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 6 Jul 2022 17:08:08 +0530 Subject: [PATCH 18/34] iommu/amd: Update amd_irte_ops functions commit c4649a45f613a34012be82c700f9be5292893752 upstream. Pass amd_iommu structure as one of the parameter to amd_irte_ops functions since its needed to activate/deactivate the iommu. [Backport change] The commits c0d05dbac9d and 13809bd2d6, which introduced the name change for the __modify_irte_ga and modify_irte_ga APIs in the current kernel, are more recent than this upstream patch. Therefore, we are retaining the names __modify_irte_ga and modify_irte_ga as they are, and only updating the argument types as per the upstream patch. Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20220706113825.25582-19-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: Arukonda Rahul Signed-off-by: PvsNarasimha --- drivers/iommu/amd/amd_iommu_types.h | 6 +-- drivers/iommu/amd/iommu.c | 60 +++++++++++------------------ 2 files changed, 26 insertions(+), 40 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 7a68244f32597..8d27b91be88e3 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -1004,9 +1004,9 @@ struct amd_ir_data { struct amd_irte_ops { void (*prepare)(void *, u32, bool, u8, u32, int); - void (*activate)(void *, u16, u16); - void (*deactivate)(void *, u16, u16); - void (*set_affinity)(void *, u16, u16, u8, u32); + void (*activate)(struct amd_iommu *iommu, void *, u16, u16); + void (*deactivate)(struct amd_iommu *iommu, void *, u16, u16); + void (*set_affinity)(struct amd_iommu *iommu, void *, u16, u16, u8, u32); void *(*get)(struct irq_remap_table *, int); void (*set_allocated)(struct irq_remap_table *, int); bool (*is_allocated)(struct irq_remap_table *, int); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 0c1ef57017447..46f51f5d71c5e 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2941,18 +2941,13 @@ static int alloc_irq_index(u16 devid, int count, bool align, return index; } -static int __modify_irte_ga(u16 devid, int index, struct irte_ga *irte) +static int __modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index, struct irte_ga *irte) { bool ret; struct irq_remap_table *table; - struct amd_iommu *iommu; unsigned long flags; struct irte_ga *entry; - iommu = amd_iommu_rlookup_table[devid]; - if (iommu == NULL) - return -EINVAL; - table = get_irq_table(iommu, devid); if (!table) return -ENOMEM; @@ -2978,16 +2973,11 @@ static int __modify_irte_ga(u16 devid, int index, struct irte_ga *irte) return 0; } -static int modify_irte_ga(u16 devid, int index, struct irte_ga *irte) +static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index, struct irte_ga *irte) { bool ret; - struct amd_iommu *iommu; - - iommu = amd_iommu_rlookup_table[devid]; - if (iommu == NULL) - return -EINVAL; - ret = __modify_irte_ga(devid, index, irte); + ret = __modify_irte_ga(iommu, devid, index, irte); if (ret) return ret; @@ -2996,16 +2986,12 @@ static int modify_irte_ga(u16 devid, int index, struct irte_ga *irte) return 0; } -static int modify_irte(u16 devid, int index, union irte *irte) +static int modify_irte(struct amd_iommu *iommu, + u16 devid, int index, union irte *irte) { struct irq_remap_table *table; - struct amd_iommu *iommu; unsigned long flags; - iommu = amd_iommu_rlookup_table[devid]; - if (iommu == NULL) - return -EINVAL; - table = get_irq_table(iommu, devid); if (!table) return -ENOMEM; @@ -3065,49 +3051,49 @@ static void irte_ga_prepare(void *entry, irte->lo.fields_remap.valid = 1; } -static void irte_activate(void *entry, u16 devid, u16 index) +static void irte_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index) { union irte *irte = (union irte *) entry; irte->fields.valid = 1; - modify_irte(devid, index, irte); + modify_irte(iommu, devid, index, irte); } -static void irte_ga_activate(void *entry, u16 devid, u16 index) +static void irte_ga_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index) { struct irte_ga *irte = (struct irte_ga *) entry; irte->lo.fields_remap.valid = 1; - modify_irte_ga(devid, index, irte); + modify_irte_ga(iommu, devid, index, irte); } -static void irte_deactivate(void *entry, u16 devid, u16 index) +static void irte_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index) { union irte *irte = (union irte *) entry; irte->fields.valid = 0; - modify_irte(devid, index, irte); + modify_irte(iommu, devid, index, irte); } -static void irte_ga_deactivate(void *entry, u16 devid, u16 index) +static void irte_ga_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index) { struct irte_ga *irte = (struct irte_ga *) entry; irte->lo.fields_remap.valid = 0; - modify_irte_ga(devid, index, irte); + modify_irte_ga(iommu, devid, index, irte); } -static void irte_set_affinity(void *entry, u16 devid, u16 index, +static void irte_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index, u8 vector, u32 dest_apicid) { union irte *irte = (union irte *) entry; irte->fields.vector = vector; irte->fields.destination = dest_apicid; - modify_irte(devid, index, irte); + modify_irte(iommu, devid, index, irte); } -static void irte_ga_set_affinity(void *entry, u16 devid, u16 index, +static void irte_ga_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index, u8 vector, u32 dest_apicid) { struct irte_ga *irte = (struct irte_ga *) entry; @@ -3118,7 +3104,7 @@ static void irte_ga_set_affinity(void *entry, u16 devid, u16 index, APICID_TO_IRTE_DEST_LO(dest_apicid); irte->hi.fields.destination = APICID_TO_IRTE_DEST_HI(dest_apicid); - modify_irte_ga(devid, index, irte); + modify_irte_ga(iommu, devid, index, irte); } } @@ -3407,7 +3393,7 @@ static int irq_remapping_activate(struct irq_domain *domain, if (!iommu) return 0; - iommu->irte_ops->activate(data->entry, irte_info->devid, + iommu->irte_ops->activate(iommu, data->entry, irte_info->devid, irte_info->index); amd_ir_update_irte(irq_data, iommu, data, irte_info, cfg); return 0; @@ -3421,7 +3407,7 @@ static void irq_remapping_deactivate(struct irq_domain *domain, struct amd_iommu *iommu = data->iommu; if (iommu) - iommu->irte_ops->deactivate(data->entry, irte_info->devid, + iommu->irte_ops->deactivate(iommu, data->entry, irte_info->devid, irte_info->index); } @@ -3475,7 +3461,7 @@ int amd_iommu_activate_guest_mode(void *data) entry->hi.fields.vector = ir_data->ga_vector; entry->lo.fields_vapic.ga_tag = ir_data->ga_tag; - return modify_irte_ga(ir_data->irq_2_irte.devid, + return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid, ir_data->irq_2_irte.index, entry); } EXPORT_SYMBOL(amd_iommu_activate_guest_mode); @@ -3505,7 +3491,7 @@ int amd_iommu_deactivate_guest_mode(void *data) entry->hi.fields.destination = APICID_TO_IRTE_DEST_HI(cfg->dest_apicid); - return modify_irte_ga(ir_data->irq_2_irte.devid, + return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid, ir_data->irq_2_irte.index, entry); } EXPORT_SYMBOL(amd_iommu_deactivate_guest_mode); @@ -3577,7 +3563,7 @@ static void amd_ir_update_irte(struct irq_data *irqd, struct amd_iommu *iommu, * Atomically updates the IRTE with the new destination, vector * and flushes the interrupt entry cache. */ - iommu->irte_ops->set_affinity(ir_data->entry, irte_info->devid, + iommu->irte_ops->set_affinity(iommu, ir_data->entry, irte_info->devid, irte_info->index, cfg->vector, cfg->dest_apicid); } @@ -3667,7 +3653,7 @@ int amd_iommu_update_ga(int cpu, bool is_run, void *data) } entry->lo.fields_vapic.is_run = is_run; - return __modify_irte_ga(ir_data->irq_2_irte.devid, + return __modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid, ir_data->irq_2_irte.index, entry); } EXPORT_SYMBOL(amd_iommu_update_ga); From 554e90e7680d84e400b90df9e2611f4ee6bf3f68 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 6 Jul 2022 17:08:09 +0530 Subject: [PATCH 19/34] iommu/amd: Update alloc_irq_table and alloc_irq_index commit e6457d7cfca14e8f7f2199cd724c4f03abe493c2 upstream. Pass amd_iommu structure as one of the parameter to these functions as its needed to retrieve variable tables inside these functions. Co-developed-by: Vasant Hegde Signed-off-by: Vasant Hegde Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20220706113825.25582-20-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: Arukonda Rahul Signed-off-by: PvsNarasimha --- drivers/iommu/amd/iommu.c | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 46f51f5d71c5e..7cae2835134ea 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2821,21 +2821,17 @@ static int set_remap_table_entry_alias(struct pci_dev *pdev, u16 alias, return 0; } -static struct irq_remap_table *alloc_irq_table(u16 devid, struct pci_dev *pdev) +static struct irq_remap_table *alloc_irq_table(struct amd_iommu *iommu, + u16 devid, struct pci_dev *pdev) { struct irq_remap_table *table = NULL; struct irq_remap_table *new_table = NULL; struct amd_iommu_pci_seg *pci_seg; - struct amd_iommu *iommu; unsigned long flags; u16 alias; spin_lock_irqsave(&iommu_table_lock, flags); - iommu = amd_iommu_rlookup_table[devid]; - if (!iommu) - goto out_unlock; - pci_seg = iommu->pci_seg; table = pci_seg->irq_lookup_table[devid]; if (table) @@ -2891,18 +2887,14 @@ static struct irq_remap_table *alloc_irq_table(u16 devid, struct pci_dev *pdev) return table; } -static int alloc_irq_index(u16 devid, int count, bool align, - struct pci_dev *pdev) +static int alloc_irq_index(struct amd_iommu *iommu, u16 devid, int count, + bool align, struct pci_dev *pdev) { struct irq_remap_table *table; int index, c, alignment = 1; unsigned long flags; - struct amd_iommu *iommu = amd_iommu_rlookup_table[devid]; - - if (!iommu) - return -ENODEV; - table = alloc_irq_table(devid, pdev); + table = alloc_irq_table(iommu, devid, pdev); if (!table) return -ENODEV; @@ -3278,7 +3270,7 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC) { struct irq_remap_table *table; - table = alloc_irq_table(devid, NULL); + table = alloc_irq_table(iommu, devid, NULL); if (table) { if (!table->min_index) { /* @@ -3298,10 +3290,10 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, info->type == X86_IRQ_ALLOC_TYPE_PCI_MSIX) { bool align = (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI); - index = alloc_irq_index(devid, nr_irqs, align, + index = alloc_irq_index(iommu, devid, nr_irqs, align, msi_desc_to_pci_dev(info->desc)); } else { - index = alloc_irq_index(devid, nr_irqs, false, NULL); + index = alloc_irq_index(iommu, devid, nr_irqs, false, NULL); } if (index < 0) { @@ -3427,8 +3419,8 @@ static int irq_remapping_select(struct irq_domain *d, struct irq_fwspec *fwspec, if (devid < 0) return 0; + iommu = __rlookup_amd_iommu((devid >> 16), (devid & 0xffff)); - iommu = amd_iommu_rlookup_table[devid]; return iommu && iommu->ir_domain == d; } From b72d3710406e3bec4e25bd31d565c31b800d6ec2 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Wed, 6 Jul 2022 17:08:10 +0530 Subject: [PATCH 20/34] iommu/amd: Convert to use per PCI segment rlookup_table commit ccacd94fdacabe77f5a887c3b75334982b1ce9ca upstream. Then, remove the global amd_iommu_rlookup_table and rlookup_table_size. Co-developed-by: Suravee Suthikulpanit Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20220706113825.25582-21-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: Arukonda Rahul Signed-off-by: PvsNarasimha --- drivers/iommu/amd/amd_iommu_types.h | 5 ----- drivers/iommu/amd/init.c | 23 ++--------------------- drivers/iommu/amd/iommu.c | 19 +++++++++---------- 3 files changed, 11 insertions(+), 36 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 8d27b91be88e3..f48f3122c1a7e 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -844,11 +844,6 @@ extern struct dev_table_entry *amd_iommu_dev_table; */ extern u16 *amd_iommu_alias_table; -/* - * Reverse lookup table to find the IOMMU which translates a specific device. - */ -extern struct amd_iommu **amd_iommu_rlookup_table; - /* size of the dma_ops aperture as power of 2 */ extern unsigned amd_iommu_aperture_order; diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 472354b21e78a..62ec5725302ba 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -203,12 +203,6 @@ struct dev_table_entry *amd_iommu_dev_table; */ u16 *amd_iommu_alias_table; -/* - * The rlookup table is used to find the IOMMU which is responsible - * for a specific device. It is also indexed by the PCI device id. - */ -struct amd_iommu **amd_iommu_rlookup_table; - /* * AMD IOMMU allows up to 2^16 different protection domains. This is a bitmap * to know which ones are already in use. @@ -217,7 +211,6 @@ unsigned long *amd_iommu_pd_alloc_bitmap; static u32 dev_table_size; /* size of the device table */ static u32 alias_table_size; /* size of the alias table */ -static u32 rlookup_table_size; /* size if the rlookup table */ enum iommu_init_state { IOMMU_START_STATE, @@ -1170,7 +1163,7 @@ void amd_iommu_apply_erratum_63(u16 devid) /* Writes the specific IOMMU for a device into the rlookup table */ static void __init set_iommu_for_device(struct amd_iommu *iommu, u16 devid) { - amd_iommu_rlookup_table[devid] = iommu; + iommu->pci_seg->rlookup_table[devid] = iommu; } /* @@ -1851,7 +1844,7 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, * Make sure IOMMU is not considered to translate itself. The IVRS * table tells us so, but this is a lie! */ - amd_iommu_rlookup_table[iommu->devid] = NULL; + pci_seg->rlookup_table[iommu->devid] = NULL; return 0; } @@ -2831,10 +2824,6 @@ static void __init free_iommu_resources(void) kmem_cache_destroy(amd_iommu_irq_cache); amd_iommu_irq_cache = NULL; - free_pages((unsigned long)amd_iommu_rlookup_table, - get_order(rlookup_table_size)); - amd_iommu_rlookup_table = NULL; - free_pages((unsigned long)amd_iommu_alias_table, get_order(alias_table_size)); amd_iommu_alias_table = NULL; @@ -2973,7 +2962,6 @@ static int __init early_amd_iommu_init(void) dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE); alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE); - rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE); /* Device table - directly used by all IOMMUs */ ret = -ENOMEM; @@ -2992,13 +2980,6 @@ static int __init early_amd_iommu_init(void) if (amd_iommu_alias_table == NULL) goto out; - /* IOMMU rlookup table - find the IOMMU for a specific device */ - amd_iommu_rlookup_table = (void *)__get_free_pages( - GFP_KERNEL | __GFP_ZERO, - get_order(rlookup_table_size)); - if (amd_iommu_rlookup_table == NULL) - goto out; - amd_iommu_pd_alloc_bitmap = (void *)__get_free_pages( GFP_KERNEL | __GFP_ZERO, get_order(MAX_DOMAIN_ID/8)); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 7cae2835134ea..15dfa382a6f1f 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -286,10 +286,9 @@ static void setup_aliases(struct amd_iommu *iommu, struct device *dev) clone_aliases(iommu, dev); } -static struct iommu_dev_data *find_dev_data(u16 devid) +static struct iommu_dev_data *find_dev_data(struct amd_iommu *iommu, u16 devid) { struct iommu_dev_data *dev_data; - struct amd_iommu *iommu = amd_iommu_rlookup_table[devid]; dev_data = search_dev_data(iommu, devid); @@ -387,7 +386,7 @@ static int iommu_init_device(struct amd_iommu *iommu, struct device *dev) if (devid < 0) return devid; - dev_data = find_dev_data(devid); + dev_data = find_dev_data(iommu, devid); if (!dev_data) return -ENOMEM; @@ -402,9 +401,6 @@ static int iommu_init_device(struct amd_iommu *iommu, struct device *dev) */ if ((iommu_default_passthrough() || !amd_iommu_force_isolation) && dev_is_pci(dev) && pci_iommuv2_capable(to_pci_dev(dev))) { - struct amd_iommu *iommu; - - iommu = amd_iommu_rlookup_table[dev_data->devid]; dev_data->iommu_v2 = iommu->is_iommu_v2; } @@ -415,13 +411,15 @@ static int iommu_init_device(struct amd_iommu *iommu, struct device *dev) static void iommu_ignore_device(struct amd_iommu *iommu, struct device *dev) { + struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; int devid; devid = get_device_id(dev); if (devid < 0) return; - amd_iommu_rlookup_table[devid] = NULL; + + pci_seg->rlookup_table[devid] = NULL; memset(&amd_iommu_dev_table[devid], 0, sizeof(struct dev_table_entry)); setup_aliases(iommu, dev); @@ -2756,8 +2754,9 @@ static struct irq_remap_table *get_irq_table(struct amd_iommu *iommu, u16 devid) struct irq_remap_table *table; struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; - if (WARN_ONCE(!amd_iommu_rlookup_table[devid], - "%s: no iommu for devid %x\n", __func__, devid)) + if (WARN_ONCE(!pci_seg->rlookup_table[devid], + "%s: no iommu for devid %x:%x\n", + __func__, pci_seg->id, devid)) return NULL; table = pci_seg->irq_lookup_table[devid]; @@ -2816,7 +2815,7 @@ static int set_remap_table_entry_alias(struct pci_dev *pdev, u16 alias, pci_seg->irq_lookup_table[alias] = table; set_dte_irq_entry(alias, table); - iommu_flush_dte(amd_iommu_rlookup_table[alias], alias); + iommu_flush_dte(pci_seg->rlookup_table[alias], alias); return 0; } From 32fb75ec9cee2bf3a1e64cbb853c095ac550254d Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 6 Jul 2022 17:08:11 +0530 Subject: [PATCH 21/34] iommu/amd: Update set_dte_entry and clear_dte_entry commit 54625ef1db1ce7e57f356d8f355095a4fbb71efb upstream. Start using per PCI segment data structures instead of global data structures. Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20220706113825.25582-22-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: Malathi Signed-off-by: PvsNarasimha --- drivers/iommu/amd/iommu.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 15dfa382a6f1f..74496c709e563 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1552,6 +1552,7 @@ static void set_dte_entry(struct amd_iommu *iommu, u16 devid, u64 pte_root = 0; u64 flags = 0; u32 old_domid; + struct dev_table_entry *dev_table = get_dev_table(iommu); if (domain->iop.mode != PAGE_MODE_NONE) pte_root = iommu_virt_to_phys(domain->iop.root); @@ -1560,7 +1561,7 @@ static void set_dte_entry(struct amd_iommu *iommu, u16 devid, << DEV_ENTRY_MODE_SHIFT; pte_root |= DTE_FLAG_IR | DTE_FLAG_IW | DTE_FLAG_V | DTE_FLAG_TV; - flags = amd_iommu_dev_table[devid].data[1]; + flags = dev_table[devid].data[1]; if (ats) flags |= DTE_FLAG_IOTLB; @@ -1599,9 +1600,9 @@ static void set_dte_entry(struct amd_iommu *iommu, u16 devid, flags &= ~DEV_DOMID_MASK; flags |= domain->id; - old_domid = amd_iommu_dev_table[devid].data[1] & DEV_DOMID_MASK; - amd_iommu_dev_table[devid].data[1] = flags; - amd_iommu_dev_table[devid].data[0] = pte_root; + old_domid = dev_table[devid].data[1] & DEV_DOMID_MASK; + dev_table[devid].data[1] = flags; + dev_table[devid].data[0] = pte_root; /* * A kdump kernel might be replacing a domain ID that was copied from @@ -1613,11 +1614,13 @@ static void set_dte_entry(struct amd_iommu *iommu, u16 devid, } } -static void clear_dte_entry(u16 devid) +static void clear_dte_entry(struct amd_iommu *iommu, u16 devid) { + struct dev_table_entry *dev_table = get_dev_table(iommu); + /* remove entry from the device table seen by the hardware */ - amd_iommu_dev_table[devid].data[0] = DTE_FLAG_V | DTE_FLAG_TV; - amd_iommu_dev_table[devid].data[1] &= DTE_FLAG_MASK; + dev_table[devid].data[0] = DTE_FLAG_V | DTE_FLAG_TV; + dev_table[devid].data[1] &= DTE_FLAG_MASK; amd_iommu_apply_erratum_63(devid); } @@ -1661,7 +1664,7 @@ static void do_detach(struct iommu_dev_data *dev_data) /* Update data structures */ dev_data->domain = NULL; list_del(&dev_data->list); - clear_dte_entry(dev_data->devid); + clear_dte_entry(iommu, dev_data->devid); clone_aliases(iommu, dev_data->dev); /* Flush the DTE entry */ From e964fd3feeec6c0fc758d4f6952b8671d531c962 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 6 Jul 2022 17:08:12 +0530 Subject: [PATCH 22/34] iommu/amd: Update iommu_ignore_device commit ccbb091f3f2019c803480bb4ed63c84869da68dc upstream. Start using per PCI segment device table instead of global device table. Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20220706113825.25582-23-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: Malathi Signed-off-by: PvsNarasimha --- drivers/iommu/amd/iommu.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 74496c709e563..cba5ebb46e71c 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -412,15 +412,15 @@ static int iommu_init_device(struct amd_iommu *iommu, struct device *dev) static void iommu_ignore_device(struct amd_iommu *iommu, struct device *dev) { struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; + struct dev_table_entry *dev_table = get_dev_table(iommu); int devid; - devid = get_device_id(dev); + devid = (get_device_id(dev)) & 0xffff; if (devid < 0) return; - pci_seg->rlookup_table[devid] = NULL; - memset(&amd_iommu_dev_table[devid], 0, sizeof(struct dev_table_entry)); + memset(&dev_table[devid], 0, sizeof(struct dev_table_entry)); setup_aliases(iommu, dev); } From ce23b3de523eb97c93054bfeb6c2669343a807f9 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 6 Jul 2022 17:08:13 +0530 Subject: [PATCH 23/34] iommu/amd: Update dump_dte_entry commit 4cc053d7aea778a959a9300e8acff7854de1be7f upstream. Start using per PCI segment device table instead of global device table. Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20220706113825.25582-24-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: Malathi Signed-off-by: PvsNarasimha --- drivers/iommu/amd/iommu.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index cba5ebb46e71c..c6c8725d4df74 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -450,13 +450,13 @@ static void amd_iommu_uninit_device(struct device *dev) * ****************************************************************************/ -static void dump_dte_entry(u16 devid) +static void dump_dte_entry(struct amd_iommu *iommu, u16 devid) { int i; + struct dev_table_entry *dev_table = get_dev_table(iommu); for (i = 0; i < 4; ++i) - pr_err("DTE[%d]: %016llx\n", i, - amd_iommu_dev_table[devid].data[i]); + pr_err("DTE[%d]: %016llx\n", i, dev_table[devid].data[i]); } static void dump_command(unsigned long phys_addr) @@ -596,7 +596,7 @@ static void iommu_print_event(struct amd_iommu *iommu, void *__evt) dev_err(dev, "Event logged [ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n", PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), pasid, address, flags); - dump_dte_entry(devid); + dump_dte_entry(iommu, devid); break; case EVENT_TYPE_DEV_TAB_ERR: dev_err(dev, "Event logged [DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x " From 3c978f6b4c9c161c63e9c462f4ea285d74529e3e Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 6 Jul 2022 17:08:14 +0530 Subject: [PATCH 24/34] iommu/amd: Update set_dte_irq_entry commit c7d311247b1b03906bc54d578db53f0bc2112674 upstream. Start using per PCI segment device table instead of global device table. Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20220706113825.25582-25-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: Malathi Signed-off-by: PvsNarasimha --- drivers/iommu/amd/iommu.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index c6c8725d4df74..bf4b83ff112ef 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2738,18 +2738,20 @@ EXPORT_SYMBOL(amd_iommu_device_info); static struct irq_chip amd_ir_chip; static DEFINE_SPINLOCK(iommu_table_lock); -static void set_dte_irq_entry(u16 devid, struct irq_remap_table *table) +static void set_dte_irq_entry(struct amd_iommu *iommu, u16 devid, + struct irq_remap_table *table) { u64 dte; + struct dev_table_entry *dev_table = get_dev_table(iommu); - dte = amd_iommu_dev_table[devid].data[2]; + dte = dev_table[devid].data[2]; dte &= ~DTE_IRQ_PHYS_ADDR_MASK; dte |= iommu_virt_to_phys(table->table); dte |= DTE_IRQ_REMAP_INTCTL; dte |= DTE_INTTABLEN; dte |= DTE_IRQ_REMAP_ENABLE; - amd_iommu_dev_table[devid].data[2] = dte; + dev_table[devid].data[2] = dte; } static struct irq_remap_table *get_irq_table(struct amd_iommu *iommu, u16 devid) @@ -2800,7 +2802,7 @@ static void set_remap_table_entry(struct amd_iommu *iommu, u16 devid, struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; pci_seg->irq_lookup_table[devid] = table; - set_dte_irq_entry(devid, table); + set_dte_irq_entry(iommu, devid, table); iommu_flush_dte(iommu, devid); } @@ -2816,8 +2818,7 @@ static int set_remap_table_entry_alias(struct pci_dev *pdev, u16 alias, pci_seg = iommu->pci_seg; pci_seg->irq_lookup_table[alias] = table; - set_dte_irq_entry(alias, table); - + set_dte_irq_entry(iommu, alias, table); iommu_flush_dte(pci_seg->rlookup_table[alias], alias); return 0; From af5c99c35484a6b6f7437b5310835eeb80eb7b15 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 6 Jul 2022 17:08:15 +0530 Subject: [PATCH 25/34] iommu/amd: Update (un)init_device_table_dma() commit 1ab5a15334529d3980a85abb2e06498c8e5ac8cc upstream. Include struct amd_iommu_pci_seg as a function parameter since we need to access per PCI segment device table. Co-developed-by: Vasant Hegde Signed-off-by: Vasant Hegde Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20220706113825.25582-26-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: Malathi Signed-off-by: PvsNarasimha --- drivers/iommu/amd/init.c | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 62ec5725302ba..4af819e675df8 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -241,7 +241,7 @@ static enum iommu_init_state init_state = IOMMU_START_STATE; static int amd_iommu_enable_interrupts(void); static int __init iommu_go_to_state(enum iommu_init_state state); -static void init_device_table_dma(void); +static void init_device_table_dma(struct amd_iommu_pci_seg *pci_seg); static bool amd_iommu_pre_enabled = true; @@ -2141,6 +2141,7 @@ static void print_iommu_info(void) static int __init amd_iommu_init_pci(void) { struct amd_iommu *iommu; + struct amd_iommu_pci_seg *pci_seg; int ret; for_each_iommu(iommu) { @@ -2164,7 +2165,8 @@ static int __init amd_iommu_init_pci(void) */ ret = amd_iommu_init_api(); - init_device_table_dma(); + for_each_pci_segment(pci_seg) + init_device_table_dma(pci_seg); for_each_iommu(iommu) iommu_flush_all_caches(iommu); @@ -2526,9 +2528,13 @@ static int __init init_memory_definitions(struct acpi_table_header *table) /* * Init the device table to not allow DMA access for devices */ -static void init_device_table_dma(void) +static void init_device_table_dma(struct amd_iommu_pci_seg *pci_seg) { u32 devid; + struct dev_table_entry *dev_table = pci_seg->dev_table; + + if (dev_table == NULL) + return; for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) { set_dev_entry_bit(devid, DEV_ENTRY_VALID); @@ -2536,13 +2542,17 @@ static void init_device_table_dma(void) } } -static void __init uninit_device_table_dma(void) +static void __init uninit_device_table_dma(struct amd_iommu_pci_seg *pci_seg) { u32 devid; + struct dev_table_entry *dev_table = pci_seg->dev_table; + + if (dev_table == NULL) + return; for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) { - amd_iommu_dev_table[devid].data[0] = 0ULL; - amd_iommu_dev_table[devid].data[1] = 0ULL; + dev_table[devid].data[0] = 0ULL; + dev_table[devid].data[1] = 0ULL; } } @@ -3165,8 +3175,11 @@ static int __init state_next(void) free_iommu_resources(); } else { struct amd_iommu *iommu; + struct amd_iommu_pci_seg *pci_seg; + + for_each_pci_segment(pci_seg) + uninit_device_table_dma(pci_seg); - uninit_device_table_dma(); for_each_iommu(iommu) iommu_flush_all_caches(iommu); } From 9f97b12cf1485b33d6cd3c903fa779383baaa133 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 6 Jul 2022 17:08:16 +0530 Subject: [PATCH 26/34] iommu/amd: Update set_dev_entry_bit() and get_dev_entry_bit() commit 56fb79514c52947107001ff9313870d76c4c8008 upstream. To include a pointer to per PCI segment device table. Also include struct amd_iommu as one of the function parameter to amd_iommu_apply_erratum_63() since it is needed when setting up DTE. Co-developed-by: Vasant Hegde Signed-off-by: Vasant Hegde Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20220706113825.25582-27-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: chaithanyaLagisetty Signed-off-by: PvsNarasimha --- drivers/iommu/amd/amd_iommu.h | 2 +- drivers/iommu/amd/init.c | 59 +++++++++++++++++++++++------------ drivers/iommu/amd/iommu.c | 2 +- 3 files changed, 41 insertions(+), 22 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 53f554a11b774..f4033825312f3 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -13,7 +13,7 @@ extern irqreturn_t amd_iommu_int_thread(int irq, void *data); extern irqreturn_t amd_iommu_int_handler(int irq, void *data); -extern void amd_iommu_apply_erratum_63(u16 devid); +extern void amd_iommu_apply_erratum_63(struct amd_iommu *iommu, u16 devid); extern void amd_iommu_restart_event_logging(struct amd_iommu *iommu); extern void amd_iommu_restart_ga_log(struct amd_iommu *iommu); extern void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu); diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 4af819e675df8..31c6af6042b8c 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1017,22 +1017,37 @@ static void iommu_enable_gt(struct amd_iommu *iommu) } /* sets a specific bit in the device table entry. */ -static void set_dev_entry_bit(u16 devid, u8 bit) +static void __set_dev_entry_bit(struct dev_table_entry *dev_table, + u16 devid, u8 bit) { int i = (bit >> 6) & 0x03; int _bit = bit & 0x3f; - amd_iommu_dev_table[devid].data[i] |= (1UL << _bit); + dev_table[devid].data[i] |= (1UL << _bit); } -static int get_dev_entry_bit(u16 devid, u8 bit) +static void set_dev_entry_bit(struct amd_iommu *iommu, u16 devid, u8 bit) +{ + struct dev_table_entry *dev_table = get_dev_table(iommu); + + return __set_dev_entry_bit(dev_table, devid, bit); +} + +static int __get_dev_entry_bit(struct dev_table_entry *dev_table, + u16 devid, u8 bit) { int i = (bit >> 6) & 0x03; int _bit = bit & 0x3f; - return (amd_iommu_dev_table[devid].data[i] & (1UL << _bit)) >> _bit; + return (dev_table[devid].data[i] & (1UL << _bit)) >> _bit; } +static int get_dev_entry_bit(struct amd_iommu *iommu, u16 devid, u8 bit) +{ + struct dev_table_entry *dev_table = get_dev_table(iommu); + + return __get_dev_entry_bit(dev_table, devid, bit); +} static bool __copy_device_table(struct amd_iommu *iommu) { @@ -1149,15 +1164,15 @@ static bool copy_device_table(void) return true; } -void amd_iommu_apply_erratum_63(u16 devid) +void amd_iommu_apply_erratum_63(struct amd_iommu *iommu, u16 devid) { int sysmgt; - sysmgt = get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1) | - (get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2) << 1); + sysmgt = get_dev_entry_bit(iommu, devid, DEV_ENTRY_SYSMGT1) | + (get_dev_entry_bit(iommu, devid, DEV_ENTRY_SYSMGT2) << 1); if (sysmgt == 0x01) - set_dev_entry_bit(devid, DEV_ENTRY_IW); + set_dev_entry_bit(iommu, devid, DEV_ENTRY_IW); } /* Writes the specific IOMMU for a device into the rlookup table */ @@ -1174,21 +1189,21 @@ static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu, u16 devid, u32 flags, u32 ext_flags) { if (flags & ACPI_DEVFLAG_INITPASS) - set_dev_entry_bit(devid, DEV_ENTRY_INIT_PASS); + set_dev_entry_bit(iommu, devid, DEV_ENTRY_INIT_PASS); if (flags & ACPI_DEVFLAG_EXTINT) - set_dev_entry_bit(devid, DEV_ENTRY_EINT_PASS); + set_dev_entry_bit(iommu, devid, DEV_ENTRY_EINT_PASS); if (flags & ACPI_DEVFLAG_NMI) - set_dev_entry_bit(devid, DEV_ENTRY_NMI_PASS); + set_dev_entry_bit(iommu, devid, DEV_ENTRY_NMI_PASS); if (flags & ACPI_DEVFLAG_SYSMGT1) - set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1); + set_dev_entry_bit(iommu, devid, DEV_ENTRY_SYSMGT1); if (flags & ACPI_DEVFLAG_SYSMGT2) - set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2); + set_dev_entry_bit(iommu, devid, DEV_ENTRY_SYSMGT2); if (flags & ACPI_DEVFLAG_LINT0) - set_dev_entry_bit(devid, DEV_ENTRY_LINT0_PASS); + set_dev_entry_bit(iommu, devid, DEV_ENTRY_LINT0_PASS); if (flags & ACPI_DEVFLAG_LINT1) - set_dev_entry_bit(devid, DEV_ENTRY_LINT1_PASS); + set_dev_entry_bit(iommu, devid, DEV_ENTRY_LINT1_PASS); - amd_iommu_apply_erratum_63(devid); + amd_iommu_apply_erratum_63(iommu, devid); set_iommu_for_device(iommu, devid); } @@ -2537,8 +2552,8 @@ static void init_device_table_dma(struct amd_iommu_pci_seg *pci_seg) return; for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) { - set_dev_entry_bit(devid, DEV_ENTRY_VALID); - set_dev_entry_bit(devid, DEV_ENTRY_TRANSLATION); + __set_dev_entry_bit(dev_table, devid, DEV_ENTRY_VALID); + __set_dev_entry_bit(dev_table, devid, DEV_ENTRY_TRANSLATION); } } @@ -2558,13 +2573,17 @@ static void __init uninit_device_table_dma(struct amd_iommu_pci_seg *pci_seg) static void init_device_table(void) { + struct amd_iommu_pci_seg *pci_seg; u32 devid; if (!amd_iommu_irq_remap) return; - for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) - set_dev_entry_bit(devid, DEV_ENTRY_IRQ_TBL_EN); + for_each_pci_segment(pci_seg) { + for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) + __set_dev_entry_bit(pci_seg->dev_table, + devid, DEV_ENTRY_IRQ_TBL_EN); + } } static void iommu_init_flags(struct amd_iommu *iommu) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index bf4b83ff112ef..44f37707bc5a4 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1622,7 +1622,7 @@ static void clear_dte_entry(struct amd_iommu *iommu, u16 devid) dev_table[devid].data[0] = DTE_FLAG_V | DTE_FLAG_TV; dev_table[devid].data[1] &= DTE_FLAG_MASK; - amd_iommu_apply_erratum_63(devid); + amd_iommu_apply_erratum_63(iommu, devid); } static void do_attach(struct iommu_dev_data *dev_data, From 4451a139f8871842b52811d78754c108da8d3b4c Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 6 Jul 2022 17:08:17 +0530 Subject: [PATCH 27/34] iommu/amd: Remove global amd_iommu_[dev_table/alias_table/last_bdf] commit 401360ec98c61bf91a107f292bfbfc9245795eea upstream. Replace them with per PCI segment device table. Also remove dev_table_size, alias_table_size, amd_iommu_last_bdf variables. Co-developed-by: Vasant Hegde Signed-off-by: Vasant Hegde Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20220706113825.25582-28-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: chaithanyaLagisetty Signed-off-by: PvsNarasimha --- drivers/iommu/amd/amd_iommu_types.h | 15 ----- drivers/iommu/amd/init.c | 89 +++++------------------------ drivers/iommu/amd/iommu.c | 18 ++++-- 3 files changed, 27 insertions(+), 95 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index f48f3122c1a7e..6f3bd5484e883 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -832,24 +832,9 @@ struct unity_map_entry { * Data structures for device handling */ -/* - * Device table used by hardware. Read and write accesses by software are - * locked with the amd_iommu_pd_table lock. - */ -extern struct dev_table_entry *amd_iommu_dev_table; - -/* - * Alias table to find requestor ids to device ids. Not locked because only - * read on runtime. - */ -extern u16 *amd_iommu_alias_table; - /* size of the dma_ops aperture as power of 2 */ extern unsigned amd_iommu_aperture_order; -/* largest PCI device id we expect translation requests for */ -extern u16 amd_iommu_last_bdf; - /* allocation bitmap for domain ids */ extern unsigned long *amd_iommu_pd_alloc_bitmap; diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 31c6af6042b8c..2f95e4d4bf561 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -164,9 +164,6 @@ static bool amd_iommu_force_enable __initdata; static bool amd_iommu_irtcachedis; static int amd_iommu_target_ivhd_type; -u16 amd_iommu_last_bdf; /* largest PCI device id we have - to handle */ - LIST_HEAD(amd_iommu_pci_seg_list); /* list of all PCI segments */ LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the system */ @@ -188,30 +185,12 @@ static bool amd_iommu_pc_present __read_mostly; bool amd_iommu_force_isolation __read_mostly; -/* - * Pointer to the device table which is shared by all AMD IOMMUs - * it is indexed by the PCI device id or the HT unit id and contains - * information about the domain the device belongs to as well as the - * page table root pointer. - */ -struct dev_table_entry *amd_iommu_dev_table; - -/* - * The alias table is a driver specific data structure which contains the - * mappings of the PCI device ids to the actual requestor ids on the IOMMU. - * More than one device can share the same requestor id. - */ -u16 *amd_iommu_alias_table; - /* * AMD IOMMU allows up to 2^16 different protection domains. This is a bitmap * to know which ones are already in use. */ unsigned long *amd_iommu_pd_alloc_bitmap; -static u32 dev_table_size; /* size of the device table */ -static u32 alias_table_size; /* size of the alias table */ - enum iommu_init_state { IOMMU_START_STATE, IOMMU_IVRS_DETECTED, @@ -266,16 +245,10 @@ static void init_translation_status(struct amd_iommu *iommu) iommu->flags |= AMD_IOMMU_FLAG_TRANS_PRE_ENABLED; } -static inline void update_last_devid(u16 devid) -{ - if (devid > amd_iommu_last_bdf) - amd_iommu_last_bdf = devid; -} - -static inline unsigned long tbl_size(int entry_size) +static inline unsigned long tbl_size(int entry_size, int last_bdf) { unsigned shift = PAGE_SHIFT + - get_order(((int)amd_iommu_last_bdf + 1) * entry_size); + get_order((last_bdf + 1) * entry_size); return 1UL << shift; } @@ -405,10 +378,11 @@ static void iommu_set_device_table(struct amd_iommu *iommu) { u64 entry; u32 dev_table_size = iommu->pci_seg->dev_table_size; + void *dev_table = (void *)get_dev_table(iommu); BUG_ON(iommu->mmio_base == NULL); - entry = iommu_virt_to_phys(amd_iommu_dev_table); + entry = iommu_virt_to_phys(dev_table); entry |= (dev_table_size >> 12) - 1; memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET, &entry, sizeof(entry)); @@ -561,14 +535,12 @@ static int __init find_last_devid_from_ivhd(struct ivhd_header *h) switch (dev->type) { case IVHD_DEV_ALL: /* Use maximum BDF value for DEV_ALL */ - update_last_devid(0xffff); return 0xffff; case IVHD_DEV_SELECT: case IVHD_DEV_RANGE_END: case IVHD_DEV_ALIAS: case IVHD_DEV_EXT_SELECT: /* all the above subfield types refer to device ids */ - update_last_devid(dev->devid); if (dev->devid > last_devid) last_devid = dev->devid; break; @@ -710,7 +682,7 @@ static int __init alloc_alias_table(struct amd_iommu_pci_seg *pci_seg) /* * let all alias entries point to itself */ - for (i = 0; i <= amd_iommu_last_bdf; ++i) + for (i = 0; i <= pci_seg->last_bdf; ++i) pci_seg->alias_table[i] = i; return 0; @@ -1099,7 +1071,7 @@ static bool __copy_device_table(struct amd_iommu *iommu) return false; } - for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) { + for (devid = 0; devid <= pci_seg->last_bdf; ++devid) { pci_seg->old_dev_tbl_cpy[devid] = old_devtb[devid]; dom_id = old_devtb[devid].data[1] & DEV_DOMID_MASK; dte_v = old_devtb[devid].data[0] & DTE_FLAG_V; @@ -1175,12 +1147,6 @@ void amd_iommu_apply_erratum_63(struct amd_iommu *iommu, u16 devid) set_dev_entry_bit(iommu, devid, DEV_ENTRY_IW); } -/* Writes the specific IOMMU for a device into the rlookup table */ -static void __init set_iommu_for_device(struct amd_iommu *iommu, u16 devid) -{ - iommu->pci_seg->rlookup_table[devid] = iommu; -} - /* * This function takes the device specific flags read from the ACPI * table and sets up the device table entry with that information @@ -1205,7 +1171,7 @@ static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu, amd_iommu_apply_erratum_63(iommu, devid); - set_iommu_for_device(iommu, devid); + amd_iommu_set_rlookup_table(iommu, devid); } int __init add_special_device(u8 type, u8 id, u16 *devid, bool cmd_line) @@ -1365,7 +1331,7 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, DUMP_printk(" DEV_ALL\t\t\tflags: %02x\n", e->flags); - for (dev_i = 0; dev_i <= amd_iommu_last_bdf; ++dev_i) + for (dev_i = 0; dev_i <= pci_seg->last_bdf; ++dev_i) set_dev_entry_from_acpi(iommu, dev_i, e->flags, 0); break; case IVHD_DEV_SELECT: @@ -1609,9 +1575,9 @@ static struct amd_iommu_pci_seg *__init alloc_pci_segment(u16 id, pci_seg->last_bdf = last_bdf; DUMP_printk("PCI segment : 0x%0x, last bdf : 0x%04x\n", id, last_bdf); - pci_seg->dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE); - pci_seg->alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE); - pci_seg->rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE); + pci_seg->dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE, last_bdf); + pci_seg->alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE, last_bdf); + pci_seg->rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE, last_bdf); pci_seg->id = id; init_llist_head(&pci_seg->dev_data_list); @@ -2484,7 +2450,7 @@ static int __init init_unity_map_range(struct ivmd_header *m, case ACPI_IVMD_TYPE_ALL: s = "IVMD_TYPE_ALL\t\t"; e->devid_start = 0; - e->devid_end = amd_iommu_last_bdf; + e->devid_end = pci_seg->last_bdf; break; case ACPI_IVMD_TYPE_RANGE: s = "IVMD_TYPE_RANGE\t\t"; @@ -2551,7 +2517,7 @@ static void init_device_table_dma(struct amd_iommu_pci_seg *pci_seg) if (dev_table == NULL) return; - for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) { + for (devid = 0; devid <= pci_seg->last_bdf; ++devid) { __set_dev_entry_bit(dev_table, devid, DEV_ENTRY_VALID); __set_dev_entry_bit(dev_table, devid, DEV_ENTRY_TRANSLATION); } @@ -2565,7 +2531,7 @@ static void __init uninit_device_table_dma(struct amd_iommu_pci_seg *pci_seg) if (dev_table == NULL) return; - for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) { + for (devid = 0; devid <= pci_seg->last_bdf; ++devid) { dev_table[devid].data[0] = 0ULL; dev_table[devid].data[1] = 0ULL; } @@ -2580,7 +2546,7 @@ static void init_device_table(void) return; for_each_pci_segment(pci_seg) { - for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) + for (devid = 0; devid <= pci_seg->last_bdf; ++devid) __set_dev_entry_bit(pci_seg->dev_table, devid, DEV_ENTRY_IRQ_TBL_EN); } @@ -2853,14 +2819,6 @@ static void __init free_iommu_resources(void) kmem_cache_destroy(amd_iommu_irq_cache); amd_iommu_irq_cache = NULL; - free_pages((unsigned long)amd_iommu_alias_table, - get_order(alias_table_size)); - amd_iommu_alias_table = NULL; - - free_pages((unsigned long)amd_iommu_dev_table, - get_order(dev_table_size)); - amd_iommu_dev_table = NULL; - free_iommu_all(); free_pci_segments(); } @@ -2989,25 +2947,8 @@ static int __init early_amd_iommu_init(void) amd_iommu_target_ivhd_type = get_highest_supported_ivhd_type(ivrs_base); DUMP_printk("Using IVHD type %#x\n", amd_iommu_target_ivhd_type); - dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE); - alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE); - /* Device table - directly used by all IOMMUs */ ret = -ENOMEM; - amd_iommu_dev_table = (void *)__get_free_pages( - GFP_KERNEL | __GFP_ZERO | GFP_DMA32, - get_order(dev_table_size)); - if (amd_iommu_dev_table == NULL) - goto out; - - /* - * Alias table - map PCI Bus/Dev/Func to Bus/Dev/Func the - * IOMMU see for that device - */ - amd_iommu_alias_table = (void *)__get_free_pages(GFP_KERNEL, - get_order(alias_table_size)); - if (amd_iommu_alias_table == NULL) - goto out; amd_iommu_pd_alloc_bitmap = (void *)__get_free_pages( GFP_KERNEL | __GFP_ZERO, diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 44f37707bc5a4..9345db52bd7d6 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -229,6 +229,7 @@ static struct iommu_dev_data *search_dev_data(struct amd_iommu *iommu, u16 devid static int clone_alias(struct pci_dev *pdev, u16 alias, void *data) { struct amd_iommu *iommu; + struct dev_table_entry *dev_table; u16 devid = pci_dev_id(pdev); if (devid == alias) @@ -239,9 +240,10 @@ static int clone_alias(struct pci_dev *pdev, u16 alias, void *data) return 0; amd_iommu_set_rlookup_table(iommu, alias); - memcpy(amd_iommu_dev_table[alias].data, - amd_iommu_dev_table[devid].data, - sizeof(amd_iommu_dev_table[alias].data)); + dev_table = get_dev_table(iommu); + memcpy(dev_table[alias].data, + dev_table[devid].data, + sizeof(dev_table[alias].data)); return 0; } @@ -355,6 +357,8 @@ static bool pci_iommuv2_capable(struct pci_dev *pdev) */ static bool check_device(struct device *dev) { + struct amd_iommu_pci_seg *pci_seg; + struct amd_iommu *iommu; int devid; if (!dev) @@ -364,11 +368,13 @@ static bool check_device(struct device *dev) if (devid < 0) return false; - /* Out of our scope? */ - if (devid > amd_iommu_last_bdf) + iommu = rlookup_amd_iommu(dev); + if (!iommu) return false; - if (rlookup_amd_iommu(dev) == NULL) + /* Out of our scope? */ + pci_seg = iommu->pci_seg; + if ((devid & 0xffff) > pci_seg->last_bdf) return false; return true; From c3a95fa954c7919137d040e202bc03b25e90ed0c Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Wed, 6 Jul 2022 17:08:18 +0530 Subject: [PATCH 28/34] iommu/amd: Flush upto last_bdf only commit a3cf6ab35751ea77a2bd47636dca761e835b282d upstream. Fix amd_iommu_flush_dte_all() and amd_iommu_flush_tlb_all() to flush upto last_bdf only. Co-developed-by: Suravee Suthikulpanit Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20220706113825.25582-29-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: chaithanyaLagisetty Signed-off-by: PvsNarasimha --- drivers/iommu/amd/iommu.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 9345db52bd7d6..bacd3fab7251e 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1177,8 +1177,9 @@ static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid) static void amd_iommu_flush_dte_all(struct amd_iommu *iommu) { u32 devid; + u16 last_bdf = iommu->pci_seg->last_bdf; - for (devid = 0; devid <= 0xffff; ++devid) + for (devid = 0; devid <= last_bdf; ++devid) iommu_flush_dte(iommu, devid); iommu_completion_wait(iommu); @@ -1191,8 +1192,9 @@ static void amd_iommu_flush_dte_all(struct amd_iommu *iommu) static void amd_iommu_flush_tlb_all(struct amd_iommu *iommu) { u32 dom_id; + u16 last_bdf = iommu->pci_seg->last_bdf; - for (dom_id = 0; dom_id <= 0xffff; ++dom_id) { + for (dom_id = 0; dom_id <= last_bdf; ++dom_id) { struct iommu_cmd cmd; build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, dom_id, 1); @@ -1235,11 +1237,12 @@ static void iommu_flush_irt(struct amd_iommu *iommu, u16 devid) static void amd_iommu_flush_irt_all(struct amd_iommu *iommu) { u32 devid; + u16 last_bdf = iommu->pci_seg->last_bdf; if (iommu->irtcachedis_enabled) return; - for (devid = 0; devid <= MAX_DEV_TABLE_ENTRIES; devid++) + for (devid = 0; devid <= last_bdf; devid++) iommu_flush_irt(iommu, devid); iommu_completion_wait(iommu); From d238ec701fe0205a97922e4855cd7944288bb4e6 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 6 Jul 2022 17:08:19 +0530 Subject: [PATCH 29/34] iommu/amd: Introduce get_device_sbdf_id() helper function commit bf87972ca664863dcfd38ab581589c1d87677cb6 upstream. Current get_device_id() only provide 16-bit PCI device ID (i.e. BDF). With multiple PCI segment support, we need to extend the helper function to include PCI segment ID. So, introduce a new helper function get_device_sbdf_id() to replace the current get_pci_device_id(). Co-developed-by: Vasant Hegde Signed-off-by: Vasant Hegde Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20220706113825.25582-30-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: chaithanyaLagisetty Signed-off-by: PvsNarasimha --- drivers/iommu/amd/amd_iommu.h | 7 ++++ drivers/iommu/amd/amd_iommu_types.h | 2 + drivers/iommu/amd/iommu.c | 58 ++++++++++++++--------------- 3 files changed, 38 insertions(+), 29 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index f4033825312f3..14c90f2c7d986 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -117,6 +117,13 @@ void amd_iommu_domain_clr_pt_root(struct protection_domain *domain) amd_iommu_domain_set_pt_root(domain, 0); } +static inline int get_pci_sbdf_id(struct pci_dev *pdev) +{ + int seg = pci_domain_nr(pdev->bus); + u16 devid = pci_dev_id(pdev); + + return PCI_SEG_DEVID_TO_SBDF(seg, devid); +} extern bool translation_pre_enabled(struct amd_iommu *iommu); extern bool amd_iommu_is_attach_deferred(struct iommu_domain *domain, diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 6f3bd5484e883..46f96357117e4 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -451,6 +451,8 @@ extern struct kmem_cache *amd_iommu_irq_cache; #define PCI_SBDF_TO_SEGID(sbdf) (((sbdf) >> 16) & 0xffff) #define PCI_SBDF_TO_DEVID(sbdf) ((sbdf) & 0xffff) +#define PCI_SEG_DEVID_TO_SBDF(seg, devid) ((((u32)(seg) & 0xffff) << 16) | \ + ((devid) & 0xffff)) /* Make iterating over all pci segment easier */ #define for_each_pci_segment(pci_seg) \ diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index bacd3fab7251e..0e04c76202c6b 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -91,13 +91,6 @@ static void detach_device(struct device *dev); * ****************************************************************************/ -static inline u16 get_pci_device_id(struct device *dev) -{ - struct pci_dev *pdev = to_pci_dev(dev); - - return pci_dev_id(pdev); -} - static inline int get_acpihid_device_id(struct device *dev, struct acpihid_map_entry **entry) { @@ -118,16 +111,16 @@ static inline int get_acpihid_device_id(struct device *dev, return -EINVAL; } -static inline int get_device_id(struct device *dev) +static inline int get_device_sbdf_id(struct device *dev) { - int devid; + int sbdf; if (dev_is_pci(dev)) - devid = get_pci_device_id(dev); + sbdf = get_pci_sbdf_id(to_pci_dev(dev)); else - devid = get_acpihid_device_id(dev, NULL); + sbdf = get_acpihid_device_id(dev, NULL); - return devid; + return sbdf; } struct dev_table_entry *get_dev_table(struct amd_iommu *iommu) @@ -181,9 +174,11 @@ static struct amd_iommu *__rlookup_amd_iommu(u16 seg, u16 devid) static struct amd_iommu *rlookup_amd_iommu(struct device *dev) { u16 seg = get_device_segment(dev); - u16 devid = get_device_id(dev); + int devid = get_device_sbdf_id(dev); - return __rlookup_amd_iommu(seg, devid); + if (devid < 0) + return NULL; + return __rlookup_amd_iommu(seg, PCI_SBDF_TO_DEVID(devid)); } static struct protection_domain *to_pdomain(struct iommu_domain *dom) @@ -359,14 +354,15 @@ static bool check_device(struct device *dev) { struct amd_iommu_pci_seg *pci_seg; struct amd_iommu *iommu; - int devid; + int devid, sbdf; if (!dev) return false; - devid = get_device_id(dev); - if (devid < 0) + sbdf = get_device_sbdf_id(dev); + if (sbdf < 0) return false; + devid = PCI_SBDF_TO_DEVID(sbdf); iommu = rlookup_amd_iommu(dev); if (!iommu) @@ -374,7 +370,7 @@ static bool check_device(struct device *dev) /* Out of our scope? */ pci_seg = iommu->pci_seg; - if ((devid & 0xffff) > pci_seg->last_bdf) + if (devid > pci_seg->last_bdf) return false; return true; @@ -383,15 +379,16 @@ static bool check_device(struct device *dev) static int iommu_init_device(struct amd_iommu *iommu, struct device *dev) { struct iommu_dev_data *dev_data; - int devid; + int devid, sbdf; if (dev_iommu_priv_get(dev)) return 0; - devid = get_device_id(dev); - if (devid < 0) - return devid; + sbdf = get_device_sbdf_id(dev); + if (sbdf < 0) + return sbdf; + devid = PCI_SBDF_TO_DEVID(sbdf); dev_data = find_dev_data(iommu, devid); if (!dev_data) return -ENOMEM; @@ -419,12 +416,13 @@ static void iommu_ignore_device(struct amd_iommu *iommu, struct device *dev) { struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; struct dev_table_entry *dev_table = get_dev_table(iommu); - int devid; + int devid, sbdf; - devid = (get_device_id(dev)) & 0xffff; - if (devid < 0) + sbdf = get_device_sbdf_id(dev); + if (sbdf < 0) return; + devid = PCI_SBDF_TO_DEVID(sbdf); pci_seg->rlookup_table[devid] = NULL; memset(&dev_table[devid], 0, sizeof(struct dev_table_entry)); @@ -2273,11 +2271,13 @@ static void amd_iommu_get_resv_regions(struct device *dev, struct unity_map_entry *entry; struct amd_iommu *iommu; struct amd_iommu_pci_seg *pci_seg; - int devid; + int devid, sbdf; - devid = get_device_id(dev); - if (devid < 0) + sbdf = get_device_sbdf_id(dev); + if (sbdf < 0) return; + + devid = PCI_SBDF_TO_DEVID(sbdf); iommu = rlookup_amd_iommu(dev); if (!iommu) return; @@ -3167,7 +3167,7 @@ static int get_devid(struct irq_alloc_info *info) return get_hpet_devid(info->devid); case X86_IRQ_ALLOC_TYPE_PCI_MSI: case X86_IRQ_ALLOC_TYPE_PCI_MSIX: - return get_device_id(msi_desc_to_dev(info->desc)); + return get_device_sbdf_id(msi_desc_to_dev(info->desc)); default: WARN_ON_ONCE(1); return -1; From fdb0a394212b03f741d47944e1983a7ba6cdcb96 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 6 Jul 2022 17:08:20 +0530 Subject: [PATCH 30/34] iommu/amd: Include PCI segment ID when initialize IOMMU commit a45627baa7bc56d633ade47d5503bd2be53f1baa upstream. Extend current device ID variables to 32-bit to include the 16-bit segment ID when parsing device information from IVRS table to initialize each IOMMU. Co-developed-by: Vasant Hegde Signed-off-by: Vasant Hegde Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20220706113825.25582-31-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: chaithanyaLagisetty Signed-off-by: PvsNarasimha --- drivers/iommu/amd/amd_iommu.h | 2 +- drivers/iommu/amd/amd_iommu_types.h | 6 ++-- drivers/iommu/amd/init.c | 56 +++++++++++++++-------------- drivers/iommu/amd/quirks.c | 4 +-- 4 files changed, 35 insertions(+), 33 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index 14c90f2c7d986..134d2a9c0f7ed 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -128,7 +128,7 @@ static inline int get_pci_sbdf_id(struct pci_dev *pdev) extern bool translation_pre_enabled(struct amd_iommu *iommu); extern bool amd_iommu_is_attach_deferred(struct iommu_domain *domain, struct device *dev); -extern int __init add_special_device(u8 type, u8 id, u16 *devid, +extern int __init add_special_device(u8 type, u8 id, u32 *devid, bool cmd_line); #ifdef CONFIG_DMI diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 46f96357117e4..d9fdf01ea0c17 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -742,8 +742,8 @@ struct acpihid_map_entry { struct list_head list; u8 uid[ACPIHID_UID_LEN]; u8 hid[ACPIHID_HID_LEN]; - u16 devid; - u16 root_devid; + u32 devid; + u32 root_devid; bool cmd_line; struct iommu_group *group; }; @@ -751,7 +751,7 @@ struct acpihid_map_entry { struct devid_map { struct list_head list; u8 id; - u16 devid; + u32 devid; bool cmd_line; }; diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 2f95e4d4bf561..f661909a622ef 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1174,7 +1174,7 @@ static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu, amd_iommu_set_rlookup_table(iommu, devid); } -int __init add_special_device(u8 type, u8 id, u16 *devid, bool cmd_line) +int __init add_special_device(u8 type, u8 id, u32 *devid, bool cmd_line) { struct devid_map *entry; struct list_head *list; @@ -1211,7 +1211,7 @@ int __init add_special_device(u8 type, u8 id, u16 *devid, bool cmd_line) return 0; } -static int __init add_acpi_hid_device(u8 *hid, u8 *uid, u16 *devid, +static int __init add_acpi_hid_device(u8 *hid, u8 *uid, u32 *devid, bool cmd_line) { struct acpihid_map_entry *entry; @@ -1290,7 +1290,7 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, { u8 *p = (u8 *)h; u8 *end = p, flags = 0; - u16 devid = 0, devid_start = 0, devid_to = 0; + u16 devid = 0, devid_start = 0, devid_to = 0, seg_id; u32 dev_i, ext_flags = 0; bool alias = false; struct ivhd_entry *e; @@ -1326,6 +1326,8 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, while (p < end) { e = (struct ivhd_entry *)p; + seg_id = pci_seg->id; + switch (e->type) { case IVHD_DEV_ALL: @@ -1336,9 +1338,9 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, break; case IVHD_DEV_SELECT: - DUMP_printk(" DEV_SELECT\t\t\t devid: %02x:%02x.%x " + DUMP_printk(" DEV_SELECT\t\t\t devid: %04x:%02x:%02x.%x " "flags: %02x\n", - PCI_BUS_NUM(e->devid), + seg_id, PCI_BUS_NUM(e->devid), PCI_SLOT(e->devid), PCI_FUNC(e->devid), e->flags); @@ -1349,8 +1351,8 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, case IVHD_DEV_SELECT_RANGE_START: DUMP_printk(" DEV_SELECT_RANGE_START\t " - "devid: %02x:%02x.%x flags: %02x\n", - PCI_BUS_NUM(e->devid), + "devid: %04x:%02x:%02x.%x flags: %02x\n", + seg_id, PCI_BUS_NUM(e->devid), PCI_SLOT(e->devid), PCI_FUNC(e->devid), e->flags); @@ -1362,9 +1364,9 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, break; case IVHD_DEV_ALIAS: - DUMP_printk(" DEV_ALIAS\t\t\t devid: %02x:%02x.%x " + DUMP_printk(" DEV_ALIAS\t\t\t devid: %04x:%02x:%02x.%x " "flags: %02x devid_to: %02x:%02x.%x\n", - PCI_BUS_NUM(e->devid), + seg_id, PCI_BUS_NUM(e->devid), PCI_SLOT(e->devid), PCI_FUNC(e->devid), e->flags, @@ -1381,13 +1383,13 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, case IVHD_DEV_ALIAS_RANGE: DUMP_printk(" DEV_ALIAS_RANGE\t\t " - "devid: %02x:%02x.%x flags: %02x " - "devid_to: %02x:%02x.%x\n", - PCI_BUS_NUM(e->devid), + "devid: %04x:%02x:%02x.%x flags: %02x " + "devid_to: %04x:%02x:%02x.%x\n", + seg_id, PCI_BUS_NUM(e->devid), PCI_SLOT(e->devid), PCI_FUNC(e->devid), e->flags, - PCI_BUS_NUM(e->ext >> 8), + seg_id, PCI_BUS_NUM(e->ext >> 8), PCI_SLOT(e->ext >> 8), PCI_FUNC(e->ext >> 8)); @@ -1399,9 +1401,9 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, break; case IVHD_DEV_EXT_SELECT: - DUMP_printk(" DEV_EXT_SELECT\t\t devid: %02x:%02x.%x " + DUMP_printk(" DEV_EXT_SELECT\t\t devid: %04x:%02x:%02x.%x " "flags: %02x ext: %08x\n", - PCI_BUS_NUM(e->devid), + seg_id, PCI_BUS_NUM(e->devid), PCI_SLOT(e->devid), PCI_FUNC(e->devid), e->flags, e->ext); @@ -1413,8 +1415,8 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, case IVHD_DEV_EXT_SELECT_RANGE: DUMP_printk(" DEV_EXT_SELECT_RANGE\t devid: " - "%02x:%02x.%x flags: %02x ext: %08x\n", - PCI_BUS_NUM(e->devid), + "%04x:%02x:%02x.%x flags: %02x ext: %08x\n", + seg_id, PCI_BUS_NUM(e->devid), PCI_SLOT(e->devid), PCI_FUNC(e->devid), e->flags, e->ext); @@ -1426,8 +1428,8 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, break; case IVHD_DEV_RANGE_END: - DUMP_printk(" DEV_RANGE_END\t\t devid: %02x:%02x.%x\n", - PCI_BUS_NUM(e->devid), + DUMP_printk(" DEV_RANGE_END\t\t devid: %04x:%02x:%02x.%x\n", + seg_id, PCI_BUS_NUM(e->devid), PCI_SLOT(e->devid), PCI_FUNC(e->devid)); @@ -1445,11 +1447,11 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, case IVHD_DEV_SPECIAL: { u8 handle, type; const char *var; - u16 devid; + u32 devid; int ret; handle = e->ext & 0xff; - devid = (e->ext >> 8) & 0xffff; + devid = PCI_SEG_DEVID_TO_SBDF(seg_id, (e->ext >> 8)); type = (e->ext >> 24) & 0xff; if (type == IVHD_SPECIAL_IOAPIC) @@ -1459,9 +1461,9 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, else var = "UNKNOWN"; - DUMP_printk(" DEV_SPECIAL(%s[%d])\t\tdevid: %02x:%02x.%x\n", + DUMP_printk(" DEV_SPECIAL(%s[%d])\t\tdevid: %04x:%02x:%02x.%x\n", var, (int)handle, - PCI_BUS_NUM(devid), + seg_id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid)); @@ -1479,7 +1481,7 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, break; } case IVHD_DEV_ACPI_HID: { - u16 devid; + u32 devid; u8 hid[ACPIHID_HID_LEN]; u8 uid[ACPIHID_UID_LEN]; int ret; @@ -1521,9 +1523,9 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, break; } - devid = e->devid; - DUMP_printk(" DEV_ACPI_HID(%s[%s])\t\tdevid: %02x:%02x.%x\n", - hid, uid, + devid = PCI_SEG_DEVID_TO_SBDF(seg_id, e->devid); + DUMP_printk(" DEV_ACPI_HID(%s[%s])\t\tdevid: %04x:%02x:%02x.%x\n", + hid, uid, seg_id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid)); diff --git a/drivers/iommu/amd/quirks.c b/drivers/iommu/amd/quirks.c index 5120ce4fdce32..79dbb8f33b473 100644 --- a/drivers/iommu/amd/quirks.c +++ b/drivers/iommu/amd/quirks.c @@ -15,7 +15,7 @@ struct ivrs_quirk_entry { u8 id; - u16 devid; + u32 devid; }; enum { @@ -49,7 +49,7 @@ static int __init ivrs_ioapic_quirk_cb(const struct dmi_system_id *d) const struct ivrs_quirk_entry *i; for (i = d->driver_data; i->id != 0 && i->devid != 0; i++) - add_special_device(IVHD_SPECIAL_IOAPIC, i->id, (u16 *)&i->devid, 0); + add_special_device(IVHD_SPECIAL_IOAPIC, i->id, (u32 *)&i->devid, 0); return 0; } From e390ab0734682314938bb7a652c9c900acbcedb5 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Wed, 6 Jul 2022 17:08:21 +0530 Subject: [PATCH 31/34] iommu/amd: Specify PCI segment ID when getting pci device commit e5670e1822cf2858e4c9133fc4e834a1d4a8a4d2 upstream. Upcoming AMD systems can have multiple PCI segments. Hence pass PCI segment ID to pci_get_domain_bus_and_slot() instead of '0'. Co-developed-by: Vasant Hegde Signed-off-by: Vasant Hegde Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20220706113825.25582-32-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: Arukonda Rahul Signed-off-by: PvsNarasimha --- drivers/iommu/amd/init.c | 6 ++++-- drivers/iommu/amd/iommu.c | 19 ++++++++++--------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index f661909a622ef..a514476b79ce2 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1987,7 +1987,8 @@ static int __init iommu_init_pci(struct amd_iommu *iommu) int cap_ptr = iommu->cap_ptr; int ret; - iommu->dev = pci_get_domain_bus_and_slot(0, PCI_BUS_NUM(iommu->devid), + iommu->dev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, + PCI_BUS_NUM(iommu->devid), iommu->devid & 0xff); if (!iommu->dev) return -ENODEV; @@ -2050,7 +2051,8 @@ static int __init iommu_init_pci(struct amd_iommu *iommu) int i, j; iommu->root_pdev = - pci_get_domain_bus_and_slot(0, iommu->dev->bus->number, + pci_get_domain_bus_and_slot(iommu->pci_seg->id, + iommu->dev->bus->number, PCI_DEVFN(0, 0)); /* diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 0e04c76202c6b..a6449fc325569 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -472,7 +472,7 @@ static void dump_command(unsigned long phys_addr) pr_err("CMD[%d]: %08x\n", i, cmd->data[i]); } -static void amd_iommu_report_rmp_hw_error(volatile u32 *event) +static void amd_iommu_report_rmp_hw_error(struct amd_iommu *iommu, volatile u32 *event) { struct iommu_dev_data *dev_data = NULL; int devid, vmg_tag, flags; @@ -484,7 +484,7 @@ static void amd_iommu_report_rmp_hw_error(volatile u32 *event) flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK; spa = ((u64)event[3] << 32) | (event[2] & 0xFFFFFFF8); - pdev = pci_get_domain_bus_and_slot(0, PCI_BUS_NUM(devid), + pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid), devid & 0xff); if (pdev) dev_data = dev_iommu_priv_get(&pdev->dev); @@ -504,7 +504,7 @@ static void amd_iommu_report_rmp_hw_error(volatile u32 *event) pci_dev_put(pdev); } -static void amd_iommu_report_rmp_fault(volatile u32 *event) +static void amd_iommu_report_rmp_fault(struct amd_iommu *iommu, volatile u32 *event) { struct iommu_dev_data *dev_data = NULL; int devid, flags_rmp, vmg_tag, flags; @@ -517,7 +517,7 @@ static void amd_iommu_report_rmp_fault(volatile u32 *event) flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK; gpa = ((u64)event[3] << 32) | event[2]; - pdev = pci_get_domain_bus_and_slot(0, PCI_BUS_NUM(devid), + pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid), devid & 0xff); if (pdev) dev_data = dev_iommu_priv_get(&pdev->dev); @@ -537,13 +537,14 @@ static void amd_iommu_report_rmp_fault(volatile u32 *event) pci_dev_put(pdev); } -static void amd_iommu_report_page_fault(u16 devid, u16 domain_id, +static void amd_iommu_report_page_fault(struct amd_iommu *iommu, + u16 devid, u16 domain_id, u64 address, int flags) { struct iommu_dev_data *dev_data = NULL; struct pci_dev *pdev; - pdev = pci_get_domain_bus_and_slot(0, PCI_BUS_NUM(devid), + pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid), devid & 0xff); if (pdev) dev_data = dev_iommu_priv_get(&pdev->dev); @@ -591,7 +592,7 @@ static void iommu_print_event(struct amd_iommu *iommu, void *__evt) } if (type == EVENT_TYPE_IO_FAULT) { - amd_iommu_report_page_fault(devid, pasid, address, flags); + amd_iommu_report_page_fault(iommu, devid, pasid, address, flags); return; } @@ -632,10 +633,10 @@ static void iommu_print_event(struct amd_iommu *iommu, void *__evt) pasid, address, flags); break; case EVENT_TYPE_RMP_FAULT: - amd_iommu_report_rmp_fault(event); + amd_iommu_report_rmp_fault(iommu, event); break; case EVENT_TYPE_RMP_HW_ERR: - amd_iommu_report_rmp_hw_error(event); + amd_iommu_report_rmp_hw_error(iommu, event); break; case EVENT_TYPE_INV_PPR_REQ: pasid = PPR_PASID(*((u64 *)__evt)); From 04a114819307ed6cbab1ce76e2a5dab9dc235f1d Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Wed, 6 Jul 2022 17:08:23 +0530 Subject: [PATCH 32/34] iommu/amd: Print PCI segment ID in error log messages commit b36a5b0f1cedbb2de2f527883d4eda8a5025591d upstream. Print pci segment ID along with bdf. Useful for debugging. Co-developed-by: Suravee Suthikulpaint Signed-off-by: Suravee Suthikulpaint Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20220706113825.25582-34-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: Arukonda Rahul Signed-off-by: PvsNarasimha --- drivers/iommu/amd/init.c | 10 +++++----- drivers/iommu/amd/iommu.c | 36 ++++++++++++++++++------------------ 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index a514476b79ce2..85e001743b8a5 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1876,11 +1876,11 @@ static int __init init_iommu_all(struct acpi_table_header *table) h = (struct ivhd_header *)p; if (*p == amd_iommu_target_ivhd_type) { - DUMP_printk("device: %02x:%02x.%01x cap: %04x " - "seg: %d flags: %01x info %04x\n", - PCI_BUS_NUM(h->devid), PCI_SLOT(h->devid), - PCI_FUNC(h->devid), h->cap_ptr, - h->pci_seg, h->flags, h->info); + DUMP_printk("device: %04x:%02x:%02x.%01x cap: %04x " + "flags: %01x info %04x\n", + h->pci_seg, PCI_BUS_NUM(h->devid), + PCI_SLOT(h->devid), PCI_FUNC(h->devid), + h->cap_ptr, h->flags, h->info); DUMP_printk(" mmio-addr: %016llx\n", h->mmio_phys); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index a6449fc325569..04bbe11eb3324 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -495,8 +495,8 @@ static void amd_iommu_report_rmp_hw_error(struct amd_iommu *iommu, volatile u32 vmg_tag, spa, flags); } } else { - pr_err_ratelimited("Event logged [RMP_HW_ERROR device=%02x:%02x.%x, vmg_tag=0x%04x, spa=0x%llx, flags=0x%04x]\n", - PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), + pr_err_ratelimited("Event logged [RMP_HW_ERROR device=%04x:%02x:%02x.%x, vmg_tag=0x%04x, spa=0x%llx, flags=0x%04x]\n", + iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), vmg_tag, spa, flags); } @@ -528,8 +528,8 @@ static void amd_iommu_report_rmp_fault(struct amd_iommu *iommu, volatile u32 *ev vmg_tag, gpa, flags_rmp, flags); } } else { - pr_err_ratelimited("Event logged [RMP_PAGE_FAULT device=%02x:%02x.%x, vmg_tag=0x%04x, gpa=0x%llx, flags_rmp=0x%04x, flags=0x%04x]\n", - PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), + pr_err_ratelimited("Event logged [RMP_PAGE_FAULT device=%04x:%02x:%02x.%x, vmg_tag=0x%04x, gpa=0x%llx, flags_rmp=0x%04x, flags=0x%04x]\n", + iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), vmg_tag, gpa, flags_rmp, flags); } @@ -555,8 +555,8 @@ static void amd_iommu_report_page_fault(struct amd_iommu *iommu, domain_id, address, flags); } } else { - pr_err_ratelimited("Event logged [IO_PAGE_FAULT device=%02x:%02x.%x domain=0x%04x address=0x%llx flags=0x%04x]\n", - PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), + pr_err_ratelimited("Event logged [IO_PAGE_FAULT device=%04x:%02x:%02x.%x domain=0x%04x address=0x%llx flags=0x%04x]\n", + iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), domain_id, address, flags); } @@ -598,20 +598,20 @@ static void iommu_print_event(struct amd_iommu *iommu, void *__evt) switch (type) { case EVENT_TYPE_ILL_DEV: - dev_err(dev, "Event logged [ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n", - PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), + dev_err(dev, "Event logged [ILLEGAL_DEV_TABLE_ENTRY device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n", + iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), pasid, address, flags); dump_dte_entry(iommu, devid); break; case EVENT_TYPE_DEV_TAB_ERR: - dev_err(dev, "Event logged [DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x " + dev_err(dev, "Event logged [DEV_TAB_HARDWARE_ERROR device=%04x:%02x:%02x.%x " "address=0x%llx flags=0x%04x]\n", - PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), + iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), address, flags); break; case EVENT_TYPE_PAGE_TAB_ERR: - dev_err(dev, "Event logged [PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x pasid=0x%04x address=0x%llx flags=0x%04x]\n", - PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), + dev_err(dev, "Event logged [PAGE_TAB_HARDWARE_ERROR device=%04x:%02x:%02x.%x pasid=0x%04x address=0x%llx flags=0x%04x]\n", + iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), pasid, address, flags); break; case EVENT_TYPE_ILL_CMD: @@ -623,13 +623,13 @@ static void iommu_print_event(struct amd_iommu *iommu, void *__evt) address, flags); break; case EVENT_TYPE_IOTLB_INV_TO: - dev_err(dev, "Event logged [IOTLB_INV_TIMEOUT device=%02x:%02x.%x address=0x%llx]\n", - PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), + dev_err(dev, "Event logged [IOTLB_INV_TIMEOUT device=%04x:%02x:%02x.%x address=0x%llx]\n", + iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), address); break; case EVENT_TYPE_INV_DEV_REQ: - dev_err(dev, "Event logged [INVALID_DEVICE_REQUEST device=%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n", - PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), + dev_err(dev, "Event logged [INVALID_DEVICE_REQUEST device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x]\n", + iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), pasid, address, flags); break; case EVENT_TYPE_RMP_FAULT: @@ -641,8 +641,8 @@ static void iommu_print_event(struct amd_iommu *iommu, void *__evt) case EVENT_TYPE_INV_PPR_REQ: pasid = PPR_PASID(*((u64 *)__evt)); tag = event[1] & 0x03FF; - dev_err(dev, "Event logged [INVALID_PPR_REQUEST device=%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x tag=0x%03x]\n", - PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), + dev_err(dev, "Event logged [INVALID_PPR_REQUEST device=%04x:%02x:%02x.%x pasid=0x%05x address=0x%llx flags=0x%04x tag=0x%03x]\n", + iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), pasid, address, flags, tag); break; default: From c841ec28ed6481ceda4c82d24cd1a039ff5febb2 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Wed, 6 Jul 2022 17:08:24 +0530 Subject: [PATCH 33/34] iommu/amd: Update device_state structure to include PCI seg ID commit 196dff712ea2c86f4957ad1a99bc7d65d4b01c5d upstream. Rename struct device_state.devid variable to struct device_state.sbdf and extend it to 32-bit to include the 16-bit PCI segment ID via the helper function get_pci_sbdf_id(). Co-developed-by: Suravee Suthikulpanit Signed-off-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20220706113825.25582-35-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: Arukonda Rahul Signed-off-by: PvsNarasimha --- drivers/iommu/amd/iommu_v2.c | 58 +++++++++++++++--------------------- 1 file changed, 24 insertions(+), 34 deletions(-) diff --git a/drivers/iommu/amd/iommu_v2.c b/drivers/iommu/amd/iommu_v2.c index 29a3a62b7a3ac..192ad7026aecf 100644 --- a/drivers/iommu/amd/iommu_v2.c +++ b/drivers/iommu/amd/iommu_v2.c @@ -51,7 +51,7 @@ struct pasid_state { struct device_state { struct list_head list; - u16 devid; + u32 sbdf; atomic_t count; struct pci_dev *pdev; struct pasid_state **states; @@ -84,35 +84,25 @@ static struct workqueue_struct *iommu_wq; static void free_pasid_states(struct device_state *dev_state); -static u16 device_id(struct pci_dev *pdev) -{ - u16 devid; - - devid = pdev->bus->number; - devid = (devid << 8) | pdev->devfn; - - return devid; -} - -static struct device_state *__get_device_state(u16 devid) +static struct device_state *__get_device_state(u32 sbdf) { struct device_state *dev_state; list_for_each_entry(dev_state, &state_list, list) { - if (dev_state->devid == devid) + if (dev_state->sbdf == sbdf) return dev_state; } return NULL; } -static struct device_state *get_device_state(u16 devid) +static struct device_state *get_device_state(u32 sbdf) { struct device_state *dev_state; unsigned long flags; spin_lock_irqsave(&state_lock, flags); - dev_state = __get_device_state(devid); + dev_state = __get_device_state(sbdf); if (dev_state != NULL) atomic_inc(&dev_state->count); spin_unlock_irqrestore(&state_lock, flags); @@ -602,7 +592,7 @@ int amd_iommu_bind_pasid(struct pci_dev *pdev, u32 pasid, struct pasid_state *pasid_state; struct device_state *dev_state; struct mm_struct *mm; - u16 devid; + u32 sbdf; int ret; might_sleep(); @@ -610,8 +600,8 @@ int amd_iommu_bind_pasid(struct pci_dev *pdev, u32 pasid, if (!amd_iommu_v2_supported()) return -ENODEV; - devid = device_id(pdev); - dev_state = get_device_state(devid); + sbdf = get_pci_sbdf_id(pdev); + dev_state = get_device_state(sbdf); if (dev_state == NULL) return -EINVAL; @@ -685,15 +675,15 @@ void amd_iommu_unbind_pasid(struct pci_dev *pdev, u32 pasid) { struct pasid_state *pasid_state; struct device_state *dev_state; - u16 devid; + u32 sbdf; might_sleep(); if (!amd_iommu_v2_supported()) return; - devid = device_id(pdev); - dev_state = get_device_state(devid); + sbdf = get_pci_sbdf_id(pdev); + dev_state = get_device_state(sbdf); if (dev_state == NULL) return; @@ -735,7 +725,7 @@ int amd_iommu_init_device(struct pci_dev *pdev, int pasids) struct iommu_group *group; unsigned long flags; int ret, tmp; - u16 devid; + u32 sbdf; might_sleep(); @@ -752,7 +742,7 @@ int amd_iommu_init_device(struct pci_dev *pdev, int pasids) if (pasids <= 0 || pasids > (PASID_MASK + 1)) return -EINVAL; - devid = device_id(pdev); + sbdf = get_pci_sbdf_id(pdev); dev_state = kzalloc(sizeof(*dev_state), GFP_KERNEL); if (dev_state == NULL) @@ -761,7 +751,7 @@ int amd_iommu_init_device(struct pci_dev *pdev, int pasids) spin_lock_init(&dev_state->lock); init_waitqueue_head(&dev_state->wq); dev_state->pdev = pdev; - dev_state->devid = devid; + dev_state->sbdf = sbdf; tmp = pasids; for (dev_state->pasid_levels = 0; (tmp - 1) & ~0x1ff; tmp >>= 9) @@ -799,7 +789,7 @@ int amd_iommu_init_device(struct pci_dev *pdev, int pasids) spin_lock_irqsave(&state_lock, flags); - if (__get_device_state(devid) != NULL) { + if (__get_device_state(sbdf) != NULL) { spin_unlock_irqrestore(&state_lock, flags); ret = -EBUSY; goto out_free_domain; @@ -831,16 +821,16 @@ void amd_iommu_free_device(struct pci_dev *pdev) { struct device_state *dev_state; unsigned long flags; - u16 devid; + u32 sbdf; if (!amd_iommu_v2_supported()) return; - devid = device_id(pdev); + sbdf = get_pci_sbdf_id(pdev); spin_lock_irqsave(&state_lock, flags); - dev_state = __get_device_state(devid); + dev_state = __get_device_state(sbdf); if (dev_state == NULL) { spin_unlock_irqrestore(&state_lock, flags); return; @@ -868,18 +858,18 @@ int amd_iommu_set_invalid_ppr_cb(struct pci_dev *pdev, { struct device_state *dev_state; unsigned long flags; - u16 devid; + u32 sbdf; int ret; if (!amd_iommu_v2_supported()) return -ENODEV; - devid = device_id(pdev); + sbdf = get_pci_sbdf_id(pdev); spin_lock_irqsave(&state_lock, flags); ret = -EINVAL; - dev_state = __get_device_state(devid); + dev_state = __get_device_state(sbdf); if (dev_state == NULL) goto out_unlock; @@ -899,18 +889,18 @@ int amd_iommu_set_invalidate_ctx_cb(struct pci_dev *pdev, { struct device_state *dev_state; unsigned long flags; - u16 devid; + u32 sbdf; int ret; if (!amd_iommu_v2_supported()) return -ENODEV; - devid = device_id(pdev); + sbdf = get_pci_sbdf_id(pdev); spin_lock_irqsave(&state_lock, flags); ret = -EINVAL; - dev_state = __get_device_state(devid); + dev_state = __get_device_state(sbdf); if (dev_state == NULL) goto out_unlock; From f9147999e037e387209e93f1768355c5f41c3e57 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Wed, 6 Jul 2022 17:08:25 +0530 Subject: [PATCH 34/34] iommu/amd: Update amd_iommu_fault structure to include PCI seg ID commit 214a05c1c2314b92c9a2e5c594b8646930d788e4 upstream. Rename 'device_id' as 'sbdf' and extend it to 32bit so that we can pass PCI segment ID to ppr_notifier(). Also pass PCI segment ID to pci_get_domain_bus_and_slot() instead of default value. Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20220706113825.25582-36-vasant.hegde@amd.com Signed-off-by: Joerg Roedel Signed-off-by: Arukonda Rahul Signed-off-by: PvsNarasimha --- drivers/iommu/amd/amd_iommu_types.h | 2 +- drivers/iommu/amd/iommu.c | 2 +- drivers/iommu/amd/iommu_v2.c | 9 +++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index d9fdf01ea0c17..dae145780ed5f 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -481,7 +481,7 @@ extern struct kmem_cache *amd_iommu_irq_cache; struct amd_iommu_fault { u64 address; /* IO virtual address of the fault*/ u32 pasid; /* Address space identifier */ - u16 device_id; /* Originating PCI device id */ + u32 sbdf; /* Originating PCI device id */ u16 tag; /* PPR tag */ u16 flags; /* Fault flags */ diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 04bbe11eb3324..7f07631ee5f14 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -679,7 +679,7 @@ static void iommu_handle_ppr_entry(struct amd_iommu *iommu, u64 *raw) fault.address = raw[1]; fault.pasid = PPR_PASID(raw[0]); - fault.device_id = PPR_DEVID(raw[0]); + fault.sbdf = PCI_SEG_DEVID_TO_SBDF(iommu->pci_seg->id, PPR_DEVID(raw[0])); fault.tag = PPR_TAG(raw[0]); fault.flags = PPR_FLAGS(raw[0]); diff --git a/drivers/iommu/amd/iommu_v2.c b/drivers/iommu/amd/iommu_v2.c index 192ad7026aecf..865153d7f917c 100644 --- a/drivers/iommu/amd/iommu_v2.c +++ b/drivers/iommu/amd/iommu_v2.c @@ -510,15 +510,16 @@ static int ppr_notifier(struct notifier_block *nb, unsigned long e, void *data) unsigned long flags; struct fault *fault; bool finish; - u16 tag, devid; + u16 tag, devid, seg_id; int ret; iommu_fault = data; tag = iommu_fault->tag & 0x1ff; finish = (iommu_fault->tag >> 9) & 1; - devid = iommu_fault->device_id; - pdev = pci_get_domain_bus_and_slot(0, PCI_BUS_NUM(devid), + seg_id = PCI_SBDF_TO_SEGID(iommu_fault->sbdf); + devid = PCI_SBDF_TO_DEVID(iommu_fault->sbdf); + pdev = pci_get_domain_bus_and_slot(seg_id, PCI_BUS_NUM(devid), devid & 0xff); if (!pdev) return -ENODEV; @@ -532,7 +533,7 @@ static int ppr_notifier(struct notifier_block *nb, unsigned long e, void *data) goto out; } - dev_state = get_device_state(iommu_fault->device_id); + dev_state = get_device_state(iommu_fault->sbdf); if (dev_state == NULL) goto out;