From 85a6a15d9e41fa68a42bf3df621ce24e15ea89ae Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Tue, 28 Jun 2022 01:07:20 +0900 Subject: [PATCH] iommu: dart: Support >64 stream IDs T8110 DARTs have up to 256 SIDs, so we need to switch to a bitmap to handle them properly. Signed-off-by: Hector Martin --- drivers/iommu/apple-dart.c | 113 +++++++++++++++++++++++-------------- 1 file changed, 70 insertions(+), 43 deletions(-) diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 1422abd69f106b..61f65a0a206e1b 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -33,11 +33,10 @@ #include #include -#define DART_MAX_STREAMS 16 +#define DART_MAX_STREAMS 256 #define DART_MAX_TTBR 4 #define MAX_DARTS_PER_DEVICE 2 -#define DART_STREAM_ALL 0xffff #define DART_PARAMS1 0x00 #define DART_PARAMS_PAGE_SHIFT GENMASK(27, 24) @@ -84,6 +83,8 @@ struct apple_dart_hw { u32 oas; enum io_pgtable_fmt fmt; + + int max_sid_count; }; /* @@ -115,6 +116,7 @@ struct apple_dart { spinlock_t lock; u32 pgsize; + u32 num_streams; u32 supports_bypass : 1; u32 force_bypass : 1; @@ -142,11 +144,11 @@ struct apple_dart { */ struct apple_dart_stream_map { struct apple_dart *dart; - unsigned long sidmap; + DECLARE_BITMAP(sidmap, DART_MAX_STREAMS); }; struct apple_dart_atomic_stream_map { struct apple_dart *dart; - atomic64_t sidmap; + atomic_long_t sidmap[BITS_TO_LONGS(DART_MAX_STREAMS)]; }; /* @@ -204,50 +206,55 @@ static struct apple_dart_domain *to_dart_domain(struct iommu_domain *dom) static void apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map) { + struct apple_dart *dart = stream_map->dart; int sid; - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) writel(DART_TCR_TRANSLATE_ENABLE, - stream_map->dart->regs + DART_TCR(sid)); + dart->regs + DART_TCR(sid)); } static void apple_dart_hw_disable_dma(struct apple_dart_stream_map *stream_map) { + struct apple_dart *dart = stream_map->dart; int sid; - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) - writel(0, stream_map->dart->regs + DART_TCR(sid)); + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) + writel(0, dart->regs + DART_TCR(sid)); } static void apple_dart_hw_enable_bypass(struct apple_dart_stream_map *stream_map) { + struct apple_dart *dart = stream_map->dart; int sid; WARN_ON(!stream_map->dart->supports_bypass); - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) writel(DART_TCR_BYPASS0_ENABLE | DART_TCR_BYPASS1_ENABLE, - stream_map->dart->regs + DART_TCR(sid)); + dart->regs + DART_TCR(sid)); } static void apple_dart_hw_set_ttbr(struct apple_dart_stream_map *stream_map, u8 idx, phys_addr_t paddr) { + struct apple_dart *dart = stream_map->dart; int sid; WARN_ON(paddr & ((1 << DART_TTBR_SHIFT) - 1)); - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) writel(DART_TTBR_VALID | (paddr >> DART_TTBR_SHIFT), - stream_map->dart->regs + DART_TTBR(sid, idx)); + dart->regs + DART_TTBR(sid, idx)); } static void apple_dart_hw_clear_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) { + struct apple_dart *dart = stream_map->dart; int sid; - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) - writel(0, stream_map->dart->regs + DART_TTBR(sid, idx)); + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) + writel(0, dart->regs + DART_TTBR(sid, idx)); } static void @@ -269,7 +276,7 @@ apple_dart_hw_stream_command(struct apple_dart_stream_map *stream_map, spin_lock_irqsave(&stream_map->dart->lock, flags); - writel(stream_map->sidmap, stream_map->dart->regs + DART_STREAM_SELECT); + writel(stream_map->sidmap[0], stream_map->dart->regs + DART_STREAM_SELECT); writel(command, stream_map->dart->regs + DART_STREAM_COMMAND); ret = readl_poll_timeout_atomic( @@ -282,7 +289,7 @@ apple_dart_hw_stream_command(struct apple_dart_stream_map *stream_map, if (ret) { dev_err(stream_map->dart->dev, "busy bit did not clear after command %x for streams %lx\n", - command, stream_map->sidmap); + command, stream_map->sidmap[0]); return ret; } @@ -300,6 +307,7 @@ static int apple_dart_hw_reset(struct apple_dart *dart) { u32 config; struct apple_dart_stream_map stream_map; + int i; config = readl(dart->regs + DART_CONFIG); if (config & DART_CONFIG_LOCK) { @@ -309,12 +317,14 @@ static int apple_dart_hw_reset(struct apple_dart *dart) } stream_map.dart = dart; - stream_map.sidmap = DART_STREAM_ALL; + bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS); + bitmap_set(stream_map.sidmap, 0, dart->num_streams); apple_dart_hw_disable_dma(&stream_map); apple_dart_hw_clear_all_ttbrs(&stream_map); /* enable all streams globally since TCR is used to control isolation */ - writel(DART_STREAM_ALL, dart->regs + DART_STREAMS_ENABLE); + for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) + writel(U32_MAX, dart->regs + DART_STREAMS_ENABLE); /* clear any pending errors before the interrupt is unmasked */ writel(readl(dart->regs + DART_ERROR), dart->regs + DART_ERROR); @@ -324,13 +334,16 @@ static int apple_dart_hw_reset(struct apple_dart *dart) static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain) { - int i; + int i, j; struct apple_dart_atomic_stream_map *domain_stream_map; struct apple_dart_stream_map stream_map; for_each_stream_map(i, domain, domain_stream_map) { stream_map.dart = domain_stream_map->dart; - stream_map.sidmap = atomic64_read(&domain_stream_map->sidmap); + + for (j = 0; j < BITS_TO_LONGS(stream_map.dart->num_streams); j++) + stream_map.sidmap[j] = atomic64_read(&domain_stream_map->sidmap[j]); + apple_dart_hw_invalidate_tlb(&stream_map); } } @@ -415,7 +428,7 @@ static int apple_dart_finalize_domain(struct iommu_domain *domain, struct apple_dart *dart = cfg->stream_maps[0].dart; struct io_pgtable_cfg pgtbl_cfg; int ret = 0; - int i; + int i, j; mutex_lock(&dart_domain->init_lock); @@ -424,8 +437,9 @@ static int apple_dart_finalize_domain(struct iommu_domain *domain, for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { dart_domain->stream_maps[i].dart = cfg->stream_maps[i].dart; - atomic64_set(&dart_domain->stream_maps[i].sidmap, - cfg->stream_maps[i].sidmap); + for (j = 0; j < BITS_TO_LONGS(dart->num_streams); j++) + atomic64_set(&dart_domain->stream_maps[i].sidmap[j], + cfg->stream_maps[i].sidmap[j]); } pgtbl_cfg = (struct io_pgtable_cfg){ @@ -460,7 +474,7 @@ apple_dart_mod_streams(struct apple_dart_atomic_stream_map *domain_maps, struct apple_dart_stream_map *master_maps, bool add_streams) { - int i; + int i, j; for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { if (domain_maps[i].dart != master_maps[i].dart) @@ -470,12 +484,14 @@ apple_dart_mod_streams(struct apple_dart_atomic_stream_map *domain_maps, for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { if (!domain_maps[i].dart) break; - if (add_streams) - atomic64_or(master_maps[i].sidmap, - &domain_maps[i].sidmap); - else - atomic64_and(~master_maps[i].sidmap, - &domain_maps[i].sidmap); + for (j = 0; j < BITS_TO_LONGS(domain_maps[i].dart->num_streams); j++) { + if (add_streams) + atomic64_or(master_maps[i].sidmap[j], + &domain_maps[i].sidmap[j]); + else + atomic64_and(~master_maps[i].sidmap[j], + &domain_maps[i].sidmap[j]); + } } return 0; @@ -639,14 +655,14 @@ static int apple_dart_of_xlate(struct device *dev, struct of_phandle_args *args) for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { if (cfg->stream_maps[i].dart == dart) { - cfg->stream_maps[i].sidmap |= 1 << sid; + set_bit(sid, cfg->stream_maps[i].sidmap); return 0; } } for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { if (!cfg->stream_maps[i].dart) { cfg->stream_maps[i].dart = dart; - cfg->stream_maps[i].sidmap = 1 << sid; + set_bit(sid, cfg->stream_maps[i].sidmap); return 0; } } @@ -665,7 +681,7 @@ static void apple_dart_release_group(void *iommu_data) mutex_lock(&apple_dart_groups_lock); for_each_stream_map(i, group_master_cfg, stream_map) - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) + for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) stream_map->dart->sid2group[sid] = NULL; kfree(iommu_data); @@ -684,7 +700,7 @@ static struct iommu_group *apple_dart_device_group(struct device *dev) mutex_lock(&apple_dart_groups_lock); for_each_stream_map(i, cfg, stream_map) { - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) { + for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) { struct iommu_group *stream_group = stream_map->dart->sid2group[sid]; @@ -723,7 +739,7 @@ static struct iommu_group *apple_dart_device_group(struct device *dev) apple_dart_release_group); for_each_stream_map(i, cfg, stream_map) - for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) + for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) stream_map->dart->sid2group[sid] = group; res = group; @@ -889,16 +905,25 @@ static int apple_dart_probe(struct platform_device *pdev) if (ret) return ret; - ret = apple_dart_hw_reset(dart); - if (ret) - goto err_clk_disable; - dart_params[0] = readl(dart->regs + DART_PARAMS1); dart_params[1] = readl(dart->regs + DART_PARAMS2); dart->pgsize = 1 << FIELD_GET(DART_PARAMS_PAGE_SHIFT, dart_params[0]); dart->supports_bypass = dart_params[1] & DART_PARAMS_BYPASS_SUPPORT; + + dart->num_streams = dart->hw->max_sid_count; + + if (dart->num_streams > DART_MAX_STREAMS) { + dev_err(&pdev->dev, "Too many streams (%d > %d)\n", + dart->num_streams, DART_MAX_STREAMS); + goto err_clk_disable; + } + dart->force_bypass = dart->pgsize > PAGE_SIZE; + ret = apple_dart_hw_reset(dart); + if (ret) + goto err_clk_disable; + ret = request_irq(dart->irq, apple_dart_irq, IRQF_SHARED, "apple-dart fault handler", dart); if (ret) @@ -921,8 +946,8 @@ static int apple_dart_probe(struct platform_device *pdev) dev_info( &pdev->dev, - "DART [pagesize %x, bypass support: %d, bypass forced: %d] initialized\n", - dart->pgsize, dart->supports_bypass, dart->force_bypass); + "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d] initialized\n", + dart->pgsize, dart->num_streams, dart->supports_bypass, dart->force_bypass); return 0; err_sysfs_remove: @@ -956,10 +981,12 @@ static int apple_dart_remove(struct platform_device *pdev) static const struct apple_dart_hw apple_dart_hw_t8103 = { .oas = 36, .fmt = APPLE_DART, + .max_sid_count = 16, }; static const struct apple_dart_hw apple_dart_hw_t6000 = { .oas = 42, .fmt = APPLE_DART2, + .max_sid_count = 16, }; #ifdef CONFIG_PM_SLEEP @@ -968,7 +995,7 @@ static int apple_dart_suspend(struct device *dev) struct apple_dart *dart = dev_get_drvdata(dev); unsigned int sid, idx; - for (sid = 0; sid < DART_MAX_STREAMS; sid++) { + for (sid = 0; sid < dart->num_streams; sid++) { dart->save_tcr[sid] = readl_relaxed(dart->regs + DART_TCR(sid)); for (idx = 0; idx < DART_MAX_TTBR; idx++) dart->save_ttbr[sid][idx] = @@ -990,7 +1017,7 @@ static int apple_dart_resume(struct device *dev) return ret; } - for (sid = 0; sid < DART_MAX_STREAMS; sid++) { + for (sid = 0; sid < dart->num_streams; sid++) { for (idx = 0; idx < DART_MAX_TTBR; idx++) writel_relaxed(dart->save_ttbr[sid][idx], dart->regs + DART_TTBR(sid, idx));