mt76: switch to page_pool allocator

In order to reduce possible memory allocation failures due to to memory
fragmentation caused by page_frag_cache allocator, switch to page_pool
allocator for dma and usb mt76 drivers.
Remove per rx-queue page_frag_cache

drivers/net/wireless/mediatek/mt76/Kconfig

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 config MT76_CORE
 	tristate
+	select PAGE_POOL
 
 config MT76_LEDS
 	bool

drivers/net/wireless/mediatek/mt76/dma.c

@@ -173,7 +173,7 @@ mt76_free_pending_rxwi(struct mt76_dev *dev)
 	local_bh_disable();
 	while ((t = __mt76_get_rxwi(dev)) != NULL) {
 		if (t->ptr)
-			skb_free_frag(t->ptr);
+			mt76_put_page_pool_buf(dev, t->ptr, false);
 		kfree(t);
 	}
 	local_bh_enable();
@@ -579,7 +579,8 @@ mt76_dma_tx_queue_skb(struct mt76_dev *dev, struct mt76_queue *q,
 }
 
 static int
-mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q)
+mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q,
+		 bool allow_direct)
 {
 	int len = SKB_WITH_OVERHEAD(q->buf_size);
 	int frames = 0, offset = q->buf_offset;
@@ -592,15 +593,15 @@ mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q)
 
 	while (q->queued < q->ndesc - 1) {
 		struct mt76_queue_buf qbuf;
-		void *buf = NULL;
+		void *buf;
 
-		buf = page_frag_alloc(&q->rx_page, q->buf_size, GFP_ATOMIC);
+		buf = mt76_get_page_pool_buf(dev, q->buf_size);
 		if (!buf)
 			break;
 
 		addr = dma_map_single(dev->dma_dev, buf, len, DMA_FROM_DEVICE);
 		if (unlikely(dma_mapping_error(dev->dma_dev, addr))) {
-			skb_free_frag(buf);
+			mt76_put_page_pool_buf(dev, buf, allow_direct);
 			break;
 		}
 
@@ -610,7 +611,7 @@ mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q)
 		if (mt76_dma_add_rx_buf(dev, q, &qbuf, buf) < 0) {
 			dma_unmap_single(dev->dma_dev, addr, len,
 					 DMA_FROM_DEVICE);
-			skb_free_frag(buf);
+			mt76_put_page_pool_buf(dev, buf, allow_direct);
 			break;
 		}
 		frames++;
@@ -654,7 +655,7 @@ int mt76_dma_wed_setup(struct mt76_dev *dev, struct mt76_queue *q, bool reset)
 		/* WED txfree queue needs ring to be initialized before setup */
 		q->flags = 0;
 		mt76_dma_queue_reset(dev, q);
-		mt76_dma_rx_fill(dev, q);
+		mt76_dma_rx_fill(dev, q, false);
 		q->flags = flags;
 
 		ret = mtk_wed_device_txfree_ring_setup(wed, q->regs);
@@ -715,7 +716,6 @@ mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q,
 static void
 mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q)
 {
-	struct page *page;
 	void *buf;
 	bool more;
 
@@ -728,16 +728,9 @@ mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q)
 		if (!buf)
 			break;
 
-		skb_free_frag(buf);
+		mt76_put_page_pool_buf(dev, buf, false);
 	} while (1);
 	spin_unlock_bh(&q->lock);
-
-	if (!q->rx_page.va)
-		return;
-
-	page = virt_to_page(q->rx_page.va);
-	__page_frag_cache_drain(page, q->rx_page.pagecnt_bias);
-	memset(&q->rx_page, 0, sizeof(q->rx_page));
 }
 
 static void
@@ -754,7 +747,7 @@ mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid)
 
 	mt76_dma_rx_cleanup(dev, q);
 	mt76_dma_sync_idx(dev, q);
-	mt76_dma_rx_fill(dev, q);
+	mt76_dma_rx_fill(dev, q, false);
 
 	if (!q->rx_head)
 		return;
@@ -777,7 +770,7 @@ mt76_add_fragment(struct mt76_dev *dev, struct mt76_queue *q, void *data,
 
 		skb_add_rx_frag(skb, nr_frags, page, offset, len, q->buf_size);
 	} else {
-		skb_free_frag(data);
+		mt76_put_page_pool_buf(dev, data, true);
 	}
 
 	if (more)
@@ -850,6 +843,7 @@ mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget)
 			goto free_frag;
 
 		skb_reserve(skb, q->buf_offset);
+		skb_mark_for_recycle(skb);
 
 		*(u32 *)skb->cb = info;
 
@@ -865,10 +859,10 @@ mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget)
 		continue;
 
 free_frag:
-		skb_free_frag(data);
+		mt76_put_page_pool_buf(dev, data, true);
 	}
 
-	mt76_dma_rx_fill(dev, q);
+	mt76_dma_rx_fill(dev, q, true);
 	return done;
 }
 
@@ -911,7 +905,7 @@ mt76_dma_init(struct mt76_dev *dev,
 
 	mt76_for_each_q_rx(dev, i) {
 		netif_napi_add(&dev->napi_dev, &dev->napi[i], poll);
-		mt76_dma_rx_fill(dev, &dev->q_rx[i]);
+		mt76_dma_rx_fill(dev, &dev->q_rx[i], false);
 		napi_enable(&dev->napi[i]);
 	}
 

drivers/net/wireless/mediatek/mt76/mac80211.c

@@ -4,6 +4,7 @@
  */
 #include <linux/sched.h>
 #include <linux/of.h>
+#include <net/page_pool.h>
 #include "mt76.h"
 
 #define CHAN2G(_idx, _freq) {			\
@@ -556,6 +557,62 @@ void mt76_unregister_phy(struct mt76_phy *phy)
 }
 EXPORT_SYMBOL_GPL(mt76_unregister_phy);
 
+static int
+mt76_create_page_pool(struct mt76_dev *dev)
+{
+	struct page_pool_params pp_params = {
+		.order = 0,
+		.pool_size = 32768, /* max page pool ring size */
+		.flags = PP_FLAG_PAGE_FRAG,
+		.nid = NUMA_NO_NODE,
+		.dev = dev->dev,
+		.dma_dir = DMA_FROM_DEVICE,
+	};
+
+	spin_lock_init(&dev->pp.lock);
+	dev->pp.pool = page_pool_create(&pp_params);
+	if (IS_ERR(dev->pp.pool)) {
+		int err = PTR_ERR(dev->pp.pool);
+
+		dev->pp.pool = NULL;
+		return err;
+	}
+
+	return 0;
+}
+
+void *mt76_get_page_pool_buf(struct mt76_dev *dev, u32 size)
+{
+	struct page *page;
+	u32 offset;
+
+	if (!dev->pp.pool)
+		return NULL;
+
+	spin_lock_bh(&dev->pp.lock);
+	page = page_pool_dev_alloc_frag(dev->pp.pool, &offset, size);
+	spin_unlock_bh(&dev->pp.lock);
+
+	return page ? page_address(page) + offset : NULL;
+}
+EXPORT_SYMBOL_GPL(mt76_get_page_pool_buf);
+
+void mt76_put_page_pool_buf(struct mt76_dev *dev, void *buf,
+			    bool allow_direct)
+{
+	struct page *page = virt_to_head_page(buf);
+
+	if (!dev->pp.pool || (page->pp_magic & ~0x3UL) != PP_SIGNATURE) {
+		skb_free_frag(buf);
+		return;
+	}
+
+	spin_lock_bh(&dev->pp.lock);
+	page_pool_put_full_page(dev->pp.pool, page, allow_direct);
+	spin_unlock_bh(&dev->pp.lock);
+}
+EXPORT_SYMBOL_GPL(mt76_put_page_pool_buf);
+
 struct mt76_dev *
 mt76_alloc_device(struct device *pdev, unsigned int size,
 		  const struct ieee80211_ops *ops,
@@ -627,6 +684,11 @@ mt76_alloc_device(struct device *pdev, unsigned int size,
 		return NULL;
 	}
 
+	if (mt76_create_page_pool(dev)) {
+		ieee80211_free_hw(hw);
+		return NULL;
+	}
+
 	return dev;
 }
 EXPORT_SYMBOL_GPL(mt76_alloc_device);
@@ -701,6 +763,7 @@ void mt76_free_device(struct mt76_dev *dev)
 		destroy_workqueue(dev->wq);
 		dev->wq = NULL;
 	}
+	page_pool_destroy(dev->pp.pool);
 	ieee80211_free_hw(dev->hw);
 }
 EXPORT_SYMBOL_GPL(mt76_free_device);

drivers/net/wireless/mediatek/mt76/mt76.h

@@ -202,7 +202,6 @@ struct mt76_queue {
 
 	dma_addr_t desc_dma;
 	struct sk_buff *rx_head;
-	struct page_frag_cache rx_page;
 };
 
 struct mt76_mcu_ops {
@@ -837,6 +836,11 @@ struct mt76_dev {
 		struct mt76_usb usb;
 		struct mt76_sdio sdio;
 	};
+
+	struct {
+		spinlock_t lock;
+		struct page_pool *pool;
+	} pp;
 };
 
 struct mt76_power_limits {
@@ -1422,6 +1426,9 @@ void __mt76_set_tx_blocked(struct mt76_dev *dev, bool blocked);
 struct mt76_txwi_cache *mt76_rx_token_release(struct mt76_dev *dev, int token);
 int mt76_rx_token_consume(struct mt76_dev *dev, void *ptr,
 			  struct mt76_txwi_cache *r, dma_addr_t phys);
+void *mt76_get_page_pool_buf(struct mt76_dev *dev, u32 size);
+void mt76_put_page_pool_buf(struct mt76_dev *dev, void *buf,
+			    bool allow_direct);
 
 static inline void mt76_set_tx_blocked(struct mt76_dev *dev, bool blocked)
 {

drivers/net/wireless/mediatek/mt76/mt7915/mmio.c

@@ -594,13 +594,9 @@ static void mt7915_mmio_wed_offload_disable(struct mtk_wed_device *wed)
 static void mt7915_mmio_wed_release_rx_buf(struct mtk_wed_device *wed)
 {
 	struct mt7915_dev *dev;
-	u32 length;
 	int i;
 
 	dev = container_of(wed, struct mt7915_dev, mt76.mmio.wed);
-	length = SKB_DATA_ALIGN(NET_SKB_PAD + wed->wlan.rx_size +
-				sizeof(struct skb_shared_info));
-
 	for (i = 0; i < dev->mt76.rx_token_size; i++) {
 		struct mt76_txwi_cache *t;
 
@@ -610,7 +606,7 @@ static void mt7915_mmio_wed_release_rx_buf(struct mtk_wed_device *wed)
 
 		dma_unmap_single(dev->mt76.dma_dev, t->dma_addr,
 				 wed->wlan.rx_size, DMA_FROM_DEVICE);
-		__free_pages(virt_to_page(t->ptr), get_order(length));
+		mt76_put_page_pool_buf(&dev->mt76, t->ptr, false);
 		t->ptr = NULL;
 
 		mt76_put_rxwi(&dev->mt76, t);
@@ -631,20 +627,18 @@ static u32 mt7915_mmio_wed_init_rx_buf(struct mtk_wed_device *wed, int size)
 	for (i = 0; i < size; i++) {
 		struct mt76_txwi_cache *t = mt76_get_rxwi(&dev->mt76);
 		dma_addr_t phy_addr;
-		struct page *page;
 		int token;
 		void *ptr;
 
-		page = __dev_alloc_pages(GFP_KERNEL, get_order(length));
-		if (!page)
+		ptr = mt76_get_page_pool_buf(&dev->mt76, length);
+		if (!ptr)
 			goto unmap;
 
-		ptr = page_address(page);
 		phy_addr = dma_map_single(dev->mt76.dma_dev, ptr,
 					  wed->wlan.rx_size,
 					  DMA_TO_DEVICE);
 		if (unlikely(dma_mapping_error(dev->mt76.dev, phy_addr))) {
-			__free_pages(page, get_order(length));
+			mt76_put_page_pool_buf(&dev->mt76, ptr, false);
 			goto unmap;
 		}
 
@@ -653,7 +647,7 @@ static u32 mt7915_mmio_wed_init_rx_buf(struct mtk_wed_device *wed, int size)
 		if (token < 0) {
 			dma_unmap_single(dev->mt76.dma_dev, phy_addr,
 					 wed->wlan.rx_size, DMA_TO_DEVICE);
-			__free_pages(page, get_order(length));
+			mt76_put_page_pool_buf(&dev->mt76, ptr, false);
 			goto unmap;
 		}