2891 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/* Copyright (C) 2021 Felix Fietkau <nbd@nbd.name> */3 4#include <linux/kernel.h>5#include <linux/platform_device.h>6#include <linux/slab.h>7#include <linux/module.h>8#include <linux/bitfield.h>9#include <linux/dma-mapping.h>10#include <linux/skbuff.h>11#include <linux/of_platform.h>12#include <linux/of_address.h>13#include <linux/of_reserved_mem.h>14#include <linux/mfd/syscon.h>15#include <linux/debugfs.h>16#include <linux/soc/mediatek/mtk_wed.h>17#include <net/flow_offload.h>18#include <net/pkt_cls.h>19#include "mtk_eth_soc.h"20#include "mtk_wed.h"21#include "mtk_ppe.h"22#include "mtk_wed_wo.h"23 24#define MTK_PCIE_BASE(n) (0x1a143000 + (n) * 0x2000)25 26#define MTK_WED_PKT_SIZE 192027#define MTK_WED_BUF_SIZE 204828#define MTK_WED_PAGE_BUF_SIZE 12829#define MTK_WED_BUF_PER_PAGE (PAGE_SIZE / 2048)30#define MTK_WED_RX_BUF_PER_PAGE (PAGE_SIZE / MTK_WED_PAGE_BUF_SIZE)31#define MTK_WED_RX_RING_SIZE 153632#define MTK_WED_RX_PG_BM_CNT 819233#define MTK_WED_AMSDU_BUF_SIZE (PAGE_SIZE << 4)34#define MTK_WED_AMSDU_NPAGES 3235 36#define MTK_WED_TX_RING_SIZE 204837#define MTK_WED_WDMA_RING_SIZE 102438#define MTK_WED_MAX_GROUP_SIZE 0x10039#define MTK_WED_VLD_GROUP_SIZE 0x4040#define MTK_WED_PER_GROUP_PKT 12841 42#define MTK_WED_FBUF_SIZE 12843#define MTK_WED_MIOD_CNT 1644#define MTK_WED_FB_CMD_CNT 102445#define MTK_WED_RRO_QUE_CNT 819246#define MTK_WED_MIOD_ENTRY_CNT 12847 48#define MTK_WED_TX_BM_DMA_SIZE 6553649#define MTK_WED_TX_BM_PKT_CNT 3276850 51static struct mtk_wed_hw *hw_list[3];52static DEFINE_MUTEX(hw_lock);53 54struct mtk_wed_flow_block_priv {55 struct mtk_wed_hw *hw;56 struct net_device *dev;57};58 59static const struct mtk_wed_soc_data mt7622_data = {60 .regmap = {61 .tx_bm_tkid = 0x088,62 .wpdma_rx_ring0 = 0x770,63 .reset_idx_tx_mask = GENMASK(3, 0),64 .reset_idx_rx_mask = GENMASK(17, 16),65 },66 .tx_ring_desc_size = sizeof(struct mtk_wdma_desc),67 .wdma_desc_size = sizeof(struct mtk_wdma_desc),68};69 70static const struct mtk_wed_soc_data mt7986_data = {71 .regmap = {72 .tx_bm_tkid = 0x0c8,73 .wpdma_rx_ring0 = 0x770,74 .reset_idx_tx_mask = GENMASK(1, 0),75 .reset_idx_rx_mask = GENMASK(7, 6),76 },77 .tx_ring_desc_size = sizeof(struct mtk_wdma_desc),78 .wdma_desc_size = 2 * sizeof(struct mtk_wdma_desc),79};80 81static const struct mtk_wed_soc_data mt7988_data = {82 .regmap = {83 .tx_bm_tkid = 0x0c8,84 .wpdma_rx_ring0 = 0x7d0,85 .reset_idx_tx_mask = GENMASK(1, 0),86 .reset_idx_rx_mask = GENMASK(7, 6),87 },88 .tx_ring_desc_size = sizeof(struct mtk_wed_bm_desc),89 .wdma_desc_size = 2 * sizeof(struct mtk_wdma_desc),90};91 92static void93wed_m32(struct mtk_wed_device *dev, u32 reg, u32 mask, u32 val)94{95 regmap_update_bits(dev->hw->regs, reg, mask | val, val);96}97 98static void99wed_set(struct mtk_wed_device *dev, u32 reg, u32 mask)100{101 return wed_m32(dev, reg, 0, mask);102}103 104static void105wed_clr(struct mtk_wed_device *dev, u32 reg, u32 mask)106{107 return wed_m32(dev, reg, mask, 0);108}109 110static void111wdma_m32(struct mtk_wed_device *dev, u32 reg, u32 mask, u32 val)112{113 wdma_w32(dev, reg, (wdma_r32(dev, reg) & ~mask) | val);114}115 116static void117wdma_set(struct mtk_wed_device *dev, u32 reg, u32 mask)118{119 wdma_m32(dev, reg, 0, mask);120}121 122static void123wdma_clr(struct mtk_wed_device *dev, u32 reg, u32 mask)124{125 wdma_m32(dev, reg, mask, 0);126}127 128static u32129wifi_r32(struct mtk_wed_device *dev, u32 reg)130{131 return readl(dev->wlan.base + reg);132}133 134static void135wifi_w32(struct mtk_wed_device *dev, u32 reg, u32 val)136{137 writel(val, dev->wlan.base + reg);138}139 140static u32141mtk_wed_read_reset(struct mtk_wed_device *dev)142{143 return wed_r32(dev, MTK_WED_RESET);144}145 146static u32147mtk_wdma_read_reset(struct mtk_wed_device *dev)148{149 return wdma_r32(dev, MTK_WDMA_GLO_CFG);150}151 152static void153mtk_wdma_v3_rx_reset(struct mtk_wed_device *dev)154{155 u32 status;156 157 if (!mtk_wed_is_v3_or_greater(dev->hw))158 return;159 160 wdma_clr(dev, MTK_WDMA_PREF_TX_CFG, MTK_WDMA_PREF_TX_CFG_PREF_EN);161 wdma_clr(dev, MTK_WDMA_PREF_RX_CFG, MTK_WDMA_PREF_RX_CFG_PREF_EN);162 163 if (read_poll_timeout(wdma_r32, status,164 !(status & MTK_WDMA_PREF_TX_CFG_PREF_BUSY),165 0, 10000, false, dev, MTK_WDMA_PREF_TX_CFG))166 dev_err(dev->hw->dev, "rx reset failed\n");167 168 if (read_poll_timeout(wdma_r32, status,169 !(status & MTK_WDMA_PREF_RX_CFG_PREF_BUSY),170 0, 10000, false, dev, MTK_WDMA_PREF_RX_CFG))171 dev_err(dev->hw->dev, "rx reset failed\n");172 173 wdma_clr(dev, MTK_WDMA_WRBK_TX_CFG, MTK_WDMA_WRBK_TX_CFG_WRBK_EN);174 wdma_clr(dev, MTK_WDMA_WRBK_RX_CFG, MTK_WDMA_WRBK_RX_CFG_WRBK_EN);175 176 if (read_poll_timeout(wdma_r32, status,177 !(status & MTK_WDMA_WRBK_TX_CFG_WRBK_BUSY),178 0, 10000, false, dev, MTK_WDMA_WRBK_TX_CFG))179 dev_err(dev->hw->dev, "rx reset failed\n");180 181 if (read_poll_timeout(wdma_r32, status,182 !(status & MTK_WDMA_WRBK_RX_CFG_WRBK_BUSY),183 0, 10000, false, dev, MTK_WDMA_WRBK_RX_CFG))184 dev_err(dev->hw->dev, "rx reset failed\n");185 186 /* prefetch FIFO */187 wdma_w32(dev, MTK_WDMA_PREF_RX_FIFO_CFG,188 MTK_WDMA_PREF_RX_FIFO_CFG_RING0_CLEAR |189 MTK_WDMA_PREF_RX_FIFO_CFG_RING1_CLEAR);190 wdma_clr(dev, MTK_WDMA_PREF_RX_FIFO_CFG,191 MTK_WDMA_PREF_RX_FIFO_CFG_RING0_CLEAR |192 MTK_WDMA_PREF_RX_FIFO_CFG_RING1_CLEAR);193 194 /* core FIFO */195 wdma_w32(dev, MTK_WDMA_XDMA_RX_FIFO_CFG,196 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_PAR_FIFO_CLEAR |197 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_CMD_FIFO_CLEAR |198 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_DMAD_FIFO_CLEAR |199 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_ARR_FIFO_CLEAR |200 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_LEN_FIFO_CLEAR |201 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_WID_FIFO_CLEAR |202 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_BID_FIFO_CLEAR);203 wdma_clr(dev, MTK_WDMA_XDMA_RX_FIFO_CFG,204 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_PAR_FIFO_CLEAR |205 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_CMD_FIFO_CLEAR |206 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_DMAD_FIFO_CLEAR |207 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_ARR_FIFO_CLEAR |208 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_LEN_FIFO_CLEAR |209 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_WID_FIFO_CLEAR |210 MTK_WDMA_XDMA_RX_FIFO_CFG_RX_BID_FIFO_CLEAR);211 212 /* writeback FIFO */213 wdma_w32(dev, MTK_WDMA_WRBK_RX_FIFO_CFG(0),214 MTK_WDMA_WRBK_RX_FIFO_CFG_RING_CLEAR);215 wdma_w32(dev, MTK_WDMA_WRBK_RX_FIFO_CFG(1),216 MTK_WDMA_WRBK_RX_FIFO_CFG_RING_CLEAR);217 218 wdma_clr(dev, MTK_WDMA_WRBK_RX_FIFO_CFG(0),219 MTK_WDMA_WRBK_RX_FIFO_CFG_RING_CLEAR);220 wdma_clr(dev, MTK_WDMA_WRBK_RX_FIFO_CFG(1),221 MTK_WDMA_WRBK_RX_FIFO_CFG_RING_CLEAR);222 223 /* prefetch ring status */224 wdma_w32(dev, MTK_WDMA_PREF_SIDX_CFG,225 MTK_WDMA_PREF_SIDX_CFG_RX_RING_CLEAR);226 wdma_clr(dev, MTK_WDMA_PREF_SIDX_CFG,227 MTK_WDMA_PREF_SIDX_CFG_RX_RING_CLEAR);228 229 /* writeback ring status */230 wdma_w32(dev, MTK_WDMA_WRBK_SIDX_CFG,231 MTK_WDMA_WRBK_SIDX_CFG_RX_RING_CLEAR);232 wdma_clr(dev, MTK_WDMA_WRBK_SIDX_CFG,233 MTK_WDMA_WRBK_SIDX_CFG_RX_RING_CLEAR);234}235 236static int237mtk_wdma_rx_reset(struct mtk_wed_device *dev)238{239 u32 status, mask = MTK_WDMA_GLO_CFG_RX_DMA_BUSY;240 int i, ret;241 242 wdma_clr(dev, MTK_WDMA_GLO_CFG, MTK_WDMA_GLO_CFG_RX_DMA_EN);243 ret = readx_poll_timeout(mtk_wdma_read_reset, dev, status,244 !(status & mask), 0, 10000);245 if (ret)246 dev_err(dev->hw->dev, "rx reset failed\n");247 248 mtk_wdma_v3_rx_reset(dev);249 wdma_w32(dev, MTK_WDMA_RESET_IDX, MTK_WDMA_RESET_IDX_RX);250 wdma_w32(dev, MTK_WDMA_RESET_IDX, 0);251 252 for (i = 0; i < ARRAY_SIZE(dev->rx_wdma); i++) {253 if (dev->rx_wdma[i].desc)254 continue;255 256 wdma_w32(dev,257 MTK_WDMA_RING_RX(i) + MTK_WED_RING_OFS_CPU_IDX, 0);258 }259 260 return ret;261}262 263static u32264mtk_wed_check_busy(struct mtk_wed_device *dev, u32 reg, u32 mask)265{266 return !!(wed_r32(dev, reg) & mask);267}268 269static int270mtk_wed_poll_busy(struct mtk_wed_device *dev, u32 reg, u32 mask)271{272 int sleep = 15000;273 int timeout = 100 * sleep;274 u32 val;275 276 return read_poll_timeout(mtk_wed_check_busy, val, !val, sleep,277 timeout, false, dev, reg, mask);278}279 280static void281mtk_wdma_v3_tx_reset(struct mtk_wed_device *dev)282{283 u32 status;284 285 if (!mtk_wed_is_v3_or_greater(dev->hw))286 return;287 288 wdma_clr(dev, MTK_WDMA_PREF_TX_CFG, MTK_WDMA_PREF_TX_CFG_PREF_EN);289 wdma_clr(dev, MTK_WDMA_PREF_RX_CFG, MTK_WDMA_PREF_RX_CFG_PREF_EN);290 291 if (read_poll_timeout(wdma_r32, status,292 !(status & MTK_WDMA_PREF_TX_CFG_PREF_BUSY),293 0, 10000, false, dev, MTK_WDMA_PREF_TX_CFG))294 dev_err(dev->hw->dev, "tx reset failed\n");295 296 if (read_poll_timeout(wdma_r32, status,297 !(status & MTK_WDMA_PREF_RX_CFG_PREF_BUSY),298 0, 10000, false, dev, MTK_WDMA_PREF_RX_CFG))299 dev_err(dev->hw->dev, "tx reset failed\n");300 301 wdma_clr(dev, MTK_WDMA_WRBK_TX_CFG, MTK_WDMA_WRBK_TX_CFG_WRBK_EN);302 wdma_clr(dev, MTK_WDMA_WRBK_RX_CFG, MTK_WDMA_WRBK_RX_CFG_WRBK_EN);303 304 if (read_poll_timeout(wdma_r32, status,305 !(status & MTK_WDMA_WRBK_TX_CFG_WRBK_BUSY),306 0, 10000, false, dev, MTK_WDMA_WRBK_TX_CFG))307 dev_err(dev->hw->dev, "tx reset failed\n");308 309 if (read_poll_timeout(wdma_r32, status,310 !(status & MTK_WDMA_WRBK_RX_CFG_WRBK_BUSY),311 0, 10000, false, dev, MTK_WDMA_WRBK_RX_CFG))312 dev_err(dev->hw->dev, "tx reset failed\n");313 314 /* prefetch FIFO */315 wdma_w32(dev, MTK_WDMA_PREF_TX_FIFO_CFG,316 MTK_WDMA_PREF_TX_FIFO_CFG_RING0_CLEAR |317 MTK_WDMA_PREF_TX_FIFO_CFG_RING1_CLEAR);318 wdma_clr(dev, MTK_WDMA_PREF_TX_FIFO_CFG,319 MTK_WDMA_PREF_TX_FIFO_CFG_RING0_CLEAR |320 MTK_WDMA_PREF_TX_FIFO_CFG_RING1_CLEAR);321 322 /* core FIFO */323 wdma_w32(dev, MTK_WDMA_XDMA_TX_FIFO_CFG,324 MTK_WDMA_XDMA_TX_FIFO_CFG_TX_PAR_FIFO_CLEAR |325 MTK_WDMA_XDMA_TX_FIFO_CFG_TX_CMD_FIFO_CLEAR |326 MTK_WDMA_XDMA_TX_FIFO_CFG_TX_DMAD_FIFO_CLEAR |327 MTK_WDMA_XDMA_TX_FIFO_CFG_TX_ARR_FIFO_CLEAR);328 wdma_clr(dev, MTK_WDMA_XDMA_TX_FIFO_CFG,329 MTK_WDMA_XDMA_TX_FIFO_CFG_TX_PAR_FIFO_CLEAR |330 MTK_WDMA_XDMA_TX_FIFO_CFG_TX_CMD_FIFO_CLEAR |331 MTK_WDMA_XDMA_TX_FIFO_CFG_TX_DMAD_FIFO_CLEAR |332 MTK_WDMA_XDMA_TX_FIFO_CFG_TX_ARR_FIFO_CLEAR);333 334 /* writeback FIFO */335 wdma_w32(dev, MTK_WDMA_WRBK_TX_FIFO_CFG(0),336 MTK_WDMA_WRBK_TX_FIFO_CFG_RING_CLEAR);337 wdma_w32(dev, MTK_WDMA_WRBK_TX_FIFO_CFG(1),338 MTK_WDMA_WRBK_TX_FIFO_CFG_RING_CLEAR);339 340 wdma_clr(dev, MTK_WDMA_WRBK_TX_FIFO_CFG(0),341 MTK_WDMA_WRBK_TX_FIFO_CFG_RING_CLEAR);342 wdma_clr(dev, MTK_WDMA_WRBK_TX_FIFO_CFG(1),343 MTK_WDMA_WRBK_TX_FIFO_CFG_RING_CLEAR);344 345 /* prefetch ring status */346 wdma_w32(dev, MTK_WDMA_PREF_SIDX_CFG,347 MTK_WDMA_PREF_SIDX_CFG_TX_RING_CLEAR);348 wdma_clr(dev, MTK_WDMA_PREF_SIDX_CFG,349 MTK_WDMA_PREF_SIDX_CFG_TX_RING_CLEAR);350 351 /* writeback ring status */352 wdma_w32(dev, MTK_WDMA_WRBK_SIDX_CFG,353 MTK_WDMA_WRBK_SIDX_CFG_TX_RING_CLEAR);354 wdma_clr(dev, MTK_WDMA_WRBK_SIDX_CFG,355 MTK_WDMA_WRBK_SIDX_CFG_TX_RING_CLEAR);356}357 358static void359mtk_wdma_tx_reset(struct mtk_wed_device *dev)360{361 u32 status, mask = MTK_WDMA_GLO_CFG_TX_DMA_BUSY;362 int i;363 364 wdma_clr(dev, MTK_WDMA_GLO_CFG, MTK_WDMA_GLO_CFG_TX_DMA_EN);365 if (readx_poll_timeout(mtk_wdma_read_reset, dev, status,366 !(status & mask), 0, 10000))367 dev_err(dev->hw->dev, "tx reset failed\n");368 369 mtk_wdma_v3_tx_reset(dev);370 wdma_w32(dev, MTK_WDMA_RESET_IDX, MTK_WDMA_RESET_IDX_TX);371 wdma_w32(dev, MTK_WDMA_RESET_IDX, 0);372 373 for (i = 0; i < ARRAY_SIZE(dev->tx_wdma); i++)374 wdma_w32(dev,375 MTK_WDMA_RING_TX(i) + MTK_WED_RING_OFS_CPU_IDX, 0);376}377 378static void379mtk_wed_reset(struct mtk_wed_device *dev, u32 mask)380{381 u32 status;382 383 wed_w32(dev, MTK_WED_RESET, mask);384 if (readx_poll_timeout(mtk_wed_read_reset, dev, status,385 !(status & mask), 0, 1000))386 WARN_ON_ONCE(1);387}388 389static u32390mtk_wed_wo_read_status(struct mtk_wed_device *dev)391{392 return wed_r32(dev, MTK_WED_SCR0 + 4 * MTK_WED_DUMMY_CR_WO_STATUS);393}394 395static void396mtk_wed_wo_reset(struct mtk_wed_device *dev)397{398 struct mtk_wed_wo *wo = dev->hw->wed_wo;399 u8 state = MTK_WED_WO_STATE_DISABLE;400 void __iomem *reg;401 u32 val;402 403 mtk_wdma_tx_reset(dev);404 mtk_wed_reset(dev, MTK_WED_RESET_WED);405 406 if (mtk_wed_mcu_send_msg(wo, MTK_WED_MODULE_ID_WO,407 MTK_WED_WO_CMD_CHANGE_STATE, &state,408 sizeof(state), false))409 return;410 411 if (readx_poll_timeout(mtk_wed_wo_read_status, dev, val,412 val == MTK_WED_WOIF_DISABLE_DONE,413 100, MTK_WOCPU_TIMEOUT))414 dev_err(dev->hw->dev, "failed to disable wed-wo\n");415 416 reg = ioremap(MTK_WED_WO_CPU_MCUSYS_RESET_ADDR, 4);417 418 val = readl(reg);419 switch (dev->hw->index) {420 case 0:421 val |= MTK_WED_WO_CPU_WO0_MCUSYS_RESET_MASK;422 writel(val, reg);423 val &= ~MTK_WED_WO_CPU_WO0_MCUSYS_RESET_MASK;424 writel(val, reg);425 break;426 case 1:427 val |= MTK_WED_WO_CPU_WO1_MCUSYS_RESET_MASK;428 writel(val, reg);429 val &= ~MTK_WED_WO_CPU_WO1_MCUSYS_RESET_MASK;430 writel(val, reg);431 break;432 default:433 break;434 }435 iounmap(reg);436}437 438void mtk_wed_fe_reset(void)439{440 int i;441 442 mutex_lock(&hw_lock);443 444 for (i = 0; i < ARRAY_SIZE(hw_list); i++) {445 struct mtk_wed_hw *hw = hw_list[i];446 struct mtk_wed_device *dev;447 int err;448 449 if (!hw)450 break;451 452 dev = hw->wed_dev;453 if (!dev || !dev->wlan.reset)454 continue;455 456 /* reset callback blocks until WLAN reset is completed */457 err = dev->wlan.reset(dev);458 if (err)459 dev_err(dev->dev, "wlan reset failed: %d\n", err);460 }461 462 mutex_unlock(&hw_lock);463}464 465void mtk_wed_fe_reset_complete(void)466{467 int i;468 469 mutex_lock(&hw_lock);470 471 for (i = 0; i < ARRAY_SIZE(hw_list); i++) {472 struct mtk_wed_hw *hw = hw_list[i];473 struct mtk_wed_device *dev;474 475 if (!hw)476 break;477 478 dev = hw->wed_dev;479 if (!dev || !dev->wlan.reset_complete)480 continue;481 482 dev->wlan.reset_complete(dev);483 }484 485 mutex_unlock(&hw_lock);486}487 488static struct mtk_wed_hw *489mtk_wed_assign(struct mtk_wed_device *dev)490{491 struct mtk_wed_hw *hw;492 int i;493 494 if (dev->wlan.bus_type == MTK_WED_BUS_PCIE) {495 hw = hw_list[pci_domain_nr(dev->wlan.pci_dev->bus)];496 if (!hw)497 return NULL;498 499 if (!hw->wed_dev)500 goto out;501 502 if (mtk_wed_is_v1(hw))503 return NULL;504 505 /* MT7986 WED devices do not have any pcie slot restrictions */506 }507 /* MT7986 PCIE or AXI */508 for (i = 0; i < ARRAY_SIZE(hw_list); i++) {509 hw = hw_list[i];510 if (hw && !hw->wed_dev)511 goto out;512 }513 514 return NULL;515 516out:517 hw->wed_dev = dev;518 return hw;519}520 521static int522mtk_wed_amsdu_buffer_alloc(struct mtk_wed_device *dev)523{524 struct mtk_wed_hw *hw = dev->hw;525 struct mtk_wed_amsdu *wed_amsdu;526 int i;527 528 if (!mtk_wed_is_v3_or_greater(hw))529 return 0;530 531 wed_amsdu = devm_kcalloc(hw->dev, MTK_WED_AMSDU_NPAGES,532 sizeof(*wed_amsdu), GFP_KERNEL);533 if (!wed_amsdu)534 return -ENOMEM;535 536 for (i = 0; i < MTK_WED_AMSDU_NPAGES; i++) {537 void *ptr;538 539 /* each segment is 64K */540 ptr = (void *)__get_free_pages(GFP_KERNEL | __GFP_NOWARN |541 __GFP_ZERO | __GFP_COMP |542 GFP_DMA32,543 get_order(MTK_WED_AMSDU_BUF_SIZE));544 if (!ptr)545 goto error;546 547 wed_amsdu[i].txd = ptr;548 wed_amsdu[i].txd_phy = dma_map_single(hw->dev, ptr,549 MTK_WED_AMSDU_BUF_SIZE,550 DMA_TO_DEVICE);551 if (dma_mapping_error(hw->dev, wed_amsdu[i].txd_phy))552 goto error;553 }554 dev->hw->wed_amsdu = wed_amsdu;555 556 return 0;557 558error:559 for (i--; i >= 0; i--)560 dma_unmap_single(hw->dev, wed_amsdu[i].txd_phy,561 MTK_WED_AMSDU_BUF_SIZE, DMA_TO_DEVICE);562 return -ENOMEM;563}564 565static void566mtk_wed_amsdu_free_buffer(struct mtk_wed_device *dev)567{568 struct mtk_wed_amsdu *wed_amsdu = dev->hw->wed_amsdu;569 int i;570 571 if (!wed_amsdu)572 return;573 574 for (i = 0; i < MTK_WED_AMSDU_NPAGES; i++) {575 dma_unmap_single(dev->hw->dev, wed_amsdu[i].txd_phy,576 MTK_WED_AMSDU_BUF_SIZE, DMA_TO_DEVICE);577 free_pages((unsigned long)wed_amsdu[i].txd,578 get_order(MTK_WED_AMSDU_BUF_SIZE));579 }580}581 582static int583mtk_wed_amsdu_init(struct mtk_wed_device *dev)584{585 struct mtk_wed_amsdu *wed_amsdu = dev->hw->wed_amsdu;586 int i, ret;587 588 if (!wed_amsdu)589 return 0;590 591 for (i = 0; i < MTK_WED_AMSDU_NPAGES; i++)592 wed_w32(dev, MTK_WED_AMSDU_HIFTXD_BASE_L(i),593 wed_amsdu[i].txd_phy);594 595 /* init all sta parameter */596 wed_w32(dev, MTK_WED_AMSDU_STA_INFO_INIT, MTK_WED_AMSDU_STA_RMVL |597 MTK_WED_AMSDU_STA_WTBL_HDRT_MODE |598 FIELD_PREP(MTK_WED_AMSDU_STA_MAX_AMSDU_LEN,599 dev->wlan.amsdu_max_len >> 8) |600 FIELD_PREP(MTK_WED_AMSDU_STA_MAX_AMSDU_NUM,601 dev->wlan.amsdu_max_subframes));602 603 wed_w32(dev, MTK_WED_AMSDU_STA_INFO, MTK_WED_AMSDU_STA_INFO_DO_INIT);604 605 ret = mtk_wed_poll_busy(dev, MTK_WED_AMSDU_STA_INFO,606 MTK_WED_AMSDU_STA_INFO_DO_INIT);607 if (ret) {608 dev_err(dev->hw->dev, "amsdu initialization failed\n");609 return ret;610 }611 612 /* init partial amsdu offload txd src */613 wed_set(dev, MTK_WED_AMSDU_HIFTXD_CFG,614 FIELD_PREP(MTK_WED_AMSDU_HIFTXD_SRC, dev->hw->index));615 616 /* init qmem */617 wed_set(dev, MTK_WED_AMSDU_PSE, MTK_WED_AMSDU_PSE_RESET);618 ret = mtk_wed_poll_busy(dev, MTK_WED_MON_AMSDU_QMEM_STS1, BIT(29));619 if (ret) {620 pr_info("%s: amsdu qmem initialization failed\n", __func__);621 return ret;622 }623 624 /* eagle E1 PCIE1 tx ring 22 flow control issue */625 if (dev->wlan.id == 0x7991)626 wed_clr(dev, MTK_WED_AMSDU_FIFO, MTK_WED_AMSDU_IS_PRIOR0_RING);627 628 wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_TX_AMSDU_EN);629 630 return 0;631}632 633static int634mtk_wed_tx_buffer_alloc(struct mtk_wed_device *dev)635{636 u32 desc_size = dev->hw->soc->tx_ring_desc_size;637 int i, page_idx = 0, n_pages, ring_size;638 int token = dev->wlan.token_start;639 struct mtk_wed_buf *page_list;640 dma_addr_t desc_phys;641 void *desc_ptr;642 643 if (!mtk_wed_is_v3_or_greater(dev->hw)) {644 ring_size = dev->wlan.nbuf & ~(MTK_WED_BUF_PER_PAGE - 1);645 dev->tx_buf_ring.size = ring_size;646 } else {647 dev->tx_buf_ring.size = MTK_WED_TX_BM_DMA_SIZE;648 ring_size = MTK_WED_TX_BM_PKT_CNT;649 }650 n_pages = dev->tx_buf_ring.size / MTK_WED_BUF_PER_PAGE;651 652 page_list = kcalloc(n_pages, sizeof(*page_list), GFP_KERNEL);653 if (!page_list)654 return -ENOMEM;655 656 dev->tx_buf_ring.pages = page_list;657 658 desc_ptr = dma_alloc_coherent(dev->hw->dev,659 dev->tx_buf_ring.size * desc_size,660 &desc_phys, GFP_KERNEL);661 if (!desc_ptr)662 return -ENOMEM;663 664 dev->tx_buf_ring.desc = desc_ptr;665 dev->tx_buf_ring.desc_phys = desc_phys;666 667 for (i = 0; i < ring_size; i += MTK_WED_BUF_PER_PAGE) {668 dma_addr_t page_phys, buf_phys;669 struct page *page;670 void *buf;671 int s;672 673 page = __dev_alloc_page(GFP_KERNEL);674 if (!page)675 return -ENOMEM;676 677 page_phys = dma_map_page(dev->hw->dev, page, 0, PAGE_SIZE,678 DMA_BIDIRECTIONAL);679 if (dma_mapping_error(dev->hw->dev, page_phys)) {680 __free_page(page);681 return -ENOMEM;682 }683 684 page_list[page_idx].p = page;685 page_list[page_idx++].phy_addr = page_phys;686 dma_sync_single_for_cpu(dev->hw->dev, page_phys, PAGE_SIZE,687 DMA_BIDIRECTIONAL);688 689 buf = page_to_virt(page);690 buf_phys = page_phys;691 692 for (s = 0; s < MTK_WED_BUF_PER_PAGE; s++) {693 struct mtk_wdma_desc *desc = desc_ptr;694 u32 ctrl;695 696 desc->buf0 = cpu_to_le32(buf_phys);697 if (!mtk_wed_is_v3_or_greater(dev->hw)) {698 u32 txd_size;699 700 txd_size = dev->wlan.init_buf(buf, buf_phys,701 token++);702 desc->buf1 = cpu_to_le32(buf_phys + txd_size);703 ctrl = FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN0, txd_size);704 if (mtk_wed_is_v1(dev->hw))705 ctrl |= MTK_WDMA_DESC_CTRL_LAST_SEG1 |706 FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN1,707 MTK_WED_BUF_SIZE - txd_size);708 else709 ctrl |= MTK_WDMA_DESC_CTRL_LAST_SEG0 |710 FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN1_V2,711 MTK_WED_BUF_SIZE - txd_size);712 desc->info = 0;713 } else {714 ctrl = token << 16 | TX_DMA_PREP_ADDR64(buf_phys);715 }716 desc->ctrl = cpu_to_le32(ctrl);717 718 desc_ptr += desc_size;719 buf += MTK_WED_BUF_SIZE;720 buf_phys += MTK_WED_BUF_SIZE;721 }722 723 dma_sync_single_for_device(dev->hw->dev, page_phys, PAGE_SIZE,724 DMA_BIDIRECTIONAL);725 }726 727 return 0;728}729 730static void731mtk_wed_free_tx_buffer(struct mtk_wed_device *dev)732{733 struct mtk_wed_buf *page_list = dev->tx_buf_ring.pages;734 struct mtk_wed_hw *hw = dev->hw;735 int i, page_idx = 0;736 737 if (!page_list)738 return;739 740 if (!dev->tx_buf_ring.desc)741 goto free_pagelist;742 743 for (i = 0; i < dev->tx_buf_ring.size; i += MTK_WED_BUF_PER_PAGE) {744 dma_addr_t page_phy = page_list[page_idx].phy_addr;745 void *page = page_list[page_idx++].p;746 747 if (!page)748 break;749 750 dma_unmap_page(dev->hw->dev, page_phy, PAGE_SIZE,751 DMA_BIDIRECTIONAL);752 __free_page(page);753 }754 755 dma_free_coherent(dev->hw->dev,756 dev->tx_buf_ring.size * hw->soc->tx_ring_desc_size,757 dev->tx_buf_ring.desc,758 dev->tx_buf_ring.desc_phys);759 760free_pagelist:761 kfree(page_list);762}763 764static int765mtk_wed_hwrro_buffer_alloc(struct mtk_wed_device *dev)766{767 int n_pages = MTK_WED_RX_PG_BM_CNT / MTK_WED_RX_BUF_PER_PAGE;768 struct mtk_wed_buf *page_list;769 struct mtk_wed_bm_desc *desc;770 dma_addr_t desc_phys;771 int i, page_idx = 0;772 773 if (!dev->wlan.hw_rro)774 return 0;775 776 page_list = kcalloc(n_pages, sizeof(*page_list), GFP_KERNEL);777 if (!page_list)778 return -ENOMEM;779 780 dev->hw_rro.size = dev->wlan.rx_nbuf & ~(MTK_WED_BUF_PER_PAGE - 1);781 dev->hw_rro.pages = page_list;782 desc = dma_alloc_coherent(dev->hw->dev,783 dev->wlan.rx_nbuf * sizeof(*desc),784 &desc_phys, GFP_KERNEL);785 if (!desc)786 return -ENOMEM;787 788 dev->hw_rro.desc = desc;789 dev->hw_rro.desc_phys = desc_phys;790 791 for (i = 0; i < MTK_WED_RX_PG_BM_CNT; i += MTK_WED_RX_BUF_PER_PAGE) {792 dma_addr_t page_phys, buf_phys;793 struct page *page;794 int s;795 796 page = __dev_alloc_page(GFP_KERNEL);797 if (!page)798 return -ENOMEM;799 800 page_phys = dma_map_page(dev->hw->dev, page, 0, PAGE_SIZE,801 DMA_BIDIRECTIONAL);802 if (dma_mapping_error(dev->hw->dev, page_phys)) {803 __free_page(page);804 return -ENOMEM;805 }806 807 page_list[page_idx].p = page;808 page_list[page_idx++].phy_addr = page_phys;809 dma_sync_single_for_cpu(dev->hw->dev, page_phys, PAGE_SIZE,810 DMA_BIDIRECTIONAL);811 812 buf_phys = page_phys;813 for (s = 0; s < MTK_WED_RX_BUF_PER_PAGE; s++) {814 desc->buf0 = cpu_to_le32(buf_phys);815 desc->token = cpu_to_le32(RX_DMA_PREP_ADDR64(buf_phys));816 buf_phys += MTK_WED_PAGE_BUF_SIZE;817 desc++;818 }819 820 dma_sync_single_for_device(dev->hw->dev, page_phys, PAGE_SIZE,821 DMA_BIDIRECTIONAL);822 }823 824 return 0;825}826 827static int828mtk_wed_rx_buffer_alloc(struct mtk_wed_device *dev)829{830 struct mtk_wed_bm_desc *desc;831 dma_addr_t desc_phys;832 833 dev->rx_buf_ring.size = dev->wlan.rx_nbuf;834 desc = dma_alloc_coherent(dev->hw->dev,835 dev->wlan.rx_nbuf * sizeof(*desc),836 &desc_phys, GFP_KERNEL);837 if (!desc)838 return -ENOMEM;839 840 dev->rx_buf_ring.desc = desc;841 dev->rx_buf_ring.desc_phys = desc_phys;842 dev->wlan.init_rx_buf(dev, dev->wlan.rx_npkt);843 844 return mtk_wed_hwrro_buffer_alloc(dev);845}846 847static void848mtk_wed_hwrro_free_buffer(struct mtk_wed_device *dev)849{850 struct mtk_wed_buf *page_list = dev->hw_rro.pages;851 struct mtk_wed_bm_desc *desc = dev->hw_rro.desc;852 int i, page_idx = 0;853 854 if (!dev->wlan.hw_rro)855 return;856 857 if (!page_list)858 return;859 860 if (!desc)861 goto free_pagelist;862 863 for (i = 0; i < MTK_WED_RX_PG_BM_CNT; i += MTK_WED_RX_BUF_PER_PAGE) {864 dma_addr_t buf_addr = page_list[page_idx].phy_addr;865 void *page = page_list[page_idx++].p;866 867 if (!page)868 break;869 870 dma_unmap_page(dev->hw->dev, buf_addr, PAGE_SIZE,871 DMA_BIDIRECTIONAL);872 __free_page(page);873 }874 875 dma_free_coherent(dev->hw->dev, dev->hw_rro.size * sizeof(*desc),876 desc, dev->hw_rro.desc_phys);877 878free_pagelist:879 kfree(page_list);880}881 882static void883mtk_wed_free_rx_buffer(struct mtk_wed_device *dev)884{885 struct mtk_wed_bm_desc *desc = dev->rx_buf_ring.desc;886 887 if (!desc)888 return;889 890 dev->wlan.release_rx_buf(dev);891 dma_free_coherent(dev->hw->dev, dev->rx_buf_ring.size * sizeof(*desc),892 desc, dev->rx_buf_ring.desc_phys);893 894 mtk_wed_hwrro_free_buffer(dev);895}896 897static void898mtk_wed_hwrro_init(struct mtk_wed_device *dev)899{900 if (!mtk_wed_get_rx_capa(dev) || !dev->wlan.hw_rro)901 return;902 903 wed_set(dev, MTK_WED_RRO_PG_BM_RX_DMAM,904 FIELD_PREP(MTK_WED_RRO_PG_BM_RX_SDL0, 128));905 906 wed_w32(dev, MTK_WED_RRO_PG_BM_BASE, dev->hw_rro.desc_phys);907 908 wed_w32(dev, MTK_WED_RRO_PG_BM_INIT_PTR,909 MTK_WED_RRO_PG_BM_INIT_SW_TAIL_IDX |910 FIELD_PREP(MTK_WED_RRO_PG_BM_SW_TAIL_IDX,911 MTK_WED_RX_PG_BM_CNT));912 913 /* enable rx_page_bm to fetch dmad */914 wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_RX_PG_BM_EN);915}916 917static void918mtk_wed_rx_buffer_hw_init(struct mtk_wed_device *dev)919{920 wed_w32(dev, MTK_WED_RX_BM_RX_DMAD,921 FIELD_PREP(MTK_WED_RX_BM_RX_DMAD_SDL0, dev->wlan.rx_size));922 wed_w32(dev, MTK_WED_RX_BM_BASE, dev->rx_buf_ring.desc_phys);923 wed_w32(dev, MTK_WED_RX_BM_INIT_PTR, MTK_WED_RX_BM_INIT_SW_TAIL |924 FIELD_PREP(MTK_WED_RX_BM_SW_TAIL, dev->wlan.rx_npkt));925 wed_w32(dev, MTK_WED_RX_BM_DYN_ALLOC_TH,926 FIELD_PREP(MTK_WED_RX_BM_DYN_ALLOC_TH_H, 0xffff));927 wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_RX_BM_EN);928 929 mtk_wed_hwrro_init(dev);930}931 932static void933mtk_wed_free_ring(struct mtk_wed_device *dev, struct mtk_wed_ring *ring)934{935 if (!ring->desc)936 return;937 938 dma_free_coherent(dev->hw->dev, ring->size * ring->desc_size,939 ring->desc, ring->desc_phys);940}941 942static void943mtk_wed_free_rx_rings(struct mtk_wed_device *dev)944{945 mtk_wed_free_rx_buffer(dev);946 mtk_wed_free_ring(dev, &dev->rro.ring);947}948 949static void950mtk_wed_free_tx_rings(struct mtk_wed_device *dev)951{952 int i;953 954 for (i = 0; i < ARRAY_SIZE(dev->tx_ring); i++)955 mtk_wed_free_ring(dev, &dev->tx_ring[i]);956 for (i = 0; i < ARRAY_SIZE(dev->rx_wdma); i++)957 mtk_wed_free_ring(dev, &dev->rx_wdma[i]);958}959 960static void961mtk_wed_set_ext_int(struct mtk_wed_device *dev, bool en)962{963 u32 mask = MTK_WED_EXT_INT_STATUS_ERROR_MASK;964 965 switch (dev->hw->version) {966 case 1:967 mask |= MTK_WED_EXT_INT_STATUS_TX_DRV_R_RESP_ERR;968 break;969 case 2:970 mask |= MTK_WED_EXT_INT_STATUS_RX_FBUF_LO_TH |971 MTK_WED_EXT_INT_STATUS_RX_FBUF_HI_TH |972 MTK_WED_EXT_INT_STATUS_RX_DRV_COHERENT |973 MTK_WED_EXT_INT_STATUS_TX_DMA_W_RESP_ERR;974 break;975 case 3:976 mask = MTK_WED_EXT_INT_STATUS_RX_DRV_COHERENT |977 MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD;978 break;979 default:980 break;981 }982 983 if (!dev->hw->num_flows)984 mask &= ~MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD;985 986 wed_w32(dev, MTK_WED_EXT_INT_MASK, en ? mask : 0);987 wed_r32(dev, MTK_WED_EXT_INT_MASK);988}989 990static void991mtk_wed_set_512_support(struct mtk_wed_device *dev, bool enable)992{993 if (!mtk_wed_is_v2(dev->hw))994 return;995 996 if (enable) {997 wed_w32(dev, MTK_WED_TXDP_CTRL, MTK_WED_TXDP_DW9_OVERWR);998 wed_w32(dev, MTK_WED_TXP_DW1,999 FIELD_PREP(MTK_WED_WPDMA_WRITE_TXP, 0x0103));1000 } else {1001 wed_w32(dev, MTK_WED_TXP_DW1,1002 FIELD_PREP(MTK_WED_WPDMA_WRITE_TXP, 0x0100));1003 wed_clr(dev, MTK_WED_TXDP_CTRL, MTK_WED_TXDP_DW9_OVERWR);1004 }1005}1006 1007static int1008mtk_wed_check_wfdma_rx_fill(struct mtk_wed_device *dev,1009 struct mtk_wed_ring *ring)1010{1011 int i;1012 1013 for (i = 0; i < 3; i++) {1014 u32 cur_idx = readl(ring->wpdma + MTK_WED_RING_OFS_CPU_IDX);1015 1016 if (cur_idx == MTK_WED_RX_RING_SIZE - 1)1017 break;1018 1019 usleep_range(100000, 200000);1020 }1021 1022 if (i == 3) {1023 dev_err(dev->hw->dev, "rx dma enable failed\n");1024 return -ETIMEDOUT;1025 }1026 1027 return 0;1028}1029 1030static void1031mtk_wed_dma_disable(struct mtk_wed_device *dev)1032{1033 wed_clr(dev, MTK_WED_WPDMA_GLO_CFG,1034 MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN |1035 MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN);1036 1037 wed_clr(dev, MTK_WED_WDMA_GLO_CFG, MTK_WED_WDMA_GLO_CFG_RX_DRV_EN);1038 1039 wed_clr(dev, MTK_WED_GLO_CFG,1040 MTK_WED_GLO_CFG_TX_DMA_EN |1041 MTK_WED_GLO_CFG_RX_DMA_EN);1042 1043 wdma_clr(dev, MTK_WDMA_GLO_CFG,1044 MTK_WDMA_GLO_CFG_TX_DMA_EN |1045 MTK_WDMA_GLO_CFG_RX_INFO1_PRERES |1046 MTK_WDMA_GLO_CFG_RX_INFO2_PRERES);1047 1048 if (mtk_wed_is_v1(dev->hw)) {1049 regmap_write(dev->hw->mirror, dev->hw->index * 4, 0);1050 wdma_clr(dev, MTK_WDMA_GLO_CFG,1051 MTK_WDMA_GLO_CFG_RX_INFO3_PRERES);1052 } else {1053 wed_clr(dev, MTK_WED_WPDMA_GLO_CFG,1054 MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_PKT_PROC |1055 MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_CRX_SYNC);1056 1057 wed_clr(dev, MTK_WED_WPDMA_RX_D_GLO_CFG,1058 MTK_WED_WPDMA_RX_D_RX_DRV_EN);1059 wed_clr(dev, MTK_WED_WDMA_GLO_CFG,1060 MTK_WED_WDMA_GLO_CFG_TX_DDONE_CHK);1061 1062 if (mtk_wed_is_v3_or_greater(dev->hw) &&1063 mtk_wed_get_rx_capa(dev)) {1064 wdma_clr(dev, MTK_WDMA_PREF_TX_CFG,1065 MTK_WDMA_PREF_TX_CFG_PREF_EN);1066 wdma_clr(dev, MTK_WDMA_PREF_RX_CFG,1067 MTK_WDMA_PREF_RX_CFG_PREF_EN);1068 }1069 }1070 1071 mtk_wed_set_512_support(dev, false);1072}1073 1074static void1075mtk_wed_stop(struct mtk_wed_device *dev)1076{1077 mtk_wed_dma_disable(dev);1078 mtk_wed_set_ext_int(dev, false);1079 1080 wed_w32(dev, MTK_WED_WPDMA_INT_TRIGGER, 0);1081 wed_w32(dev, MTK_WED_WDMA_INT_TRIGGER, 0);1082 wdma_w32(dev, MTK_WDMA_INT_MASK, 0);1083 wdma_w32(dev, MTK_WDMA_INT_GRP2, 0);1084 1085 if (!mtk_wed_get_rx_capa(dev))1086 return;1087 1088 wed_w32(dev, MTK_WED_EXT_INT_MASK1, 0);1089 wed_w32(dev, MTK_WED_EXT_INT_MASK2, 0);1090}1091 1092static void1093mtk_wed_deinit(struct mtk_wed_device *dev)1094{1095 mtk_wed_stop(dev);1096 1097 wed_clr(dev, MTK_WED_CTRL,1098 MTK_WED_CTRL_WDMA_INT_AGENT_EN |1099 MTK_WED_CTRL_WPDMA_INT_AGENT_EN |1100 MTK_WED_CTRL_WED_TX_BM_EN |1101 MTK_WED_CTRL_WED_TX_FREE_AGENT_EN);1102 1103 if (mtk_wed_is_v1(dev->hw))1104 return;1105 1106 wed_clr(dev, MTK_WED_CTRL,1107 MTK_WED_CTRL_RX_ROUTE_QM_EN |1108 MTK_WED_CTRL_WED_RX_BM_EN |1109 MTK_WED_CTRL_RX_RRO_QM_EN);1110 1111 if (mtk_wed_is_v3_or_greater(dev->hw)) {1112 wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_TX_AMSDU_EN);1113 wed_clr(dev, MTK_WED_RESET, MTK_WED_RESET_TX_AMSDU);1114 wed_clr(dev, MTK_WED_PCIE_INT_CTRL,1115 MTK_WED_PCIE_INT_CTRL_MSK_EN_POLA |1116 MTK_WED_PCIE_INT_CTRL_MSK_IRQ_FILTER);1117 }1118}1119 1120static void1121__mtk_wed_detach(struct mtk_wed_device *dev)1122{1123 struct mtk_wed_hw *hw = dev->hw;1124 1125 mtk_wed_deinit(dev);1126 1127 mtk_wdma_rx_reset(dev);1128 mtk_wed_reset(dev, MTK_WED_RESET_WED);1129 mtk_wed_amsdu_free_buffer(dev);1130 mtk_wed_free_tx_buffer(dev);1131 mtk_wed_free_tx_rings(dev);1132 1133 if (mtk_wed_get_rx_capa(dev)) {1134 if (hw->wed_wo)1135 mtk_wed_wo_reset(dev);1136 mtk_wed_free_rx_rings(dev);1137 if (hw->wed_wo)1138 mtk_wed_wo_deinit(hw);1139 }1140 1141 if (dev->wlan.bus_type == MTK_WED_BUS_PCIE) {1142 struct device_node *wlan_node;1143 1144 wlan_node = dev->wlan.pci_dev->dev.of_node;1145 if (of_dma_is_coherent(wlan_node) && hw->hifsys)1146 regmap_update_bits(hw->hifsys, HIFSYS_DMA_AG_MAP,1147 BIT(hw->index), BIT(hw->index));1148 }1149 1150 if ((!hw_list[!hw->index] || !hw_list[!hw->index]->wed_dev) &&1151 hw->eth->dma_dev != hw->eth->dev)1152 mtk_eth_set_dma_device(hw->eth, hw->eth->dev);1153 1154 memset(dev, 0, sizeof(*dev));1155 module_put(THIS_MODULE);1156 1157 hw->wed_dev = NULL;1158}1159 1160static void1161mtk_wed_detach(struct mtk_wed_device *dev)1162{1163 mutex_lock(&hw_lock);1164 __mtk_wed_detach(dev);1165 mutex_unlock(&hw_lock);1166}1167 1168static void1169mtk_wed_bus_init(struct mtk_wed_device *dev)1170{1171 switch (dev->wlan.bus_type) {1172 case MTK_WED_BUS_PCIE: {1173 struct device_node *np = dev->hw->eth->dev->of_node;1174 1175 if (mtk_wed_is_v2(dev->hw)) {1176 struct regmap *regs;1177 1178 regs = syscon_regmap_lookup_by_phandle(np,1179 "mediatek,wed-pcie");1180 if (IS_ERR(regs))1181 break;1182 1183 regmap_update_bits(regs, 0, BIT(0), BIT(0));1184 }1185 1186 if (dev->wlan.msi) {1187 wed_w32(dev, MTK_WED_PCIE_CFG_INTM,1188 dev->hw->pcie_base | 0xc08);1189 wed_w32(dev, MTK_WED_PCIE_CFG_BASE,1190 dev->hw->pcie_base | 0xc04);1191 wed_w32(dev, MTK_WED_PCIE_INT_TRIGGER, BIT(8));1192 } else {1193 wed_w32(dev, MTK_WED_PCIE_CFG_INTM,1194 dev->hw->pcie_base | 0x180);1195 wed_w32(dev, MTK_WED_PCIE_CFG_BASE,1196 dev->hw->pcie_base | 0x184);1197 wed_w32(dev, MTK_WED_PCIE_INT_TRIGGER, BIT(24));1198 }1199 1200 wed_w32(dev, MTK_WED_PCIE_INT_CTRL,1201 FIELD_PREP(MTK_WED_PCIE_INT_CTRL_POLL_EN, 2));1202 1203 /* pcie interrupt control: pola/source selection */1204 wed_set(dev, MTK_WED_PCIE_INT_CTRL,1205 MTK_WED_PCIE_INT_CTRL_MSK_EN_POLA |1206 MTK_WED_PCIE_INT_CTRL_MSK_IRQ_FILTER |1207 FIELD_PREP(MTK_WED_PCIE_INT_CTRL_SRC_SEL,1208 dev->hw->index));1209 break;1210 }1211 case MTK_WED_BUS_AXI:1212 wed_set(dev, MTK_WED_WPDMA_INT_CTRL,1213 MTK_WED_WPDMA_INT_CTRL_SIG_SRC |1214 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_SRC_SEL, 0));1215 break;1216 default:1217 break;1218 }1219}1220 1221static void1222mtk_wed_set_wpdma(struct mtk_wed_device *dev)1223{1224 int i;1225 1226 if (mtk_wed_is_v1(dev->hw)) {1227 wed_w32(dev, MTK_WED_WPDMA_CFG_BASE, dev->wlan.wpdma_phys);1228 return;1229 }1230 1231 mtk_wed_bus_init(dev);1232 1233 wed_w32(dev, MTK_WED_WPDMA_CFG_BASE, dev->wlan.wpdma_int);1234 wed_w32(dev, MTK_WED_WPDMA_CFG_INT_MASK, dev->wlan.wpdma_mask);1235 wed_w32(dev, MTK_WED_WPDMA_CFG_TX, dev->wlan.wpdma_tx);1236 wed_w32(dev, MTK_WED_WPDMA_CFG_TX_FREE, dev->wlan.wpdma_txfree);1237 1238 if (!mtk_wed_get_rx_capa(dev))1239 return;1240 1241 wed_w32(dev, MTK_WED_WPDMA_RX_GLO_CFG, dev->wlan.wpdma_rx_glo);1242 wed_w32(dev, dev->hw->soc->regmap.wpdma_rx_ring0, dev->wlan.wpdma_rx);1243 1244 if (!dev->wlan.hw_rro)1245 return;1246 1247 wed_w32(dev, MTK_WED_RRO_RX_D_CFG(0), dev->wlan.wpdma_rx_rro[0]);1248 wed_w32(dev, MTK_WED_RRO_RX_D_CFG(1), dev->wlan.wpdma_rx_rro[1]);1249 for (i = 0; i < MTK_WED_RX_PAGE_QUEUES; i++)1250 wed_w32(dev, MTK_WED_RRO_MSDU_PG_RING_CFG(i),1251 dev->wlan.wpdma_rx_pg + i * 0x10);1252}1253 1254static void1255mtk_wed_hw_init_early(struct mtk_wed_device *dev)1256{1257 u32 set = FIELD_PREP(MTK_WED_WDMA_GLO_CFG_BT_SIZE, 2);1258 u32 mask = MTK_WED_WDMA_GLO_CFG_BT_SIZE;1259 1260 mtk_wed_deinit(dev);1261 mtk_wed_reset(dev, MTK_WED_RESET_WED);1262 mtk_wed_set_wpdma(dev);1263 1264 if (!mtk_wed_is_v3_or_greater(dev->hw)) {1265 mask |= MTK_WED_WDMA_GLO_CFG_DYNAMIC_DMAD_RECYCLE |1266 MTK_WED_WDMA_GLO_CFG_RX_DIS_FSM_AUTO_IDLE;1267 set |= MTK_WED_WDMA_GLO_CFG_DYNAMIC_SKIP_DMAD_PREP |1268 MTK_WED_WDMA_GLO_CFG_IDLE_DMAD_SUPPLY;1269 }1270 wed_m32(dev, MTK_WED_WDMA_GLO_CFG, mask, set);1271 1272 if (mtk_wed_is_v1(dev->hw)) {1273 u32 offset = dev->hw->index ? 0x04000400 : 0;1274 1275 wdma_set(dev, MTK_WDMA_GLO_CFG,1276 MTK_WDMA_GLO_CFG_RX_INFO1_PRERES |1277 MTK_WDMA_GLO_CFG_RX_INFO2_PRERES |1278 MTK_WDMA_GLO_CFG_RX_INFO3_PRERES);1279 1280 wed_w32(dev, MTK_WED_WDMA_OFFSET0, 0x2a042a20 + offset);1281 wed_w32(dev, MTK_WED_WDMA_OFFSET1, 0x29002800 + offset);1282 wed_w32(dev, MTK_WED_PCIE_CFG_BASE,1283 MTK_PCIE_BASE(dev->hw->index));1284 } else {1285 wed_w32(dev, MTK_WED_WDMA_CFG_BASE, dev->hw->wdma_phy);1286 wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_ETH_DMAD_FMT);1287 wed_w32(dev, MTK_WED_WDMA_OFFSET0,1288 FIELD_PREP(MTK_WED_WDMA_OFST0_GLO_INTS,1289 MTK_WDMA_INT_STATUS) |1290 FIELD_PREP(MTK_WED_WDMA_OFST0_GLO_CFG,1291 MTK_WDMA_GLO_CFG));1292 1293 wed_w32(dev, MTK_WED_WDMA_OFFSET1,1294 FIELD_PREP(MTK_WED_WDMA_OFST1_TX_CTRL,1295 MTK_WDMA_RING_TX(0)) |1296 FIELD_PREP(MTK_WED_WDMA_OFST1_RX_CTRL,1297 MTK_WDMA_RING_RX(0)));1298 }1299}1300 1301static int1302mtk_wed_rro_ring_alloc(struct mtk_wed_device *dev, struct mtk_wed_ring *ring,1303 int size)1304{1305 ring->desc = dma_alloc_coherent(dev->hw->dev,1306 size * sizeof(*ring->desc),1307 &ring->desc_phys, GFP_KERNEL);1308 if (!ring->desc)1309 return -ENOMEM;1310 1311 ring->desc_size = sizeof(*ring->desc);1312 ring->size = size;1313 1314 return 0;1315}1316 1317#define MTK_WED_MIOD_COUNT (MTK_WED_MIOD_ENTRY_CNT * MTK_WED_MIOD_CNT)1318static int1319mtk_wed_rro_alloc(struct mtk_wed_device *dev)1320{1321 struct reserved_mem *rmem;1322 struct device_node *np;1323 int index;1324 1325 index = of_property_match_string(dev->hw->node, "memory-region-names",1326 "wo-dlm");1327 if (index < 0)1328 return index;1329 1330 np = of_parse_phandle(dev->hw->node, "memory-region", index);1331 if (!np)1332 return -ENODEV;1333 1334 rmem = of_reserved_mem_lookup(np);1335 of_node_put(np);1336 1337 if (!rmem)1338 return -ENODEV;1339 1340 dev->rro.miod_phys = rmem->base;1341 dev->rro.fdbk_phys = MTK_WED_MIOD_COUNT + dev->rro.miod_phys;1342 1343 return mtk_wed_rro_ring_alloc(dev, &dev->rro.ring,1344 MTK_WED_RRO_QUE_CNT);1345}1346 1347static int1348mtk_wed_rro_cfg(struct mtk_wed_device *dev)1349{1350 struct mtk_wed_wo *wo = dev->hw->wed_wo;1351 struct {1352 struct {1353 __le32 base;1354 __le32 cnt;1355 __le32 unit;1356 } ring[2];1357 __le32 wed;1358 u8 version;1359 } req = {1360 .ring[0] = {1361 .base = cpu_to_le32(MTK_WED_WOCPU_VIEW_MIOD_BASE),1362 .cnt = cpu_to_le32(MTK_WED_MIOD_CNT),1363 .unit = cpu_to_le32(MTK_WED_MIOD_ENTRY_CNT),1364 },1365 .ring[1] = {1366 .base = cpu_to_le32(MTK_WED_WOCPU_VIEW_MIOD_BASE +1367 MTK_WED_MIOD_COUNT),1368 .cnt = cpu_to_le32(MTK_WED_FB_CMD_CNT),1369 .unit = cpu_to_le32(4),1370 },1371 };1372 1373 return mtk_wed_mcu_send_msg(wo, MTK_WED_MODULE_ID_WO,1374 MTK_WED_WO_CMD_WED_CFG,1375 &req, sizeof(req), true);1376}1377 1378static void1379mtk_wed_rro_hw_init(struct mtk_wed_device *dev)1380{1381 wed_w32(dev, MTK_WED_RROQM_MIOD_CFG,1382 FIELD_PREP(MTK_WED_RROQM_MIOD_MID_DW, 0x70 >> 2) |1383 FIELD_PREP(MTK_WED_RROQM_MIOD_MOD_DW, 0x10 >> 2) |1384 FIELD_PREP(MTK_WED_RROQM_MIOD_ENTRY_DW,1385 MTK_WED_MIOD_ENTRY_CNT >> 2));1386 1387 wed_w32(dev, MTK_WED_RROQM_MIOD_CTRL0, dev->rro.miod_phys);1388 wed_w32(dev, MTK_WED_RROQM_MIOD_CTRL1,1389 FIELD_PREP(MTK_WED_RROQM_MIOD_CNT, MTK_WED_MIOD_CNT));1390 wed_w32(dev, MTK_WED_RROQM_FDBK_CTRL0, dev->rro.fdbk_phys);1391 wed_w32(dev, MTK_WED_RROQM_FDBK_CTRL1,1392 FIELD_PREP(MTK_WED_RROQM_FDBK_CNT, MTK_WED_FB_CMD_CNT));1393 wed_w32(dev, MTK_WED_RROQM_FDBK_CTRL2, 0);1394 wed_w32(dev, MTK_WED_RROQ_BASE_L, dev->rro.ring.desc_phys);1395 1396 wed_set(dev, MTK_WED_RROQM_RST_IDX,1397 MTK_WED_RROQM_RST_IDX_MIOD |1398 MTK_WED_RROQM_RST_IDX_FDBK);1399 1400 wed_w32(dev, MTK_WED_RROQM_RST_IDX, 0);1401 wed_w32(dev, MTK_WED_RROQM_MIOD_CTRL2, MTK_WED_MIOD_CNT - 1);1402 wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_RX_RRO_QM_EN);1403}1404 1405static void1406mtk_wed_route_qm_hw_init(struct mtk_wed_device *dev)1407{1408 wed_w32(dev, MTK_WED_RESET, MTK_WED_RESET_RX_ROUTE_QM);1409 1410 for (;;) {1411 usleep_range(100, 200);1412 if (!(wed_r32(dev, MTK_WED_RESET) & MTK_WED_RESET_RX_ROUTE_QM))1413 break;1414 }1415 1416 /* configure RX_ROUTE_QM */1417 if (mtk_wed_is_v2(dev->hw)) {1418 wed_clr(dev, MTK_WED_RTQM_GLO_CFG, MTK_WED_RTQM_Q_RST);1419 wed_clr(dev, MTK_WED_RTQM_GLO_CFG, MTK_WED_RTQM_TXDMAD_FPORT);1420 wed_set(dev, MTK_WED_RTQM_GLO_CFG,1421 FIELD_PREP(MTK_WED_RTQM_TXDMAD_FPORT,1422 0x3 + dev->hw->index));1423 wed_clr(dev, MTK_WED_RTQM_GLO_CFG, MTK_WED_RTQM_Q_RST);1424 } else {1425 wed_set(dev, MTK_WED_RTQM_ENQ_CFG0,1426 FIELD_PREP(MTK_WED_RTQM_ENQ_CFG_TXDMAD_FPORT,1427 0x3 + dev->hw->index));1428 }1429 /* enable RX_ROUTE_QM */1430 wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_RX_ROUTE_QM_EN);1431}1432 1433static void1434mtk_wed_hw_init(struct mtk_wed_device *dev)1435{1436 if (dev->init_done)1437 return;1438 1439 dev->init_done = true;1440 mtk_wed_set_ext_int(dev, false);1441 1442 wed_w32(dev, MTK_WED_TX_BM_BASE, dev->tx_buf_ring.desc_phys);1443 wed_w32(dev, MTK_WED_TX_BM_BUF_LEN, MTK_WED_PKT_SIZE);1444 1445 if (mtk_wed_is_v1(dev->hw)) {1446 wed_w32(dev, MTK_WED_TX_BM_CTRL,1447 MTK_WED_TX_BM_CTRL_PAUSE |1448 FIELD_PREP(MTK_WED_TX_BM_CTRL_VLD_GRP_NUM,1449 dev->tx_buf_ring.size / 128) |1450 FIELD_PREP(MTK_WED_TX_BM_CTRL_RSV_GRP_NUM,1451 MTK_WED_TX_RING_SIZE / 256));1452 wed_w32(dev, MTK_WED_TX_BM_DYN_THR,1453 FIELD_PREP(MTK_WED_TX_BM_DYN_THR_LO, 1) |1454 MTK_WED_TX_BM_DYN_THR_HI);1455 } else if (mtk_wed_is_v2(dev->hw)) {1456 wed_w32(dev, MTK_WED_TX_BM_CTRL,1457 MTK_WED_TX_BM_CTRL_PAUSE |1458 FIELD_PREP(MTK_WED_TX_BM_CTRL_VLD_GRP_NUM,1459 dev->tx_buf_ring.size / 128) |1460 FIELD_PREP(MTK_WED_TX_BM_CTRL_RSV_GRP_NUM,1461 MTK_WED_TX_RING_SIZE / 256));1462 wed_w32(dev, MTK_WED_TX_TKID_DYN_THR,1463 FIELD_PREP(MTK_WED_TX_TKID_DYN_THR_LO, 0) |1464 MTK_WED_TX_TKID_DYN_THR_HI);1465 wed_w32(dev, MTK_WED_TX_BM_DYN_THR,1466 FIELD_PREP(MTK_WED_TX_BM_DYN_THR_LO_V2, 0) |1467 MTK_WED_TX_BM_DYN_THR_HI_V2);1468 wed_w32(dev, MTK_WED_TX_TKID_CTRL,1469 MTK_WED_TX_TKID_CTRL_PAUSE |1470 FIELD_PREP(MTK_WED_TX_TKID_CTRL_VLD_GRP_NUM,1471 dev->tx_buf_ring.size / 128) |1472 FIELD_PREP(MTK_WED_TX_TKID_CTRL_RSV_GRP_NUM,1473 dev->tx_buf_ring.size / 128));1474 }1475 1476 wed_w32(dev, dev->hw->soc->regmap.tx_bm_tkid,1477 FIELD_PREP(MTK_WED_TX_BM_TKID_START, dev->wlan.token_start) |1478 FIELD_PREP(MTK_WED_TX_BM_TKID_END,1479 dev->wlan.token_start + dev->wlan.nbuf - 1));1480 1481 mtk_wed_reset(dev, MTK_WED_RESET_TX_BM);1482 1483 if (mtk_wed_is_v3_or_greater(dev->hw)) {1484 /* switch to new bm architecture */1485 wed_clr(dev, MTK_WED_TX_BM_CTRL,1486 MTK_WED_TX_BM_CTRL_LEGACY_EN);1487 1488 wed_w32(dev, MTK_WED_TX_TKID_CTRL,1489 MTK_WED_TX_TKID_CTRL_PAUSE |1490 FIELD_PREP(MTK_WED_TX_TKID_CTRL_VLD_GRP_NUM_V3,1491 dev->wlan.nbuf / 128) |1492 FIELD_PREP(MTK_WED_TX_TKID_CTRL_RSV_GRP_NUM_V3,1493 dev->wlan.nbuf / 128));1494 /* return SKBID + SDP back to bm */1495 wed_set(dev, MTK_WED_TX_TKID_CTRL,1496 MTK_WED_TX_TKID_CTRL_FREE_FORMAT);1497 1498 wed_w32(dev, MTK_WED_TX_BM_INIT_PTR,1499 MTK_WED_TX_BM_PKT_CNT |1500 MTK_WED_TX_BM_INIT_SW_TAIL_IDX);1501 }1502 1503 if (mtk_wed_is_v1(dev->hw)) {1504 wed_set(dev, MTK_WED_CTRL,1505 MTK_WED_CTRL_WED_TX_BM_EN |1506 MTK_WED_CTRL_WED_TX_FREE_AGENT_EN);1507 } else if (mtk_wed_get_rx_capa(dev)) {1508 /* rx hw init */1509 wed_w32(dev, MTK_WED_WPDMA_RX_D_RST_IDX,1510 MTK_WED_WPDMA_RX_D_RST_CRX_IDX |1511 MTK_WED_WPDMA_RX_D_RST_DRV_IDX);1512 wed_w32(dev, MTK_WED_WPDMA_RX_D_RST_IDX, 0);1513 1514 /* reset prefetch index of ring */1515 wed_set(dev, MTK_WED_WPDMA_RX_D_PREF_RX0_SIDX,1516 MTK_WED_WPDMA_RX_D_PREF_SIDX_IDX_CLR);1517 wed_clr(dev, MTK_WED_WPDMA_RX_D_PREF_RX0_SIDX,1518 MTK_WED_WPDMA_RX_D_PREF_SIDX_IDX_CLR);1519 1520 wed_set(dev, MTK_WED_WPDMA_RX_D_PREF_RX1_SIDX,1521 MTK_WED_WPDMA_RX_D_PREF_SIDX_IDX_CLR);1522 wed_clr(dev, MTK_WED_WPDMA_RX_D_PREF_RX1_SIDX,1523 MTK_WED_WPDMA_RX_D_PREF_SIDX_IDX_CLR);1524 1525 /* reset prefetch FIFO of ring */1526 wed_set(dev, MTK_WED_WPDMA_RX_D_PREF_FIFO_CFG,1527 MTK_WED_WPDMA_RX_D_PREF_FIFO_CFG_R0_CLR |1528 MTK_WED_WPDMA_RX_D_PREF_FIFO_CFG_R1_CLR);1529 wed_w32(dev, MTK_WED_WPDMA_RX_D_PREF_FIFO_CFG, 0);1530 1531 mtk_wed_rx_buffer_hw_init(dev);1532 mtk_wed_rro_hw_init(dev);1533 mtk_wed_route_qm_hw_init(dev);1534 }1535 1536 wed_clr(dev, MTK_WED_TX_BM_CTRL, MTK_WED_TX_BM_CTRL_PAUSE);1537 if (!mtk_wed_is_v1(dev->hw))1538 wed_clr(dev, MTK_WED_TX_TKID_CTRL, MTK_WED_TX_TKID_CTRL_PAUSE);1539}1540 1541static void1542mtk_wed_ring_reset(struct mtk_wed_ring *ring, int size, bool tx)1543{1544 void *head = (void *)ring->desc;1545 int i;1546 1547 for (i = 0; i < size; i++) {1548 struct mtk_wdma_desc *desc;1549 1550 desc = (struct mtk_wdma_desc *)(head + i * ring->desc_size);1551 desc->buf0 = 0;1552 if (tx)1553 desc->ctrl = cpu_to_le32(MTK_WDMA_DESC_CTRL_DMA_DONE);1554 else1555 desc->ctrl = cpu_to_le32(MTK_WFDMA_DESC_CTRL_TO_HOST);1556 desc->buf1 = 0;1557 desc->info = 0;1558 }1559}1560 1561static int1562mtk_wed_rx_reset(struct mtk_wed_device *dev)1563{1564 struct mtk_wed_wo *wo = dev->hw->wed_wo;1565 u8 val = MTK_WED_WO_STATE_SER_RESET;1566 int i, ret;1567 1568 ret = mtk_wed_mcu_send_msg(wo, MTK_WED_MODULE_ID_WO,1569 MTK_WED_WO_CMD_CHANGE_STATE, &val,1570 sizeof(val), true);1571 if (ret)1572 return ret;1573 1574 if (dev->wlan.hw_rro) {1575 wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_RX_IND_CMD_EN);1576 mtk_wed_poll_busy(dev, MTK_WED_RRO_RX_HW_STS,1577 MTK_WED_RX_IND_CMD_BUSY);1578 mtk_wed_reset(dev, MTK_WED_RESET_RRO_RX_TO_PG);1579 }1580 1581 wed_clr(dev, MTK_WED_WPDMA_RX_D_GLO_CFG, MTK_WED_WPDMA_RX_D_RX_DRV_EN);1582 ret = mtk_wed_poll_busy(dev, MTK_WED_WPDMA_RX_D_GLO_CFG,1583 MTK_WED_WPDMA_RX_D_RX_DRV_BUSY);1584 if (!ret && mtk_wed_is_v3_or_greater(dev->hw))1585 ret = mtk_wed_poll_busy(dev, MTK_WED_WPDMA_RX_D_PREF_CFG,1586 MTK_WED_WPDMA_RX_D_PREF_BUSY);1587 if (ret) {1588 mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_INT_AGENT);1589 mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_RX_D_DRV);1590 } else {1591 if (mtk_wed_is_v3_or_greater(dev->hw)) {1592 /* 1.a. disable prefetch HW */1593 wed_clr(dev, MTK_WED_WPDMA_RX_D_PREF_CFG,1594 MTK_WED_WPDMA_RX_D_PREF_EN);1595 mtk_wed_poll_busy(dev, MTK_WED_WPDMA_RX_D_PREF_CFG,1596 MTK_WED_WPDMA_RX_D_PREF_BUSY);1597 wed_w32(dev, MTK_WED_WPDMA_RX_D_RST_IDX,1598 MTK_WED_WPDMA_RX_D_RST_DRV_IDX_ALL);1599 }1600 1601 wed_w32(dev, MTK_WED_WPDMA_RX_D_RST_IDX,1602 MTK_WED_WPDMA_RX_D_RST_CRX_IDX |1603 MTK_WED_WPDMA_RX_D_RST_DRV_IDX);1604 1605 wed_set(dev, MTK_WED_WPDMA_RX_D_GLO_CFG,1606 MTK_WED_WPDMA_RX_D_RST_INIT_COMPLETE |1607 MTK_WED_WPDMA_RX_D_FSM_RETURN_IDLE);1608 wed_clr(dev, MTK_WED_WPDMA_RX_D_GLO_CFG,1609 MTK_WED_WPDMA_RX_D_RST_INIT_COMPLETE |1610 MTK_WED_WPDMA_RX_D_FSM_RETURN_IDLE);1611 1612 wed_w32(dev, MTK_WED_WPDMA_RX_D_RST_IDX, 0);1613 }1614 1615 /* reset rro qm */1616 wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_RX_RRO_QM_EN);1617 ret = mtk_wed_poll_busy(dev, MTK_WED_CTRL,1618 MTK_WED_CTRL_RX_RRO_QM_BUSY);1619 if (ret) {1620 mtk_wed_reset(dev, MTK_WED_RESET_RX_RRO_QM);1621 } else {1622 wed_set(dev, MTK_WED_RROQM_RST_IDX,1623 MTK_WED_RROQM_RST_IDX_MIOD |1624 MTK_WED_RROQM_RST_IDX_FDBK);1625 wed_w32(dev, MTK_WED_RROQM_RST_IDX, 0);1626 }1627 1628 if (dev->wlan.hw_rro) {1629 /* disable rro msdu page drv */1630 wed_clr(dev, MTK_WED_RRO_MSDU_PG_RING2_CFG,1631 MTK_WED_RRO_MSDU_PG_DRV_EN);1632 1633 /* disable rro data drv */1634 wed_clr(dev, MTK_WED_RRO_RX_D_CFG(2), MTK_WED_RRO_RX_D_DRV_EN);1635 1636 /* rro msdu page drv reset */1637 wed_w32(dev, MTK_WED_RRO_MSDU_PG_RING2_CFG,1638 MTK_WED_RRO_MSDU_PG_DRV_CLR);1639 mtk_wed_poll_busy(dev, MTK_WED_RRO_MSDU_PG_RING2_CFG,1640 MTK_WED_RRO_MSDU_PG_DRV_CLR);1641 1642 /* rro data drv reset */1643 wed_w32(dev, MTK_WED_RRO_RX_D_CFG(2),1644 MTK_WED_RRO_RX_D_DRV_CLR);1645 mtk_wed_poll_busy(dev, MTK_WED_RRO_RX_D_CFG(2),1646 MTK_WED_RRO_RX_D_DRV_CLR);1647 }1648 1649 /* reset route qm */1650 wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_RX_ROUTE_QM_EN);1651 ret = mtk_wed_poll_busy(dev, MTK_WED_CTRL,1652 MTK_WED_CTRL_RX_ROUTE_QM_BUSY);1653 if (ret) {1654 mtk_wed_reset(dev, MTK_WED_RESET_RX_ROUTE_QM);1655 } else if (mtk_wed_is_v3_or_greater(dev->hw)) {1656 wed_set(dev, MTK_WED_RTQM_RST, BIT(0));1657 wed_clr(dev, MTK_WED_RTQM_RST, BIT(0));1658 mtk_wed_reset(dev, MTK_WED_RESET_RX_ROUTE_QM);1659 } else {1660 wed_set(dev, MTK_WED_RTQM_GLO_CFG, MTK_WED_RTQM_Q_RST);1661 }1662 1663 /* reset tx wdma */1664 mtk_wdma_tx_reset(dev);1665 1666 /* reset tx wdma drv */1667 wed_clr(dev, MTK_WED_WDMA_GLO_CFG, MTK_WED_WDMA_GLO_CFG_TX_DRV_EN);1668 if (mtk_wed_is_v3_or_greater(dev->hw))1669 mtk_wed_poll_busy(dev, MTK_WED_WPDMA_STATUS,1670 MTK_WED_WPDMA_STATUS_TX_DRV);1671 else1672 mtk_wed_poll_busy(dev, MTK_WED_CTRL,1673 MTK_WED_CTRL_WDMA_INT_AGENT_BUSY);1674 mtk_wed_reset(dev, MTK_WED_RESET_WDMA_TX_DRV);1675 1676 /* reset wed rx dma */1677 ret = mtk_wed_poll_busy(dev, MTK_WED_GLO_CFG,1678 MTK_WED_GLO_CFG_RX_DMA_BUSY);1679 wed_clr(dev, MTK_WED_GLO_CFG, MTK_WED_GLO_CFG_RX_DMA_EN);1680 if (ret) {1681 mtk_wed_reset(dev, MTK_WED_RESET_WED_RX_DMA);1682 } else {1683 wed_set(dev, MTK_WED_RESET_IDX,1684 dev->hw->soc->regmap.reset_idx_rx_mask);1685 wed_w32(dev, MTK_WED_RESET_IDX, 0);1686 }1687 1688 /* reset rx bm */1689 wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_RX_BM_EN);1690 mtk_wed_poll_busy(dev, MTK_WED_CTRL,1691 MTK_WED_CTRL_WED_RX_BM_BUSY);1692 mtk_wed_reset(dev, MTK_WED_RESET_RX_BM);1693 1694 if (dev->wlan.hw_rro) {1695 wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_RX_PG_BM_EN);1696 mtk_wed_poll_busy(dev, MTK_WED_CTRL,1697 MTK_WED_CTRL_WED_RX_PG_BM_BUSY);1698 wed_set(dev, MTK_WED_RESET, MTK_WED_RESET_RX_PG_BM);1699 wed_clr(dev, MTK_WED_RESET, MTK_WED_RESET_RX_PG_BM);1700 }1701 1702 /* wo change to enable state */1703 val = MTK_WED_WO_STATE_ENABLE;1704 ret = mtk_wed_mcu_send_msg(wo, MTK_WED_MODULE_ID_WO,1705 MTK_WED_WO_CMD_CHANGE_STATE, &val,1706 sizeof(val), true);1707 if (ret)1708 return ret;1709 1710 /* wed_rx_ring_reset */1711 for (i = 0; i < ARRAY_SIZE(dev->rx_ring); i++) {1712 if (!dev->rx_ring[i].desc)1713 continue;1714 1715 mtk_wed_ring_reset(&dev->rx_ring[i], MTK_WED_RX_RING_SIZE,1716 false);1717 }1718 mtk_wed_free_rx_buffer(dev);1719 mtk_wed_hwrro_free_buffer(dev);1720 1721 return 0;1722}1723 1724static void1725mtk_wed_reset_dma(struct mtk_wed_device *dev)1726{1727 bool busy = false;1728 u32 val;1729 int i;1730 1731 for (i = 0; i < ARRAY_SIZE(dev->tx_ring); i++) {1732 if (!dev->tx_ring[i].desc)1733 continue;1734 1735 mtk_wed_ring_reset(&dev->tx_ring[i], MTK_WED_TX_RING_SIZE,1736 true);1737 }1738 1739 /* 1. reset WED tx DMA */1740 wed_clr(dev, MTK_WED_GLO_CFG, MTK_WED_GLO_CFG_TX_DMA_EN);1741 busy = mtk_wed_poll_busy(dev, MTK_WED_GLO_CFG,1742 MTK_WED_GLO_CFG_TX_DMA_BUSY);1743 if (busy) {1744 mtk_wed_reset(dev, MTK_WED_RESET_WED_TX_DMA);1745 } else {1746 wed_w32(dev, MTK_WED_RESET_IDX,1747 dev->hw->soc->regmap.reset_idx_tx_mask);1748 wed_w32(dev, MTK_WED_RESET_IDX, 0);1749 }1750 1751 /* 2. reset WDMA rx DMA */1752 busy = !!mtk_wdma_rx_reset(dev);1753 if (mtk_wed_is_v3_or_greater(dev->hw)) {1754 val = MTK_WED_WDMA_GLO_CFG_RX_DIS_FSM_AUTO_IDLE |1755 wed_r32(dev, MTK_WED_WDMA_GLO_CFG);1756 val &= ~MTK_WED_WDMA_GLO_CFG_RX_DRV_EN;1757 wed_w32(dev, MTK_WED_WDMA_GLO_CFG, val);1758 } else {1759 wed_clr(dev, MTK_WED_WDMA_GLO_CFG,1760 MTK_WED_WDMA_GLO_CFG_RX_DRV_EN);1761 }1762 1763 if (!busy)1764 busy = mtk_wed_poll_busy(dev, MTK_WED_WDMA_GLO_CFG,1765 MTK_WED_WDMA_GLO_CFG_RX_DRV_BUSY);1766 if (!busy && mtk_wed_is_v3_or_greater(dev->hw))1767 busy = mtk_wed_poll_busy(dev, MTK_WED_WDMA_RX_PREF_CFG,1768 MTK_WED_WDMA_RX_PREF_BUSY);1769 1770 if (busy) {1771 mtk_wed_reset(dev, MTK_WED_RESET_WDMA_INT_AGENT);1772 mtk_wed_reset(dev, MTK_WED_RESET_WDMA_RX_DRV);1773 } else {1774 if (mtk_wed_is_v3_or_greater(dev->hw)) {1775 /* 1.a. disable prefetch HW */1776 wed_clr(dev, MTK_WED_WDMA_RX_PREF_CFG,1777 MTK_WED_WDMA_RX_PREF_EN);1778 mtk_wed_poll_busy(dev, MTK_WED_WDMA_RX_PREF_CFG,1779 MTK_WED_WDMA_RX_PREF_BUSY);1780 wed_clr(dev, MTK_WED_WDMA_RX_PREF_CFG,1781 MTK_WED_WDMA_RX_PREF_DDONE2_EN);1782 1783 /* 2. Reset dma index */1784 wed_w32(dev, MTK_WED_WDMA_RESET_IDX,1785 MTK_WED_WDMA_RESET_IDX_RX_ALL);1786 }1787 1788 wed_w32(dev, MTK_WED_WDMA_RESET_IDX,1789 MTK_WED_WDMA_RESET_IDX_RX | MTK_WED_WDMA_RESET_IDX_DRV);1790 wed_w32(dev, MTK_WED_WDMA_RESET_IDX, 0);1791 1792 wed_set(dev, MTK_WED_WDMA_GLO_CFG,1793 MTK_WED_WDMA_GLO_CFG_RST_INIT_COMPLETE);1794 1795 wed_clr(dev, MTK_WED_WDMA_GLO_CFG,1796 MTK_WED_WDMA_GLO_CFG_RST_INIT_COMPLETE);1797 }1798 1799 /* 3. reset WED WPDMA tx */1800 wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_TX_FREE_AGENT_EN);1801 1802 for (i = 0; i < 100; i++) {1803 if (mtk_wed_is_v1(dev->hw))1804 val = FIELD_GET(MTK_WED_TX_BM_INTF_TKFIFO_FDEP,1805 wed_r32(dev, MTK_WED_TX_BM_INTF));1806 else1807 val = FIELD_GET(MTK_WED_TX_TKID_INTF_TKFIFO_FDEP,1808 wed_r32(dev, MTK_WED_TX_TKID_INTF));1809 if (val == 0x40)1810 break;1811 }1812 1813 mtk_wed_reset(dev, MTK_WED_RESET_TX_FREE_AGENT);1814 wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_TX_BM_EN);1815 mtk_wed_reset(dev, MTK_WED_RESET_TX_BM);1816 1817 /* 4. reset WED WPDMA tx */1818 busy = mtk_wed_poll_busy(dev, MTK_WED_WPDMA_GLO_CFG,1819 MTK_WED_WPDMA_GLO_CFG_TX_DRV_BUSY);1820 wed_clr(dev, MTK_WED_WPDMA_GLO_CFG,1821 MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN |1822 MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN);1823 if (!busy)1824 busy = mtk_wed_poll_busy(dev, MTK_WED_WPDMA_GLO_CFG,1825 MTK_WED_WPDMA_GLO_CFG_RX_DRV_BUSY);1826 1827 if (busy) {1828 mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_INT_AGENT);1829 mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_TX_DRV);1830 mtk_wed_reset(dev, MTK_WED_RESET_WPDMA_RX_DRV);1831 if (mtk_wed_is_v3_or_greater(dev->hw))1832 wed_w32(dev, MTK_WED_RX1_CTRL2, 0);1833 } else {1834 wed_w32(dev, MTK_WED_WPDMA_RESET_IDX,1835 MTK_WED_WPDMA_RESET_IDX_TX |1836 MTK_WED_WPDMA_RESET_IDX_RX);1837 wed_w32(dev, MTK_WED_WPDMA_RESET_IDX, 0);1838 }1839 1840 dev->init_done = false;1841 if (mtk_wed_is_v1(dev->hw))1842 return;1843 1844 if (!busy) {1845 wed_w32(dev, MTK_WED_RESET_IDX, MTK_WED_RESET_WPDMA_IDX_RX);1846 wed_w32(dev, MTK_WED_RESET_IDX, 0);1847 }1848 1849 if (mtk_wed_is_v3_or_greater(dev->hw)) {1850 /* reset amsdu engine */1851 wed_clr(dev, MTK_WED_CTRL, MTK_WED_CTRL_TX_AMSDU_EN);1852 mtk_wed_reset(dev, MTK_WED_RESET_TX_AMSDU);1853 }1854 1855 if (mtk_wed_get_rx_capa(dev))1856 mtk_wed_rx_reset(dev);1857}1858 1859static int1860mtk_wed_ring_alloc(struct mtk_wed_device *dev, struct mtk_wed_ring *ring,1861 int size, u32 desc_size, bool tx)1862{1863 ring->desc = dma_alloc_coherent(dev->hw->dev, size * desc_size,1864 &ring->desc_phys, GFP_KERNEL);1865 if (!ring->desc)1866 return -ENOMEM;1867 1868 ring->desc_size = desc_size;1869 ring->size = size;1870 mtk_wed_ring_reset(ring, size, tx);1871 1872 return 0;1873}1874 1875static int1876mtk_wed_wdma_rx_ring_setup(struct mtk_wed_device *dev, int idx, int size,1877 bool reset)1878{1879 struct mtk_wed_ring *wdma;1880 1881 if (idx >= ARRAY_SIZE(dev->rx_wdma))1882 return -EINVAL;1883 1884 wdma = &dev->rx_wdma[idx];1885 if (!reset && mtk_wed_ring_alloc(dev, wdma, MTK_WED_WDMA_RING_SIZE,1886 dev->hw->soc->wdma_desc_size, true))1887 return -ENOMEM;1888 1889 wdma_w32(dev, MTK_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_BASE,1890 wdma->desc_phys);1891 wdma_w32(dev, MTK_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_COUNT,1892 size);1893 wdma_w32(dev, MTK_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_CPU_IDX, 0);1894 1895 wed_w32(dev, MTK_WED_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_BASE,1896 wdma->desc_phys);1897 wed_w32(dev, MTK_WED_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_COUNT,1898 size);1899 1900 return 0;1901}1902 1903static int1904mtk_wed_wdma_tx_ring_setup(struct mtk_wed_device *dev, int idx, int size,1905 bool reset)1906{1907 struct mtk_wed_ring *wdma;1908 1909 if (idx >= ARRAY_SIZE(dev->tx_wdma))1910 return -EINVAL;1911 1912 wdma = &dev->tx_wdma[idx];1913 if (!reset && mtk_wed_ring_alloc(dev, wdma, MTK_WED_WDMA_RING_SIZE,1914 dev->hw->soc->wdma_desc_size, true))1915 return -ENOMEM;1916 1917 if (mtk_wed_is_v3_or_greater(dev->hw)) {1918 struct mtk_wdma_desc *desc = wdma->desc;1919 int i;1920 1921 for (i = 0; i < MTK_WED_WDMA_RING_SIZE; i++) {1922 desc->buf0 = 0;1923 desc->ctrl = cpu_to_le32(MTK_WDMA_DESC_CTRL_DMA_DONE);1924 desc->buf1 = 0;1925 desc->info = cpu_to_le32(MTK_WDMA_TXD0_DESC_INFO_DMA_DONE);1926 desc++;1927 desc->buf0 = 0;1928 desc->ctrl = cpu_to_le32(MTK_WDMA_DESC_CTRL_DMA_DONE);1929 desc->buf1 = 0;1930 desc->info = cpu_to_le32(MTK_WDMA_TXD1_DESC_INFO_DMA_DONE);1931 desc++;1932 }1933 }1934 1935 wdma_w32(dev, MTK_WDMA_RING_TX(idx) + MTK_WED_RING_OFS_BASE,1936 wdma->desc_phys);1937 wdma_w32(dev, MTK_WDMA_RING_TX(idx) + MTK_WED_RING_OFS_COUNT,1938 size);1939 wdma_w32(dev, MTK_WDMA_RING_TX(idx) + MTK_WED_RING_OFS_CPU_IDX, 0);1940 wdma_w32(dev, MTK_WDMA_RING_TX(idx) + MTK_WED_RING_OFS_DMA_IDX, 0);1941 1942 if (reset)1943 mtk_wed_ring_reset(wdma, MTK_WED_WDMA_RING_SIZE, true);1944 1945 if (!idx) {1946 wed_w32(dev, MTK_WED_WDMA_RING_TX + MTK_WED_RING_OFS_BASE,1947 wdma->desc_phys);1948 wed_w32(dev, MTK_WED_WDMA_RING_TX + MTK_WED_RING_OFS_COUNT,1949 size);1950 wed_w32(dev, MTK_WED_WDMA_RING_TX + MTK_WED_RING_OFS_CPU_IDX,1951 0);1952 wed_w32(dev, MTK_WED_WDMA_RING_TX + MTK_WED_RING_OFS_DMA_IDX,1953 0);1954 }1955 1956 return 0;1957}1958 1959static void1960mtk_wed_ppe_check(struct mtk_wed_device *dev, struct sk_buff *skb,1961 u32 reason, u32 hash)1962{1963 struct mtk_eth *eth = dev->hw->eth;1964 struct ethhdr *eh;1965 1966 if (!skb)1967 return;1968 1969 if (reason != MTK_PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED)1970 return;1971 1972 skb_set_mac_header(skb, 0);1973 eh = eth_hdr(skb);1974 skb->protocol = eh->h_proto;1975 mtk_ppe_check_skb(eth->ppe[dev->hw->index], skb, hash);1976}1977 1978static void1979mtk_wed_configure_irq(struct mtk_wed_device *dev, u32 irq_mask)1980{1981 u32 wdma_mask = FIELD_PREP(MTK_WDMA_INT_MASK_RX_DONE, GENMASK(1, 0));1982 1983 /* wed control cr set */1984 wed_set(dev, MTK_WED_CTRL,1985 MTK_WED_CTRL_WDMA_INT_AGENT_EN |1986 MTK_WED_CTRL_WPDMA_INT_AGENT_EN |1987 MTK_WED_CTRL_WED_TX_BM_EN |1988 MTK_WED_CTRL_WED_TX_FREE_AGENT_EN);1989 1990 if (mtk_wed_is_v1(dev->hw)) {1991 wed_w32(dev, MTK_WED_PCIE_INT_TRIGGER,1992 MTK_WED_PCIE_INT_TRIGGER_STATUS);1993 1994 wed_w32(dev, MTK_WED_WPDMA_INT_TRIGGER,1995 MTK_WED_WPDMA_INT_TRIGGER_RX_DONE |1996 MTK_WED_WPDMA_INT_TRIGGER_TX_DONE);1997 1998 wed_clr(dev, MTK_WED_WDMA_INT_CTRL, wdma_mask);1999 } else {2000 if (mtk_wed_is_v3_or_greater(dev->hw))2001 wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_TX_TKID_ALI_EN);2002 2003 /* initail tx interrupt trigger */2004 wed_w32(dev, MTK_WED_WPDMA_INT_CTRL_TX,2005 MTK_WED_WPDMA_INT_CTRL_TX0_DONE_EN |2006 MTK_WED_WPDMA_INT_CTRL_TX0_DONE_CLR |2007 MTK_WED_WPDMA_INT_CTRL_TX1_DONE_EN |2008 MTK_WED_WPDMA_INT_CTRL_TX1_DONE_CLR |2009 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_TX0_DONE_TRIG,2010 dev->wlan.tx_tbit[0]) |2011 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_TX1_DONE_TRIG,2012 dev->wlan.tx_tbit[1]));2013 2014 /* initail txfree interrupt trigger */2015 wed_w32(dev, MTK_WED_WPDMA_INT_CTRL_TX_FREE,2016 MTK_WED_WPDMA_INT_CTRL_TX_FREE_DONE_EN |2017 MTK_WED_WPDMA_INT_CTRL_TX_FREE_DONE_CLR |2018 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_TX_FREE_DONE_TRIG,2019 dev->wlan.txfree_tbit));2020 2021 if (mtk_wed_get_rx_capa(dev)) {2022 wed_w32(dev, MTK_WED_WPDMA_INT_CTRL_RX,2023 MTK_WED_WPDMA_INT_CTRL_RX0_EN |2024 MTK_WED_WPDMA_INT_CTRL_RX0_CLR |2025 MTK_WED_WPDMA_INT_CTRL_RX1_EN |2026 MTK_WED_WPDMA_INT_CTRL_RX1_CLR |2027 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_RX0_DONE_TRIG,2028 dev->wlan.rx_tbit[0]) |2029 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_RX1_DONE_TRIG,2030 dev->wlan.rx_tbit[1]));2031 2032 wdma_mask |= FIELD_PREP(MTK_WDMA_INT_MASK_TX_DONE,2033 GENMASK(1, 0));2034 }2035 2036 wed_w32(dev, MTK_WED_WDMA_INT_CLR, wdma_mask);2037 wed_set(dev, MTK_WED_WDMA_INT_CTRL,2038 FIELD_PREP(MTK_WED_WDMA_INT_CTRL_POLL_SRC_SEL,2039 dev->wdma_idx));2040 }2041 2042 wed_w32(dev, MTK_WED_WDMA_INT_TRIGGER, wdma_mask);2043 2044 wdma_w32(dev, MTK_WDMA_INT_MASK, wdma_mask);2045 wdma_w32(dev, MTK_WDMA_INT_GRP2, wdma_mask);2046 wed_w32(dev, MTK_WED_WPDMA_INT_MASK, irq_mask);2047 wed_w32(dev, MTK_WED_INT_MASK, irq_mask);2048}2049 2050#define MTK_WFMDA_RX_DMA_EN BIT(2)2051static void2052mtk_wed_dma_enable(struct mtk_wed_device *dev)2053{2054 int i;2055 2056 if (!mtk_wed_is_v3_or_greater(dev->hw)) {2057 wed_set(dev, MTK_WED_WPDMA_INT_CTRL,2058 MTK_WED_WPDMA_INT_CTRL_SUBRT_ADV);2059 wed_set(dev, MTK_WED_WPDMA_GLO_CFG,2060 MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN |2061 MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN);2062 wdma_set(dev, MTK_WDMA_GLO_CFG,2063 MTK_WDMA_GLO_CFG_TX_DMA_EN |2064 MTK_WDMA_GLO_CFG_RX_INFO1_PRERES |2065 MTK_WDMA_GLO_CFG_RX_INFO2_PRERES);2066 wed_set(dev, MTK_WED_WPDMA_CTRL, MTK_WED_WPDMA_CTRL_SDL1_FIXED);2067 } else {2068 wed_set(dev, MTK_WED_WPDMA_GLO_CFG,2069 MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN |2070 MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN |2071 MTK_WED_WPDMA_GLO_CFG_RX_DDONE2_WR);2072 wdma_set(dev, MTK_WDMA_GLO_CFG, MTK_WDMA_GLO_CFG_TX_DMA_EN);2073 }2074 2075 wed_set(dev, MTK_WED_GLO_CFG,2076 MTK_WED_GLO_CFG_TX_DMA_EN |2077 MTK_WED_GLO_CFG_RX_DMA_EN);2078 2079 wed_set(dev, MTK_WED_WDMA_GLO_CFG,2080 MTK_WED_WDMA_GLO_CFG_RX_DRV_EN);2081 2082 if (mtk_wed_is_v1(dev->hw)) {2083 wdma_set(dev, MTK_WDMA_GLO_CFG,2084 MTK_WDMA_GLO_CFG_RX_INFO3_PRERES);2085 return;2086 }2087 2088 wed_set(dev, MTK_WED_WPDMA_GLO_CFG,2089 MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_PKT_PROC |2090 MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_CRX_SYNC);2091 2092 if (mtk_wed_is_v3_or_greater(dev->hw)) {2093 wed_set(dev, MTK_WED_WDMA_RX_PREF_CFG,2094 FIELD_PREP(MTK_WED_WDMA_RX_PREF_BURST_SIZE, 0x10) |2095 FIELD_PREP(MTK_WED_WDMA_RX_PREF_LOW_THRES, 0x8));2096 wed_clr(dev, MTK_WED_WDMA_RX_PREF_CFG,2097 MTK_WED_WDMA_RX_PREF_DDONE2_EN);2098 wed_set(dev, MTK_WED_WDMA_RX_PREF_CFG, MTK_WED_WDMA_RX_PREF_EN);2099 2100 wed_clr(dev, MTK_WED_WPDMA_GLO_CFG,2101 MTK_WED_WPDMA_GLO_CFG_TX_DDONE_CHK_LAST);2102 wed_set(dev, MTK_WED_WPDMA_GLO_CFG,2103 MTK_WED_WPDMA_GLO_CFG_TX_DDONE_CHK |2104 MTK_WED_WPDMA_GLO_CFG_RX_DRV_EVENT_PKT_FMT_CHK |2105 MTK_WED_WPDMA_GLO_CFG_RX_DRV_UNS_VER_FORCE_4);2106 2107 wdma_set(dev, MTK_WDMA_PREF_RX_CFG, MTK_WDMA_PREF_RX_CFG_PREF_EN);2108 wdma_set(dev, MTK_WDMA_WRBK_RX_CFG, MTK_WDMA_WRBK_RX_CFG_WRBK_EN);2109 }2110 2111 wed_clr(dev, MTK_WED_WPDMA_GLO_CFG,2112 MTK_WED_WPDMA_GLO_CFG_TX_TKID_KEEP |2113 MTK_WED_WPDMA_GLO_CFG_TX_DMAD_DW3_PREV);2114 2115 if (!mtk_wed_get_rx_capa(dev))2116 return;2117 2118 wed_set(dev, MTK_WED_WDMA_GLO_CFG,2119 MTK_WED_WDMA_GLO_CFG_TX_DRV_EN |2120 MTK_WED_WDMA_GLO_CFG_TX_DDONE_CHK);2121 2122 wed_clr(dev, MTK_WED_WPDMA_RX_D_GLO_CFG, MTK_WED_WPDMA_RX_D_RXD_READ_LEN);2123 wed_set(dev, MTK_WED_WPDMA_RX_D_GLO_CFG,2124 MTK_WED_WPDMA_RX_D_RX_DRV_EN |2125 FIELD_PREP(MTK_WED_WPDMA_RX_D_RXD_READ_LEN, 0x18) |2126 FIELD_PREP(MTK_WED_WPDMA_RX_D_INIT_PHASE_RXEN_SEL, 0x2));2127 2128 if (mtk_wed_is_v3_or_greater(dev->hw)) {2129 wed_set(dev, MTK_WED_WPDMA_RX_D_PREF_CFG,2130 MTK_WED_WPDMA_RX_D_PREF_EN |2131 FIELD_PREP(MTK_WED_WPDMA_RX_D_PREF_BURST_SIZE, 0x10) |2132 FIELD_PREP(MTK_WED_WPDMA_RX_D_PREF_LOW_THRES, 0x8));2133 2134 wed_set(dev, MTK_WED_RRO_RX_D_CFG(2), MTK_WED_RRO_RX_D_DRV_EN);2135 wdma_set(dev, MTK_WDMA_PREF_TX_CFG, MTK_WDMA_PREF_TX_CFG_PREF_EN);2136 wdma_set(dev, MTK_WDMA_WRBK_TX_CFG, MTK_WDMA_WRBK_TX_CFG_WRBK_EN);2137 }2138 2139 for (i = 0; i < MTK_WED_RX_QUEUES; i++) {2140 struct mtk_wed_ring *ring = &dev->rx_ring[i];2141 u32 val;2142 2143 if (!(ring->flags & MTK_WED_RING_CONFIGURED))2144 continue; /* queue is not configured by mt76 */2145 2146 if (mtk_wed_check_wfdma_rx_fill(dev, ring)) {2147 dev_err(dev->hw->dev,2148 "rx_ring(%d) dma enable failed\n", i);2149 continue;2150 }2151 2152 val = wifi_r32(dev,2153 dev->wlan.wpdma_rx_glo -2154 dev->wlan.phy_base) | MTK_WFMDA_RX_DMA_EN;2155 wifi_w32(dev,2156 dev->wlan.wpdma_rx_glo - dev->wlan.phy_base,2157 val);2158 }2159}2160 2161static void2162mtk_wed_start_hw_rro(struct mtk_wed_device *dev, u32 irq_mask, bool reset)2163{2164 int i;2165 2166 wed_w32(dev, MTK_WED_WPDMA_INT_MASK, irq_mask);2167 wed_w32(dev, MTK_WED_INT_MASK, irq_mask);2168 2169 if (!mtk_wed_get_rx_capa(dev) || !dev->wlan.hw_rro)2170 return;2171 2172 if (reset) {2173 wed_set(dev, MTK_WED_RRO_MSDU_PG_RING2_CFG,2174 MTK_WED_RRO_MSDU_PG_DRV_EN);2175 return;2176 }2177 2178 wed_set(dev, MTK_WED_RRO_RX_D_CFG(2), MTK_WED_RRO_MSDU_PG_DRV_CLR);2179 wed_w32(dev, MTK_WED_RRO_MSDU_PG_RING2_CFG,2180 MTK_WED_RRO_MSDU_PG_DRV_CLR);2181 2182 wed_w32(dev, MTK_WED_WPDMA_INT_CTRL_RRO_RX,2183 MTK_WED_WPDMA_INT_CTRL_RRO_RX0_EN |2184 MTK_WED_WPDMA_INT_CTRL_RRO_RX0_CLR |2185 MTK_WED_WPDMA_INT_CTRL_RRO_RX1_EN |2186 MTK_WED_WPDMA_INT_CTRL_RRO_RX1_CLR |2187 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_RRO_RX0_DONE_TRIG,2188 dev->wlan.rro_rx_tbit[0]) |2189 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_RRO_RX1_DONE_TRIG,2190 dev->wlan.rro_rx_tbit[1]));2191 2192 wed_w32(dev, MTK_WED_WPDMA_INT_CTRL_RRO_MSDU_PG,2193 MTK_WED_WPDMA_INT_CTRL_RRO_PG0_EN |2194 MTK_WED_WPDMA_INT_CTRL_RRO_PG0_CLR |2195 MTK_WED_WPDMA_INT_CTRL_RRO_PG1_EN |2196 MTK_WED_WPDMA_INT_CTRL_RRO_PG1_CLR |2197 MTK_WED_WPDMA_INT_CTRL_RRO_PG2_EN |2198 MTK_WED_WPDMA_INT_CTRL_RRO_PG2_CLR |2199 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_RRO_PG0_DONE_TRIG,2200 dev->wlan.rx_pg_tbit[0]) |2201 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_RRO_PG1_DONE_TRIG,2202 dev->wlan.rx_pg_tbit[1]) |2203 FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_RRO_PG2_DONE_TRIG,2204 dev->wlan.rx_pg_tbit[2]));2205 2206 /* RRO_MSDU_PG_RING2_CFG1_FLD_DRV_EN should be enabled after2207 * WM FWDL completed, otherwise RRO_MSDU_PG ring may broken2208 */2209 wed_set(dev, MTK_WED_RRO_MSDU_PG_RING2_CFG,2210 MTK_WED_RRO_MSDU_PG_DRV_EN);2211 2212 for (i = 0; i < MTK_WED_RX_QUEUES; i++) {2213 struct mtk_wed_ring *ring = &dev->rx_rro_ring[i];2214 2215 if (!(ring->flags & MTK_WED_RING_CONFIGURED))2216 continue;2217 2218 if (mtk_wed_check_wfdma_rx_fill(dev, ring))2219 dev_err(dev->hw->dev,2220 "rx_rro_ring(%d) initialization failed\n", i);2221 }2222 2223 for (i = 0; i < MTK_WED_RX_PAGE_QUEUES; i++) {2224 struct mtk_wed_ring *ring = &dev->rx_page_ring[i];2225 2226 if (!(ring->flags & MTK_WED_RING_CONFIGURED))2227 continue;2228 2229 if (mtk_wed_check_wfdma_rx_fill(dev, ring))2230 dev_err(dev->hw->dev,2231 "rx_page_ring(%d) initialization failed\n", i);2232 }2233}2234 2235static void2236mtk_wed_rro_rx_ring_setup(struct mtk_wed_device *dev, int idx,2237 void __iomem *regs)2238{2239 struct mtk_wed_ring *ring = &dev->rx_rro_ring[idx];2240 2241 ring->wpdma = regs;2242 wed_w32(dev, MTK_WED_RRO_RX_D_RX(idx) + MTK_WED_RING_OFS_BASE,2243 readl(regs));2244 wed_w32(dev, MTK_WED_RRO_RX_D_RX(idx) + MTK_WED_RING_OFS_COUNT,2245 readl(regs + MTK_WED_RING_OFS_COUNT));2246 ring->flags |= MTK_WED_RING_CONFIGURED;2247}2248 2249static void2250mtk_wed_msdu_pg_rx_ring_setup(struct mtk_wed_device *dev, int idx, void __iomem *regs)2251{2252 struct mtk_wed_ring *ring = &dev->rx_page_ring[idx];2253 2254 ring->wpdma = regs;2255 wed_w32(dev, MTK_WED_RRO_MSDU_PG_CTRL0(idx) + MTK_WED_RING_OFS_BASE,2256 readl(regs));2257 wed_w32(dev, MTK_WED_RRO_MSDU_PG_CTRL0(idx) + MTK_WED_RING_OFS_COUNT,2258 readl(regs + MTK_WED_RING_OFS_COUNT));2259 ring->flags |= MTK_WED_RING_CONFIGURED;2260}2261 2262static int2263mtk_wed_ind_rx_ring_setup(struct mtk_wed_device *dev, void __iomem *regs)2264{2265 struct mtk_wed_ring *ring = &dev->ind_cmd_ring;2266 u32 val = readl(regs + MTK_WED_RING_OFS_COUNT);2267 int i, count = 0;2268 2269 ring->wpdma = regs;2270 wed_w32(dev, MTK_WED_IND_CMD_RX_CTRL1 + MTK_WED_RING_OFS_BASE,2271 readl(regs) & 0xfffffff0);2272 2273 wed_w32(dev, MTK_WED_IND_CMD_RX_CTRL1 + MTK_WED_RING_OFS_COUNT,2274 readl(regs + MTK_WED_RING_OFS_COUNT));2275 2276 /* ack sn cr */2277 wed_w32(dev, MTK_WED_RRO_CFG0, dev->wlan.phy_base +2278 dev->wlan.ind_cmd.ack_sn_addr);2279 wed_w32(dev, MTK_WED_RRO_CFG1,2280 FIELD_PREP(MTK_WED_RRO_CFG1_MAX_WIN_SZ,2281 dev->wlan.ind_cmd.win_size) |2282 FIELD_PREP(MTK_WED_RRO_CFG1_PARTICL_SE_ID,2283 dev->wlan.ind_cmd.particular_sid));2284 2285 /* particular session addr element */2286 wed_w32(dev, MTK_WED_ADDR_ELEM_CFG0,2287 dev->wlan.ind_cmd.particular_se_phys);2288 2289 for (i = 0; i < dev->wlan.ind_cmd.se_group_nums; i++) {2290 wed_w32(dev, MTK_WED_RADDR_ELEM_TBL_WDATA,2291 dev->wlan.ind_cmd.addr_elem_phys[i] >> 4);2292 wed_w32(dev, MTK_WED_ADDR_ELEM_TBL_CFG,2293 MTK_WED_ADDR_ELEM_TBL_WR | (i & 0x7f));2294 2295 val = wed_r32(dev, MTK_WED_ADDR_ELEM_TBL_CFG);2296 while (!(val & MTK_WED_ADDR_ELEM_TBL_WR_RDY) && count++ < 100)2297 val = wed_r32(dev, MTK_WED_ADDR_ELEM_TBL_CFG);2298 if (count >= 100)2299 dev_err(dev->hw->dev,2300 "write ba session base failed\n");2301 }2302 2303 /* pn check init */2304 for (i = 0; i < dev->wlan.ind_cmd.particular_sid; i++) {2305 wed_w32(dev, MTK_WED_PN_CHECK_WDATA_M,2306 MTK_WED_PN_CHECK_IS_FIRST);2307 2308 wed_w32(dev, MTK_WED_PN_CHECK_CFG, MTK_WED_PN_CHECK_WR |2309 FIELD_PREP(MTK_WED_PN_CHECK_SE_ID, i));2310 2311 count = 0;2312 val = wed_r32(dev, MTK_WED_PN_CHECK_CFG);2313 while (!(val & MTK_WED_PN_CHECK_WR_RDY) && count++ < 100)2314 val = wed_r32(dev, MTK_WED_PN_CHECK_CFG);2315 if (count >= 100)2316 dev_err(dev->hw->dev,2317 "session(%d) initialization failed\n", i);2318 }2319 2320 wed_w32(dev, MTK_WED_RX_IND_CMD_CNT0, MTK_WED_RX_IND_CMD_DBG_CNT_EN);2321 wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_WED_RX_IND_CMD_EN);2322 2323 return 0;2324}2325 2326static void2327mtk_wed_start(struct mtk_wed_device *dev, u32 irq_mask)2328{2329 int i;2330 2331 if (mtk_wed_get_rx_capa(dev) && mtk_wed_rx_buffer_alloc(dev))2332 return;2333 2334 for (i = 0; i < ARRAY_SIZE(dev->rx_wdma); i++)2335 if (!dev->rx_wdma[i].desc)2336 mtk_wed_wdma_rx_ring_setup(dev, i, 16, false);2337 2338 mtk_wed_hw_init(dev);2339 mtk_wed_configure_irq(dev, irq_mask);2340 2341 mtk_wed_set_ext_int(dev, true);2342 2343 if (mtk_wed_is_v1(dev->hw)) {2344 u32 val = dev->wlan.wpdma_phys | MTK_PCIE_MIRROR_MAP_EN |2345 FIELD_PREP(MTK_PCIE_MIRROR_MAP_WED_ID,2346 dev->hw->index);2347 2348 val |= BIT(0) | (BIT(1) * !!dev->hw->index);2349 regmap_write(dev->hw->mirror, dev->hw->index * 4, val);2350 } else if (mtk_wed_get_rx_capa(dev)) {2351 /* driver set mid ready and only once */2352 wed_w32(dev, MTK_WED_EXT_INT_MASK1,2353 MTK_WED_EXT_INT_STATUS_WPDMA_MID_RDY);2354 wed_w32(dev, MTK_WED_EXT_INT_MASK2,2355 MTK_WED_EXT_INT_STATUS_WPDMA_MID_RDY);2356 2357 wed_r32(dev, MTK_WED_EXT_INT_MASK1);2358 wed_r32(dev, MTK_WED_EXT_INT_MASK2);2359 2360 if (mtk_wed_is_v3_or_greater(dev->hw)) {2361 wed_w32(dev, MTK_WED_EXT_INT_MASK3,2362 MTK_WED_EXT_INT_STATUS_WPDMA_MID_RDY);2363 wed_r32(dev, MTK_WED_EXT_INT_MASK3);2364 }2365 2366 if (mtk_wed_rro_cfg(dev))2367 return;2368 }2369 2370 mtk_wed_set_512_support(dev, dev->wlan.wcid_512);2371 mtk_wed_amsdu_init(dev);2372 2373 mtk_wed_dma_enable(dev);2374 dev->running = true;2375}2376 2377static int2378mtk_wed_attach(struct mtk_wed_device *dev)2379 __releases(RCU)2380{2381 struct mtk_wed_hw *hw;2382 struct device *device;2383 int ret = 0;2384 2385 RCU_LOCKDEP_WARN(!rcu_read_lock_held(),2386 "mtk_wed_attach without holding the RCU read lock");2387 2388 if ((dev->wlan.bus_type == MTK_WED_BUS_PCIE &&2389 pci_domain_nr(dev->wlan.pci_dev->bus) > 1) ||2390 !try_module_get(THIS_MODULE))2391 ret = -ENODEV;2392 2393 rcu_read_unlock();2394 2395 if (ret)2396 return ret;2397 2398 mutex_lock(&hw_lock);2399 2400 hw = mtk_wed_assign(dev);2401 if (!hw) {2402 module_put(THIS_MODULE);2403 ret = -ENODEV;2404 goto unlock;2405 }2406 2407 device = dev->wlan.bus_type == MTK_WED_BUS_PCIE2408 ? &dev->wlan.pci_dev->dev2409 : &dev->wlan.platform_dev->dev;2410 dev_info(device, "attaching wed device %d version %d\n",2411 hw->index, hw->version);2412 2413 dev->hw = hw;2414 dev->dev = hw->dev;2415 dev->irq = hw->irq;2416 dev->wdma_idx = hw->index;2417 dev->version = hw->version;2418 dev->hw->pcie_base = mtk_wed_get_pcie_base(dev);2419 2420 if (hw->eth->dma_dev == hw->eth->dev &&2421 of_dma_is_coherent(hw->eth->dev->of_node))2422 mtk_eth_set_dma_device(hw->eth, hw->dev);2423 2424 ret = mtk_wed_tx_buffer_alloc(dev);2425 if (ret)2426 goto out;2427 2428 ret = mtk_wed_amsdu_buffer_alloc(dev);2429 if (ret)2430 goto out;2431 2432 if (mtk_wed_get_rx_capa(dev)) {2433 ret = mtk_wed_rro_alloc(dev);2434 if (ret)2435 goto out;2436 }2437 2438 mtk_wed_hw_init_early(dev);2439 if (mtk_wed_is_v1(hw))2440 regmap_update_bits(hw->hifsys, HIFSYS_DMA_AG_MAP,2441 BIT(hw->index), 0);2442 else2443 dev->rev_id = wed_r32(dev, MTK_WED_REV_ID);2444 2445 if (mtk_wed_get_rx_capa(dev))2446 ret = mtk_wed_wo_init(hw);2447out:2448 if (ret) {2449 dev_err(dev->hw->dev, "failed to attach wed device\n");2450 __mtk_wed_detach(dev);2451 }2452unlock:2453 mutex_unlock(&hw_lock);2454 2455 return ret;2456}2457 2458static int2459mtk_wed_tx_ring_setup(struct mtk_wed_device *dev, int idx, void __iomem *regs,2460 bool reset)2461{2462 struct mtk_wed_ring *ring = &dev->tx_ring[idx];2463 2464 /*2465 * Tx ring redirection:2466 * Instead of configuring the WLAN PDMA TX ring directly, the WLAN2467 * driver allocated DMA ring gets configured into WED MTK_WED_RING_TX(n)2468 * registers.2469 *2470 * WED driver posts its own DMA ring as WLAN PDMA TX and configures it2471 * into MTK_WED_WPDMA_RING_TX(n) registers.2472 * It gets filled with packets picked up from WED TX ring and from2473 * WDMA RX.2474 */2475 2476 if (WARN_ON(idx >= ARRAY_SIZE(dev->tx_ring)))2477 return -EINVAL;2478 2479 if (!reset && mtk_wed_ring_alloc(dev, ring, MTK_WED_TX_RING_SIZE,2480 sizeof(*ring->desc), true))2481 return -ENOMEM;2482 2483 if (mtk_wed_wdma_rx_ring_setup(dev, idx, MTK_WED_WDMA_RING_SIZE,2484 reset))2485 return -ENOMEM;2486 2487 ring->reg_base = MTK_WED_RING_TX(idx);2488 ring->wpdma = regs;2489 2490 if (mtk_wed_is_v3_or_greater(dev->hw) && idx == 1) {2491 /* reset prefetch index */2492 wed_set(dev, MTK_WED_WDMA_RX_PREF_CFG,2493 MTK_WED_WDMA_RX_PREF_RX0_SIDX_CLR |2494 MTK_WED_WDMA_RX_PREF_RX1_SIDX_CLR);2495 2496 wed_clr(dev, MTK_WED_WDMA_RX_PREF_CFG,2497 MTK_WED_WDMA_RX_PREF_RX0_SIDX_CLR |2498 MTK_WED_WDMA_RX_PREF_RX1_SIDX_CLR);2499 2500 /* reset prefetch FIFO */2501 wed_w32(dev, MTK_WED_WDMA_RX_PREF_FIFO_CFG,2502 MTK_WED_WDMA_RX_PREF_FIFO_RX0_CLR |2503 MTK_WED_WDMA_RX_PREF_FIFO_RX1_CLR);2504 wed_w32(dev, MTK_WED_WDMA_RX_PREF_FIFO_CFG, 0);2505 }2506 2507 /* WED -> WPDMA */2508 wpdma_tx_w32(dev, idx, MTK_WED_RING_OFS_BASE, ring->desc_phys);2509 wpdma_tx_w32(dev, idx, MTK_WED_RING_OFS_COUNT, MTK_WED_TX_RING_SIZE);2510 wpdma_tx_w32(dev, idx, MTK_WED_RING_OFS_CPU_IDX, 0);2511 2512 wed_w32(dev, MTK_WED_WPDMA_RING_TX(idx) + MTK_WED_RING_OFS_BASE,2513 ring->desc_phys);2514 wed_w32(dev, MTK_WED_WPDMA_RING_TX(idx) + MTK_WED_RING_OFS_COUNT,2515 MTK_WED_TX_RING_SIZE);2516 wed_w32(dev, MTK_WED_WPDMA_RING_TX(idx) + MTK_WED_RING_OFS_CPU_IDX, 0);2517 2518 return 0;2519}2520 2521static int2522mtk_wed_txfree_ring_setup(struct mtk_wed_device *dev, void __iomem *regs)2523{2524 struct mtk_wed_ring *ring = &dev->txfree_ring;2525 int i, index = mtk_wed_is_v1(dev->hw);2526 2527 /*2528 * For txfree event handling, the same DMA ring is shared between WED2529 * and WLAN. The WLAN driver accesses the ring index registers through2530 * WED2531 */2532 ring->reg_base = MTK_WED_RING_RX(index);2533 ring->wpdma = regs;2534 2535 for (i = 0; i < 12; i += 4) {2536 u32 val = readl(regs + i);2537 2538 wed_w32(dev, MTK_WED_RING_RX(index) + i, val);2539 wed_w32(dev, MTK_WED_WPDMA_RING_RX(index) + i, val);2540 }2541 2542 return 0;2543}2544 2545static int2546mtk_wed_rx_ring_setup(struct mtk_wed_device *dev, int idx, void __iomem *regs,2547 bool reset)2548{2549 struct mtk_wed_ring *ring = &dev->rx_ring[idx];2550 2551 if (WARN_ON(idx >= ARRAY_SIZE(dev->rx_ring)))2552 return -EINVAL;2553 2554 if (!reset && mtk_wed_ring_alloc(dev, ring, MTK_WED_RX_RING_SIZE,2555 sizeof(*ring->desc), false))2556 return -ENOMEM;2557 2558 if (mtk_wed_wdma_tx_ring_setup(dev, idx, MTK_WED_WDMA_RING_SIZE,2559 reset))2560 return -ENOMEM;2561 2562 ring->reg_base = MTK_WED_RING_RX_DATA(idx);2563 ring->wpdma = regs;2564 ring->flags |= MTK_WED_RING_CONFIGURED;2565 2566 /* WPDMA -> WED */2567 wpdma_rx_w32(dev, idx, MTK_WED_RING_OFS_BASE, ring->desc_phys);2568 wpdma_rx_w32(dev, idx, MTK_WED_RING_OFS_COUNT, MTK_WED_RX_RING_SIZE);2569 2570 wed_w32(dev, MTK_WED_WPDMA_RING_RX_DATA(idx) + MTK_WED_RING_OFS_BASE,2571 ring->desc_phys);2572 wed_w32(dev, MTK_WED_WPDMA_RING_RX_DATA(idx) + MTK_WED_RING_OFS_COUNT,2573 MTK_WED_RX_RING_SIZE);2574 2575 return 0;2576}2577 2578static u322579mtk_wed_irq_get(struct mtk_wed_device *dev, u32 mask)2580{2581 u32 val, ext_mask;2582 2583 if (mtk_wed_is_v3_or_greater(dev->hw))2584 ext_mask = MTK_WED_EXT_INT_STATUS_RX_DRV_COHERENT |2585 MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD;2586 else2587 ext_mask = MTK_WED_EXT_INT_STATUS_ERROR_MASK;2588 2589 val = wed_r32(dev, MTK_WED_EXT_INT_STATUS);2590 wed_w32(dev, MTK_WED_EXT_INT_STATUS, val);2591 val &= ext_mask;2592 if (!dev->hw->num_flows)2593 val &= ~MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD;2594 if (val && net_ratelimit())2595 pr_err("mtk_wed%d: error status=%08x\n", dev->hw->index, val);2596 2597 val = wed_r32(dev, MTK_WED_INT_STATUS);2598 val &= mask;2599 wed_w32(dev, MTK_WED_INT_STATUS, val); /* ACK */2600 2601 return val;2602}2603 2604static void2605mtk_wed_irq_set_mask(struct mtk_wed_device *dev, u32 mask)2606{2607 mtk_wed_set_ext_int(dev, !!mask);2608 wed_w32(dev, MTK_WED_INT_MASK, mask);2609}2610 2611int mtk_wed_flow_add(int index)2612{2613 struct mtk_wed_hw *hw = hw_list[index];2614 int ret = 0;2615 2616 mutex_lock(&hw_lock);2617 2618 if (!hw || !hw->wed_dev) {2619 ret = -ENODEV;2620 goto out;2621 }2622 2623 if (!hw->wed_dev->wlan.offload_enable)2624 goto out;2625 2626 if (hw->num_flows) {2627 hw->num_flows++;2628 goto out;2629 }2630 2631 ret = hw->wed_dev->wlan.offload_enable(hw->wed_dev);2632 if (!ret)2633 hw->num_flows++;2634 mtk_wed_set_ext_int(hw->wed_dev, true);2635 2636out:2637 mutex_unlock(&hw_lock);2638 2639 return ret;2640}2641 2642void mtk_wed_flow_remove(int index)2643{2644 struct mtk_wed_hw *hw = hw_list[index];2645 2646 mutex_lock(&hw_lock);2647 2648 if (!hw || !hw->wed_dev)2649 goto out;2650 2651 if (!hw->wed_dev->wlan.offload_disable)2652 goto out;2653 2654 if (--hw->num_flows)2655 goto out;2656 2657 hw->wed_dev->wlan.offload_disable(hw->wed_dev);2658 mtk_wed_set_ext_int(hw->wed_dev, true);2659 2660out:2661 mutex_unlock(&hw_lock);2662}2663 2664static int2665mtk_wed_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv)2666{2667 struct mtk_wed_flow_block_priv *priv = cb_priv;2668 struct flow_cls_offload *cls = type_data;2669 struct mtk_wed_hw *hw = NULL;2670 2671 if (!priv || !tc_can_offload(priv->dev))2672 return -EOPNOTSUPP;2673 2674 if (type != TC_SETUP_CLSFLOWER)2675 return -EOPNOTSUPP;2676 2677 hw = priv->hw;2678 return mtk_flow_offload_cmd(hw->eth, cls, hw->index);2679}2680 2681static int2682mtk_wed_setup_tc_block(struct mtk_wed_hw *hw, struct net_device *dev,2683 struct flow_block_offload *f)2684{2685 struct mtk_wed_flow_block_priv *priv;2686 static LIST_HEAD(block_cb_list);2687 struct flow_block_cb *block_cb;2688 struct mtk_eth *eth = hw->eth;2689 flow_setup_cb_t *cb;2690 2691 if (!eth->soc->offload_version)2692 return -EOPNOTSUPP;2693 2694 if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)2695 return -EOPNOTSUPP;2696 2697 cb = mtk_wed_setup_tc_block_cb;2698 f->driver_block_list = &block_cb_list;2699 2700 switch (f->command) {2701 case FLOW_BLOCK_BIND:2702 block_cb = flow_block_cb_lookup(f->block, cb, dev);2703 if (block_cb) {2704 flow_block_cb_incref(block_cb);2705 return 0;2706 }2707 2708 priv = kzalloc(sizeof(*priv), GFP_KERNEL);2709 if (!priv)2710 return -ENOMEM;2711 2712 priv->hw = hw;2713 priv->dev = dev;2714 block_cb = flow_block_cb_alloc(cb, dev, priv, NULL);2715 if (IS_ERR(block_cb)) {2716 kfree(priv);2717 return PTR_ERR(block_cb);2718 }2719 2720 flow_block_cb_incref(block_cb);2721 flow_block_cb_add(block_cb, f);2722 list_add_tail(&block_cb->driver_list, &block_cb_list);2723 return 0;2724 case FLOW_BLOCK_UNBIND:2725 block_cb = flow_block_cb_lookup(f->block, cb, dev);2726 if (!block_cb)2727 return -ENOENT;2728 2729 if (!flow_block_cb_decref(block_cb)) {2730 flow_block_cb_remove(block_cb, f);2731 list_del(&block_cb->driver_list);2732 kfree(block_cb->cb_priv);2733 block_cb->cb_priv = NULL;2734 }2735 return 0;2736 default:2737 return -EOPNOTSUPP;2738 }2739}2740 2741static int2742mtk_wed_setup_tc(struct mtk_wed_device *wed, struct net_device *dev,2743 enum tc_setup_type type, void *type_data)2744{2745 struct mtk_wed_hw *hw = wed->hw;2746 2747 if (mtk_wed_is_v1(hw))2748 return -EOPNOTSUPP;2749 2750 switch (type) {2751 case TC_SETUP_BLOCK:2752 case TC_SETUP_FT:2753 return mtk_wed_setup_tc_block(hw, dev, type_data);2754 default:2755 return -EOPNOTSUPP;2756 }2757}2758 2759void mtk_wed_add_hw(struct device_node *np, struct mtk_eth *eth,2760 void __iomem *wdma, phys_addr_t wdma_phy,2761 int index)2762{2763 static const struct mtk_wed_ops wed_ops = {2764 .attach = mtk_wed_attach,2765 .tx_ring_setup = mtk_wed_tx_ring_setup,2766 .rx_ring_setup = mtk_wed_rx_ring_setup,2767 .txfree_ring_setup = mtk_wed_txfree_ring_setup,2768 .msg_update = mtk_wed_mcu_msg_update,2769 .start = mtk_wed_start,2770 .stop = mtk_wed_stop,2771 .reset_dma = mtk_wed_reset_dma,2772 .reg_read = wed_r32,2773 .reg_write = wed_w32,2774 .irq_get = mtk_wed_irq_get,2775 .irq_set_mask = mtk_wed_irq_set_mask,2776 .detach = mtk_wed_detach,2777 .ppe_check = mtk_wed_ppe_check,2778 .setup_tc = mtk_wed_setup_tc,2779 .start_hw_rro = mtk_wed_start_hw_rro,2780 .rro_rx_ring_setup = mtk_wed_rro_rx_ring_setup,2781 .msdu_pg_rx_ring_setup = mtk_wed_msdu_pg_rx_ring_setup,2782 .ind_rx_ring_setup = mtk_wed_ind_rx_ring_setup,2783 };2784 struct device_node *eth_np = eth->dev->of_node;2785 struct platform_device *pdev;2786 struct mtk_wed_hw *hw;2787 struct regmap *regs;2788 int irq;2789 2790 if (!np)2791 return;2792 2793 pdev = of_find_device_by_node(np);2794 if (!pdev)2795 goto err_of_node_put;2796 2797 get_device(&pdev->dev);2798 irq = platform_get_irq(pdev, 0);2799 if (irq < 0)2800 goto err_put_device;2801 2802 regs = syscon_regmap_lookup_by_phandle(np, NULL);2803 if (IS_ERR(regs))2804 goto err_put_device;2805 2806 rcu_assign_pointer(mtk_soc_wed_ops, &wed_ops);2807 2808 mutex_lock(&hw_lock);2809 2810 if (WARN_ON(hw_list[index]))2811 goto unlock;2812 2813 hw = kzalloc(sizeof(*hw), GFP_KERNEL);2814 if (!hw)2815 goto unlock;2816 2817 hw->node = np;2818 hw->regs = regs;2819 hw->eth = eth;2820 hw->dev = &pdev->dev;2821 hw->wdma_phy = wdma_phy;2822 hw->wdma = wdma;2823 hw->index = index;2824 hw->irq = irq;2825 hw->version = eth->soc->version;2826 2827 switch (hw->version) {2828 case 2:2829 hw->soc = &mt7986_data;2830 break;2831 case 3:2832 hw->soc = &mt7988_data;2833 break;2834 default:2835 case 1:2836 hw->mirror = syscon_regmap_lookup_by_phandle(eth_np,2837 "mediatek,pcie-mirror");2838 hw->hifsys = syscon_regmap_lookup_by_phandle(eth_np,2839 "mediatek,hifsys");2840 if (IS_ERR(hw->mirror) || IS_ERR(hw->hifsys)) {2841 kfree(hw);2842 goto unlock;2843 }2844 2845 if (!index) {2846 regmap_write(hw->mirror, 0, 0);2847 regmap_write(hw->mirror, 4, 0);2848 }2849 hw->soc = &mt7622_data;2850 break;2851 }2852 2853 mtk_wed_hw_add_debugfs(hw);2854 2855 hw_list[index] = hw;2856 2857 mutex_unlock(&hw_lock);2858 2859 return;2860 2861unlock:2862 mutex_unlock(&hw_lock);2863err_put_device:2864 put_device(&pdev->dev);2865err_of_node_put:2866 of_node_put(np);2867}2868 2869void mtk_wed_exit(void)2870{2871 int i;2872 2873 rcu_assign_pointer(mtk_soc_wed_ops, NULL);2874 2875 synchronize_rcu();2876 2877 for (i = 0; i < ARRAY_SIZE(hw_list); i++) {2878 struct mtk_wed_hw *hw;2879 2880 hw = hw_list[i];2881 if (!hw)2882 continue;2883 2884 hw_list[i] = NULL;2885 debugfs_remove(hw->debugfs_dir);2886 put_device(hw->dev);2887 of_node_put(hw->node);2888 kfree(hw);2889 }2890}2891