1237 lines · c
1// SPDX-License-Identifier: GPL-2.0+2/* Copyright (c) 2018 Quantenna Communications */3 4#include <linux/kernel.h>5#include <linux/firmware.h>6#include <linux/pci.h>7#include <linux/vmalloc.h>8#include <linux/delay.h>9#include <linux/interrupt.h>10#include <linux/sched.h>11#include <linux/crc32.h>12#include <linux/completion.h>13#include <linux/spinlock.h>14#include <linux/circ_buf.h>15 16#include "pcie_priv.h"17#include "topaz_pcie_regs.h"18#include "topaz_pcie_ipc.h"19#include "qtn_hw_ids.h"20#include "core.h"21#include "bus.h"22#include "shm_ipc.h"23#include "debug.h"24 25#define TOPAZ_TX_BD_SIZE_DEFAULT 12826#define TOPAZ_RX_BD_SIZE_DEFAULT 25627 28struct qtnf_topaz_tx_bd {29 __le32 addr;30 __le32 info;31} __packed;32 33struct qtnf_topaz_rx_bd {34 __le32 addr;35 __le32 info;36} __packed;37 38struct qtnf_extra_bd_params {39 __le32 param1;40 __le32 param2;41 __le32 param3;42 __le32 param4;43} __packed;44 45#define QTNF_BD_PARAM_OFFSET(n) offsetof(struct qtnf_extra_bd_params, param##n)46 47struct vmac_pkt_info {48 __le32 addr;49 __le32 info;50};51 52struct qtnf_topaz_bda {53 __le16 bda_len;54 __le16 bda_version;55 __le32 bda_bootstate;56 __le32 bda_dma_mask;57 __le32 bda_dma_offset;58 __le32 bda_flags;59 __le32 bda_img;60 __le32 bda_img_size;61 __le32 bda_ep2h_irqstatus;62 __le32 bda_h2ep_irqstatus;63 __le32 bda_msi_addr;64 u8 reserved1[56];65 __le32 bda_flashsz;66 u8 bda_boardname[PCIE_BDA_NAMELEN];67 __le32 bda_pci_pre_status;68 __le32 bda_pci_endian;69 __le32 bda_pci_post_status;70 __le32 bda_h2ep_txd_budget;71 __le32 bda_ep2h_txd_budget;72 __le32 bda_rc_rx_bd_base;73 __le32 bda_rc_rx_bd_num;74 __le32 bda_rc_tx_bd_base;75 __le32 bda_rc_tx_bd_num;76 u8 bda_ep_link_state;77 u8 bda_rc_link_state;78 u8 bda_rc_msi_enabled;79 u8 reserved2;80 __le32 bda_ep_next_pkt;81 struct vmac_pkt_info request[QTN_PCIE_RC_TX_QUEUE_LEN];82 struct qtnf_shm_ipc_region bda_shm_reg1 __aligned(4096);83 struct qtnf_shm_ipc_region bda_shm_reg2 __aligned(4096);84} __packed;85 86struct qtnf_pcie_topaz_state {87 struct qtnf_pcie_bus_priv base;88 struct qtnf_topaz_bda __iomem *bda;89 90 dma_addr_t dma_msi_dummy;91 u32 dma_msi_imwr;92 93 struct qtnf_topaz_tx_bd *tx_bd_vbase;94 struct qtnf_topaz_rx_bd *rx_bd_vbase;95 96 __le32 __iomem *ep_next_rx_pkt;97 __le32 __iomem *txqueue_wake;98 __le32 __iomem *ep_pmstate;99 100 unsigned long rx_pkt_count;101};102 103static void qtnf_deassert_intx(struct qtnf_pcie_topaz_state *ts)104{105 void __iomem *reg = ts->base.sysctl_bar + TOPAZ_PCIE_CFG0_OFFSET;106 u32 cfg;107 108 cfg = readl(reg);109 cfg &= ~TOPAZ_ASSERT_INTX;110 qtnf_non_posted_write(cfg, reg);111}112 113static inline int qtnf_topaz_intx_asserted(struct qtnf_pcie_topaz_state *ts)114{115 void __iomem *reg = ts->base.sysctl_bar + TOPAZ_PCIE_CFG0_OFFSET;116 u32 cfg = readl(reg);117 118 return !!(cfg & TOPAZ_ASSERT_INTX);119}120 121static void qtnf_topaz_reset_ep(struct qtnf_pcie_topaz_state *ts)122{123 writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_RST_EP_IRQ),124 TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar));125 msleep(QTN_EP_RESET_WAIT_MS);126 pci_restore_state(ts->base.pdev);127}128 129static void setup_rx_irqs(struct qtnf_pcie_topaz_state *ts)130{131 void __iomem *reg = PCIE_DMA_WR_DONE_IMWR_ADDR_LOW(ts->base.dmareg_bar);132 133 ts->dma_msi_imwr = readl(reg);134}135 136static void enable_rx_irqs(struct qtnf_pcie_topaz_state *ts)137{138 void __iomem *reg = PCIE_DMA_WR_DONE_IMWR_ADDR_LOW(ts->base.dmareg_bar);139 140 qtnf_non_posted_write(ts->dma_msi_imwr, reg);141}142 143static void disable_rx_irqs(struct qtnf_pcie_topaz_state *ts)144{145 void __iomem *reg = PCIE_DMA_WR_DONE_IMWR_ADDR_LOW(ts->base.dmareg_bar);146 147 qtnf_non_posted_write(QTN_HOST_LO32(ts->dma_msi_dummy), reg);148}149 150static void qtnf_topaz_ipc_gen_ep_int(void *arg)151{152 struct qtnf_pcie_topaz_state *ts = arg;153 154 writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_CTRL_IRQ),155 TOPAZ_CTL_M2L_INT(ts->base.sysctl_bar));156}157 158static int qtnf_is_state(__le32 __iomem *reg, u32 state)159{160 u32 s = readl(reg);161 162 return (s == state);163}164 165static void qtnf_set_state(__le32 __iomem *reg, u32 state)166{167 qtnf_non_posted_write(state, reg);168}169 170static int qtnf_poll_state(__le32 __iomem *reg, u32 state, u32 delay_in_ms)171{172 u32 timeout = 0;173 174 while ((qtnf_is_state(reg, state) == 0)) {175 usleep_range(1000, 1200);176 if (++timeout > delay_in_ms)177 return -1;178 }179 180 return 0;181}182 183static int topaz_alloc_bd_table(struct qtnf_pcie_topaz_state *ts,184 struct qtnf_topaz_bda __iomem *bda)185{186 struct qtnf_extra_bd_params __iomem *extra_params;187 struct qtnf_pcie_bus_priv *priv = &ts->base;188 dma_addr_t paddr;189 void *vaddr;190 int len;191 int i;192 193 /* bd table */194 195 len = priv->tx_bd_num * sizeof(struct qtnf_topaz_tx_bd) +196 priv->rx_bd_num * sizeof(struct qtnf_topaz_rx_bd) +197 sizeof(struct qtnf_extra_bd_params);198 199 vaddr = dmam_alloc_coherent(&priv->pdev->dev, len, &paddr, GFP_KERNEL);200 if (!vaddr)201 return -ENOMEM;202 203 /* tx bd */204 205 ts->tx_bd_vbase = vaddr;206 qtnf_non_posted_write(paddr, &bda->bda_rc_tx_bd_base);207 208 for (i = 0; i < priv->tx_bd_num; i++)209 ts->tx_bd_vbase[i].info |= cpu_to_le32(QTN_BD_EMPTY);210 211 pr_debug("TX descriptor table: vaddr=0x%p paddr=%pad\n", vaddr, &paddr);212 213 priv->tx_bd_r_index = 0;214 priv->tx_bd_w_index = 0;215 216 /* rx bd */217 218 vaddr = ((struct qtnf_topaz_tx_bd *)vaddr) + priv->tx_bd_num;219 paddr += priv->tx_bd_num * sizeof(struct qtnf_topaz_tx_bd);220 221 ts->rx_bd_vbase = vaddr;222 qtnf_non_posted_write(paddr, &bda->bda_rc_rx_bd_base);223 224 pr_debug("RX descriptor table: vaddr=0x%p paddr=%pad\n", vaddr, &paddr);225 226 /* extra shared params */227 228 vaddr = ((struct qtnf_topaz_rx_bd *)vaddr) + priv->rx_bd_num;229 paddr += priv->rx_bd_num * sizeof(struct qtnf_topaz_rx_bd);230 231 extra_params = (struct qtnf_extra_bd_params __iomem *)vaddr;232 233 ts->ep_next_rx_pkt = &extra_params->param1;234 qtnf_non_posted_write(paddr + QTNF_BD_PARAM_OFFSET(1),235 &bda->bda_ep_next_pkt);236 ts->txqueue_wake = &extra_params->param2;237 ts->ep_pmstate = &extra_params->param3;238 ts->dma_msi_dummy = paddr + QTNF_BD_PARAM_OFFSET(4);239 240 return 0;241}242 243static int244topaz_skb2rbd_attach(struct qtnf_pcie_topaz_state *ts, u16 index, u32 wrap)245{246 struct qtnf_topaz_rx_bd *rxbd = &ts->rx_bd_vbase[index];247 struct sk_buff *skb;248 dma_addr_t paddr;249 250 skb = netdev_alloc_skb_ip_align(NULL, SKB_BUF_SIZE);251 if (!skb) {252 ts->base.rx_skb[index] = NULL;253 return -ENOMEM;254 }255 256 ts->base.rx_skb[index] = skb;257 258 paddr = dma_map_single(&ts->base.pdev->dev, skb->data, SKB_BUF_SIZE,259 DMA_FROM_DEVICE);260 if (dma_mapping_error(&ts->base.pdev->dev, paddr)) {261 pr_err("skb mapping error: %pad\n", &paddr);262 return -ENOMEM;263 }264 265 rxbd->addr = cpu_to_le32(QTN_HOST_LO32(paddr));266 rxbd->info = cpu_to_le32(QTN_BD_EMPTY | wrap);267 268 ts->base.rx_bd_w_index = index;269 270 return 0;271}272 273static int topaz_alloc_rx_buffers(struct qtnf_pcie_topaz_state *ts)274{275 u16 i;276 int ret = 0;277 278 memset(ts->rx_bd_vbase, 0x0,279 ts->base.rx_bd_num * sizeof(struct qtnf_topaz_rx_bd));280 281 for (i = 0; i < ts->base.rx_bd_num; i++) {282 ret = topaz_skb2rbd_attach(ts, i, 0);283 if (ret)284 break;285 }286 287 ts->rx_bd_vbase[ts->base.rx_bd_num - 1].info |=288 cpu_to_le32(QTN_BD_WRAP);289 290 return ret;291}292 293/* all rx/tx activity should have ceased before calling this function */294static void qtnf_topaz_free_xfer_buffers(struct qtnf_pcie_topaz_state *ts)295{296 struct qtnf_pcie_bus_priv *priv = &ts->base;297 struct qtnf_topaz_rx_bd *rxbd;298 struct qtnf_topaz_tx_bd *txbd;299 struct sk_buff *skb;300 dma_addr_t paddr;301 int i;302 303 /* free rx buffers */304 for (i = 0; i < priv->rx_bd_num; i++) {305 if (priv->rx_skb && priv->rx_skb[i]) {306 rxbd = &ts->rx_bd_vbase[i];307 skb = priv->rx_skb[i];308 paddr = QTN_HOST_ADDR(0x0, le32_to_cpu(rxbd->addr));309 dma_unmap_single(&priv->pdev->dev, paddr,310 SKB_BUF_SIZE, DMA_FROM_DEVICE);311 dev_kfree_skb_any(skb);312 priv->rx_skb[i] = NULL;313 rxbd->addr = 0;314 rxbd->info = 0;315 }316 }317 318 /* free tx buffers */319 for (i = 0; i < priv->tx_bd_num; i++) {320 if (priv->tx_skb && priv->tx_skb[i]) {321 txbd = &ts->tx_bd_vbase[i];322 skb = priv->tx_skb[i];323 paddr = QTN_HOST_ADDR(0x0, le32_to_cpu(txbd->addr));324 dma_unmap_single(&priv->pdev->dev, paddr,325 SKB_BUF_SIZE, DMA_TO_DEVICE);326 dev_kfree_skb_any(skb);327 priv->tx_skb[i] = NULL;328 txbd->addr = 0;329 txbd->info = 0;330 }331 }332}333 334static int qtnf_pcie_topaz_init_xfer(struct qtnf_pcie_topaz_state *ts,335 unsigned int tx_bd_size,336 unsigned int rx_bd_size)337{338 struct qtnf_topaz_bda __iomem *bda = ts->bda;339 struct qtnf_pcie_bus_priv *priv = &ts->base;340 int ret;341 342 if (tx_bd_size == 0)343 tx_bd_size = TOPAZ_TX_BD_SIZE_DEFAULT;344 345 /* check TX BD queue max length according to struct qtnf_topaz_bda */346 if (tx_bd_size > QTN_PCIE_RC_TX_QUEUE_LEN) {347 pr_warn("TX BD queue cannot exceed %d\n",348 QTN_PCIE_RC_TX_QUEUE_LEN);349 tx_bd_size = QTN_PCIE_RC_TX_QUEUE_LEN;350 }351 352 priv->tx_bd_num = tx_bd_size;353 qtnf_non_posted_write(priv->tx_bd_num, &bda->bda_rc_tx_bd_num);354 355 if (rx_bd_size == 0)356 rx_bd_size = TOPAZ_RX_BD_SIZE_DEFAULT;357 358 if (rx_bd_size > TOPAZ_RX_BD_SIZE_DEFAULT) {359 pr_warn("RX BD queue cannot exceed %d\n",360 TOPAZ_RX_BD_SIZE_DEFAULT);361 rx_bd_size = TOPAZ_RX_BD_SIZE_DEFAULT;362 }363 364 priv->rx_bd_num = rx_bd_size;365 qtnf_non_posted_write(priv->rx_bd_num, &bda->bda_rc_rx_bd_num);366 367 priv->rx_bd_w_index = 0;368 priv->rx_bd_r_index = 0;369 370 ret = qtnf_pcie_alloc_skb_array(priv);371 if (ret) {372 pr_err("failed to allocate skb array\n");373 return ret;374 }375 376 ret = topaz_alloc_bd_table(ts, bda);377 if (ret) {378 pr_err("failed to allocate bd table\n");379 return ret;380 }381 382 ret = topaz_alloc_rx_buffers(ts);383 if (ret) {384 pr_err("failed to allocate rx buffers\n");385 return ret;386 }387 388 return ret;389}390 391static void qtnf_topaz_data_tx_reclaim(struct qtnf_pcie_topaz_state *ts)392{393 struct qtnf_pcie_bus_priv *priv = &ts->base;394 struct qtnf_topaz_tx_bd *txbd;395 struct sk_buff *skb;396 unsigned long flags;397 dma_addr_t paddr;398 u32 tx_done_index;399 int count = 0;400 int i;401 402 spin_lock_irqsave(&priv->tx_reclaim_lock, flags);403 404 tx_done_index = readl(ts->ep_next_rx_pkt);405 i = priv->tx_bd_r_index;406 407 if (CIRC_CNT(priv->tx_bd_w_index, tx_done_index, priv->tx_bd_num))408 writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_TX_DONE_IRQ),409 TOPAZ_LH_IPC4_INT(priv->sysctl_bar));410 411 while (CIRC_CNT(tx_done_index, i, priv->tx_bd_num)) {412 skb = priv->tx_skb[i];413 414 if (likely(skb)) {415 txbd = &ts->tx_bd_vbase[i];416 paddr = QTN_HOST_ADDR(0x0, le32_to_cpu(txbd->addr));417 dma_unmap_single(&priv->pdev->dev, paddr, skb->len,418 DMA_TO_DEVICE);419 420 if (skb->dev) {421 dev_sw_netstats_tx_add(skb->dev, 1, skb->len);422 if (unlikely(priv->tx_stopped)) {423 qtnf_wake_all_queues(skb->dev);424 priv->tx_stopped = 0;425 }426 }427 428 dev_kfree_skb_any(skb);429 }430 431 priv->tx_skb[i] = NULL;432 count++;433 434 if (++i >= priv->tx_bd_num)435 i = 0;436 }437 438 priv->tx_reclaim_done += count;439 priv->tx_reclaim_req++;440 priv->tx_bd_r_index = i;441 442 spin_unlock_irqrestore(&priv->tx_reclaim_lock, flags);443}444 445static void qtnf_try_stop_xmit(struct qtnf_bus *bus, struct net_device *ndev)446{447 struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);448 449 if (ndev) {450 netif_tx_stop_all_queues(ndev);451 ts->base.tx_stopped = 1;452 }453 454 writel(0x0, ts->txqueue_wake);455 456 /* sync up tx queue status before generating interrupt */457 dma_wmb();458 459 /* send irq to card: tx stopped */460 writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_TX_STOP_IRQ),461 TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar));462 463 /* schedule reclaim attempt */464 tasklet_hi_schedule(&ts->base.reclaim_tq);465}466 467static void qtnf_try_wake_xmit(struct qtnf_bus *bus, struct net_device *ndev)468{469 struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);470 int ready;471 472 ready = readl(ts->txqueue_wake);473 if (ready) {474 netif_wake_queue(ndev);475 } else {476 /* re-send irq to card: tx stopped */477 writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_TX_STOP_IRQ),478 TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar));479 }480}481 482static int qtnf_tx_queue_ready(struct qtnf_pcie_topaz_state *ts)483{484 struct qtnf_pcie_bus_priv *priv = &ts->base;485 486 if (!CIRC_SPACE(priv->tx_bd_w_index, priv->tx_bd_r_index,487 priv->tx_bd_num)) {488 qtnf_topaz_data_tx_reclaim(ts);489 490 if (!CIRC_SPACE(priv->tx_bd_w_index, priv->tx_bd_r_index,491 priv->tx_bd_num)) {492 priv->tx_full_count++;493 return 0;494 }495 }496 497 return 1;498}499 500static int qtnf_pcie_data_tx(struct qtnf_bus *bus, struct sk_buff *skb,501 unsigned int macid, unsigned int vifid)502{503 struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);504 struct qtnf_pcie_bus_priv *priv = &ts->base;505 struct qtnf_topaz_bda __iomem *bda = ts->bda;506 struct qtnf_topaz_tx_bd *txbd;507 dma_addr_t skb_paddr;508 unsigned long flags;509 int ret = 0;510 int len;511 int i;512 513 spin_lock_irqsave(&priv->tx_lock, flags);514 515 if (!qtnf_tx_queue_ready(ts)) {516 qtnf_try_stop_xmit(bus, skb->dev);517 spin_unlock_irqrestore(&priv->tx_lock, flags);518 return NETDEV_TX_BUSY;519 }520 521 i = priv->tx_bd_w_index;522 priv->tx_skb[i] = skb;523 len = skb->len;524 525 skb_paddr = dma_map_single(&priv->pdev->dev, skb->data, skb->len,526 DMA_TO_DEVICE);527 if (dma_mapping_error(&priv->pdev->dev, skb_paddr)) {528 ret = -ENOMEM;529 goto tx_done;530 }531 532 txbd = &ts->tx_bd_vbase[i];533 txbd->addr = cpu_to_le32(QTN_HOST_LO32(skb_paddr));534 535 writel(QTN_HOST_LO32(skb_paddr), &bda->request[i].addr);536 writel(len | QTN_PCIE_TX_VALID_PKT, &bda->request[i].info);537 538 /* sync up descriptor updates before generating interrupt */539 dma_wmb();540 541 /* generate irq to card: tx done */542 writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_TX_DONE_IRQ),543 TOPAZ_LH_IPC4_INT(priv->sysctl_bar));544 545 if (++i >= priv->tx_bd_num)546 i = 0;547 548 priv->tx_bd_w_index = i;549 550tx_done:551 if (ret) {552 if (skb->dev)553 skb->dev->stats.tx_dropped++;554 dev_kfree_skb_any(skb);555 }556 557 priv->tx_done_count++;558 spin_unlock_irqrestore(&priv->tx_lock, flags);559 560 qtnf_topaz_data_tx_reclaim(ts);561 562 return NETDEV_TX_OK;563}564 565static irqreturn_t qtnf_pcie_topaz_interrupt(int irq, void *data)566{567 struct qtnf_bus *bus = (struct qtnf_bus *)data;568 struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);569 struct qtnf_pcie_bus_priv *priv = &ts->base;570 571 if (!priv->msi_enabled && !qtnf_topaz_intx_asserted(ts))572 return IRQ_NONE;573 574 if (!priv->msi_enabled)575 qtnf_deassert_intx(ts);576 577 priv->pcie_irq_count++;578 579 qtnf_shm_ipc_irq_handler(&priv->shm_ipc_ep_in);580 qtnf_shm_ipc_irq_handler(&priv->shm_ipc_ep_out);581 582 if (napi_schedule_prep(&bus->mux_napi)) {583 disable_rx_irqs(ts);584 __napi_schedule(&bus->mux_napi);585 }586 587 tasklet_hi_schedule(&priv->reclaim_tq);588 589 return IRQ_HANDLED;590}591 592static int qtnf_rx_data_ready(struct qtnf_pcie_topaz_state *ts)593{594 u16 index = ts->base.rx_bd_r_index;595 struct qtnf_topaz_rx_bd *rxbd;596 u32 descw;597 598 rxbd = &ts->rx_bd_vbase[index];599 descw = le32_to_cpu(rxbd->info);600 601 if (descw & QTN_BD_EMPTY)602 return 0;603 604 return 1;605}606 607static int qtnf_topaz_rx_poll(struct napi_struct *napi, int budget)608{609 struct qtnf_bus *bus = container_of(napi, struct qtnf_bus, mux_napi);610 struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);611 struct qtnf_pcie_bus_priv *priv = &ts->base;612 struct net_device *ndev = NULL;613 struct sk_buff *skb = NULL;614 int processed = 0;615 struct qtnf_topaz_rx_bd *rxbd;616 dma_addr_t skb_paddr;617 int consume;618 u32 descw;619 u32 poffset;620 u32 psize;621 u16 r_idx;622 u16 w_idx;623 int ret;624 625 while (processed < budget) {626 if (!qtnf_rx_data_ready(ts))627 goto rx_out;628 629 r_idx = priv->rx_bd_r_index;630 rxbd = &ts->rx_bd_vbase[r_idx];631 descw = le32_to_cpu(rxbd->info);632 633 skb = priv->rx_skb[r_idx];634 poffset = QTN_GET_OFFSET(descw);635 psize = QTN_GET_LEN(descw);636 consume = 1;637 638 if (descw & QTN_BD_EMPTY) {639 pr_warn("skip invalid rxbd[%d]\n", r_idx);640 consume = 0;641 }642 643 if (!skb) {644 pr_warn("skip missing rx_skb[%d]\n", r_idx);645 consume = 0;646 }647 648 if (skb && (skb_tailroom(skb) < psize)) {649 pr_err("skip packet with invalid length: %u > %u\n",650 psize, skb_tailroom(skb));651 consume = 0;652 }653 654 if (skb) {655 skb_paddr = QTN_HOST_ADDR(0x0, le32_to_cpu(rxbd->addr));656 dma_unmap_single(&priv->pdev->dev, skb_paddr,657 SKB_BUF_SIZE, DMA_FROM_DEVICE);658 }659 660 if (consume) {661 skb_reserve(skb, poffset);662 skb_put(skb, psize);663 ndev = qtnf_classify_skb(bus, skb);664 if (likely(ndev)) {665 dev_sw_netstats_rx_add(ndev, skb->len);666 skb->protocol = eth_type_trans(skb, ndev);667 netif_receive_skb(skb);668 } else {669 pr_debug("drop untagged skb\n");670 bus->mux_dev->stats.rx_dropped++;671 dev_kfree_skb_any(skb);672 }673 } else {674 if (skb) {675 bus->mux_dev->stats.rx_dropped++;676 dev_kfree_skb_any(skb);677 }678 }679 680 /* notify card about recv packets once per several packets */681 if (((++ts->rx_pkt_count) & RX_DONE_INTR_MSK) == 0)682 writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_RX_DONE_IRQ),683 TOPAZ_LH_IPC4_INT(priv->sysctl_bar));684 685 priv->rx_skb[r_idx] = NULL;686 if (++r_idx >= priv->rx_bd_num)687 r_idx = 0;688 689 priv->rx_bd_r_index = r_idx;690 691 /* repalce processed buffer by a new one */692 w_idx = priv->rx_bd_w_index;693 while (CIRC_SPACE(priv->rx_bd_w_index, priv->rx_bd_r_index,694 priv->rx_bd_num) > 0) {695 if (++w_idx >= priv->rx_bd_num)696 w_idx = 0;697 698 ret = topaz_skb2rbd_attach(ts, w_idx,699 descw & QTN_BD_WRAP);700 if (ret) {701 pr_err("failed to allocate new rx_skb[%d]\n",702 w_idx);703 break;704 }705 }706 707 processed++;708 }709 710rx_out:711 if (processed < budget) {712 napi_complete(napi);713 enable_rx_irqs(ts);714 }715 716 return processed;717}718 719static void720qtnf_pcie_data_tx_timeout(struct qtnf_bus *bus, struct net_device *ndev)721{722 struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);723 724 qtnf_try_wake_xmit(bus, ndev);725 tasklet_hi_schedule(&ts->base.reclaim_tq);726}727 728static void qtnf_pcie_data_rx_start(struct qtnf_bus *bus)729{730 struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);731 732 napi_enable(&bus->mux_napi);733 enable_rx_irqs(ts);734}735 736static void qtnf_pcie_data_rx_stop(struct qtnf_bus *bus)737{738 struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);739 740 disable_rx_irqs(ts);741 napi_disable(&bus->mux_napi);742}743 744static struct qtnf_bus_ops qtnf_pcie_topaz_bus_ops = {745 /* control path methods */746 .control_tx = qtnf_pcie_control_tx,747 748 /* data path methods */749 .data_tx = qtnf_pcie_data_tx,750 .data_tx_timeout = qtnf_pcie_data_tx_timeout,751 .data_rx_start = qtnf_pcie_data_rx_start,752 .data_rx_stop = qtnf_pcie_data_rx_stop,753};754 755static int qtnf_dbg_irq_stats(struct seq_file *s, void *data)756{757 struct qtnf_bus *bus = dev_get_drvdata(s->private);758 struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);759 760 seq_printf(s, "pcie_irq_count(%u)\n", ts->base.pcie_irq_count);761 762 return 0;763}764 765static int qtnf_dbg_pkt_stats(struct seq_file *s, void *data)766{767 struct qtnf_bus *bus = dev_get_drvdata(s->private);768 struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);769 struct qtnf_pcie_bus_priv *priv = &ts->base;770 u32 tx_done_index = readl(ts->ep_next_rx_pkt);771 772 seq_printf(s, "tx_full_count(%u)\n", priv->tx_full_count);773 seq_printf(s, "tx_done_count(%u)\n", priv->tx_done_count);774 seq_printf(s, "tx_reclaim_done(%u)\n", priv->tx_reclaim_done);775 seq_printf(s, "tx_reclaim_req(%u)\n", priv->tx_reclaim_req);776 777 seq_printf(s, "tx_bd_r_index(%u)\n", priv->tx_bd_r_index);778 seq_printf(s, "tx_done_index(%u)\n", tx_done_index);779 seq_printf(s, "tx_bd_w_index(%u)\n", priv->tx_bd_w_index);780 781 seq_printf(s, "tx host queue len(%u)\n",782 CIRC_CNT(priv->tx_bd_w_index, priv->tx_bd_r_index,783 priv->tx_bd_num));784 seq_printf(s, "tx reclaim queue len(%u)\n",785 CIRC_CNT(tx_done_index, priv->tx_bd_r_index,786 priv->tx_bd_num));787 seq_printf(s, "tx card queue len(%u)\n",788 CIRC_CNT(priv->tx_bd_w_index, tx_done_index,789 priv->tx_bd_num));790 791 seq_printf(s, "rx_bd_r_index(%u)\n", priv->rx_bd_r_index);792 seq_printf(s, "rx_bd_w_index(%u)\n", priv->rx_bd_w_index);793 seq_printf(s, "rx alloc queue len(%u)\n",794 CIRC_SPACE(priv->rx_bd_w_index, priv->rx_bd_r_index,795 priv->rx_bd_num));796 797 return 0;798}799 800static void qtnf_reset_dma_offset(struct qtnf_pcie_topaz_state *ts)801{802 struct qtnf_topaz_bda __iomem *bda = ts->bda;803 u32 offset = readl(&bda->bda_dma_offset);804 805 if ((offset & PCIE_DMA_OFFSET_ERROR_MASK) != PCIE_DMA_OFFSET_ERROR)806 return;807 808 writel(0x0, &bda->bda_dma_offset);809}810 811static int qtnf_pcie_endian_detect(struct qtnf_pcie_topaz_state *ts)812{813 struct qtnf_topaz_bda __iomem *bda = ts->bda;814 u32 timeout = 0;815 u32 endian;816 int ret = 0;817 818 writel(QTN_PCI_ENDIAN_DETECT_DATA, &bda->bda_pci_endian);819 820 /* flush endian modifications before status update */821 dma_wmb();822 823 writel(QTN_PCI_ENDIAN_VALID_STATUS, &bda->bda_pci_pre_status);824 825 while (readl(&bda->bda_pci_post_status) !=826 QTN_PCI_ENDIAN_VALID_STATUS) {827 usleep_range(1000, 1200);828 if (++timeout > QTN_FW_DL_TIMEOUT_MS) {829 pr_err("card endianness detection timed out\n");830 ret = -ETIMEDOUT;831 goto endian_out;832 }833 }834 835 /* do not read before status is updated */836 dma_rmb();837 838 endian = readl(&bda->bda_pci_endian);839 WARN(endian != QTN_PCI_LITTLE_ENDIAN,840 "%s: unexpected card endianness", __func__);841 842endian_out:843 writel(0, &bda->bda_pci_pre_status);844 writel(0, &bda->bda_pci_post_status);845 writel(0, &bda->bda_pci_endian);846 847 return ret;848}849 850static int qtnf_pre_init_ep(struct qtnf_bus *bus)851{852 struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);853 struct qtnf_topaz_bda __iomem *bda = ts->bda;854 u32 flags;855 int ret;856 857 ret = qtnf_pcie_endian_detect(ts);858 if (ret < 0) {859 pr_err("failed to detect card endianness\n");860 return ret;861 }862 863 writeb(ts->base.msi_enabled, &ts->bda->bda_rc_msi_enabled);864 qtnf_reset_dma_offset(ts);865 866 /* notify card about driver type and boot mode */867 flags = readl(&bda->bda_flags) | QTN_BDA_HOST_QLINK_DRV;868 869 if (ts->base.flashboot)870 flags |= QTN_BDA_FLASH_BOOT;871 else872 flags &= ~QTN_BDA_FLASH_BOOT;873 874 writel(flags, &bda->bda_flags);875 876 qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_HOST_RDY);877 if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_TARGET_RDY,878 QTN_FW_DL_TIMEOUT_MS)) {879 pr_err("card is not ready to boot...\n");880 return -ETIMEDOUT;881 }882 883 return ret;884}885 886static int qtnf_post_init_ep(struct qtnf_pcie_topaz_state *ts)887{888 struct pci_dev *pdev = ts->base.pdev;889 890 setup_rx_irqs(ts);891 disable_rx_irqs(ts);892 893 if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_QLINK_DONE,894 QTN_FW_QLINK_TIMEOUT_MS))895 return -ETIMEDOUT;896 897 enable_irq(pdev->irq);898 return 0;899}900 901static int902qtnf_ep_fw_load(struct qtnf_pcie_topaz_state *ts, const u8 *fw, u32 fw_size)903{904 struct qtnf_topaz_bda __iomem *bda = ts->bda;905 struct pci_dev *pdev = ts->base.pdev;906 u32 remaining = fw_size;907 u8 *curr = (u8 *)fw;908 u32 blksize;909 u32 nblocks;910 u32 offset;911 u32 count;912 u32 size;913 dma_addr_t paddr;914 void *data;915 int ret = 0;916 917 pr_debug("FW upload started: fw_addr = 0x%p, size=%d\n", fw, fw_size);918 919 blksize = ts->base.fw_blksize;920 921 if (blksize < PAGE_SIZE)922 blksize = PAGE_SIZE;923 924 while (blksize >= PAGE_SIZE) {925 pr_debug("allocating %u bytes to upload FW\n", blksize);926 data = dma_alloc_coherent(&pdev->dev, blksize,927 &paddr, GFP_KERNEL);928 if (data)929 break;930 blksize /= 2;931 }932 933 if (!data) {934 pr_err("failed to allocate DMA buffer for FW upload\n");935 ret = -ENOMEM;936 goto fw_load_out;937 }938 939 nblocks = NBLOCKS(fw_size, blksize);940 offset = readl(&bda->bda_dma_offset);941 942 qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_HOST_LOAD);943 if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_EP_RDY,944 QTN_FW_DL_TIMEOUT_MS)) {945 pr_err("card is not ready to download FW\n");946 ret = -ETIMEDOUT;947 goto fw_load_map;948 }949 950 for (count = 0 ; count < nblocks; count++) {951 size = (remaining > blksize) ? blksize : remaining;952 953 memcpy(data, curr, size);954 qtnf_non_posted_write(paddr + offset, &bda->bda_img);955 qtnf_non_posted_write(size, &bda->bda_img_size);956 957 pr_debug("chunk[%u] VA[0x%p] PA[%pad] sz[%u]\n",958 count, (void *)curr, &paddr, size);959 960 qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_BLOCK_RDY);961 if (qtnf_poll_state(&ts->bda->bda_bootstate,962 QTN_BDA_FW_BLOCK_DONE,963 QTN_FW_DL_TIMEOUT_MS)) {964 pr_err("confirmation for block #%d timed out\n", count);965 ret = -ETIMEDOUT;966 goto fw_load_map;967 }968 969 remaining = (remaining < size) ? remaining : (remaining - size);970 curr += size;971 }972 973 /* upload completion mark: zero-sized block */974 qtnf_non_posted_write(0, &bda->bda_img);975 qtnf_non_posted_write(0, &bda->bda_img_size);976 977 qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_BLOCK_RDY);978 if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_BLOCK_DONE,979 QTN_FW_DL_TIMEOUT_MS)) {980 pr_err("confirmation for the last block timed out\n");981 ret = -ETIMEDOUT;982 goto fw_load_map;983 }984 985 /* RC is done */986 qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_BLOCK_END);987 if (qtnf_poll_state(&ts->bda->bda_bootstate, QTN_BDA_FW_LOAD_DONE,988 QTN_FW_DL_TIMEOUT_MS)) {989 pr_err("confirmation for FW upload completion timed out\n");990 ret = -ETIMEDOUT;991 goto fw_load_map;992 }993 994 pr_debug("FW upload completed: totally sent %d blocks\n", count);995 996fw_load_map:997 dma_free_coherent(&pdev->dev, blksize, data, paddr);998 999fw_load_out:1000 return ret;1001}1002 1003static int qtnf_topaz_fw_upload(struct qtnf_pcie_topaz_state *ts,1004 const char *fwname)1005{1006 const struct firmware *fw;1007 struct pci_dev *pdev = ts->base.pdev;1008 int ret;1009 1010 if (qtnf_poll_state(&ts->bda->bda_bootstate,1011 QTN_BDA_FW_LOAD_RDY,1012 QTN_FW_DL_TIMEOUT_MS)) {1013 pr_err("%s: card is not ready\n", fwname);1014 return -1;1015 }1016 1017 pr_info("starting firmware upload: %s\n", fwname);1018 1019 ret = request_firmware(&fw, fwname, &pdev->dev);1020 if (ret < 0) {1021 pr_err("%s: request_firmware error %d\n", fwname, ret);1022 return -1;1023 }1024 1025 ret = qtnf_ep_fw_load(ts, fw->data, fw->size);1026 release_firmware(fw);1027 1028 if (ret)1029 pr_err("%s: FW upload error\n", fwname);1030 1031 return ret;1032}1033 1034static void qtnf_topaz_fw_work_handler(struct work_struct *work)1035{1036 struct qtnf_bus *bus = container_of(work, struct qtnf_bus, fw_work);1037 struct qtnf_pcie_topaz_state *ts = (void *)get_bus_priv(bus);1038 int bootloader_needed = readl(&ts->bda->bda_flags) & QTN_BDA_XMIT_UBOOT;1039 struct pci_dev *pdev = ts->base.pdev;1040 int ret;1041 1042 qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_TARGET_BOOT);1043 1044 if (bootloader_needed) {1045 ret = qtnf_topaz_fw_upload(ts, QTN_PCI_TOPAZ_BOOTLD_NAME);1046 if (ret)1047 goto fw_load_exit;1048 1049 ret = qtnf_pre_init_ep(bus);1050 if (ret)1051 goto fw_load_exit;1052 1053 qtnf_set_state(&ts->bda->bda_bootstate,1054 QTN_BDA_FW_TARGET_BOOT);1055 }1056 1057 if (ts->base.flashboot) {1058 pr_info("booting firmware from flash\n");1059 1060 ret = qtnf_poll_state(&ts->bda->bda_bootstate,1061 QTN_BDA_FW_FLASH_BOOT,1062 QTN_FW_DL_TIMEOUT_MS);1063 if (ret)1064 goto fw_load_exit;1065 } else {1066 ret = qtnf_topaz_fw_upload(ts, QTN_PCI_TOPAZ_FW_NAME);1067 if (ret)1068 goto fw_load_exit;1069 1070 qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_START);1071 ret = qtnf_poll_state(&ts->bda->bda_bootstate,1072 QTN_BDA_FW_CONFIG,1073 QTN_FW_QLINK_TIMEOUT_MS);1074 if (ret) {1075 pr_err("FW bringup timed out\n");1076 goto fw_load_exit;1077 }1078 1079 qtnf_set_state(&ts->bda->bda_bootstate, QTN_BDA_FW_RUN);1080 ret = qtnf_poll_state(&ts->bda->bda_bootstate,1081 QTN_BDA_FW_RUNNING,1082 QTN_FW_QLINK_TIMEOUT_MS);1083 if (ret) {1084 pr_err("card bringup timed out\n");1085 goto fw_load_exit;1086 }1087 }1088 1089 ret = qtnf_post_init_ep(ts);1090 if (ret) {1091 pr_err("FW runtime failure\n");1092 goto fw_load_exit;1093 }1094 1095 pr_info("firmware is up and running\n");1096 1097 ret = qtnf_pcie_fw_boot_done(bus);1098 if (ret)1099 goto fw_load_exit;1100 1101 qtnf_debugfs_add_entry(bus, "pkt_stats", qtnf_dbg_pkt_stats);1102 qtnf_debugfs_add_entry(bus, "irq_stats", qtnf_dbg_irq_stats);1103 1104fw_load_exit:1105 put_device(&pdev->dev);1106}1107 1108static void qtnf_reclaim_tasklet_fn(struct tasklet_struct *t)1109{1110 struct qtnf_pcie_topaz_state *ts = from_tasklet(ts, t, base.reclaim_tq);1111 1112 qtnf_topaz_data_tx_reclaim(ts);1113}1114 1115static u64 qtnf_topaz_dma_mask_get(void)1116{1117 return DMA_BIT_MASK(32);1118}1119 1120static int qtnf_pcie_topaz_probe(struct qtnf_bus *bus,1121 unsigned int tx_bd_num, unsigned int rx_bd_num)1122{1123 struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);1124 struct pci_dev *pdev = ts->base.pdev;1125 struct qtnf_shm_ipc_int ipc_int;1126 unsigned long irqflags;1127 int ret;1128 1129 bus->bus_ops = &qtnf_pcie_topaz_bus_ops;1130 INIT_WORK(&bus->fw_work, qtnf_topaz_fw_work_handler);1131 ts->bda = ts->base.epmem_bar;1132 1133 /* assign host msi irq before card init */1134 if (ts->base.msi_enabled)1135 irqflags = IRQF_NOBALANCING;1136 else1137 irqflags = IRQF_NOBALANCING | IRQF_SHARED;1138 1139 ret = devm_request_irq(&pdev->dev, pdev->irq,1140 &qtnf_pcie_topaz_interrupt,1141 irqflags, "qtnf_topaz_irq", (void *)bus);1142 if (ret) {1143 pr_err("failed to request pcie irq %d\n", pdev->irq);1144 return ret;1145 }1146 1147 disable_irq(pdev->irq);1148 1149 ret = qtnf_pre_init_ep(bus);1150 if (ret) {1151 pr_err("failed to init card\n");1152 return ret;1153 }1154 1155 ret = qtnf_pcie_topaz_init_xfer(ts, tx_bd_num, rx_bd_num);1156 if (ret) {1157 pr_err("PCIE xfer init failed\n");1158 return ret;1159 }1160 1161 tasklet_setup(&ts->base.reclaim_tq, qtnf_reclaim_tasklet_fn);1162 netif_napi_add_weight(bus->mux_dev, &bus->mux_napi,1163 qtnf_topaz_rx_poll, 10);1164 1165 ipc_int.fn = qtnf_topaz_ipc_gen_ep_int;1166 ipc_int.arg = ts;1167 qtnf_pcie_init_shm_ipc(&ts->base, &ts->bda->bda_shm_reg1,1168 &ts->bda->bda_shm_reg2, &ipc_int);1169 1170 return 0;1171}1172 1173static void qtnf_pcie_topaz_remove(struct qtnf_bus *bus)1174{1175 struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);1176 1177 qtnf_topaz_reset_ep(ts);1178 qtnf_topaz_free_xfer_buffers(ts);1179}1180 1181#ifdef CONFIG_PM_SLEEP1182static int qtnf_pcie_topaz_suspend(struct qtnf_bus *bus)1183{1184 struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);1185 struct pci_dev *pdev = ts->base.pdev;1186 1187 writel((u32 __force)PCI_D3hot, ts->ep_pmstate);1188 dma_wmb();1189 writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_PM_EP_IRQ),1190 TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar));1191 1192 pci_save_state(pdev);1193 pci_enable_wake(pdev, PCI_D3hot, 1);1194 pci_set_power_state(pdev, PCI_D3hot);1195 1196 return 0;1197}1198 1199static int qtnf_pcie_topaz_resume(struct qtnf_bus *bus)1200{1201 struct qtnf_pcie_topaz_state *ts = get_bus_priv(bus);1202 struct pci_dev *pdev = ts->base.pdev;1203 1204 pci_set_power_state(pdev, PCI_D0);1205 pci_restore_state(pdev);1206 pci_enable_wake(pdev, PCI_D0, 0);1207 1208 writel((u32 __force)PCI_D0, ts->ep_pmstate);1209 dma_wmb();1210 writel(TOPAZ_IPC_IRQ_WORD(TOPAZ_RC_PM_EP_IRQ),1211 TOPAZ_LH_IPC4_INT(ts->base.sysctl_bar));1212 1213 return 0;1214}1215#endif1216 1217struct qtnf_bus *qtnf_pcie_topaz_alloc(struct pci_dev *pdev)1218{1219 struct qtnf_bus *bus;1220 struct qtnf_pcie_topaz_state *ts;1221 1222 bus = devm_kzalloc(&pdev->dev, sizeof(*bus) + sizeof(*ts), GFP_KERNEL);1223 if (!bus)1224 return NULL;1225 1226 ts = get_bus_priv(bus);1227 ts->base.probe_cb = qtnf_pcie_topaz_probe;1228 ts->base.remove_cb = qtnf_pcie_topaz_remove;1229 ts->base.dma_mask_get_cb = qtnf_topaz_dma_mask_get;1230#ifdef CONFIG_PM_SLEEP1231 ts->base.resume_cb = qtnf_pcie_topaz_resume;1232 ts->base.suspend_cb = qtnf_pcie_topaz_suspend;1233#endif1234 1235 return bus;1236}1237