463 lines · c
1// SPDX-License-Identifier: GPL-2.0+2/* Microchip Sparx5 Switch driver3 *4 * Copyright (c) 2021 Microchip Technology Inc. and its subsidiaries.5 *6 * The Sparx5 Chip Register Model can be browsed at this location:7 * https://github.com/microchip-ung/sparx-5_reginfo8 */9 10#include <linux/types.h>11#include <linux/skbuff.h>12#include <linux/netdevice.h>13#include <linux/interrupt.h>14#include <linux/ip.h>15#include <linux/dma-mapping.h>16 17#include "sparx5_main_regs.h"18#include "sparx5_main.h"19#include "sparx5_port.h"20 21#define FDMA_XTR_CHANNEL 622#define FDMA_INJ_CHANNEL 023 24#define FDMA_XTR_BUFFER_SIZE 204825#define FDMA_WEIGHT 426 27static int sparx5_fdma_tx_dataptr_cb(struct fdma *fdma, int dcb, int db,28 u64 *dataptr)29{30 *dataptr = fdma->dma + (sizeof(struct fdma_dcb) * fdma->n_dcbs) +31 ((dcb * fdma->n_dbs + db) * fdma->db_size);32 33 return 0;34}35 36static int sparx5_fdma_rx_dataptr_cb(struct fdma *fdma, int dcb, int db,37 u64 *dataptr)38{39 struct sparx5 *sparx5 = fdma->priv;40 struct sparx5_rx *rx = &sparx5->rx;41 struct sk_buff *skb;42 43 skb = __netdev_alloc_skb(rx->ndev, fdma->db_size, GFP_ATOMIC);44 if (unlikely(!skb))45 return -ENOMEM;46 47 *dataptr = virt_to_phys(skb->data);48 49 rx->skb[dcb][db] = skb;50 51 return 0;52}53 54static void sparx5_fdma_rx_activate(struct sparx5 *sparx5, struct sparx5_rx *rx)55{56 struct fdma *fdma = &rx->fdma;57 58 /* Write the buffer address in the LLP and LLP1 regs */59 spx5_wr(((u64)fdma->dma) & GENMASK(31, 0), sparx5,60 FDMA_DCB_LLP(fdma->channel_id));61 spx5_wr(((u64)fdma->dma) >> 32, sparx5,62 FDMA_DCB_LLP1(fdma->channel_id));63 64 /* Set the number of RX DBs to be used, and DB end-of-frame interrupt */65 spx5_wr(FDMA_CH_CFG_CH_DCB_DB_CNT_SET(fdma->n_dbs) |66 FDMA_CH_CFG_CH_INTR_DB_EOF_ONLY_SET(1) |67 FDMA_CH_CFG_CH_INJ_PORT_SET(XTR_QUEUE),68 sparx5, FDMA_CH_CFG(fdma->channel_id));69 70 /* Set the RX Watermark to max */71 spx5_rmw(FDMA_XTR_CFG_XTR_FIFO_WM_SET(31), FDMA_XTR_CFG_XTR_FIFO_WM,72 sparx5,73 FDMA_XTR_CFG);74 75 /* Start RX fdma */76 spx5_rmw(FDMA_PORT_CTRL_XTR_STOP_SET(0), FDMA_PORT_CTRL_XTR_STOP,77 sparx5, FDMA_PORT_CTRL(0));78 79 /* Enable RX channel DB interrupt */80 spx5_rmw(BIT(fdma->channel_id),81 BIT(fdma->channel_id) & FDMA_INTR_DB_ENA_INTR_DB_ENA,82 sparx5, FDMA_INTR_DB_ENA);83 84 /* Activate the RX channel */85 spx5_wr(BIT(fdma->channel_id), sparx5, FDMA_CH_ACTIVATE);86}87 88static void sparx5_fdma_rx_deactivate(struct sparx5 *sparx5, struct sparx5_rx *rx)89{90 struct fdma *fdma = &rx->fdma;91 92 /* Deactivate the RX channel */93 spx5_rmw(0, BIT(fdma->channel_id) & FDMA_CH_ACTIVATE_CH_ACTIVATE,94 sparx5, FDMA_CH_ACTIVATE);95 96 /* Disable RX channel DB interrupt */97 spx5_rmw(0, BIT(fdma->channel_id) & FDMA_INTR_DB_ENA_INTR_DB_ENA,98 sparx5, FDMA_INTR_DB_ENA);99 100 /* Stop RX fdma */101 spx5_rmw(FDMA_PORT_CTRL_XTR_STOP_SET(1), FDMA_PORT_CTRL_XTR_STOP,102 sparx5, FDMA_PORT_CTRL(0));103}104 105static void sparx5_fdma_tx_activate(struct sparx5 *sparx5, struct sparx5_tx *tx)106{107 struct fdma *fdma = &tx->fdma;108 109 /* Write the buffer address in the LLP and LLP1 regs */110 spx5_wr(((u64)fdma->dma) & GENMASK(31, 0), sparx5,111 FDMA_DCB_LLP(fdma->channel_id));112 spx5_wr(((u64)fdma->dma) >> 32, sparx5,113 FDMA_DCB_LLP1(fdma->channel_id));114 115 /* Set the number of TX DBs to be used, and DB end-of-frame interrupt */116 spx5_wr(FDMA_CH_CFG_CH_DCB_DB_CNT_SET(fdma->n_dbs) |117 FDMA_CH_CFG_CH_INTR_DB_EOF_ONLY_SET(1) |118 FDMA_CH_CFG_CH_INJ_PORT_SET(INJ_QUEUE),119 sparx5, FDMA_CH_CFG(fdma->channel_id));120 121 /* Start TX fdma */122 spx5_rmw(FDMA_PORT_CTRL_INJ_STOP_SET(0), FDMA_PORT_CTRL_INJ_STOP,123 sparx5, FDMA_PORT_CTRL(0));124 125 /* Activate the channel */126 spx5_wr(BIT(fdma->channel_id), sparx5, FDMA_CH_ACTIVATE);127}128 129static void sparx5_fdma_tx_deactivate(struct sparx5 *sparx5, struct sparx5_tx *tx)130{131 /* Disable the channel */132 spx5_rmw(0, BIT(tx->fdma.channel_id) & FDMA_CH_ACTIVATE_CH_ACTIVATE,133 sparx5, FDMA_CH_ACTIVATE);134}135 136static void sparx5_fdma_reload(struct sparx5 *sparx5, struct fdma *fdma)137{138 /* Reload the RX channel */139 spx5_wr(BIT(fdma->channel_id), sparx5, FDMA_CH_RELOAD);140}141 142static bool sparx5_fdma_rx_get_frame(struct sparx5 *sparx5, struct sparx5_rx *rx)143{144 struct fdma *fdma = &rx->fdma;145 struct sparx5_port *port;146 struct fdma_db *db_hw;147 struct frame_info fi;148 struct sk_buff *skb;149 150 /* Check if the DCB is done */151 db_hw = fdma_db_next_get(fdma);152 if (unlikely(!fdma_db_is_done(db_hw)))153 return false;154 skb = rx->skb[fdma->dcb_index][fdma->db_index];155 skb_put(skb, fdma_db_len_get(db_hw));156 /* Now do the normal processing of the skb */157 sparx5_ifh_parse((u32 *)skb->data, &fi);158 /* Map to port netdev */159 port = fi.src_port < SPX5_PORTS ? sparx5->ports[fi.src_port] : NULL;160 if (!port || !port->ndev) {161 dev_err(sparx5->dev, "Data on inactive port %d\n", fi.src_port);162 sparx5_xtr_flush(sparx5, XTR_QUEUE);163 return false;164 }165 skb->dev = port->ndev;166 skb_pull(skb, IFH_LEN * sizeof(u32));167 if (likely(!(skb->dev->features & NETIF_F_RXFCS)))168 skb_trim(skb, skb->len - ETH_FCS_LEN);169 170 sparx5_ptp_rxtstamp(sparx5, skb, fi.timestamp);171 skb->protocol = eth_type_trans(skb, skb->dev);172 /* Everything we see on an interface that is in the HW bridge173 * has already been forwarded174 */175 if (test_bit(port->portno, sparx5->bridge_mask))176 skb->offload_fwd_mark = 1;177 skb->dev->stats.rx_bytes += skb->len;178 skb->dev->stats.rx_packets++;179 rx->packets++;180 netif_receive_skb(skb);181 return true;182}183 184static int sparx5_fdma_napi_callback(struct napi_struct *napi, int weight)185{186 struct sparx5_rx *rx = container_of(napi, struct sparx5_rx, napi);187 struct sparx5 *sparx5 = container_of(rx, struct sparx5, rx);188 struct fdma *fdma = &rx->fdma;189 int counter = 0;190 191 while (counter < weight && sparx5_fdma_rx_get_frame(sparx5, rx)) {192 fdma_db_advance(fdma);193 counter++;194 /* Check if the DCB can be reused */195 if (fdma_dcb_is_reusable(fdma))196 continue;197 fdma_dcb_add(fdma, fdma->dcb_index,198 FDMA_DCB_INFO_DATAL(fdma->db_size),199 FDMA_DCB_STATUS_INTR);200 fdma_db_reset(fdma);201 fdma_dcb_advance(fdma);202 }203 if (counter < weight) {204 napi_complete_done(&rx->napi, counter);205 spx5_rmw(BIT(fdma->channel_id),206 BIT(fdma->channel_id) & FDMA_INTR_DB_ENA_INTR_DB_ENA,207 sparx5, FDMA_INTR_DB_ENA);208 }209 if (counter)210 sparx5_fdma_reload(sparx5, fdma);211 return counter;212}213 214int sparx5_fdma_xmit(struct sparx5 *sparx5, u32 *ifh, struct sk_buff *skb)215{216 struct sparx5_tx *tx = &sparx5->tx;217 struct fdma *fdma = &tx->fdma;218 static bool first_time = true;219 void *virt_addr;220 221 fdma_dcb_advance(fdma);222 if (!fdma_db_is_done(fdma_db_get(fdma, fdma->dcb_index, 0)))223 return -EINVAL;224 225 /* Get the virtual address of the dataptr for the next DB */226 virt_addr = ((u8 *)fdma->dcbs +227 (sizeof(struct fdma_dcb) * fdma->n_dcbs) +228 ((fdma->dcb_index * fdma->n_dbs) * fdma->db_size));229 230 memcpy(virt_addr, ifh, IFH_LEN * 4);231 memcpy(virt_addr + IFH_LEN * 4, skb->data, skb->len);232 233 fdma_dcb_add(fdma, fdma->dcb_index, 0,234 FDMA_DCB_STATUS_SOF |235 FDMA_DCB_STATUS_EOF |236 FDMA_DCB_STATUS_BLOCKO(0) |237 FDMA_DCB_STATUS_BLOCKL(skb->len + IFH_LEN * 4 + 4));238 239 if (first_time) {240 sparx5_fdma_tx_activate(sparx5, tx);241 first_time = false;242 } else {243 sparx5_fdma_reload(sparx5, fdma);244 }245 return NETDEV_TX_OK;246}247 248static int sparx5_fdma_rx_alloc(struct sparx5 *sparx5)249{250 struct sparx5_rx *rx = &sparx5->rx;251 struct fdma *fdma = &rx->fdma;252 int err;253 254 err = fdma_alloc_phys(fdma);255 if (err)256 return err;257 258 fdma_dcbs_init(fdma, FDMA_DCB_INFO_DATAL(fdma->db_size),259 FDMA_DCB_STATUS_INTR);260 261 netif_napi_add_weight(rx->ndev, &rx->napi, sparx5_fdma_napi_callback,262 FDMA_WEIGHT);263 napi_enable(&rx->napi);264 sparx5_fdma_rx_activate(sparx5, rx);265 return 0;266}267 268static int sparx5_fdma_tx_alloc(struct sparx5 *sparx5)269{270 struct sparx5_tx *tx = &sparx5->tx;271 struct fdma *fdma = &tx->fdma;272 int err;273 274 err = fdma_alloc_phys(fdma);275 if (err)276 return err;277 278 fdma_dcbs_init(fdma, FDMA_DCB_INFO_DATAL(fdma->db_size),279 FDMA_DCB_STATUS_DONE);280 281 return 0;282}283 284static void sparx5_fdma_rx_init(struct sparx5 *sparx5,285 struct sparx5_rx *rx, int channel)286{287 struct fdma *fdma = &rx->fdma;288 int idx;289 290 fdma->channel_id = channel;291 fdma->n_dcbs = FDMA_DCB_MAX;292 fdma->n_dbs = FDMA_RX_DCB_MAX_DBS;293 fdma->priv = sparx5;294 fdma->db_size = ALIGN(FDMA_XTR_BUFFER_SIZE, PAGE_SIZE);295 fdma->size = fdma_get_size(&sparx5->rx.fdma);296 fdma->ops.dataptr_cb = &sparx5_fdma_rx_dataptr_cb;297 fdma->ops.nextptr_cb = &fdma_nextptr_cb;298 /* Fetch a netdev for SKB and NAPI use, any will do */299 for (idx = 0; idx < SPX5_PORTS; ++idx) {300 struct sparx5_port *port = sparx5->ports[idx];301 302 if (port && port->ndev) {303 rx->ndev = port->ndev;304 break;305 }306 }307}308 309static void sparx5_fdma_tx_init(struct sparx5 *sparx5,310 struct sparx5_tx *tx, int channel)311{312 struct fdma *fdma = &tx->fdma;313 314 fdma->channel_id = channel;315 fdma->n_dcbs = FDMA_DCB_MAX;316 fdma->n_dbs = FDMA_TX_DCB_MAX_DBS;317 fdma->priv = sparx5;318 fdma->db_size = ALIGN(FDMA_XTR_BUFFER_SIZE, PAGE_SIZE);319 fdma->size = fdma_get_size_contiguous(&sparx5->tx.fdma);320 fdma->ops.dataptr_cb = &sparx5_fdma_tx_dataptr_cb;321 fdma->ops.nextptr_cb = &fdma_nextptr_cb;322}323 324irqreturn_t sparx5_fdma_handler(int irq, void *args)325{326 struct sparx5 *sparx5 = args;327 u32 db = 0, err = 0;328 329 db = spx5_rd(sparx5, FDMA_INTR_DB);330 err = spx5_rd(sparx5, FDMA_INTR_ERR);331 /* Clear interrupt */332 if (db) {333 spx5_wr(0, sparx5, FDMA_INTR_DB_ENA);334 spx5_wr(db, sparx5, FDMA_INTR_DB);335 napi_schedule(&sparx5->rx.napi);336 }337 if (err) {338 u32 err_type = spx5_rd(sparx5, FDMA_ERRORS);339 340 dev_err_ratelimited(sparx5->dev,341 "ERR: int: %#x, type: %#x\n",342 err, err_type);343 spx5_wr(err, sparx5, FDMA_INTR_ERR);344 spx5_wr(err_type, sparx5, FDMA_ERRORS);345 }346 return IRQ_HANDLED;347}348 349static void sparx5_fdma_injection_mode(struct sparx5 *sparx5)350{351 const int byte_swap = 1;352 int portno;353 int urgency;354 355 /* Change mode to fdma extraction and injection */356 spx5_wr(QS_XTR_GRP_CFG_MODE_SET(2) |357 QS_XTR_GRP_CFG_STATUS_WORD_POS_SET(1) |358 QS_XTR_GRP_CFG_BYTE_SWAP_SET(byte_swap),359 sparx5, QS_XTR_GRP_CFG(XTR_QUEUE));360 spx5_wr(QS_INJ_GRP_CFG_MODE_SET(2) |361 QS_INJ_GRP_CFG_BYTE_SWAP_SET(byte_swap),362 sparx5, QS_INJ_GRP_CFG(INJ_QUEUE));363 364 /* CPU ports capture setup */365 for (portno = SPX5_PORT_CPU_0; portno <= SPX5_PORT_CPU_1; portno++) {366 /* ASM CPU port: No preamble, IFH, enable padding */367 spx5_wr(ASM_PORT_CFG_PAD_ENA_SET(1) |368 ASM_PORT_CFG_NO_PREAMBLE_ENA_SET(1) |369 ASM_PORT_CFG_INJ_FORMAT_CFG_SET(1), /* 1 = IFH */370 sparx5, ASM_PORT_CFG(portno));371 372 /* Reset WM cnt to unclog queued frames */373 spx5_rmw(DSM_DEV_TX_STOP_WM_CFG_DEV_TX_CNT_CLR_SET(1),374 DSM_DEV_TX_STOP_WM_CFG_DEV_TX_CNT_CLR,375 sparx5,376 DSM_DEV_TX_STOP_WM_CFG(portno));377 378 /* Set Disassembler Stop Watermark level */379 spx5_rmw(DSM_DEV_TX_STOP_WM_CFG_DEV_TX_STOP_WM_SET(100),380 DSM_DEV_TX_STOP_WM_CFG_DEV_TX_STOP_WM,381 sparx5,382 DSM_DEV_TX_STOP_WM_CFG(portno));383 384 /* Enable port in queue system */385 urgency = sparx5_port_fwd_urg(sparx5, SPEED_2500);386 spx5_rmw(QFWD_SWITCH_PORT_MODE_PORT_ENA_SET(1) |387 QFWD_SWITCH_PORT_MODE_FWD_URGENCY_SET(urgency),388 QFWD_SWITCH_PORT_MODE_PORT_ENA |389 QFWD_SWITCH_PORT_MODE_FWD_URGENCY,390 sparx5,391 QFWD_SWITCH_PORT_MODE(portno));392 393 /* Disable Disassembler buffer underrun watchdog394 * to avoid truncated packets in XTR395 */396 spx5_rmw(DSM_BUF_CFG_UNDERFLOW_WATCHDOG_DIS_SET(1),397 DSM_BUF_CFG_UNDERFLOW_WATCHDOG_DIS,398 sparx5,399 DSM_BUF_CFG(portno));400 401 /* Disabling frame aging */402 spx5_rmw(HSCH_PORT_MODE_AGE_DIS_SET(1),403 HSCH_PORT_MODE_AGE_DIS,404 sparx5,405 HSCH_PORT_MODE(portno));406 }407}408 409int sparx5_fdma_start(struct sparx5 *sparx5)410{411 int err;412 413 /* Reset FDMA state */414 spx5_wr(FDMA_CTRL_NRESET_SET(0), sparx5, FDMA_CTRL);415 spx5_wr(FDMA_CTRL_NRESET_SET(1), sparx5, FDMA_CTRL);416 417 /* Force ACP caching but disable read/write allocation */418 spx5_rmw(CPU_PROC_CTRL_ACP_CACHE_FORCE_ENA_SET(1) |419 CPU_PROC_CTRL_ACP_AWCACHE_SET(0) |420 CPU_PROC_CTRL_ACP_ARCACHE_SET(0),421 CPU_PROC_CTRL_ACP_CACHE_FORCE_ENA |422 CPU_PROC_CTRL_ACP_AWCACHE |423 CPU_PROC_CTRL_ACP_ARCACHE,424 sparx5, CPU_PROC_CTRL);425 426 sparx5_fdma_injection_mode(sparx5);427 sparx5_fdma_rx_init(sparx5, &sparx5->rx, FDMA_XTR_CHANNEL);428 sparx5_fdma_tx_init(sparx5, &sparx5->tx, FDMA_INJ_CHANNEL);429 err = sparx5_fdma_rx_alloc(sparx5);430 if (err) {431 dev_err(sparx5->dev, "Could not allocate RX buffers: %d\n", err);432 return err;433 }434 err = sparx5_fdma_tx_alloc(sparx5);435 if (err) {436 dev_err(sparx5->dev, "Could not allocate TX buffers: %d\n", err);437 return err;438 }439 return err;440}441 442static u32 sparx5_fdma_port_ctrl(struct sparx5 *sparx5)443{444 return spx5_rd(sparx5, FDMA_PORT_CTRL(0));445}446 447int sparx5_fdma_stop(struct sparx5 *sparx5)448{449 u32 val;450 451 napi_disable(&sparx5->rx.napi);452 /* Stop the fdma and channel interrupts */453 sparx5_fdma_rx_deactivate(sparx5, &sparx5->rx);454 sparx5_fdma_tx_deactivate(sparx5, &sparx5->tx);455 /* Wait for the RX channel to stop */456 read_poll_timeout(sparx5_fdma_port_ctrl, val,457 FDMA_PORT_CTRL_XTR_BUF_IS_EMPTY_GET(val) == 0,458 500, 10000, 0, sparx5);459 fdma_free_phys(&sparx5->rx.fdma);460 fdma_free_phys(&sparx5->tx.fdma);461 return 0;462}463