1350 lines · c
1/* Synopsys DesignWare Core Enterprise Ethernet (XLGMAC) Driver2 *3 * Copyright (c) 2017 Synopsys, Inc. (www.synopsys.com)4 *5 * This program is dual-licensed; you may select either version 2 of6 * the GNU General Public License ("GPL") or BSD license ("BSD").7 *8 * This Synopsys DWC XLGMAC software driver and associated documentation9 * (hereinafter the "Software") is an unsupported proprietary work of10 * Synopsys, Inc. unless otherwise expressly agreed to in writing between11 * Synopsys and you. The Software IS NOT an item of Licensed Software or a12 * Licensed Product under any End User Software License Agreement or13 * Agreement for Licensed Products with Synopsys or any supplement thereto.14 * Synopsys is a registered trademark of Synopsys, Inc. Other names included15 * in the SOFTWARE may be the trademarks of their respective owners.16 */17 18#include <linux/netdevice.h>19#include <linux/tcp.h>20#include <linux/interrupt.h>21 22#include "dwc-xlgmac.h"23#include "dwc-xlgmac-reg.h"24 25static int xlgmac_one_poll(struct napi_struct *, int);26static int xlgmac_all_poll(struct napi_struct *, int);27 28static inline unsigned int xlgmac_tx_avail_desc(struct xlgmac_ring *ring)29{30 return (ring->dma_desc_count - (ring->cur - ring->dirty));31}32 33static inline unsigned int xlgmac_rx_dirty_desc(struct xlgmac_ring *ring)34{35 return (ring->cur - ring->dirty);36}37 38static int xlgmac_maybe_stop_tx_queue(39 struct xlgmac_channel *channel,40 struct xlgmac_ring *ring,41 unsigned int count)42{43 struct xlgmac_pdata *pdata = channel->pdata;44 45 if (count > xlgmac_tx_avail_desc(ring)) {46 netif_info(pdata, drv, pdata->netdev,47 "Tx queue stopped, not enough descriptors available\n");48 netif_stop_subqueue(pdata->netdev, channel->queue_index);49 ring->tx.queue_stopped = 1;50 51 /* If we haven't notified the hardware because of xmit_more52 * support, tell it now53 */54 if (ring->tx.xmit_more)55 pdata->hw_ops.tx_start_xmit(channel, ring);56 57 return NETDEV_TX_BUSY;58 }59 60 return 0;61}62 63static void xlgmac_prep_vlan(struct sk_buff *skb,64 struct xlgmac_pkt_info *pkt_info)65{66 if (skb_vlan_tag_present(skb))67 pkt_info->vlan_ctag = skb_vlan_tag_get(skb);68}69 70static int xlgmac_prep_tso(struct sk_buff *skb,71 struct xlgmac_pkt_info *pkt_info)72{73 int ret;74 75 if (!XLGMAC_GET_REG_BITS(pkt_info->attributes,76 TX_PACKET_ATTRIBUTES_TSO_ENABLE_POS,77 TX_PACKET_ATTRIBUTES_TSO_ENABLE_LEN))78 return 0;79 80 ret = skb_cow_head(skb, 0);81 if (ret)82 return ret;83 84 pkt_info->header_len = skb_tcp_all_headers(skb);85 pkt_info->tcp_header_len = tcp_hdrlen(skb);86 pkt_info->tcp_payload_len = skb->len - pkt_info->header_len;87 pkt_info->mss = skb_shinfo(skb)->gso_size;88 89 XLGMAC_PR("header_len=%u\n", pkt_info->header_len);90 XLGMAC_PR("tcp_header_len=%u, tcp_payload_len=%u\n",91 pkt_info->tcp_header_len, pkt_info->tcp_payload_len);92 XLGMAC_PR("mss=%u\n", pkt_info->mss);93 94 /* Update the number of packets that will ultimately be transmitted95 * along with the extra bytes for each extra packet96 */97 pkt_info->tx_packets = skb_shinfo(skb)->gso_segs;98 pkt_info->tx_bytes += (pkt_info->tx_packets - 1) * pkt_info->header_len;99 100 return 0;101}102 103static int xlgmac_is_tso(struct sk_buff *skb)104{105 if (skb->ip_summed != CHECKSUM_PARTIAL)106 return 0;107 108 if (!skb_is_gso(skb))109 return 0;110 111 return 1;112}113 114static void xlgmac_prep_tx_pkt(struct xlgmac_pdata *pdata,115 struct xlgmac_ring *ring,116 struct sk_buff *skb,117 struct xlgmac_pkt_info *pkt_info)118{119 skb_frag_t *frag;120 unsigned int context_desc;121 unsigned int len;122 unsigned int i;123 124 pkt_info->skb = skb;125 126 context_desc = 0;127 pkt_info->desc_count = 0;128 129 pkt_info->tx_packets = 1;130 pkt_info->tx_bytes = skb->len;131 132 if (xlgmac_is_tso(skb)) {133 /* TSO requires an extra descriptor if mss is different */134 if (skb_shinfo(skb)->gso_size != ring->tx.cur_mss) {135 context_desc = 1;136 pkt_info->desc_count++;137 }138 139 /* TSO requires an extra descriptor for TSO header */140 pkt_info->desc_count++;141 142 pkt_info->attributes = XLGMAC_SET_REG_BITS(143 pkt_info->attributes,144 TX_PACKET_ATTRIBUTES_TSO_ENABLE_POS,145 TX_PACKET_ATTRIBUTES_TSO_ENABLE_LEN,146 1);147 pkt_info->attributes = XLGMAC_SET_REG_BITS(148 pkt_info->attributes,149 TX_PACKET_ATTRIBUTES_CSUM_ENABLE_POS,150 TX_PACKET_ATTRIBUTES_CSUM_ENABLE_LEN,151 1);152 } else if (skb->ip_summed == CHECKSUM_PARTIAL)153 pkt_info->attributes = XLGMAC_SET_REG_BITS(154 pkt_info->attributes,155 TX_PACKET_ATTRIBUTES_CSUM_ENABLE_POS,156 TX_PACKET_ATTRIBUTES_CSUM_ENABLE_LEN,157 1);158 159 if (skb_vlan_tag_present(skb)) {160 /* VLAN requires an extra descriptor if tag is different */161 if (skb_vlan_tag_get(skb) != ring->tx.cur_vlan_ctag)162 /* We can share with the TSO context descriptor */163 if (!context_desc) {164 context_desc = 1;165 pkt_info->desc_count++;166 }167 168 pkt_info->attributes = XLGMAC_SET_REG_BITS(169 pkt_info->attributes,170 TX_PACKET_ATTRIBUTES_VLAN_CTAG_POS,171 TX_PACKET_ATTRIBUTES_VLAN_CTAG_LEN,172 1);173 }174 175 for (len = skb_headlen(skb); len;) {176 pkt_info->desc_count++;177 len -= min_t(unsigned int, len, XLGMAC_TX_MAX_BUF_SIZE);178 }179 180 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {181 frag = &skb_shinfo(skb)->frags[i];182 for (len = skb_frag_size(frag); len; ) {183 pkt_info->desc_count++;184 len -= min_t(unsigned int, len, XLGMAC_TX_MAX_BUF_SIZE);185 }186 }187}188 189static int xlgmac_calc_rx_buf_size(struct net_device *netdev, unsigned int mtu)190{191 unsigned int rx_buf_size;192 193 if (mtu > XLGMAC_JUMBO_PACKET_MTU) {194 netdev_alert(netdev, "MTU exceeds maximum supported value\n");195 return -EINVAL;196 }197 198 rx_buf_size = mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;199 rx_buf_size = clamp_val(rx_buf_size, XLGMAC_RX_MIN_BUF_SIZE, PAGE_SIZE);200 201 rx_buf_size = (rx_buf_size + XLGMAC_RX_BUF_ALIGN - 1) &202 ~(XLGMAC_RX_BUF_ALIGN - 1);203 204 return rx_buf_size;205}206 207static void xlgmac_enable_rx_tx_ints(struct xlgmac_pdata *pdata)208{209 struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;210 struct xlgmac_channel *channel;211 enum xlgmac_int int_id;212 unsigned int i;213 214 channel = pdata->channel_head;215 for (i = 0; i < pdata->channel_count; i++, channel++) {216 if (channel->tx_ring && channel->rx_ring)217 int_id = XLGMAC_INT_DMA_CH_SR_TI_RI;218 else if (channel->tx_ring)219 int_id = XLGMAC_INT_DMA_CH_SR_TI;220 else if (channel->rx_ring)221 int_id = XLGMAC_INT_DMA_CH_SR_RI;222 else223 continue;224 225 hw_ops->enable_int(channel, int_id);226 }227}228 229static void xlgmac_disable_rx_tx_ints(struct xlgmac_pdata *pdata)230{231 struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;232 struct xlgmac_channel *channel;233 enum xlgmac_int int_id;234 unsigned int i;235 236 channel = pdata->channel_head;237 for (i = 0; i < pdata->channel_count; i++, channel++) {238 if (channel->tx_ring && channel->rx_ring)239 int_id = XLGMAC_INT_DMA_CH_SR_TI_RI;240 else if (channel->tx_ring)241 int_id = XLGMAC_INT_DMA_CH_SR_TI;242 else if (channel->rx_ring)243 int_id = XLGMAC_INT_DMA_CH_SR_RI;244 else245 continue;246 247 hw_ops->disable_int(channel, int_id);248 }249}250 251static irqreturn_t xlgmac_isr(int irq, void *data)252{253 unsigned int dma_isr, dma_ch_isr, mac_isr;254 struct xlgmac_pdata *pdata = data;255 struct xlgmac_channel *channel;256 struct xlgmac_hw_ops *hw_ops;257 unsigned int i, ti, ri;258 259 hw_ops = &pdata->hw_ops;260 261 /* The DMA interrupt status register also reports MAC and MTL262 * interrupts. So for polling mode, we just need to check for263 * this register to be non-zero264 */265 dma_isr = readl(pdata->mac_regs + DMA_ISR);266 if (!dma_isr)267 return IRQ_HANDLED;268 269 netif_dbg(pdata, intr, pdata->netdev, "DMA_ISR=%#010x\n", dma_isr);270 271 for (i = 0; i < pdata->channel_count; i++) {272 if (!(dma_isr & (1 << i)))273 continue;274 275 channel = pdata->channel_head + i;276 277 dma_ch_isr = readl(XLGMAC_DMA_REG(channel, DMA_CH_SR));278 netif_dbg(pdata, intr, pdata->netdev, "DMA_CH%u_ISR=%#010x\n",279 i, dma_ch_isr);280 281 /* The TI or RI interrupt bits may still be set even if using282 * per channel DMA interrupts. Check to be sure those are not283 * enabled before using the private data napi structure.284 */285 ti = XLGMAC_GET_REG_BITS(dma_ch_isr, DMA_CH_SR_TI_POS,286 DMA_CH_SR_TI_LEN);287 ri = XLGMAC_GET_REG_BITS(dma_ch_isr, DMA_CH_SR_RI_POS,288 DMA_CH_SR_RI_LEN);289 if (!pdata->per_channel_irq && (ti || ri)) {290 if (napi_schedule_prep(&pdata->napi)) {291 /* Disable Tx and Rx interrupts */292 xlgmac_disable_rx_tx_ints(pdata);293 294 pdata->stats.napi_poll_isr++;295 /* Turn on polling */296 __napi_schedule_irqoff(&pdata->napi);297 }298 }299 300 if (XLGMAC_GET_REG_BITS(dma_ch_isr, DMA_CH_SR_TPS_POS,301 DMA_CH_SR_TPS_LEN))302 pdata->stats.tx_process_stopped++;303 304 if (XLGMAC_GET_REG_BITS(dma_ch_isr, DMA_CH_SR_RPS_POS,305 DMA_CH_SR_RPS_LEN))306 pdata->stats.rx_process_stopped++;307 308 if (XLGMAC_GET_REG_BITS(dma_ch_isr, DMA_CH_SR_TBU_POS,309 DMA_CH_SR_TBU_LEN))310 pdata->stats.tx_buffer_unavailable++;311 312 if (XLGMAC_GET_REG_BITS(dma_ch_isr, DMA_CH_SR_RBU_POS,313 DMA_CH_SR_RBU_LEN))314 pdata->stats.rx_buffer_unavailable++;315 316 /* Restart the device on a Fatal Bus Error */317 if (XLGMAC_GET_REG_BITS(dma_ch_isr, DMA_CH_SR_FBE_POS,318 DMA_CH_SR_FBE_LEN)) {319 pdata->stats.fatal_bus_error++;320 schedule_work(&pdata->restart_work);321 }322 323 /* Clear all interrupt signals */324 writel(dma_ch_isr, XLGMAC_DMA_REG(channel, DMA_CH_SR));325 }326 327 if (XLGMAC_GET_REG_BITS(dma_isr, DMA_ISR_MACIS_POS,328 DMA_ISR_MACIS_LEN)) {329 mac_isr = readl(pdata->mac_regs + MAC_ISR);330 331 if (XLGMAC_GET_REG_BITS(mac_isr, MAC_ISR_MMCTXIS_POS,332 MAC_ISR_MMCTXIS_LEN))333 hw_ops->tx_mmc_int(pdata);334 335 if (XLGMAC_GET_REG_BITS(mac_isr, MAC_ISR_MMCRXIS_POS,336 MAC_ISR_MMCRXIS_LEN))337 hw_ops->rx_mmc_int(pdata);338 }339 340 return IRQ_HANDLED;341}342 343static irqreturn_t xlgmac_dma_isr(int irq, void *data)344{345 struct xlgmac_channel *channel = data;346 347 /* Per channel DMA interrupts are enabled, so we use the per348 * channel napi structure and not the private data napi structure349 */350 if (napi_schedule_prep(&channel->napi)) {351 /* Disable Tx and Rx interrupts */352 disable_irq_nosync(channel->dma_irq);353 354 /* Turn on polling */355 __napi_schedule_irqoff(&channel->napi);356 }357 358 return IRQ_HANDLED;359}360 361static void xlgmac_tx_timer(struct timer_list *t)362{363 struct xlgmac_channel *channel = from_timer(channel, t, tx_timer);364 struct xlgmac_pdata *pdata = channel->pdata;365 struct napi_struct *napi;366 367 napi = (pdata->per_channel_irq) ? &channel->napi : &pdata->napi;368 369 if (napi_schedule_prep(napi)) {370 /* Disable Tx and Rx interrupts */371 if (pdata->per_channel_irq)372 disable_irq_nosync(channel->dma_irq);373 else374 xlgmac_disable_rx_tx_ints(pdata);375 376 pdata->stats.napi_poll_txtimer++;377 /* Turn on polling */378 __napi_schedule(napi);379 }380 381 channel->tx_timer_active = 0;382}383 384static void xlgmac_init_timers(struct xlgmac_pdata *pdata)385{386 struct xlgmac_channel *channel;387 unsigned int i;388 389 channel = pdata->channel_head;390 for (i = 0; i < pdata->channel_count; i++, channel++) {391 if (!channel->tx_ring)392 break;393 394 timer_setup(&channel->tx_timer, xlgmac_tx_timer, 0);395 }396}397 398static void xlgmac_stop_timers(struct xlgmac_pdata *pdata)399{400 struct xlgmac_channel *channel;401 unsigned int i;402 403 channel = pdata->channel_head;404 for (i = 0; i < pdata->channel_count; i++, channel++) {405 if (!channel->tx_ring)406 break;407 408 del_timer_sync(&channel->tx_timer);409 }410}411 412static void xlgmac_napi_enable(struct xlgmac_pdata *pdata, unsigned int add)413{414 struct xlgmac_channel *channel;415 unsigned int i;416 417 if (pdata->per_channel_irq) {418 channel = pdata->channel_head;419 for (i = 0; i < pdata->channel_count; i++, channel++) {420 if (add)421 netif_napi_add(pdata->netdev, &channel->napi,422 xlgmac_one_poll);423 424 napi_enable(&channel->napi);425 }426 } else {427 if (add)428 netif_napi_add(pdata->netdev, &pdata->napi,429 xlgmac_all_poll);430 431 napi_enable(&pdata->napi);432 }433}434 435static void xlgmac_napi_disable(struct xlgmac_pdata *pdata, unsigned int del)436{437 struct xlgmac_channel *channel;438 unsigned int i;439 440 if (pdata->per_channel_irq) {441 channel = pdata->channel_head;442 for (i = 0; i < pdata->channel_count; i++, channel++) {443 napi_disable(&channel->napi);444 445 if (del)446 netif_napi_del(&channel->napi);447 }448 } else {449 napi_disable(&pdata->napi);450 451 if (del)452 netif_napi_del(&pdata->napi);453 }454}455 456static int xlgmac_request_irqs(struct xlgmac_pdata *pdata)457{458 struct net_device *netdev = pdata->netdev;459 struct xlgmac_channel *channel;460 unsigned int i;461 int ret;462 463 ret = devm_request_irq(pdata->dev, pdata->dev_irq, xlgmac_isr,464 IRQF_SHARED, netdev->name, pdata);465 if (ret) {466 netdev_alert(netdev, "error requesting irq %d\n",467 pdata->dev_irq);468 return ret;469 }470 471 if (!pdata->per_channel_irq)472 return 0;473 474 channel = pdata->channel_head;475 for (i = 0; i < pdata->channel_count; i++, channel++) {476 snprintf(channel->dma_irq_name,477 sizeof(channel->dma_irq_name) - 1,478 "%s-TxRx-%u", netdev_name(netdev),479 channel->queue_index);480 481 ret = devm_request_irq(pdata->dev, channel->dma_irq,482 xlgmac_dma_isr, 0,483 channel->dma_irq_name, channel);484 if (ret) {485 netdev_alert(netdev, "error requesting irq %d\n",486 channel->dma_irq);487 goto err_irq;488 }489 }490 491 return 0;492 493err_irq:494 /* Using an unsigned int, 'i' will go to UINT_MAX and exit */495 for (i--, channel--; i < pdata->channel_count; i--, channel--)496 devm_free_irq(pdata->dev, channel->dma_irq, channel);497 498 devm_free_irq(pdata->dev, pdata->dev_irq, pdata);499 500 return ret;501}502 503static void xlgmac_free_irqs(struct xlgmac_pdata *pdata)504{505 struct xlgmac_channel *channel;506 unsigned int i;507 508 devm_free_irq(pdata->dev, pdata->dev_irq, pdata);509 510 if (!pdata->per_channel_irq)511 return;512 513 channel = pdata->channel_head;514 for (i = 0; i < pdata->channel_count; i++, channel++)515 devm_free_irq(pdata->dev, channel->dma_irq, channel);516}517 518static void xlgmac_free_tx_data(struct xlgmac_pdata *pdata)519{520 struct xlgmac_desc_ops *desc_ops = &pdata->desc_ops;521 struct xlgmac_desc_data *desc_data;522 struct xlgmac_channel *channel;523 struct xlgmac_ring *ring;524 unsigned int i, j;525 526 channel = pdata->channel_head;527 for (i = 0; i < pdata->channel_count; i++, channel++) {528 ring = channel->tx_ring;529 if (!ring)530 break;531 532 for (j = 0; j < ring->dma_desc_count; j++) {533 desc_data = XLGMAC_GET_DESC_DATA(ring, j);534 desc_ops->unmap_desc_data(pdata, desc_data);535 }536 }537}538 539static void xlgmac_free_rx_data(struct xlgmac_pdata *pdata)540{541 struct xlgmac_desc_ops *desc_ops = &pdata->desc_ops;542 struct xlgmac_desc_data *desc_data;543 struct xlgmac_channel *channel;544 struct xlgmac_ring *ring;545 unsigned int i, j;546 547 channel = pdata->channel_head;548 for (i = 0; i < pdata->channel_count; i++, channel++) {549 ring = channel->rx_ring;550 if (!ring)551 break;552 553 for (j = 0; j < ring->dma_desc_count; j++) {554 desc_data = XLGMAC_GET_DESC_DATA(ring, j);555 desc_ops->unmap_desc_data(pdata, desc_data);556 }557 }558}559 560static int xlgmac_start(struct xlgmac_pdata *pdata)561{562 struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;563 struct net_device *netdev = pdata->netdev;564 int ret;565 566 hw_ops->init(pdata);567 xlgmac_napi_enable(pdata, 1);568 569 ret = xlgmac_request_irqs(pdata);570 if (ret)571 goto err_napi;572 573 hw_ops->enable_tx(pdata);574 hw_ops->enable_rx(pdata);575 netif_tx_start_all_queues(netdev);576 577 return 0;578 579err_napi:580 xlgmac_napi_disable(pdata, 1);581 hw_ops->exit(pdata);582 583 return ret;584}585 586static void xlgmac_stop(struct xlgmac_pdata *pdata)587{588 struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;589 struct net_device *netdev = pdata->netdev;590 struct xlgmac_channel *channel;591 struct netdev_queue *txq;592 unsigned int i;593 594 netif_tx_stop_all_queues(netdev);595 xlgmac_stop_timers(pdata);596 hw_ops->disable_tx(pdata);597 hw_ops->disable_rx(pdata);598 xlgmac_free_irqs(pdata);599 xlgmac_napi_disable(pdata, 1);600 hw_ops->exit(pdata);601 602 channel = pdata->channel_head;603 for (i = 0; i < pdata->channel_count; i++, channel++) {604 if (!channel->tx_ring)605 continue;606 607 txq = netdev_get_tx_queue(netdev, channel->queue_index);608 netdev_tx_reset_queue(txq);609 }610}611 612static void xlgmac_restart_dev(struct xlgmac_pdata *pdata)613{614 /* If not running, "restart" will happen on open */615 if (!netif_running(pdata->netdev))616 return;617 618 xlgmac_stop(pdata);619 620 xlgmac_free_tx_data(pdata);621 xlgmac_free_rx_data(pdata);622 623 xlgmac_start(pdata);624}625 626static void xlgmac_restart(struct work_struct *work)627{628 struct xlgmac_pdata *pdata = container_of(work,629 struct xlgmac_pdata,630 restart_work);631 632 rtnl_lock();633 634 xlgmac_restart_dev(pdata);635 636 rtnl_unlock();637}638 639static int xlgmac_open(struct net_device *netdev)640{641 struct xlgmac_pdata *pdata = netdev_priv(netdev);642 struct xlgmac_desc_ops *desc_ops;643 int ret;644 645 desc_ops = &pdata->desc_ops;646 647 /* TODO: Initialize the phy */648 649 /* Calculate the Rx buffer size before allocating rings */650 ret = xlgmac_calc_rx_buf_size(netdev, netdev->mtu);651 if (ret < 0)652 return ret;653 pdata->rx_buf_size = ret;654 655 /* Allocate the channels and rings */656 ret = desc_ops->alloc_channels_and_rings(pdata);657 if (ret)658 return ret;659 660 INIT_WORK(&pdata->restart_work, xlgmac_restart);661 xlgmac_init_timers(pdata);662 663 ret = xlgmac_start(pdata);664 if (ret)665 goto err_channels_and_rings;666 667 return 0;668 669err_channels_and_rings:670 desc_ops->free_channels_and_rings(pdata);671 672 return ret;673}674 675static int xlgmac_close(struct net_device *netdev)676{677 struct xlgmac_pdata *pdata = netdev_priv(netdev);678 struct xlgmac_desc_ops *desc_ops;679 680 desc_ops = &pdata->desc_ops;681 682 /* Stop the device */683 xlgmac_stop(pdata);684 685 /* Free the channels and rings */686 desc_ops->free_channels_and_rings(pdata);687 688 return 0;689}690 691static void xlgmac_tx_timeout(struct net_device *netdev, unsigned int txqueue)692{693 struct xlgmac_pdata *pdata = netdev_priv(netdev);694 695 netdev_warn(netdev, "tx timeout, device restarting\n");696 schedule_work(&pdata->restart_work);697}698 699static netdev_tx_t xlgmac_xmit(struct sk_buff *skb, struct net_device *netdev)700{701 struct xlgmac_pdata *pdata = netdev_priv(netdev);702 struct xlgmac_pkt_info *tx_pkt_info;703 struct xlgmac_desc_ops *desc_ops;704 struct xlgmac_channel *channel;705 struct xlgmac_hw_ops *hw_ops;706 struct netdev_queue *txq;707 struct xlgmac_ring *ring;708 int ret;709 710 desc_ops = &pdata->desc_ops;711 hw_ops = &pdata->hw_ops;712 713 XLGMAC_PR("skb->len = %d\n", skb->len);714 715 channel = pdata->channel_head + skb->queue_mapping;716 txq = netdev_get_tx_queue(netdev, channel->queue_index);717 ring = channel->tx_ring;718 tx_pkt_info = &ring->pkt_info;719 720 if (skb->len == 0) {721 netif_err(pdata, tx_err, netdev,722 "empty skb received from stack\n");723 dev_kfree_skb_any(skb);724 return NETDEV_TX_OK;725 }726 727 /* Prepare preliminary packet info for TX */728 memset(tx_pkt_info, 0, sizeof(*tx_pkt_info));729 xlgmac_prep_tx_pkt(pdata, ring, skb, tx_pkt_info);730 731 /* Check that there are enough descriptors available */732 ret = xlgmac_maybe_stop_tx_queue(channel, ring,733 tx_pkt_info->desc_count);734 if (ret)735 return ret;736 737 ret = xlgmac_prep_tso(skb, tx_pkt_info);738 if (ret) {739 netif_err(pdata, tx_err, netdev,740 "error processing TSO packet\n");741 dev_kfree_skb_any(skb);742 return ret;743 }744 xlgmac_prep_vlan(skb, tx_pkt_info);745 746 if (!desc_ops->map_tx_skb(channel, skb)) {747 dev_kfree_skb_any(skb);748 return NETDEV_TX_OK;749 }750 751 /* Report on the actual number of bytes (to be) sent */752 netdev_tx_sent_queue(txq, tx_pkt_info->tx_bytes);753 754 /* Configure required descriptor fields for transmission */755 hw_ops->dev_xmit(channel);756 757 if (netif_msg_pktdata(pdata))758 xlgmac_print_pkt(netdev, skb, true);759 760 /* Stop the queue in advance if there may not be enough descriptors */761 xlgmac_maybe_stop_tx_queue(channel, ring, XLGMAC_TX_MAX_DESC_NR);762 763 return NETDEV_TX_OK;764}765 766static void xlgmac_get_stats64(struct net_device *netdev,767 struct rtnl_link_stats64 *s)768{769 struct xlgmac_pdata *pdata = netdev_priv(netdev);770 struct xlgmac_stats *pstats = &pdata->stats;771 772 pdata->hw_ops.read_mmc_stats(pdata);773 774 s->rx_packets = pstats->rxframecount_gb;775 s->rx_bytes = pstats->rxoctetcount_gb;776 s->rx_errors = pstats->rxframecount_gb -777 pstats->rxbroadcastframes_g -778 pstats->rxmulticastframes_g -779 pstats->rxunicastframes_g;780 s->multicast = pstats->rxmulticastframes_g;781 s->rx_length_errors = pstats->rxlengtherror;782 s->rx_crc_errors = pstats->rxcrcerror;783 s->rx_fifo_errors = pstats->rxfifooverflow;784 785 s->tx_packets = pstats->txframecount_gb;786 s->tx_bytes = pstats->txoctetcount_gb;787 s->tx_errors = pstats->txframecount_gb - pstats->txframecount_g;788 s->tx_dropped = netdev->stats.tx_dropped;789}790 791static int xlgmac_set_mac_address(struct net_device *netdev, void *addr)792{793 struct xlgmac_pdata *pdata = netdev_priv(netdev);794 struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;795 struct sockaddr *saddr = addr;796 797 if (!is_valid_ether_addr(saddr->sa_data))798 return -EADDRNOTAVAIL;799 800 eth_hw_addr_set(netdev, saddr->sa_data);801 802 hw_ops->set_mac_address(pdata, netdev->dev_addr);803 804 return 0;805}806 807static int xlgmac_ioctl(struct net_device *netdev,808 struct ifreq *ifreq, int cmd)809{810 if (!netif_running(netdev))811 return -ENODEV;812 813 return 0;814}815 816static int xlgmac_change_mtu(struct net_device *netdev, int mtu)817{818 struct xlgmac_pdata *pdata = netdev_priv(netdev);819 int ret;820 821 ret = xlgmac_calc_rx_buf_size(netdev, mtu);822 if (ret < 0)823 return ret;824 825 pdata->rx_buf_size = ret;826 WRITE_ONCE(netdev->mtu, mtu);827 828 xlgmac_restart_dev(pdata);829 830 return 0;831}832 833static int xlgmac_vlan_rx_add_vid(struct net_device *netdev,834 __be16 proto,835 u16 vid)836{837 struct xlgmac_pdata *pdata = netdev_priv(netdev);838 struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;839 840 set_bit(vid, pdata->active_vlans);841 hw_ops->update_vlan_hash_table(pdata);842 843 return 0;844}845 846static int xlgmac_vlan_rx_kill_vid(struct net_device *netdev,847 __be16 proto,848 u16 vid)849{850 struct xlgmac_pdata *pdata = netdev_priv(netdev);851 struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;852 853 clear_bit(vid, pdata->active_vlans);854 hw_ops->update_vlan_hash_table(pdata);855 856 return 0;857}858 859#ifdef CONFIG_NET_POLL_CONTROLLER860static void xlgmac_poll_controller(struct net_device *netdev)861{862 struct xlgmac_pdata *pdata = netdev_priv(netdev);863 struct xlgmac_channel *channel;864 unsigned int i;865 866 if (pdata->per_channel_irq) {867 channel = pdata->channel_head;868 for (i = 0; i < pdata->channel_count; i++, channel++)869 xlgmac_dma_isr(channel->dma_irq, channel);870 } else {871 disable_irq(pdata->dev_irq);872 xlgmac_isr(pdata->dev_irq, pdata);873 enable_irq(pdata->dev_irq);874 }875}876#endif /* CONFIG_NET_POLL_CONTROLLER */877 878static int xlgmac_set_features(struct net_device *netdev,879 netdev_features_t features)880{881 netdev_features_t rxhash, rxcsum, rxvlan, rxvlan_filter;882 struct xlgmac_pdata *pdata = netdev_priv(netdev);883 struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;884 int ret = 0;885 886 rxhash = pdata->netdev_features & NETIF_F_RXHASH;887 rxcsum = pdata->netdev_features & NETIF_F_RXCSUM;888 rxvlan = pdata->netdev_features & NETIF_F_HW_VLAN_CTAG_RX;889 rxvlan_filter = pdata->netdev_features & NETIF_F_HW_VLAN_CTAG_FILTER;890 891 if ((features & NETIF_F_RXHASH) && !rxhash)892 ret = hw_ops->enable_rss(pdata);893 else if (!(features & NETIF_F_RXHASH) && rxhash)894 ret = hw_ops->disable_rss(pdata);895 if (ret)896 return ret;897 898 if ((features & NETIF_F_RXCSUM) && !rxcsum)899 hw_ops->enable_rx_csum(pdata);900 else if (!(features & NETIF_F_RXCSUM) && rxcsum)901 hw_ops->disable_rx_csum(pdata);902 903 if ((features & NETIF_F_HW_VLAN_CTAG_RX) && !rxvlan)904 hw_ops->enable_rx_vlan_stripping(pdata);905 else if (!(features & NETIF_F_HW_VLAN_CTAG_RX) && rxvlan)906 hw_ops->disable_rx_vlan_stripping(pdata);907 908 if ((features & NETIF_F_HW_VLAN_CTAG_FILTER) && !rxvlan_filter)909 hw_ops->enable_rx_vlan_filtering(pdata);910 else if (!(features & NETIF_F_HW_VLAN_CTAG_FILTER) && rxvlan_filter)911 hw_ops->disable_rx_vlan_filtering(pdata);912 913 pdata->netdev_features = features;914 915 return 0;916}917 918static void xlgmac_set_rx_mode(struct net_device *netdev)919{920 struct xlgmac_pdata *pdata = netdev_priv(netdev);921 struct xlgmac_hw_ops *hw_ops = &pdata->hw_ops;922 923 hw_ops->config_rx_mode(pdata);924}925 926static const struct net_device_ops xlgmac_netdev_ops = {927 .ndo_open = xlgmac_open,928 .ndo_stop = xlgmac_close,929 .ndo_start_xmit = xlgmac_xmit,930 .ndo_tx_timeout = xlgmac_tx_timeout,931 .ndo_get_stats64 = xlgmac_get_stats64,932 .ndo_change_mtu = xlgmac_change_mtu,933 .ndo_set_mac_address = xlgmac_set_mac_address,934 .ndo_validate_addr = eth_validate_addr,935 .ndo_eth_ioctl = xlgmac_ioctl,936 .ndo_vlan_rx_add_vid = xlgmac_vlan_rx_add_vid,937 .ndo_vlan_rx_kill_vid = xlgmac_vlan_rx_kill_vid,938#ifdef CONFIG_NET_POLL_CONTROLLER939 .ndo_poll_controller = xlgmac_poll_controller,940#endif941 .ndo_set_features = xlgmac_set_features,942 .ndo_set_rx_mode = xlgmac_set_rx_mode,943};944 945const struct net_device_ops *xlgmac_get_netdev_ops(void)946{947 return &xlgmac_netdev_ops;948}949 950static void xlgmac_rx_refresh(struct xlgmac_channel *channel)951{952 struct xlgmac_pdata *pdata = channel->pdata;953 struct xlgmac_ring *ring = channel->rx_ring;954 struct xlgmac_desc_data *desc_data;955 struct xlgmac_desc_ops *desc_ops;956 struct xlgmac_hw_ops *hw_ops;957 958 desc_ops = &pdata->desc_ops;959 hw_ops = &pdata->hw_ops;960 961 while (ring->dirty != ring->cur) {962 desc_data = XLGMAC_GET_DESC_DATA(ring, ring->dirty);963 964 /* Reset desc_data values */965 desc_ops->unmap_desc_data(pdata, desc_data);966 967 if (desc_ops->map_rx_buffer(pdata, ring, desc_data))968 break;969 970 hw_ops->rx_desc_reset(pdata, desc_data, ring->dirty);971 972 ring->dirty++;973 }974 975 /* Make sure everything is written before the register write */976 wmb();977 978 /* Update the Rx Tail Pointer Register with address of979 * the last cleaned entry980 */981 desc_data = XLGMAC_GET_DESC_DATA(ring, ring->dirty - 1);982 writel(lower_32_bits(desc_data->dma_desc_addr),983 XLGMAC_DMA_REG(channel, DMA_CH_RDTR_LO));984}985 986static struct sk_buff *xlgmac_create_skb(struct xlgmac_pdata *pdata,987 struct napi_struct *napi,988 struct xlgmac_desc_data *desc_data,989 unsigned int len)990{991 unsigned int copy_len;992 struct sk_buff *skb;993 u8 *packet;994 995 skb = napi_alloc_skb(napi, desc_data->rx.hdr.dma_len);996 if (!skb)997 return NULL;998 999 /* Start with the header buffer which may contain just the header1000 * or the header plus data1001 */1002 dma_sync_single_range_for_cpu(pdata->dev, desc_data->rx.hdr.dma_base,1003 desc_data->rx.hdr.dma_off,1004 desc_data->rx.hdr.dma_len,1005 DMA_FROM_DEVICE);1006 1007 packet = page_address(desc_data->rx.hdr.pa.pages) +1008 desc_data->rx.hdr.pa.pages_offset;1009 copy_len = (desc_data->rx.hdr_len) ? desc_data->rx.hdr_len : len;1010 copy_len = min(desc_data->rx.hdr.dma_len, copy_len);1011 skb_copy_to_linear_data(skb, packet, copy_len);1012 skb_put(skb, copy_len);1013 1014 len -= copy_len;1015 if (len) {1016 /* Add the remaining data as a frag */1017 dma_sync_single_range_for_cpu(pdata->dev,1018 desc_data->rx.buf.dma_base,1019 desc_data->rx.buf.dma_off,1020 desc_data->rx.buf.dma_len,1021 DMA_FROM_DEVICE);1022 1023 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,1024 desc_data->rx.buf.pa.pages,1025 desc_data->rx.buf.pa.pages_offset,1026 len, desc_data->rx.buf.dma_len);1027 desc_data->rx.buf.pa.pages = NULL;1028 }1029 1030 return skb;1031}1032 1033static int xlgmac_tx_poll(struct xlgmac_channel *channel)1034{1035 struct xlgmac_pdata *pdata = channel->pdata;1036 struct xlgmac_ring *ring = channel->tx_ring;1037 struct net_device *netdev = pdata->netdev;1038 unsigned int tx_packets = 0, tx_bytes = 0;1039 struct xlgmac_desc_data *desc_data;1040 struct xlgmac_dma_desc *dma_desc;1041 struct xlgmac_desc_ops *desc_ops;1042 struct xlgmac_hw_ops *hw_ops;1043 struct netdev_queue *txq;1044 int processed = 0;1045 unsigned int cur;1046 1047 desc_ops = &pdata->desc_ops;1048 hw_ops = &pdata->hw_ops;1049 1050 /* Nothing to do if there isn't a Tx ring for this channel */1051 if (!ring)1052 return 0;1053 1054 cur = ring->cur;1055 1056 /* Be sure we get ring->cur before accessing descriptor data */1057 smp_rmb();1058 1059 txq = netdev_get_tx_queue(netdev, channel->queue_index);1060 1061 while ((processed < XLGMAC_TX_DESC_MAX_PROC) &&1062 (ring->dirty != cur)) {1063 desc_data = XLGMAC_GET_DESC_DATA(ring, ring->dirty);1064 dma_desc = desc_data->dma_desc;1065 1066 if (!hw_ops->tx_complete(dma_desc))1067 break;1068 1069 /* Make sure descriptor fields are read after reading1070 * the OWN bit1071 */1072 dma_rmb();1073 1074 if (netif_msg_tx_done(pdata))1075 xlgmac_dump_tx_desc(pdata, ring, ring->dirty, 1, 0);1076 1077 if (hw_ops->is_last_desc(dma_desc)) {1078 tx_packets += desc_data->tx.packets;1079 tx_bytes += desc_data->tx.bytes;1080 }1081 1082 /* Free the SKB and reset the descriptor for re-use */1083 desc_ops->unmap_desc_data(pdata, desc_data);1084 hw_ops->tx_desc_reset(desc_data);1085 1086 processed++;1087 ring->dirty++;1088 }1089 1090 if (!processed)1091 return 0;1092 1093 netdev_tx_completed_queue(txq, tx_packets, tx_bytes);1094 1095 if ((ring->tx.queue_stopped == 1) &&1096 (xlgmac_tx_avail_desc(ring) > XLGMAC_TX_DESC_MIN_FREE)) {1097 ring->tx.queue_stopped = 0;1098 netif_tx_wake_queue(txq);1099 }1100 1101 XLGMAC_PR("processed=%d\n", processed);1102 1103 return processed;1104}1105 1106static int xlgmac_rx_poll(struct xlgmac_channel *channel, int budget)1107{1108 struct xlgmac_pdata *pdata = channel->pdata;1109 struct xlgmac_ring *ring = channel->rx_ring;1110 struct net_device *netdev = pdata->netdev;1111 unsigned int len, dma_desc_len, max_len;1112 unsigned int context_next, context;1113 struct xlgmac_desc_data *desc_data;1114 struct xlgmac_pkt_info *pkt_info;1115 unsigned int incomplete, error;1116 struct xlgmac_hw_ops *hw_ops;1117 unsigned int received = 0;1118 struct napi_struct *napi;1119 struct sk_buff *skb;1120 int packet_count = 0;1121 1122 hw_ops = &pdata->hw_ops;1123 1124 /* Nothing to do if there isn't a Rx ring for this channel */1125 if (!ring)1126 return 0;1127 1128 incomplete = 0;1129 context_next = 0;1130 1131 napi = (pdata->per_channel_irq) ? &channel->napi : &pdata->napi;1132 1133 desc_data = XLGMAC_GET_DESC_DATA(ring, ring->cur);1134 pkt_info = &ring->pkt_info;1135 while (packet_count < budget) {1136 /* First time in loop see if we need to restore state */1137 if (!received && desc_data->state_saved) {1138 skb = desc_data->state.skb;1139 error = desc_data->state.error;1140 len = desc_data->state.len;1141 } else {1142 memset(pkt_info, 0, sizeof(*pkt_info));1143 skb = NULL;1144 error = 0;1145 len = 0;1146 }1147 1148read_again:1149 desc_data = XLGMAC_GET_DESC_DATA(ring, ring->cur);1150 1151 if (xlgmac_rx_dirty_desc(ring) > XLGMAC_RX_DESC_MAX_DIRTY)1152 xlgmac_rx_refresh(channel);1153 1154 if (hw_ops->dev_read(channel))1155 break;1156 1157 received++;1158 ring->cur++;1159 1160 incomplete = XLGMAC_GET_REG_BITS(1161 pkt_info->attributes,1162 RX_PACKET_ATTRIBUTES_INCOMPLETE_POS,1163 RX_PACKET_ATTRIBUTES_INCOMPLETE_LEN);1164 context_next = XLGMAC_GET_REG_BITS(1165 pkt_info->attributes,1166 RX_PACKET_ATTRIBUTES_CONTEXT_NEXT_POS,1167 RX_PACKET_ATTRIBUTES_CONTEXT_NEXT_LEN);1168 context = XLGMAC_GET_REG_BITS(1169 pkt_info->attributes,1170 RX_PACKET_ATTRIBUTES_CONTEXT_POS,1171 RX_PACKET_ATTRIBUTES_CONTEXT_LEN);1172 1173 /* Earlier error, just drain the remaining data */1174 if ((incomplete || context_next) && error)1175 goto read_again;1176 1177 if (error || pkt_info->errors) {1178 if (pkt_info->errors)1179 netif_err(pdata, rx_err, netdev,1180 "error in received packet\n");1181 dev_kfree_skb(skb);1182 goto next_packet;1183 }1184 1185 if (!context) {1186 /* Length is cumulative, get this descriptor's length */1187 dma_desc_len = desc_data->rx.len - len;1188 len += dma_desc_len;1189 1190 if (dma_desc_len && !skb) {1191 skb = xlgmac_create_skb(pdata, napi, desc_data,1192 dma_desc_len);1193 if (!skb)1194 error = 1;1195 } else if (dma_desc_len) {1196 dma_sync_single_range_for_cpu(1197 pdata->dev,1198 desc_data->rx.buf.dma_base,1199 desc_data->rx.buf.dma_off,1200 desc_data->rx.buf.dma_len,1201 DMA_FROM_DEVICE);1202 1203 skb_add_rx_frag(1204 skb, skb_shinfo(skb)->nr_frags,1205 desc_data->rx.buf.pa.pages,1206 desc_data->rx.buf.pa.pages_offset,1207 dma_desc_len,1208 desc_data->rx.buf.dma_len);1209 desc_data->rx.buf.pa.pages = NULL;1210 }1211 }1212 1213 if (incomplete || context_next)1214 goto read_again;1215 1216 if (!skb)1217 goto next_packet;1218 1219 /* Be sure we don't exceed the configured MTU */1220 max_len = netdev->mtu + ETH_HLEN;1221 if (!(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&1222 (skb->protocol == htons(ETH_P_8021Q)))1223 max_len += VLAN_HLEN;1224 1225 if (skb->len > max_len) {1226 netif_err(pdata, rx_err, netdev,1227 "packet length exceeds configured MTU\n");1228 dev_kfree_skb(skb);1229 goto next_packet;1230 }1231 1232 if (netif_msg_pktdata(pdata))1233 xlgmac_print_pkt(netdev, skb, false);1234 1235 skb_checksum_none_assert(skb);1236 if (XLGMAC_GET_REG_BITS(pkt_info->attributes,1237 RX_PACKET_ATTRIBUTES_CSUM_DONE_POS,1238 RX_PACKET_ATTRIBUTES_CSUM_DONE_LEN))1239 skb->ip_summed = CHECKSUM_UNNECESSARY;1240 1241 if (XLGMAC_GET_REG_BITS(pkt_info->attributes,1242 RX_PACKET_ATTRIBUTES_VLAN_CTAG_POS,1243 RX_PACKET_ATTRIBUTES_VLAN_CTAG_LEN)) {1244 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),1245 pkt_info->vlan_ctag);1246 pdata->stats.rx_vlan_packets++;1247 }1248 1249 if (XLGMAC_GET_REG_BITS(pkt_info->attributes,1250 RX_PACKET_ATTRIBUTES_RSS_HASH_POS,1251 RX_PACKET_ATTRIBUTES_RSS_HASH_LEN))1252 skb_set_hash(skb, pkt_info->rss_hash,1253 pkt_info->rss_hash_type);1254 1255 skb->dev = netdev;1256 skb->protocol = eth_type_trans(skb, netdev);1257 skb_record_rx_queue(skb, channel->queue_index);1258 1259 napi_gro_receive(napi, skb);1260 1261next_packet:1262 packet_count++;1263 }1264 1265 /* Check if we need to save state before leaving */1266 if (received && (incomplete || context_next)) {1267 desc_data = XLGMAC_GET_DESC_DATA(ring, ring->cur);1268 desc_data->state_saved = 1;1269 desc_data->state.skb = skb;1270 desc_data->state.len = len;1271 desc_data->state.error = error;1272 }1273 1274 XLGMAC_PR("packet_count = %d\n", packet_count);1275 1276 return packet_count;1277}1278 1279static int xlgmac_one_poll(struct napi_struct *napi, int budget)1280{1281 struct xlgmac_channel *channel = container_of(napi,1282 struct xlgmac_channel,1283 napi);1284 int processed = 0;1285 1286 XLGMAC_PR("budget=%d\n", budget);1287 1288 /* Cleanup Tx ring first */1289 xlgmac_tx_poll(channel);1290 1291 /* Process Rx ring next */1292 processed = xlgmac_rx_poll(channel, budget);1293 1294 /* If we processed everything, we are done */1295 if (processed < budget) {1296 /* Turn off polling */1297 napi_complete_done(napi, processed);1298 1299 /* Enable Tx and Rx interrupts */1300 enable_irq(channel->dma_irq);1301 }1302 1303 XLGMAC_PR("received = %d\n", processed);1304 1305 return processed;1306}1307 1308static int xlgmac_all_poll(struct napi_struct *napi, int budget)1309{1310 struct xlgmac_pdata *pdata = container_of(napi,1311 struct xlgmac_pdata,1312 napi);1313 struct xlgmac_channel *channel;1314 int processed, last_processed;1315 int ring_budget;1316 unsigned int i;1317 1318 XLGMAC_PR("budget=%d\n", budget);1319 1320 processed = 0;1321 ring_budget = budget / pdata->rx_ring_count;1322 do {1323 last_processed = processed;1324 1325 channel = pdata->channel_head;1326 for (i = 0; i < pdata->channel_count; i++, channel++) {1327 /* Cleanup Tx ring first */1328 xlgmac_tx_poll(channel);1329 1330 /* Process Rx ring next */1331 if (ring_budget > (budget - processed))1332 ring_budget = budget - processed;1333 processed += xlgmac_rx_poll(channel, ring_budget);1334 }1335 } while ((processed < budget) && (processed != last_processed));1336 1337 /* If we processed everything, we are done */1338 if (processed < budget) {1339 /* Turn off polling */1340 napi_complete_done(napi, processed);1341 1342 /* Enable Tx and Rx interrupts */1343 xlgmac_enable_rx_tx_ints(pdata);1344 }1345 1346 XLGMAC_PR("received = %d\n", processed);1347 1348 return processed;1349}1350