5507 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Cadence MACB/GEM Ethernet Controller driver4 *5 * Copyright (C) 2004-2006 Atmel Corporation6 */7 8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt9#include <linux/clk.h>10#include <linux/clk-provider.h>11#include <linux/crc32.h>12#include <linux/module.h>13#include <linux/moduleparam.h>14#include <linux/kernel.h>15#include <linux/types.h>16#include <linux/circ_buf.h>17#include <linux/slab.h>18#include <linux/init.h>19#include <linux/io.h>20#include <linux/gpio.h>21#include <linux/gpio/consumer.h>22#include <linux/interrupt.h>23#include <linux/netdevice.h>24#include <linux/etherdevice.h>25#include <linux/dma-mapping.h>26#include <linux/platform_device.h>27#include <linux/phylink.h>28#include <linux/of.h>29#include <linux/of_gpio.h>30#include <linux/of_mdio.h>31#include <linux/of_net.h>32#include <linux/ip.h>33#include <linux/udp.h>34#include <linux/tcp.h>35#include <linux/iopoll.h>36#include <linux/phy/phy.h>37#include <linux/pm_runtime.h>38#include <linux/ptp_classify.h>39#include <linux/reset.h>40#include <linux/firmware/xlnx-zynqmp.h>41#include <linux/inetdevice.h>42#include "macb.h"43 44/* This structure is only used for MACB on SiFive FU540 devices */45struct sifive_fu540_macb_mgmt {46 void __iomem *reg;47 unsigned long rate;48 struct clk_hw hw;49};50 51#define MACB_RX_BUFFER_SIZE 12852#define RX_BUFFER_MULTIPLE 64 /* bytes */53 54#define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */55#define MIN_RX_RING_SIZE 6456#define MAX_RX_RING_SIZE 819257#define RX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \58 * (bp)->rx_ring_size)59 60#define DEFAULT_TX_RING_SIZE 512 /* must be power of 2 */61#define MIN_TX_RING_SIZE 6462#define MAX_TX_RING_SIZE 409663#define TX_RING_BYTES(bp) (macb_dma_desc_get_size(bp) \64 * (bp)->tx_ring_size)65 66/* level of occupied TX descriptors under which we wake up TX process */67#define MACB_TX_WAKEUP_THRESH(bp) (3 * (bp)->tx_ring_size / 4)68 69#define MACB_RX_INT_FLAGS (MACB_BIT(RCOMP) | MACB_BIT(ISR_ROVR))70#define MACB_TX_ERR_FLAGS (MACB_BIT(ISR_TUND) \71 | MACB_BIT(ISR_RLE) \72 | MACB_BIT(TXERR))73#define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP) \74 | MACB_BIT(TXUBR))75 76/* Max length of transmit frame must be a multiple of 8 bytes */77#define MACB_TX_LEN_ALIGN 878#define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1)))79/* Limit maximum TX length as per Cadence TSO errata. This is to avoid a80 * false amba_error in TX path from the DMA assuming there is not enough81 * space in the SRAM (16KB) even when there is.82 */83#define GEM_MAX_TX_LEN (unsigned int)(0x3FC0)84 85#define GEM_MTU_MIN_SIZE ETH_MIN_MTU86#define MACB_NETIF_LSO NETIF_F_TSO87 88#define MACB_WOL_ENABLED BIT(0)89 90#define HS_SPEED_10000M 491#define MACB_SERDES_RATE_10G 192 93/* Graceful stop timeouts in us. We should allow up to94 * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions)95 */96#define MACB_HALT_TIMEOUT 1400097#define MACB_PM_TIMEOUT 100 /* ms */98 99#define MACB_MDIO_TIMEOUT 1000000 /* in usecs */100 101/* DMA buffer descriptor might be different size102 * depends on hardware configuration:103 *104 * 1. dma address width 32 bits:105 * word 1: 32 bit address of Data Buffer106 * word 2: control107 *108 * 2. dma address width 64 bits:109 * word 1: 32 bit address of Data Buffer110 * word 2: control111 * word 3: upper 32 bit address of Data Buffer112 * word 4: unused113 *114 * 3. dma address width 32 bits with hardware timestamping:115 * word 1: 32 bit address of Data Buffer116 * word 2: control117 * word 3: timestamp word 1118 * word 4: timestamp word 2119 *120 * 4. dma address width 64 bits with hardware timestamping:121 * word 1: 32 bit address of Data Buffer122 * word 2: control123 * word 3: upper 32 bit address of Data Buffer124 * word 4: unused125 * word 5: timestamp word 1126 * word 6: timestamp word 2127 */128static unsigned int macb_dma_desc_get_size(struct macb *bp)129{130#ifdef MACB_EXT_DESC131 unsigned int desc_size;132 133 switch (bp->hw_dma_cap) {134 case HW_DMA_CAP_64B:135 desc_size = sizeof(struct macb_dma_desc)136 + sizeof(struct macb_dma_desc_64);137 break;138 case HW_DMA_CAP_PTP:139 desc_size = sizeof(struct macb_dma_desc)140 + sizeof(struct macb_dma_desc_ptp);141 break;142 case HW_DMA_CAP_64B_PTP:143 desc_size = sizeof(struct macb_dma_desc)144 + sizeof(struct macb_dma_desc_64)145 + sizeof(struct macb_dma_desc_ptp);146 break;147 default:148 desc_size = sizeof(struct macb_dma_desc);149 }150 return desc_size;151#endif152 return sizeof(struct macb_dma_desc);153}154 155static unsigned int macb_adj_dma_desc_idx(struct macb *bp, unsigned int desc_idx)156{157#ifdef MACB_EXT_DESC158 switch (bp->hw_dma_cap) {159 case HW_DMA_CAP_64B:160 case HW_DMA_CAP_PTP:161 desc_idx <<= 1;162 break;163 case HW_DMA_CAP_64B_PTP:164 desc_idx *= 3;165 break;166 default:167 break;168 }169#endif170 return desc_idx;171}172 173#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT174static struct macb_dma_desc_64 *macb_64b_desc(struct macb *bp, struct macb_dma_desc *desc)175{176 return (struct macb_dma_desc_64 *)((void *)desc177 + sizeof(struct macb_dma_desc));178}179#endif180 181/* Ring buffer accessors */182static unsigned int macb_tx_ring_wrap(struct macb *bp, unsigned int index)183{184 return index & (bp->tx_ring_size - 1);185}186 187static struct macb_dma_desc *macb_tx_desc(struct macb_queue *queue,188 unsigned int index)189{190 index = macb_tx_ring_wrap(queue->bp, index);191 index = macb_adj_dma_desc_idx(queue->bp, index);192 return &queue->tx_ring[index];193}194 195static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue,196 unsigned int index)197{198 return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)];199}200 201static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index)202{203 dma_addr_t offset;204 205 offset = macb_tx_ring_wrap(queue->bp, index) *206 macb_dma_desc_get_size(queue->bp);207 208 return queue->tx_ring_dma + offset;209}210 211static unsigned int macb_rx_ring_wrap(struct macb *bp, unsigned int index)212{213 return index & (bp->rx_ring_size - 1);214}215 216static struct macb_dma_desc *macb_rx_desc(struct macb_queue *queue, unsigned int index)217{218 index = macb_rx_ring_wrap(queue->bp, index);219 index = macb_adj_dma_desc_idx(queue->bp, index);220 return &queue->rx_ring[index];221}222 223static void *macb_rx_buffer(struct macb_queue *queue, unsigned int index)224{225 return queue->rx_buffers + queue->bp->rx_buffer_size *226 macb_rx_ring_wrap(queue->bp, index);227}228 229/* I/O accessors */230static u32 hw_readl_native(struct macb *bp, int offset)231{232 return __raw_readl(bp->regs + offset);233}234 235static void hw_writel_native(struct macb *bp, int offset, u32 value)236{237 __raw_writel(value, bp->regs + offset);238}239 240static u32 hw_readl(struct macb *bp, int offset)241{242 return readl_relaxed(bp->regs + offset);243}244 245static void hw_writel(struct macb *bp, int offset, u32 value)246{247 writel_relaxed(value, bp->regs + offset);248}249 250/* Find the CPU endianness by using the loopback bit of NCR register. When the251 * CPU is in big endian we need to program swapped mode for management252 * descriptor access.253 */254static bool hw_is_native_io(void __iomem *addr)255{256 u32 value = MACB_BIT(LLB);257 258 __raw_writel(value, addr + MACB_NCR);259 value = __raw_readl(addr + MACB_NCR);260 261 /* Write 0 back to disable everything */262 __raw_writel(0, addr + MACB_NCR);263 264 return value == MACB_BIT(LLB);265}266 267static bool hw_is_gem(void __iomem *addr, bool native_io)268{269 u32 id;270 271 if (native_io)272 id = __raw_readl(addr + MACB_MID);273 else274 id = readl_relaxed(addr + MACB_MID);275 276 return MACB_BFEXT(IDNUM, id) >= 0x2;277}278 279static void macb_set_hwaddr(struct macb *bp)280{281 u32 bottom;282 u16 top;283 284 bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr));285 macb_or_gem_writel(bp, SA1B, bottom);286 top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4)));287 macb_or_gem_writel(bp, SA1T, top);288 289 if (gem_has_ptp(bp)) {290 gem_writel(bp, RXPTPUNI, bottom);291 gem_writel(bp, TXPTPUNI, bottom);292 }293 294 /* Clear unused address register sets */295 macb_or_gem_writel(bp, SA2B, 0);296 macb_or_gem_writel(bp, SA2T, 0);297 macb_or_gem_writel(bp, SA3B, 0);298 macb_or_gem_writel(bp, SA3T, 0);299 macb_or_gem_writel(bp, SA4B, 0);300 macb_or_gem_writel(bp, SA4T, 0);301}302 303static void macb_get_hwaddr(struct macb *bp)304{305 u32 bottom;306 u16 top;307 u8 addr[6];308 int i;309 310 /* Check all 4 address register for valid address */311 for (i = 0; i < 4; i++) {312 bottom = macb_or_gem_readl(bp, SA1B + i * 8);313 top = macb_or_gem_readl(bp, SA1T + i * 8);314 315 addr[0] = bottom & 0xff;316 addr[1] = (bottom >> 8) & 0xff;317 addr[2] = (bottom >> 16) & 0xff;318 addr[3] = (bottom >> 24) & 0xff;319 addr[4] = top & 0xff;320 addr[5] = (top >> 8) & 0xff;321 322 if (is_valid_ether_addr(addr)) {323 eth_hw_addr_set(bp->dev, addr);324 return;325 }326 }327 328 dev_info(&bp->pdev->dev, "invalid hw address, using random\n");329 eth_hw_addr_random(bp->dev);330}331 332static int macb_mdio_wait_for_idle(struct macb *bp)333{334 u32 val;335 336 return readx_poll_timeout(MACB_READ_NSR, bp, val, val & MACB_BIT(IDLE),337 1, MACB_MDIO_TIMEOUT);338}339 340static int macb_mdio_read_c22(struct mii_bus *bus, int mii_id, int regnum)341{342 struct macb *bp = bus->priv;343 int status;344 345 status = pm_runtime_resume_and_get(&bp->pdev->dev);346 if (status < 0)347 goto mdio_pm_exit;348 349 status = macb_mdio_wait_for_idle(bp);350 if (status < 0)351 goto mdio_read_exit;352 353 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF)354 | MACB_BF(RW, MACB_MAN_C22_READ)355 | MACB_BF(PHYA, mii_id)356 | MACB_BF(REGA, regnum)357 | MACB_BF(CODE, MACB_MAN_C22_CODE)));358 359 status = macb_mdio_wait_for_idle(bp);360 if (status < 0)361 goto mdio_read_exit;362 363 status = MACB_BFEXT(DATA, macb_readl(bp, MAN));364 365mdio_read_exit:366 pm_runtime_mark_last_busy(&bp->pdev->dev);367 pm_runtime_put_autosuspend(&bp->pdev->dev);368mdio_pm_exit:369 return status;370}371 372static int macb_mdio_read_c45(struct mii_bus *bus, int mii_id, int devad,373 int regnum)374{375 struct macb *bp = bus->priv;376 int status;377 378 status = pm_runtime_get_sync(&bp->pdev->dev);379 if (status < 0) {380 pm_runtime_put_noidle(&bp->pdev->dev);381 goto mdio_pm_exit;382 }383 384 status = macb_mdio_wait_for_idle(bp);385 if (status < 0)386 goto mdio_read_exit;387 388 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF)389 | MACB_BF(RW, MACB_MAN_C45_ADDR)390 | MACB_BF(PHYA, mii_id)391 | MACB_BF(REGA, devad & 0x1F)392 | MACB_BF(DATA, regnum & 0xFFFF)393 | MACB_BF(CODE, MACB_MAN_C45_CODE)));394 395 status = macb_mdio_wait_for_idle(bp);396 if (status < 0)397 goto mdio_read_exit;398 399 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF)400 | MACB_BF(RW, MACB_MAN_C45_READ)401 | MACB_BF(PHYA, mii_id)402 | MACB_BF(REGA, devad & 0x1F)403 | MACB_BF(CODE, MACB_MAN_C45_CODE)));404 405 status = macb_mdio_wait_for_idle(bp);406 if (status < 0)407 goto mdio_read_exit;408 409 status = MACB_BFEXT(DATA, macb_readl(bp, MAN));410 411mdio_read_exit:412 pm_runtime_mark_last_busy(&bp->pdev->dev);413 pm_runtime_put_autosuspend(&bp->pdev->dev);414mdio_pm_exit:415 return status;416}417 418static int macb_mdio_write_c22(struct mii_bus *bus, int mii_id, int regnum,419 u16 value)420{421 struct macb *bp = bus->priv;422 int status;423 424 status = pm_runtime_resume_and_get(&bp->pdev->dev);425 if (status < 0)426 goto mdio_pm_exit;427 428 status = macb_mdio_wait_for_idle(bp);429 if (status < 0)430 goto mdio_write_exit;431 432 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C22_SOF)433 | MACB_BF(RW, MACB_MAN_C22_WRITE)434 | MACB_BF(PHYA, mii_id)435 | MACB_BF(REGA, regnum)436 | MACB_BF(CODE, MACB_MAN_C22_CODE)437 | MACB_BF(DATA, value)));438 439 status = macb_mdio_wait_for_idle(bp);440 if (status < 0)441 goto mdio_write_exit;442 443mdio_write_exit:444 pm_runtime_mark_last_busy(&bp->pdev->dev);445 pm_runtime_put_autosuspend(&bp->pdev->dev);446mdio_pm_exit:447 return status;448}449 450static int macb_mdio_write_c45(struct mii_bus *bus, int mii_id,451 int devad, int regnum,452 u16 value)453{454 struct macb *bp = bus->priv;455 int status;456 457 status = pm_runtime_get_sync(&bp->pdev->dev);458 if (status < 0) {459 pm_runtime_put_noidle(&bp->pdev->dev);460 goto mdio_pm_exit;461 }462 463 status = macb_mdio_wait_for_idle(bp);464 if (status < 0)465 goto mdio_write_exit;466 467 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF)468 | MACB_BF(RW, MACB_MAN_C45_ADDR)469 | MACB_BF(PHYA, mii_id)470 | MACB_BF(REGA, devad & 0x1F)471 | MACB_BF(DATA, regnum & 0xFFFF)472 | MACB_BF(CODE, MACB_MAN_C45_CODE)));473 474 status = macb_mdio_wait_for_idle(bp);475 if (status < 0)476 goto mdio_write_exit;477 478 macb_writel(bp, MAN, (MACB_BF(SOF, MACB_MAN_C45_SOF)479 | MACB_BF(RW, MACB_MAN_C45_WRITE)480 | MACB_BF(PHYA, mii_id)481 | MACB_BF(REGA, devad & 0x1F)482 | MACB_BF(CODE, MACB_MAN_C45_CODE)483 | MACB_BF(DATA, value)));484 485 status = macb_mdio_wait_for_idle(bp);486 if (status < 0)487 goto mdio_write_exit;488 489mdio_write_exit:490 pm_runtime_mark_last_busy(&bp->pdev->dev);491 pm_runtime_put_autosuspend(&bp->pdev->dev);492mdio_pm_exit:493 return status;494}495 496static void macb_init_buffers(struct macb *bp)497{498 struct macb_queue *queue;499 unsigned int q;500 501 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {502 queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma));503#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT504 if (bp->hw_dma_cap & HW_DMA_CAP_64B)505 queue_writel(queue, RBQPH,506 upper_32_bits(queue->rx_ring_dma));507#endif508 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma));509#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT510 if (bp->hw_dma_cap & HW_DMA_CAP_64B)511 queue_writel(queue, TBQPH,512 upper_32_bits(queue->tx_ring_dma));513#endif514 }515}516 517/**518 * macb_set_tx_clk() - Set a clock to a new frequency519 * @bp: pointer to struct macb520 * @speed: New frequency in Hz521 */522static void macb_set_tx_clk(struct macb *bp, int speed)523{524 long ferr, rate, rate_rounded;525 526 if (!bp->tx_clk || (bp->caps & MACB_CAPS_CLK_HW_CHG))527 return;528 529 /* In case of MII the PHY is the clock master */530 if (bp->phy_interface == PHY_INTERFACE_MODE_MII)531 return;532 533 switch (speed) {534 case SPEED_10:535 rate = 2500000;536 break;537 case SPEED_100:538 rate = 25000000;539 break;540 case SPEED_1000:541 rate = 125000000;542 break;543 default:544 return;545 }546 547 rate_rounded = clk_round_rate(bp->tx_clk, rate);548 if (rate_rounded < 0)549 return;550 551 /* RGMII allows 50 ppm frequency error. Test and warn if this limit552 * is not satisfied.553 */554 ferr = abs(rate_rounded - rate);555 ferr = DIV_ROUND_UP(ferr, rate / 100000);556 if (ferr > 5)557 netdev_warn(bp->dev,558 "unable to generate target frequency: %ld Hz\n",559 rate);560 561 if (clk_set_rate(bp->tx_clk, rate_rounded))562 netdev_err(bp->dev, "adjusting tx_clk failed.\n");563}564 565static void macb_usx_pcs_link_up(struct phylink_pcs *pcs, unsigned int neg_mode,566 phy_interface_t interface, int speed,567 int duplex)568{569 struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs);570 u32 config;571 572 config = gem_readl(bp, USX_CONTROL);573 config = GEM_BFINS(SERDES_RATE, MACB_SERDES_RATE_10G, config);574 config = GEM_BFINS(USX_CTRL_SPEED, HS_SPEED_10000M, config);575 config &= ~(GEM_BIT(TX_SCR_BYPASS) | GEM_BIT(RX_SCR_BYPASS));576 config |= GEM_BIT(TX_EN);577 gem_writel(bp, USX_CONTROL, config);578}579 580static void macb_usx_pcs_get_state(struct phylink_pcs *pcs,581 struct phylink_link_state *state)582{583 struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs);584 u32 val;585 586 state->speed = SPEED_10000;587 state->duplex = 1;588 state->an_complete = 1;589 590 val = gem_readl(bp, USX_STATUS);591 state->link = !!(val & GEM_BIT(USX_BLOCK_LOCK));592 val = gem_readl(bp, NCFGR);593 if (val & GEM_BIT(PAE))594 state->pause = MLO_PAUSE_RX;595}596 597static int macb_usx_pcs_config(struct phylink_pcs *pcs,598 unsigned int neg_mode,599 phy_interface_t interface,600 const unsigned long *advertising,601 bool permit_pause_to_mac)602{603 struct macb *bp = container_of(pcs, struct macb, phylink_usx_pcs);604 605 gem_writel(bp, USX_CONTROL, gem_readl(bp, USX_CONTROL) |606 GEM_BIT(SIGNAL_OK));607 608 return 0;609}610 611static void macb_pcs_get_state(struct phylink_pcs *pcs,612 struct phylink_link_state *state)613{614 state->link = 0;615}616 617static void macb_pcs_an_restart(struct phylink_pcs *pcs)618{619 /* Not supported */620}621 622static int macb_pcs_config(struct phylink_pcs *pcs,623 unsigned int neg_mode,624 phy_interface_t interface,625 const unsigned long *advertising,626 bool permit_pause_to_mac)627{628 return 0;629}630 631static const struct phylink_pcs_ops macb_phylink_usx_pcs_ops = {632 .pcs_get_state = macb_usx_pcs_get_state,633 .pcs_config = macb_usx_pcs_config,634 .pcs_link_up = macb_usx_pcs_link_up,635};636 637static const struct phylink_pcs_ops macb_phylink_pcs_ops = {638 .pcs_get_state = macb_pcs_get_state,639 .pcs_an_restart = macb_pcs_an_restart,640 .pcs_config = macb_pcs_config,641};642 643static void macb_mac_config(struct phylink_config *config, unsigned int mode,644 const struct phylink_link_state *state)645{646 struct net_device *ndev = to_net_dev(config->dev);647 struct macb *bp = netdev_priv(ndev);648 unsigned long flags;649 u32 old_ctrl, ctrl;650 u32 old_ncr, ncr;651 652 spin_lock_irqsave(&bp->lock, flags);653 654 old_ctrl = ctrl = macb_or_gem_readl(bp, NCFGR);655 old_ncr = ncr = macb_or_gem_readl(bp, NCR);656 657 if (bp->caps & MACB_CAPS_MACB_IS_EMAC) {658 if (state->interface == PHY_INTERFACE_MODE_RMII)659 ctrl |= MACB_BIT(RM9200_RMII);660 } else if (macb_is_gem(bp)) {661 ctrl &= ~(GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL));662 ncr &= ~GEM_BIT(ENABLE_HS_MAC);663 664 if (state->interface == PHY_INTERFACE_MODE_SGMII) {665 ctrl |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL);666 } else if (state->interface == PHY_INTERFACE_MODE_10GBASER) {667 ctrl |= GEM_BIT(PCSSEL);668 ncr |= GEM_BIT(ENABLE_HS_MAC);669 } else if (bp->caps & MACB_CAPS_MIIONRGMII &&670 bp->phy_interface == PHY_INTERFACE_MODE_MII) {671 ncr |= MACB_BIT(MIIONRGMII);672 }673 }674 675 /* Apply the new configuration, if any */676 if (old_ctrl ^ ctrl)677 macb_or_gem_writel(bp, NCFGR, ctrl);678 679 if (old_ncr ^ ncr)680 macb_or_gem_writel(bp, NCR, ncr);681 682 /* Disable AN for SGMII fixed link configuration, enable otherwise.683 * Must be written after PCSSEL is set in NCFGR,684 * otherwise writes will not take effect.685 */686 if (macb_is_gem(bp) && state->interface == PHY_INTERFACE_MODE_SGMII) {687 u32 pcsctrl, old_pcsctrl;688 689 old_pcsctrl = gem_readl(bp, PCSCNTRL);690 if (mode == MLO_AN_FIXED)691 pcsctrl = old_pcsctrl & ~GEM_BIT(PCSAUTONEG);692 else693 pcsctrl = old_pcsctrl | GEM_BIT(PCSAUTONEG);694 if (old_pcsctrl != pcsctrl)695 gem_writel(bp, PCSCNTRL, pcsctrl);696 }697 698 spin_unlock_irqrestore(&bp->lock, flags);699}700 701static void macb_mac_link_down(struct phylink_config *config, unsigned int mode,702 phy_interface_t interface)703{704 struct net_device *ndev = to_net_dev(config->dev);705 struct macb *bp = netdev_priv(ndev);706 struct macb_queue *queue;707 unsigned int q;708 u32 ctrl;709 710 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC))711 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)712 queue_writel(queue, IDR,713 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP));714 715 /* Disable Rx and Tx */716 ctrl = macb_readl(bp, NCR) & ~(MACB_BIT(RE) | MACB_BIT(TE));717 macb_writel(bp, NCR, ctrl);718 719 netif_tx_stop_all_queues(ndev);720}721 722static void macb_mac_link_up(struct phylink_config *config,723 struct phy_device *phy,724 unsigned int mode, phy_interface_t interface,725 int speed, int duplex,726 bool tx_pause, bool rx_pause)727{728 struct net_device *ndev = to_net_dev(config->dev);729 struct macb *bp = netdev_priv(ndev);730 struct macb_queue *queue;731 unsigned long flags;732 unsigned int q;733 u32 ctrl;734 735 spin_lock_irqsave(&bp->lock, flags);736 737 ctrl = macb_or_gem_readl(bp, NCFGR);738 739 ctrl &= ~(MACB_BIT(SPD) | MACB_BIT(FD));740 741 if (speed == SPEED_100)742 ctrl |= MACB_BIT(SPD);743 744 if (duplex)745 ctrl |= MACB_BIT(FD);746 747 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) {748 ctrl &= ~MACB_BIT(PAE);749 if (macb_is_gem(bp)) {750 ctrl &= ~GEM_BIT(GBE);751 752 if (speed == SPEED_1000)753 ctrl |= GEM_BIT(GBE);754 }755 756 if (rx_pause)757 ctrl |= MACB_BIT(PAE);758 759 /* Initialize rings & buffers as clearing MACB_BIT(TE) in link down760 * cleared the pipeline and control registers.761 */762 bp->macbgem_ops.mog_init_rings(bp);763 macb_init_buffers(bp);764 765 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)766 queue_writel(queue, IER,767 bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP));768 }769 770 macb_or_gem_writel(bp, NCFGR, ctrl);771 772 if (bp->phy_interface == PHY_INTERFACE_MODE_10GBASER)773 gem_writel(bp, HS_MAC_CONFIG, GEM_BFINS(HS_MAC_SPEED, HS_SPEED_10000M,774 gem_readl(bp, HS_MAC_CONFIG)));775 776 spin_unlock_irqrestore(&bp->lock, flags);777 778 if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC))779 macb_set_tx_clk(bp, speed);780 781 /* Enable Rx and Tx; Enable PTP unicast */782 ctrl = macb_readl(bp, NCR);783 if (gem_has_ptp(bp))784 ctrl |= MACB_BIT(PTPUNI);785 786 macb_writel(bp, NCR, ctrl | MACB_BIT(RE) | MACB_BIT(TE));787 788 netif_tx_wake_all_queues(ndev);789}790 791static struct phylink_pcs *macb_mac_select_pcs(struct phylink_config *config,792 phy_interface_t interface)793{794 struct net_device *ndev = to_net_dev(config->dev);795 struct macb *bp = netdev_priv(ndev);796 797 if (interface == PHY_INTERFACE_MODE_10GBASER)798 return &bp->phylink_usx_pcs;799 else if (interface == PHY_INTERFACE_MODE_SGMII)800 return &bp->phylink_sgmii_pcs;801 else802 return NULL;803}804 805static const struct phylink_mac_ops macb_phylink_ops = {806 .mac_select_pcs = macb_mac_select_pcs,807 .mac_config = macb_mac_config,808 .mac_link_down = macb_mac_link_down,809 .mac_link_up = macb_mac_link_up,810};811 812static bool macb_phy_handle_exists(struct device_node *dn)813{814 dn = of_parse_phandle(dn, "phy-handle", 0);815 of_node_put(dn);816 return dn != NULL;817}818 819static int macb_phylink_connect(struct macb *bp)820{821 struct device_node *dn = bp->pdev->dev.of_node;822 struct net_device *dev = bp->dev;823 struct phy_device *phydev;824 int ret;825 826 if (dn)827 ret = phylink_of_phy_connect(bp->phylink, dn, 0);828 829 if (!dn || (ret && !macb_phy_handle_exists(dn))) {830 phydev = phy_find_first(bp->mii_bus);831 if (!phydev) {832 netdev_err(dev, "no PHY found\n");833 return -ENXIO;834 }835 836 /* attach the mac to the phy */837 ret = phylink_connect_phy(bp->phylink, phydev);838 }839 840 if (ret) {841 netdev_err(dev, "Could not attach PHY (%d)\n", ret);842 return ret;843 }844 845 phylink_start(bp->phylink);846 847 return 0;848}849 850static void macb_get_pcs_fixed_state(struct phylink_config *config,851 struct phylink_link_state *state)852{853 struct net_device *ndev = to_net_dev(config->dev);854 struct macb *bp = netdev_priv(ndev);855 856 state->link = (macb_readl(bp, NSR) & MACB_BIT(NSR_LINK)) != 0;857}858 859/* based on au1000_eth. c*/860static int macb_mii_probe(struct net_device *dev)861{862 struct macb *bp = netdev_priv(dev);863 864 bp->phylink_sgmii_pcs.ops = &macb_phylink_pcs_ops;865 bp->phylink_sgmii_pcs.neg_mode = true;866 bp->phylink_usx_pcs.ops = &macb_phylink_usx_pcs_ops;867 bp->phylink_usx_pcs.neg_mode = true;868 869 bp->phylink_config.dev = &dev->dev;870 bp->phylink_config.type = PHYLINK_NETDEV;871 bp->phylink_config.mac_managed_pm = true;872 873 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) {874 bp->phylink_config.poll_fixed_state = true;875 bp->phylink_config.get_fixed_state = macb_get_pcs_fixed_state;876 }877 878 bp->phylink_config.mac_capabilities = MAC_ASYM_PAUSE |879 MAC_10 | MAC_100;880 881 __set_bit(PHY_INTERFACE_MODE_MII,882 bp->phylink_config.supported_interfaces);883 __set_bit(PHY_INTERFACE_MODE_RMII,884 bp->phylink_config.supported_interfaces);885 886 /* Determine what modes are supported */887 if (macb_is_gem(bp) && (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE)) {888 bp->phylink_config.mac_capabilities |= MAC_1000FD;889 if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF))890 bp->phylink_config.mac_capabilities |= MAC_1000HD;891 892 __set_bit(PHY_INTERFACE_MODE_GMII,893 bp->phylink_config.supported_interfaces);894 phy_interface_set_rgmii(bp->phylink_config.supported_interfaces);895 896 if (bp->caps & MACB_CAPS_PCS)897 __set_bit(PHY_INTERFACE_MODE_SGMII,898 bp->phylink_config.supported_interfaces);899 900 if (bp->caps & MACB_CAPS_HIGH_SPEED) {901 __set_bit(PHY_INTERFACE_MODE_10GBASER,902 bp->phylink_config.supported_interfaces);903 bp->phylink_config.mac_capabilities |= MAC_10000FD;904 }905 }906 907 bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode,908 bp->phy_interface, &macb_phylink_ops);909 if (IS_ERR(bp->phylink)) {910 netdev_err(dev, "Could not create a phylink instance (%ld)\n",911 PTR_ERR(bp->phylink));912 return PTR_ERR(bp->phylink);913 }914 915 return 0;916}917 918static int macb_mdiobus_register(struct macb *bp)919{920 struct device_node *child, *np = bp->pdev->dev.of_node;921 922 /* If we have a child named mdio, probe it instead of looking for PHYs923 * directly under the MAC node924 */925 child = of_get_child_by_name(np, "mdio");926 if (child) {927 int ret = of_mdiobus_register(bp->mii_bus, child);928 929 of_node_put(child);930 return ret;931 }932 933 /* Only create the PHY from the device tree if at least one PHY is934 * described. Otherwise scan the entire MDIO bus. We do this to support935 * old device tree that did not follow the best practices and did not936 * describe their network PHYs.937 */938 for_each_available_child_of_node(np, child)939 if (of_mdiobus_child_is_phy(child)) {940 /* The loop increments the child refcount,941 * decrement it before returning.942 */943 of_node_put(child);944 945 return of_mdiobus_register(bp->mii_bus, np);946 }947 948 return mdiobus_register(bp->mii_bus);949}950 951static int macb_mii_init(struct macb *bp)952{953 struct device_node *child, *np = bp->pdev->dev.of_node;954 int err = -ENXIO;955 956 /* With fixed-link, we don't need to register the MDIO bus,957 * except if we have a child named "mdio" in the device tree.958 * In that case, some devices may be attached to the MACB's MDIO bus.959 */960 child = of_get_child_by_name(np, "mdio");961 if (child)962 of_node_put(child);963 else if (of_phy_is_fixed_link(np))964 return macb_mii_probe(bp->dev);965 966 /* Enable management port */967 macb_writel(bp, NCR, MACB_BIT(MPE));968 969 bp->mii_bus = mdiobus_alloc();970 if (!bp->mii_bus) {971 err = -ENOMEM;972 goto err_out;973 }974 975 bp->mii_bus->name = "MACB_mii_bus";976 bp->mii_bus->read = &macb_mdio_read_c22;977 bp->mii_bus->write = &macb_mdio_write_c22;978 bp->mii_bus->read_c45 = &macb_mdio_read_c45;979 bp->mii_bus->write_c45 = &macb_mdio_write_c45;980 snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x",981 bp->pdev->name, bp->pdev->id);982 bp->mii_bus->priv = bp;983 bp->mii_bus->parent = &bp->pdev->dev;984 985 dev_set_drvdata(&bp->dev->dev, bp->mii_bus);986 987 err = macb_mdiobus_register(bp);988 if (err)989 goto err_out_free_mdiobus;990 991 err = macb_mii_probe(bp->dev);992 if (err)993 goto err_out_unregister_bus;994 995 return 0;996 997err_out_unregister_bus:998 mdiobus_unregister(bp->mii_bus);999err_out_free_mdiobus:1000 mdiobus_free(bp->mii_bus);1001err_out:1002 return err;1003}1004 1005static void macb_update_stats(struct macb *bp)1006{1007 u32 *p = &bp->hw_stats.macb.rx_pause_frames;1008 u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1;1009 int offset = MACB_PFR;1010 1011 WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4);1012 1013 for (; p < end; p++, offset += 4)1014 *p += bp->macb_reg_readl(bp, offset);1015}1016 1017static int macb_halt_tx(struct macb *bp)1018{1019 unsigned long halt_time, timeout;1020 u32 status;1021 1022 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(THALT));1023 1024 timeout = jiffies + usecs_to_jiffies(MACB_HALT_TIMEOUT);1025 do {1026 halt_time = jiffies;1027 status = macb_readl(bp, TSR);1028 if (!(status & MACB_BIT(TGO)))1029 return 0;1030 1031 udelay(250);1032 } while (time_before(halt_time, timeout));1033 1034 return -ETIMEDOUT;1035}1036 1037static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb, int budget)1038{1039 if (tx_skb->mapping) {1040 if (tx_skb->mapped_as_page)1041 dma_unmap_page(&bp->pdev->dev, tx_skb->mapping,1042 tx_skb->size, DMA_TO_DEVICE);1043 else1044 dma_unmap_single(&bp->pdev->dev, tx_skb->mapping,1045 tx_skb->size, DMA_TO_DEVICE);1046 tx_skb->mapping = 0;1047 }1048 1049 if (tx_skb->skb) {1050 napi_consume_skb(tx_skb->skb, budget);1051 tx_skb->skb = NULL;1052 }1053}1054 1055static void macb_set_addr(struct macb *bp, struct macb_dma_desc *desc, dma_addr_t addr)1056{1057#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT1058 struct macb_dma_desc_64 *desc_64;1059 1060 if (bp->hw_dma_cap & HW_DMA_CAP_64B) {1061 desc_64 = macb_64b_desc(bp, desc);1062 desc_64->addrh = upper_32_bits(addr);1063 /* The low bits of RX address contain the RX_USED bit, clearing1064 * of which allows packet RX. Make sure the high bits are also1065 * visible to HW at that point.1066 */1067 dma_wmb();1068 }1069#endif1070 desc->addr = lower_32_bits(addr);1071}1072 1073static dma_addr_t macb_get_addr(struct macb *bp, struct macb_dma_desc *desc)1074{1075 dma_addr_t addr = 0;1076#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT1077 struct macb_dma_desc_64 *desc_64;1078 1079 if (bp->hw_dma_cap & HW_DMA_CAP_64B) {1080 desc_64 = macb_64b_desc(bp, desc);1081 addr = ((u64)(desc_64->addrh) << 32);1082 }1083#endif1084 addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr));1085#ifdef CONFIG_MACB_USE_HWSTAMP1086 if (bp->hw_dma_cap & HW_DMA_CAP_PTP)1087 addr &= ~GEM_BIT(DMA_RXVALID);1088#endif1089 return addr;1090}1091 1092static void macb_tx_error_task(struct work_struct *work)1093{1094 struct macb_queue *queue = container_of(work, struct macb_queue,1095 tx_error_task);1096 bool halt_timeout = false;1097 struct macb *bp = queue->bp;1098 struct macb_tx_skb *tx_skb;1099 struct macb_dma_desc *desc;1100 struct sk_buff *skb;1101 unsigned int tail;1102 unsigned long flags;1103 1104 netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n",1105 (unsigned int)(queue - bp->queues),1106 queue->tx_tail, queue->tx_head);1107 1108 /* Prevent the queue NAPI TX poll from running, as it calls1109 * macb_tx_complete(), which in turn may call netif_wake_subqueue().1110 * As explained below, we have to halt the transmission before updating1111 * TBQP registers so we call netif_tx_stop_all_queues() to notify the1112 * network engine about the macb/gem being halted.1113 */1114 napi_disable(&queue->napi_tx);1115 spin_lock_irqsave(&bp->lock, flags);1116 1117 /* Make sure nobody is trying to queue up new packets */1118 netif_tx_stop_all_queues(bp->dev);1119 1120 /* Stop transmission now1121 * (in case we have just queued new packets)1122 * macb/gem must be halted to write TBQP register1123 */1124 if (macb_halt_tx(bp)) {1125 netdev_err(bp->dev, "BUG: halt tx timed out\n");1126 macb_writel(bp, NCR, macb_readl(bp, NCR) & (~MACB_BIT(TE)));1127 halt_timeout = true;1128 }1129 1130 /* Treat frames in TX queue including the ones that caused the error.1131 * Free transmit buffers in upper layer.1132 */1133 for (tail = queue->tx_tail; tail != queue->tx_head; tail++) {1134 u32 ctrl;1135 1136 desc = macb_tx_desc(queue, tail);1137 ctrl = desc->ctrl;1138 tx_skb = macb_tx_skb(queue, tail);1139 skb = tx_skb->skb;1140 1141 if (ctrl & MACB_BIT(TX_USED)) {1142 /* skb is set for the last buffer of the frame */1143 while (!skb) {1144 macb_tx_unmap(bp, tx_skb, 0);1145 tail++;1146 tx_skb = macb_tx_skb(queue, tail);1147 skb = tx_skb->skb;1148 }1149 1150 /* ctrl still refers to the first buffer descriptor1151 * since it's the only one written back by the hardware1152 */1153 if (!(ctrl & MACB_BIT(TX_BUF_EXHAUSTED))) {1154 netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n",1155 macb_tx_ring_wrap(bp, tail),1156 skb->data);1157 bp->dev->stats.tx_packets++;1158 queue->stats.tx_packets++;1159 bp->dev->stats.tx_bytes += skb->len;1160 queue->stats.tx_bytes += skb->len;1161 }1162 } else {1163 /* "Buffers exhausted mid-frame" errors may only happen1164 * if the driver is buggy, so complain loudly about1165 * those. Statistics are updated by hardware.1166 */1167 if (ctrl & MACB_BIT(TX_BUF_EXHAUSTED))1168 netdev_err(bp->dev,1169 "BUG: TX buffers exhausted mid-frame\n");1170 1171 desc->ctrl = ctrl | MACB_BIT(TX_USED);1172 }1173 1174 macb_tx_unmap(bp, tx_skb, 0);1175 }1176 1177 /* Set end of TX queue */1178 desc = macb_tx_desc(queue, 0);1179 macb_set_addr(bp, desc, 0);1180 desc->ctrl = MACB_BIT(TX_USED);1181 1182 /* Make descriptor updates visible to hardware */1183 wmb();1184 1185 /* Reinitialize the TX desc queue */1186 queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma));1187#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT1188 if (bp->hw_dma_cap & HW_DMA_CAP_64B)1189 queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma));1190#endif1191 /* Make TX ring reflect state of hardware */1192 queue->tx_head = 0;1193 queue->tx_tail = 0;1194 1195 /* Housework before enabling TX IRQ */1196 macb_writel(bp, TSR, macb_readl(bp, TSR));1197 queue_writel(queue, IER, MACB_TX_INT_FLAGS);1198 1199 if (halt_timeout)1200 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TE));1201 1202 /* Now we are ready to start transmission again */1203 netif_tx_start_all_queues(bp->dev);1204 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART));1205 1206 spin_unlock_irqrestore(&bp->lock, flags);1207 napi_enable(&queue->napi_tx);1208}1209 1210static bool ptp_one_step_sync(struct sk_buff *skb)1211{1212 struct ptp_header *hdr;1213 unsigned int ptp_class;1214 u8 msgtype;1215 1216 /* No need to parse packet if PTP TS is not involved */1217 if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)))1218 goto not_oss;1219 1220 /* Identify and return whether PTP one step sync is being processed */1221 ptp_class = ptp_classify_raw(skb);1222 if (ptp_class == PTP_CLASS_NONE)1223 goto not_oss;1224 1225 hdr = ptp_parse_header(skb, ptp_class);1226 if (!hdr)1227 goto not_oss;1228 1229 if (hdr->flag_field[0] & PTP_FLAG_TWOSTEP)1230 goto not_oss;1231 1232 msgtype = ptp_get_msgtype(hdr, ptp_class);1233 if (msgtype == PTP_MSGTYPE_SYNC)1234 return true;1235 1236not_oss:1237 return false;1238}1239 1240static int macb_tx_complete(struct macb_queue *queue, int budget)1241{1242 struct macb *bp = queue->bp;1243 u16 queue_index = queue - bp->queues;1244 unsigned int tail;1245 unsigned int head;1246 int packets = 0;1247 1248 spin_lock(&queue->tx_ptr_lock);1249 head = queue->tx_head;1250 for (tail = queue->tx_tail; tail != head && packets < budget; tail++) {1251 struct macb_tx_skb *tx_skb;1252 struct sk_buff *skb;1253 struct macb_dma_desc *desc;1254 u32 ctrl;1255 1256 desc = macb_tx_desc(queue, tail);1257 1258 /* Make hw descriptor updates visible to CPU */1259 rmb();1260 1261 ctrl = desc->ctrl;1262 1263 /* TX_USED bit is only set by hardware on the very first buffer1264 * descriptor of the transmitted frame.1265 */1266 if (!(ctrl & MACB_BIT(TX_USED)))1267 break;1268 1269 /* Process all buffers of the current transmitted frame */1270 for (;; tail++) {1271 tx_skb = macb_tx_skb(queue, tail);1272 skb = tx_skb->skb;1273 1274 /* First, update TX stats if needed */1275 if (skb) {1276 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&1277 !ptp_one_step_sync(skb))1278 gem_ptp_do_txstamp(bp, skb, desc);1279 1280 netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n",1281 macb_tx_ring_wrap(bp, tail),1282 skb->data);1283 bp->dev->stats.tx_packets++;1284 queue->stats.tx_packets++;1285 bp->dev->stats.tx_bytes += skb->len;1286 queue->stats.tx_bytes += skb->len;1287 packets++;1288 }1289 1290 /* Now we can safely release resources */1291 macb_tx_unmap(bp, tx_skb, budget);1292 1293 /* skb is set only for the last buffer of the frame.1294 * WARNING: at this point skb has been freed by1295 * macb_tx_unmap().1296 */1297 if (skb)1298 break;1299 }1300 }1301 1302 queue->tx_tail = tail;1303 if (__netif_subqueue_stopped(bp->dev, queue_index) &&1304 CIRC_CNT(queue->tx_head, queue->tx_tail,1305 bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp))1306 netif_wake_subqueue(bp->dev, queue_index);1307 spin_unlock(&queue->tx_ptr_lock);1308 1309 return packets;1310}1311 1312static void gem_rx_refill(struct macb_queue *queue)1313{1314 unsigned int entry;1315 struct sk_buff *skb;1316 dma_addr_t paddr;1317 struct macb *bp = queue->bp;1318 struct macb_dma_desc *desc;1319 1320 while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail,1321 bp->rx_ring_size) > 0) {1322 entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head);1323 1324 /* Make hw descriptor updates visible to CPU */1325 rmb();1326 1327 desc = macb_rx_desc(queue, entry);1328 1329 if (!queue->rx_skbuff[entry]) {1330 /* allocate sk_buff for this free entry in ring */1331 skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size);1332 if (unlikely(!skb)) {1333 netdev_err(bp->dev,1334 "Unable to allocate sk_buff\n");1335 break;1336 }1337 1338 /* now fill corresponding descriptor entry */1339 paddr = dma_map_single(&bp->pdev->dev, skb->data,1340 bp->rx_buffer_size,1341 DMA_FROM_DEVICE);1342 if (dma_mapping_error(&bp->pdev->dev, paddr)) {1343 dev_kfree_skb(skb);1344 break;1345 }1346 1347 queue->rx_skbuff[entry] = skb;1348 1349 if (entry == bp->rx_ring_size - 1)1350 paddr |= MACB_BIT(RX_WRAP);1351 desc->ctrl = 0;1352 /* Setting addr clears RX_USED and allows reception,1353 * make sure ctrl is cleared first to avoid a race.1354 */1355 dma_wmb();1356 macb_set_addr(bp, desc, paddr);1357 1358 /* properly align Ethernet header */1359 skb_reserve(skb, NET_IP_ALIGN);1360 } else {1361 desc->ctrl = 0;1362 dma_wmb();1363 desc->addr &= ~MACB_BIT(RX_USED);1364 }1365 queue->rx_prepared_head++;1366 }1367 1368 /* Make descriptor updates visible to hardware */1369 wmb();1370 1371 netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n",1372 queue, queue->rx_prepared_head, queue->rx_tail);1373}1374 1375/* Mark DMA descriptors from begin up to and not including end as unused */1376static void discard_partial_frame(struct macb_queue *queue, unsigned int begin,1377 unsigned int end)1378{1379 unsigned int frag;1380 1381 for (frag = begin; frag != end; frag++) {1382 struct macb_dma_desc *desc = macb_rx_desc(queue, frag);1383 1384 desc->addr &= ~MACB_BIT(RX_USED);1385 }1386 1387 /* Make descriptor updates visible to hardware */1388 wmb();1389 1390 /* When this happens, the hardware stats registers for1391 * whatever caused this is updated, so we don't have to record1392 * anything.1393 */1394}1395 1396static int gem_rx(struct macb_queue *queue, struct napi_struct *napi,1397 int budget)1398{1399 struct macb *bp = queue->bp;1400 unsigned int len;1401 unsigned int entry;1402 struct sk_buff *skb;1403 struct macb_dma_desc *desc;1404 int count = 0;1405 1406 while (count < budget) {1407 u32 ctrl;1408 dma_addr_t addr;1409 bool rxused;1410 1411 entry = macb_rx_ring_wrap(bp, queue->rx_tail);1412 desc = macb_rx_desc(queue, entry);1413 1414 /* Make hw descriptor updates visible to CPU */1415 rmb();1416 1417 rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false;1418 addr = macb_get_addr(bp, desc);1419 1420 if (!rxused)1421 break;1422 1423 /* Ensure ctrl is at least as up-to-date as rxused */1424 dma_rmb();1425 1426 ctrl = desc->ctrl;1427 1428 queue->rx_tail++;1429 count++;1430 1431 if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) {1432 netdev_err(bp->dev,1433 "not whole frame pointed by descriptor\n");1434 bp->dev->stats.rx_dropped++;1435 queue->stats.rx_dropped++;1436 break;1437 }1438 skb = queue->rx_skbuff[entry];1439 if (unlikely(!skb)) {1440 netdev_err(bp->dev,1441 "inconsistent Rx descriptor chain\n");1442 bp->dev->stats.rx_dropped++;1443 queue->stats.rx_dropped++;1444 break;1445 }1446 /* now everything is ready for receiving packet */1447 queue->rx_skbuff[entry] = NULL;1448 len = ctrl & bp->rx_frm_len_mask;1449 1450 netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len);1451 1452 skb_put(skb, len);1453 dma_unmap_single(&bp->pdev->dev, addr,1454 bp->rx_buffer_size, DMA_FROM_DEVICE);1455 1456 skb->protocol = eth_type_trans(skb, bp->dev);1457 skb_checksum_none_assert(skb);1458 if (bp->dev->features & NETIF_F_RXCSUM &&1459 !(bp->dev->flags & IFF_PROMISC) &&1460 GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK)1461 skb->ip_summed = CHECKSUM_UNNECESSARY;1462 1463 bp->dev->stats.rx_packets++;1464 queue->stats.rx_packets++;1465 bp->dev->stats.rx_bytes += skb->len;1466 queue->stats.rx_bytes += skb->len;1467 1468 gem_ptp_do_rxstamp(bp, skb, desc);1469 1470#if defined(DEBUG) && defined(VERBOSE_DEBUG)1471 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n",1472 skb->len, skb->csum);1473 print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1,1474 skb_mac_header(skb), 16, true);1475 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1,1476 skb->data, 32, true);1477#endif1478 1479 napi_gro_receive(napi, skb);1480 }1481 1482 gem_rx_refill(queue);1483 1484 return count;1485}1486 1487static int macb_rx_frame(struct macb_queue *queue, struct napi_struct *napi,1488 unsigned int first_frag, unsigned int last_frag)1489{1490 unsigned int len;1491 unsigned int frag;1492 unsigned int offset;1493 struct sk_buff *skb;1494 struct macb_dma_desc *desc;1495 struct macb *bp = queue->bp;1496 1497 desc = macb_rx_desc(queue, last_frag);1498 len = desc->ctrl & bp->rx_frm_len_mask;1499 1500 netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n",1501 macb_rx_ring_wrap(bp, first_frag),1502 macb_rx_ring_wrap(bp, last_frag), len);1503 1504 /* The ethernet header starts NET_IP_ALIGN bytes into the1505 * first buffer. Since the header is 14 bytes, this makes the1506 * payload word-aligned.1507 *1508 * Instead of calling skb_reserve(NET_IP_ALIGN), we just copy1509 * the two padding bytes into the skb so that we avoid hitting1510 * the slowpath in memcpy(), and pull them off afterwards.1511 */1512 skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN);1513 if (!skb) {1514 bp->dev->stats.rx_dropped++;1515 for (frag = first_frag; ; frag++) {1516 desc = macb_rx_desc(queue, frag);1517 desc->addr &= ~MACB_BIT(RX_USED);1518 if (frag == last_frag)1519 break;1520 }1521 1522 /* Make descriptor updates visible to hardware */1523 wmb();1524 1525 return 1;1526 }1527 1528 offset = 0;1529 len += NET_IP_ALIGN;1530 skb_checksum_none_assert(skb);1531 skb_put(skb, len);1532 1533 for (frag = first_frag; ; frag++) {1534 unsigned int frag_len = bp->rx_buffer_size;1535 1536 if (offset + frag_len > len) {1537 if (unlikely(frag != last_frag)) {1538 dev_kfree_skb_any(skb);1539 return -1;1540 }1541 frag_len = len - offset;1542 }1543 skb_copy_to_linear_data_offset(skb, offset,1544 macb_rx_buffer(queue, frag),1545 frag_len);1546 offset += bp->rx_buffer_size;1547 desc = macb_rx_desc(queue, frag);1548 desc->addr &= ~MACB_BIT(RX_USED);1549 1550 if (frag == last_frag)1551 break;1552 }1553 1554 /* Make descriptor updates visible to hardware */1555 wmb();1556 1557 __skb_pull(skb, NET_IP_ALIGN);1558 skb->protocol = eth_type_trans(skb, bp->dev);1559 1560 bp->dev->stats.rx_packets++;1561 bp->dev->stats.rx_bytes += skb->len;1562 netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n",1563 skb->len, skb->csum);1564 napi_gro_receive(napi, skb);1565 1566 return 0;1567}1568 1569static inline void macb_init_rx_ring(struct macb_queue *queue)1570{1571 struct macb *bp = queue->bp;1572 dma_addr_t addr;1573 struct macb_dma_desc *desc = NULL;1574 int i;1575 1576 addr = queue->rx_buffers_dma;1577 for (i = 0; i < bp->rx_ring_size; i++) {1578 desc = macb_rx_desc(queue, i);1579 macb_set_addr(bp, desc, addr);1580 desc->ctrl = 0;1581 addr += bp->rx_buffer_size;1582 }1583 desc->addr |= MACB_BIT(RX_WRAP);1584 queue->rx_tail = 0;1585}1586 1587static int macb_rx(struct macb_queue *queue, struct napi_struct *napi,1588 int budget)1589{1590 struct macb *bp = queue->bp;1591 bool reset_rx_queue = false;1592 int received = 0;1593 unsigned int tail;1594 int first_frag = -1;1595 1596 for (tail = queue->rx_tail; budget > 0; tail++) {1597 struct macb_dma_desc *desc = macb_rx_desc(queue, tail);1598 u32 ctrl;1599 1600 /* Make hw descriptor updates visible to CPU */1601 rmb();1602 1603 if (!(desc->addr & MACB_BIT(RX_USED)))1604 break;1605 1606 /* Ensure ctrl is at least as up-to-date as addr */1607 dma_rmb();1608 1609 ctrl = desc->ctrl;1610 1611 if (ctrl & MACB_BIT(RX_SOF)) {1612 if (first_frag != -1)1613 discard_partial_frame(queue, first_frag, tail);1614 first_frag = tail;1615 }1616 1617 if (ctrl & MACB_BIT(RX_EOF)) {1618 int dropped;1619 1620 if (unlikely(first_frag == -1)) {1621 reset_rx_queue = true;1622 continue;1623 }1624 1625 dropped = macb_rx_frame(queue, napi, first_frag, tail);1626 first_frag = -1;1627 if (unlikely(dropped < 0)) {1628 reset_rx_queue = true;1629 continue;1630 }1631 if (!dropped) {1632 received++;1633 budget--;1634 }1635 }1636 }1637 1638 if (unlikely(reset_rx_queue)) {1639 unsigned long flags;1640 u32 ctrl;1641 1642 netdev_err(bp->dev, "RX queue corruption: reset it\n");1643 1644 spin_lock_irqsave(&bp->lock, flags);1645 1646 ctrl = macb_readl(bp, NCR);1647 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE));1648 1649 macb_init_rx_ring(queue);1650 queue_writel(queue, RBQP, queue->rx_ring_dma);1651 1652 macb_writel(bp, NCR, ctrl | MACB_BIT(RE));1653 1654 spin_unlock_irqrestore(&bp->lock, flags);1655 return received;1656 }1657 1658 if (first_frag != -1)1659 queue->rx_tail = first_frag;1660 else1661 queue->rx_tail = tail;1662 1663 return received;1664}1665 1666static bool macb_rx_pending(struct macb_queue *queue)1667{1668 struct macb *bp = queue->bp;1669 unsigned int entry;1670 struct macb_dma_desc *desc;1671 1672 entry = macb_rx_ring_wrap(bp, queue->rx_tail);1673 desc = macb_rx_desc(queue, entry);1674 1675 /* Make hw descriptor updates visible to CPU */1676 rmb();1677 1678 return (desc->addr & MACB_BIT(RX_USED)) != 0;1679}1680 1681static int macb_rx_poll(struct napi_struct *napi, int budget)1682{1683 struct macb_queue *queue = container_of(napi, struct macb_queue, napi_rx);1684 struct macb *bp = queue->bp;1685 int work_done;1686 1687 work_done = bp->macbgem_ops.mog_rx(queue, napi, budget);1688 1689 netdev_vdbg(bp->dev, "RX poll: queue = %u, work_done = %d, budget = %d\n",1690 (unsigned int)(queue - bp->queues), work_done, budget);1691 1692 if (work_done < budget && napi_complete_done(napi, work_done)) {1693 queue_writel(queue, IER, bp->rx_intr_mask);1694 1695 /* Packet completions only seem to propagate to raise1696 * interrupts when interrupts are enabled at the time, so if1697 * packets were received while interrupts were disabled,1698 * they will not cause another interrupt to be generated when1699 * interrupts are re-enabled.1700 * Check for this case here to avoid losing a wakeup. This can1701 * potentially race with the interrupt handler doing the same1702 * actions if an interrupt is raised just after enabling them,1703 * but this should be harmless.1704 */1705 if (macb_rx_pending(queue)) {1706 queue_writel(queue, IDR, bp->rx_intr_mask);1707 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)1708 queue_writel(queue, ISR, MACB_BIT(RCOMP));1709 netdev_vdbg(bp->dev, "poll: packets pending, reschedule\n");1710 napi_schedule(napi);1711 }1712 }1713 1714 /* TODO: Handle errors */1715 1716 return work_done;1717}1718 1719static void macb_tx_restart(struct macb_queue *queue)1720{1721 struct macb *bp = queue->bp;1722 unsigned int head_idx, tbqp;1723 1724 spin_lock(&queue->tx_ptr_lock);1725 1726 if (queue->tx_head == queue->tx_tail)1727 goto out_tx_ptr_unlock;1728 1729 tbqp = queue_readl(queue, TBQP) / macb_dma_desc_get_size(bp);1730 tbqp = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, tbqp));1731 head_idx = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, queue->tx_head));1732 1733 if (tbqp == head_idx)1734 goto out_tx_ptr_unlock;1735 1736 spin_lock_irq(&bp->lock);1737 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART));1738 spin_unlock_irq(&bp->lock);1739 1740out_tx_ptr_unlock:1741 spin_unlock(&queue->tx_ptr_lock);1742}1743 1744static bool macb_tx_complete_pending(struct macb_queue *queue)1745{1746 bool retval = false;1747 1748 spin_lock(&queue->tx_ptr_lock);1749 if (queue->tx_head != queue->tx_tail) {1750 /* Make hw descriptor updates visible to CPU */1751 rmb();1752 1753 if (macb_tx_desc(queue, queue->tx_tail)->ctrl & MACB_BIT(TX_USED))1754 retval = true;1755 }1756 spin_unlock(&queue->tx_ptr_lock);1757 return retval;1758}1759 1760static int macb_tx_poll(struct napi_struct *napi, int budget)1761{1762 struct macb_queue *queue = container_of(napi, struct macb_queue, napi_tx);1763 struct macb *bp = queue->bp;1764 int work_done;1765 1766 work_done = macb_tx_complete(queue, budget);1767 1768 rmb(); // ensure txubr_pending is up to date1769 if (queue->txubr_pending) {1770 queue->txubr_pending = false;1771 netdev_vdbg(bp->dev, "poll: tx restart\n");1772 macb_tx_restart(queue);1773 }1774 1775 netdev_vdbg(bp->dev, "TX poll: queue = %u, work_done = %d, budget = %d\n",1776 (unsigned int)(queue - bp->queues), work_done, budget);1777 1778 if (work_done < budget && napi_complete_done(napi, work_done)) {1779 queue_writel(queue, IER, MACB_BIT(TCOMP));1780 1781 /* Packet completions only seem to propagate to raise1782 * interrupts when interrupts are enabled at the time, so if1783 * packets were sent while interrupts were disabled,1784 * they will not cause another interrupt to be generated when1785 * interrupts are re-enabled.1786 * Check for this case here to avoid losing a wakeup. This can1787 * potentially race with the interrupt handler doing the same1788 * actions if an interrupt is raised just after enabling them,1789 * but this should be harmless.1790 */1791 if (macb_tx_complete_pending(queue)) {1792 queue_writel(queue, IDR, MACB_BIT(TCOMP));1793 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)1794 queue_writel(queue, ISR, MACB_BIT(TCOMP));1795 netdev_vdbg(bp->dev, "TX poll: packets pending, reschedule\n");1796 napi_schedule(napi);1797 }1798 }1799 1800 return work_done;1801}1802 1803static void macb_hresp_error_task(struct work_struct *work)1804{1805 struct macb *bp = from_work(bp, work, hresp_err_bh_work);1806 struct net_device *dev = bp->dev;1807 struct macb_queue *queue;1808 unsigned int q;1809 u32 ctrl;1810 1811 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {1812 queue_writel(queue, IDR, bp->rx_intr_mask |1813 MACB_TX_INT_FLAGS |1814 MACB_BIT(HRESP));1815 }1816 ctrl = macb_readl(bp, NCR);1817 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE));1818 macb_writel(bp, NCR, ctrl);1819 1820 netif_tx_stop_all_queues(dev);1821 netif_carrier_off(dev);1822 1823 bp->macbgem_ops.mog_init_rings(bp);1824 1825 /* Initialize TX and RX buffers */1826 macb_init_buffers(bp);1827 1828 /* Enable interrupts */1829 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)1830 queue_writel(queue, IER,1831 bp->rx_intr_mask |1832 MACB_TX_INT_FLAGS |1833 MACB_BIT(HRESP));1834 1835 ctrl |= MACB_BIT(RE) | MACB_BIT(TE);1836 macb_writel(bp, NCR, ctrl);1837 1838 netif_carrier_on(dev);1839 netif_tx_start_all_queues(dev);1840}1841 1842static irqreturn_t macb_wol_interrupt(int irq, void *dev_id)1843{1844 struct macb_queue *queue = dev_id;1845 struct macb *bp = queue->bp;1846 u32 status;1847 1848 status = queue_readl(queue, ISR);1849 1850 if (unlikely(!status))1851 return IRQ_NONE;1852 1853 spin_lock(&bp->lock);1854 1855 if (status & MACB_BIT(WOL)) {1856 queue_writel(queue, IDR, MACB_BIT(WOL));1857 macb_writel(bp, WOL, 0);1858 netdev_vdbg(bp->dev, "MACB WoL: queue = %u, isr = 0x%08lx\n",1859 (unsigned int)(queue - bp->queues),1860 (unsigned long)status);1861 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)1862 queue_writel(queue, ISR, MACB_BIT(WOL));1863 pm_wakeup_event(&bp->pdev->dev, 0);1864 }1865 1866 spin_unlock(&bp->lock);1867 1868 return IRQ_HANDLED;1869}1870 1871static irqreturn_t gem_wol_interrupt(int irq, void *dev_id)1872{1873 struct macb_queue *queue = dev_id;1874 struct macb *bp = queue->bp;1875 u32 status;1876 1877 status = queue_readl(queue, ISR);1878 1879 if (unlikely(!status))1880 return IRQ_NONE;1881 1882 spin_lock(&bp->lock);1883 1884 if (status & GEM_BIT(WOL)) {1885 queue_writel(queue, IDR, GEM_BIT(WOL));1886 gem_writel(bp, WOL, 0);1887 netdev_vdbg(bp->dev, "GEM WoL: queue = %u, isr = 0x%08lx\n",1888 (unsigned int)(queue - bp->queues),1889 (unsigned long)status);1890 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)1891 queue_writel(queue, ISR, GEM_BIT(WOL));1892 pm_wakeup_event(&bp->pdev->dev, 0);1893 }1894 1895 spin_unlock(&bp->lock);1896 1897 return IRQ_HANDLED;1898}1899 1900static irqreturn_t macb_interrupt(int irq, void *dev_id)1901{1902 struct macb_queue *queue = dev_id;1903 struct macb *bp = queue->bp;1904 struct net_device *dev = bp->dev;1905 u32 status, ctrl;1906 1907 status = queue_readl(queue, ISR);1908 1909 if (unlikely(!status))1910 return IRQ_NONE;1911 1912 spin_lock(&bp->lock);1913 1914 while (status) {1915 /* close possible race with dev_close */1916 if (unlikely(!netif_running(dev))) {1917 queue_writel(queue, IDR, -1);1918 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)1919 queue_writel(queue, ISR, -1);1920 break;1921 }1922 1923 netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n",1924 (unsigned int)(queue - bp->queues),1925 (unsigned long)status);1926 1927 if (status & bp->rx_intr_mask) {1928 /* There's no point taking any more interrupts1929 * until we have processed the buffers. The1930 * scheduling call may fail if the poll routine1931 * is already scheduled, so disable interrupts1932 * now.1933 */1934 queue_writel(queue, IDR, bp->rx_intr_mask);1935 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)1936 queue_writel(queue, ISR, MACB_BIT(RCOMP));1937 1938 if (napi_schedule_prep(&queue->napi_rx)) {1939 netdev_vdbg(bp->dev, "scheduling RX softirq\n");1940 __napi_schedule(&queue->napi_rx);1941 }1942 }1943 1944 if (status & (MACB_BIT(TCOMP) |1945 MACB_BIT(TXUBR))) {1946 queue_writel(queue, IDR, MACB_BIT(TCOMP));1947 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)1948 queue_writel(queue, ISR, MACB_BIT(TCOMP) |1949 MACB_BIT(TXUBR));1950 1951 if (status & MACB_BIT(TXUBR)) {1952 queue->txubr_pending = true;1953 wmb(); // ensure softirq can see update1954 }1955 1956 if (napi_schedule_prep(&queue->napi_tx)) {1957 netdev_vdbg(bp->dev, "scheduling TX softirq\n");1958 __napi_schedule(&queue->napi_tx);1959 }1960 }1961 1962 if (unlikely(status & (MACB_TX_ERR_FLAGS))) {1963 queue_writel(queue, IDR, MACB_TX_INT_FLAGS);1964 schedule_work(&queue->tx_error_task);1965 1966 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)1967 queue_writel(queue, ISR, MACB_TX_ERR_FLAGS);1968 1969 break;1970 }1971 1972 /* Link change detection isn't possible with RMII, so we'll1973 * add that if/when we get our hands on a full-blown MII PHY.1974 */1975 1976 /* There is a hardware issue under heavy load where DMA can1977 * stop, this causes endless "used buffer descriptor read"1978 * interrupts but it can be cleared by re-enabling RX. See1979 * the at91rm9200 manual, section 41.3.1 or the Zynq manual1980 * section 16.7.4 for details. RXUBR is only enabled for1981 * these two versions.1982 */1983 if (status & MACB_BIT(RXUBR)) {1984 ctrl = macb_readl(bp, NCR);1985 macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE));1986 wmb();1987 macb_writel(bp, NCR, ctrl | MACB_BIT(RE));1988 1989 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)1990 queue_writel(queue, ISR, MACB_BIT(RXUBR));1991 }1992 1993 if (status & MACB_BIT(ISR_ROVR)) {1994 /* We missed at least one packet */1995 if (macb_is_gem(bp))1996 bp->hw_stats.gem.rx_overruns++;1997 else1998 bp->hw_stats.macb.rx_overruns++;1999 2000 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)2001 queue_writel(queue, ISR, MACB_BIT(ISR_ROVR));2002 }2003 2004 if (status & MACB_BIT(HRESP)) {2005 queue_work(system_bh_wq, &bp->hresp_err_bh_work);2006 netdev_err(dev, "DMA bus error: HRESP not OK\n");2007 2008 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)2009 queue_writel(queue, ISR, MACB_BIT(HRESP));2010 }2011 status = queue_readl(queue, ISR);2012 }2013 2014 spin_unlock(&bp->lock);2015 2016 return IRQ_HANDLED;2017}2018 2019#ifdef CONFIG_NET_POLL_CONTROLLER2020/* Polling receive - used by netconsole and other diagnostic tools2021 * to allow network i/o with interrupts disabled.2022 */2023static void macb_poll_controller(struct net_device *dev)2024{2025 struct macb *bp = netdev_priv(dev);2026 struct macb_queue *queue;2027 unsigned long flags;2028 unsigned int q;2029 2030 local_irq_save(flags);2031 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)2032 macb_interrupt(dev->irq, queue);2033 local_irq_restore(flags);2034}2035#endif2036 2037static unsigned int macb_tx_map(struct macb *bp,2038 struct macb_queue *queue,2039 struct sk_buff *skb,2040 unsigned int hdrlen)2041{2042 dma_addr_t mapping;2043 unsigned int len, entry, i, tx_head = queue->tx_head;2044 struct macb_tx_skb *tx_skb = NULL;2045 struct macb_dma_desc *desc;2046 unsigned int offset, size, count = 0;2047 unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags;2048 unsigned int eof = 1, mss_mfs = 0;2049 u32 ctrl, lso_ctrl = 0, seq_ctrl = 0;2050 2051 /* LSO */2052 if (skb_shinfo(skb)->gso_size != 0) {2053 if (ip_hdr(skb)->protocol == IPPROTO_UDP)2054 /* UDP - UFO */2055 lso_ctrl = MACB_LSO_UFO_ENABLE;2056 else2057 /* TCP - TSO */2058 lso_ctrl = MACB_LSO_TSO_ENABLE;2059 }2060 2061 /* First, map non-paged data */2062 len = skb_headlen(skb);2063 2064 /* first buffer length */2065 size = hdrlen;2066 2067 offset = 0;2068 while (len) {2069 entry = macb_tx_ring_wrap(bp, tx_head);2070 tx_skb = &queue->tx_skb[entry];2071 2072 mapping = dma_map_single(&bp->pdev->dev,2073 skb->data + offset,2074 size, DMA_TO_DEVICE);2075 if (dma_mapping_error(&bp->pdev->dev, mapping))2076 goto dma_error;2077 2078 /* Save info to properly release resources */2079 tx_skb->skb = NULL;2080 tx_skb->mapping = mapping;2081 tx_skb->size = size;2082 tx_skb->mapped_as_page = false;2083 2084 len -= size;2085 offset += size;2086 count++;2087 tx_head++;2088 2089 size = min(len, bp->max_tx_length);2090 }2091 2092 /* Then, map paged data from fragments */2093 for (f = 0; f < nr_frags; f++) {2094 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];2095 2096 len = skb_frag_size(frag);2097 offset = 0;2098 while (len) {2099 size = min(len, bp->max_tx_length);2100 entry = macb_tx_ring_wrap(bp, tx_head);2101 tx_skb = &queue->tx_skb[entry];2102 2103 mapping = skb_frag_dma_map(&bp->pdev->dev, frag,2104 offset, size, DMA_TO_DEVICE);2105 if (dma_mapping_error(&bp->pdev->dev, mapping))2106 goto dma_error;2107 2108 /* Save info to properly release resources */2109 tx_skb->skb = NULL;2110 tx_skb->mapping = mapping;2111 tx_skb->size = size;2112 tx_skb->mapped_as_page = true;2113 2114 len -= size;2115 offset += size;2116 count++;2117 tx_head++;2118 }2119 }2120 2121 /* Should never happen */2122 if (unlikely(!tx_skb)) {2123 netdev_err(bp->dev, "BUG! empty skb!\n");2124 return 0;2125 }2126 2127 /* This is the last buffer of the frame: save socket buffer */2128 tx_skb->skb = skb;2129 2130 /* Update TX ring: update buffer descriptors in reverse order2131 * to avoid race condition2132 */2133 2134 /* Set 'TX_USED' bit in buffer descriptor at tx_head position2135 * to set the end of TX queue2136 */2137 i = tx_head;2138 entry = macb_tx_ring_wrap(bp, i);2139 ctrl = MACB_BIT(TX_USED);2140 desc = macb_tx_desc(queue, entry);2141 desc->ctrl = ctrl;2142 2143 if (lso_ctrl) {2144 if (lso_ctrl == MACB_LSO_UFO_ENABLE)2145 /* include header and FCS in value given to h/w */2146 mss_mfs = skb_shinfo(skb)->gso_size +2147 skb_transport_offset(skb) +2148 ETH_FCS_LEN;2149 else /* TSO */ {2150 mss_mfs = skb_shinfo(skb)->gso_size;2151 /* TCP Sequence Number Source Select2152 * can be set only for TSO2153 */2154 seq_ctrl = 0;2155 }2156 }2157 2158 do {2159 i--;2160 entry = macb_tx_ring_wrap(bp, i);2161 tx_skb = &queue->tx_skb[entry];2162 desc = macb_tx_desc(queue, entry);2163 2164 ctrl = (u32)tx_skb->size;2165 if (eof) {2166 ctrl |= MACB_BIT(TX_LAST);2167 eof = 0;2168 }2169 if (unlikely(entry == (bp->tx_ring_size - 1)))2170 ctrl |= MACB_BIT(TX_WRAP);2171 2172 /* First descriptor is header descriptor */2173 if (i == queue->tx_head) {2174 ctrl |= MACB_BF(TX_LSO, lso_ctrl);2175 ctrl |= MACB_BF(TX_TCP_SEQ_SRC, seq_ctrl);2176 if ((bp->dev->features & NETIF_F_HW_CSUM) &&2177 skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl &&2178 !ptp_one_step_sync(skb))2179 ctrl |= MACB_BIT(TX_NOCRC);2180 } else2181 /* Only set MSS/MFS on payload descriptors2182 * (second or later descriptor)2183 */2184 ctrl |= MACB_BF(MSS_MFS, mss_mfs);2185 2186 /* Set TX buffer descriptor */2187 macb_set_addr(bp, desc, tx_skb->mapping);2188 /* desc->addr must be visible to hardware before clearing2189 * 'TX_USED' bit in desc->ctrl.2190 */2191 wmb();2192 desc->ctrl = ctrl;2193 } while (i != queue->tx_head);2194 2195 queue->tx_head = tx_head;2196 2197 return count;2198 2199dma_error:2200 netdev_err(bp->dev, "TX DMA map failed\n");2201 2202 for (i = queue->tx_head; i != tx_head; i++) {2203 tx_skb = macb_tx_skb(queue, i);2204 2205 macb_tx_unmap(bp, tx_skb, 0);2206 }2207 2208 return 0;2209}2210 2211static netdev_features_t macb_features_check(struct sk_buff *skb,2212 struct net_device *dev,2213 netdev_features_t features)2214{2215 unsigned int nr_frags, f;2216 unsigned int hdrlen;2217 2218 /* Validate LSO compatibility */2219 2220 /* there is only one buffer or protocol is not UDP */2221 if (!skb_is_nonlinear(skb) || (ip_hdr(skb)->protocol != IPPROTO_UDP))2222 return features;2223 2224 /* length of header */2225 hdrlen = skb_transport_offset(skb);2226 2227 /* For UFO only:2228 * When software supplies two or more payload buffers all payload buffers2229 * apart from the last must be a multiple of 8 bytes in size.2230 */2231 if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN))2232 return features & ~MACB_NETIF_LSO;2233 2234 nr_frags = skb_shinfo(skb)->nr_frags;2235 /* No need to check last fragment */2236 nr_frags--;2237 for (f = 0; f < nr_frags; f++) {2238 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];2239 2240 if (!IS_ALIGNED(skb_frag_size(frag), MACB_TX_LEN_ALIGN))2241 return features & ~MACB_NETIF_LSO;2242 }2243 return features;2244}2245 2246static inline int macb_clear_csum(struct sk_buff *skb)2247{2248 /* no change for packets without checksum offloading */2249 if (skb->ip_summed != CHECKSUM_PARTIAL)2250 return 0;2251 2252 /* make sure we can modify the header */2253 if (unlikely(skb_cow_head(skb, 0)))2254 return -1;2255 2256 /* initialize checksum field2257 * This is required - at least for Zynq, which otherwise calculates2258 * wrong UDP header checksums for UDP packets with UDP data len <=22259 */2260 *(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0;2261 return 0;2262}2263 2264static int macb_pad_and_fcs(struct sk_buff **skb, struct net_device *ndev)2265{2266 bool cloned = skb_cloned(*skb) || skb_header_cloned(*skb) ||2267 skb_is_nonlinear(*skb);2268 int padlen = ETH_ZLEN - (*skb)->len;2269 int tailroom = skb_tailroom(*skb);2270 struct sk_buff *nskb;2271 u32 fcs;2272 2273 if (!(ndev->features & NETIF_F_HW_CSUM) ||2274 !((*skb)->ip_summed != CHECKSUM_PARTIAL) ||2275 skb_shinfo(*skb)->gso_size || ptp_one_step_sync(*skb))2276 return 0;2277 2278 if (padlen <= 0) {2279 /* FCS could be appeded to tailroom. */2280 if (tailroom >= ETH_FCS_LEN)2281 goto add_fcs;2282 /* No room for FCS, need to reallocate skb. */2283 else2284 padlen = ETH_FCS_LEN;2285 } else {2286 /* Add room for FCS. */2287 padlen += ETH_FCS_LEN;2288 }2289 2290 if (cloned || tailroom < padlen) {2291 nskb = skb_copy_expand(*skb, 0, padlen, GFP_ATOMIC);2292 if (!nskb)2293 return -ENOMEM;2294 2295 dev_consume_skb_any(*skb);2296 *skb = nskb;2297 }2298 2299 if (padlen > ETH_FCS_LEN)2300 skb_put_zero(*skb, padlen - ETH_FCS_LEN);2301 2302add_fcs:2303 /* set FCS to packet */2304 fcs = crc32_le(~0, (*skb)->data, (*skb)->len);2305 fcs = ~fcs;2306 2307 skb_put_u8(*skb, fcs & 0xff);2308 skb_put_u8(*skb, (fcs >> 8) & 0xff);2309 skb_put_u8(*skb, (fcs >> 16) & 0xff);2310 skb_put_u8(*skb, (fcs >> 24) & 0xff);2311 2312 return 0;2313}2314 2315static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev)2316{2317 u16 queue_index = skb_get_queue_mapping(skb);2318 struct macb *bp = netdev_priv(dev);2319 struct macb_queue *queue = &bp->queues[queue_index];2320 unsigned int desc_cnt, nr_frags, frag_size, f;2321 unsigned int hdrlen;2322 bool is_lso;2323 netdev_tx_t ret = NETDEV_TX_OK;2324 2325 if (macb_clear_csum(skb)) {2326 dev_kfree_skb_any(skb);2327 return ret;2328 }2329 2330 if (macb_pad_and_fcs(&skb, dev)) {2331 dev_kfree_skb_any(skb);2332 return ret;2333 }2334 2335#ifdef CONFIG_MACB_USE_HWSTAMP2336 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&2337 (bp->hw_dma_cap & HW_DMA_CAP_PTP))2338 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;2339#endif2340 2341 is_lso = (skb_shinfo(skb)->gso_size != 0);2342 2343 if (is_lso) {2344 /* length of headers */2345 if (ip_hdr(skb)->protocol == IPPROTO_UDP)2346 /* only queue eth + ip headers separately for UDP */2347 hdrlen = skb_transport_offset(skb);2348 else2349 hdrlen = skb_tcp_all_headers(skb);2350 if (skb_headlen(skb) < hdrlen) {2351 netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n");2352 /* if this is required, would need to copy to single buffer */2353 return NETDEV_TX_BUSY;2354 }2355 } else2356 hdrlen = min(skb_headlen(skb), bp->max_tx_length);2357 2358#if defined(DEBUG) && defined(VERBOSE_DEBUG)2359 netdev_vdbg(bp->dev,2360 "start_xmit: queue %hu len %u head %p data %p tail %p end %p\n",2361 queue_index, skb->len, skb->head, skb->data,2362 skb_tail_pointer(skb), skb_end_pointer(skb));2363 print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_OFFSET, 16, 1,2364 skb->data, 16, true);2365#endif2366 2367 /* Count how many TX buffer descriptors are needed to send this2368 * socket buffer: skb fragments of jumbo frames may need to be2369 * split into many buffer descriptors.2370 */2371 if (is_lso && (skb_headlen(skb) > hdrlen))2372 /* extra header descriptor if also payload in first buffer */2373 desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1;2374 else2375 desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length);2376 nr_frags = skb_shinfo(skb)->nr_frags;2377 for (f = 0; f < nr_frags; f++) {2378 frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]);2379 desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length);2380 }2381 2382 spin_lock_bh(&queue->tx_ptr_lock);2383 2384 /* This is a hard error, log it. */2385 if (CIRC_SPACE(queue->tx_head, queue->tx_tail,2386 bp->tx_ring_size) < desc_cnt) {2387 netif_stop_subqueue(dev, queue_index);2388 netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n",2389 queue->tx_head, queue->tx_tail);2390 ret = NETDEV_TX_BUSY;2391 goto unlock;2392 }2393 2394 /* Map socket buffer for DMA transfer */2395 if (!macb_tx_map(bp, queue, skb, hdrlen)) {2396 dev_kfree_skb_any(skb);2397 goto unlock;2398 }2399 2400 /* Make newly initialized descriptor visible to hardware */2401 wmb();2402 skb_tx_timestamp(skb);2403 2404 spin_lock_irq(&bp->lock);2405 macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART));2406 spin_unlock_irq(&bp->lock);2407 2408 if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1)2409 netif_stop_subqueue(dev, queue_index);2410 2411unlock:2412 spin_unlock_bh(&queue->tx_ptr_lock);2413 2414 return ret;2415}2416 2417static void macb_init_rx_buffer_size(struct macb *bp, size_t size)2418{2419 if (!macb_is_gem(bp)) {2420 bp->rx_buffer_size = MACB_RX_BUFFER_SIZE;2421 } else {2422 bp->rx_buffer_size = size;2423 2424 if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) {2425 netdev_dbg(bp->dev,2426 "RX buffer must be multiple of %d bytes, expanding\n",2427 RX_BUFFER_MULTIPLE);2428 bp->rx_buffer_size =2429 roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE);2430 }2431 }2432 2433 netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n",2434 bp->dev->mtu, bp->rx_buffer_size);2435}2436 2437static void gem_free_rx_buffers(struct macb *bp)2438{2439 struct sk_buff *skb;2440 struct macb_dma_desc *desc;2441 struct macb_queue *queue;2442 dma_addr_t addr;2443 unsigned int q;2444 int i;2445 2446 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {2447 if (!queue->rx_skbuff)2448 continue;2449 2450 for (i = 0; i < bp->rx_ring_size; i++) {2451 skb = queue->rx_skbuff[i];2452 2453 if (!skb)2454 continue;2455 2456 desc = macb_rx_desc(queue, i);2457 addr = macb_get_addr(bp, desc);2458 2459 dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size,2460 DMA_FROM_DEVICE);2461 dev_kfree_skb_any(skb);2462 skb = NULL;2463 }2464 2465 kfree(queue->rx_skbuff);2466 queue->rx_skbuff = NULL;2467 }2468}2469 2470static void macb_free_rx_buffers(struct macb *bp)2471{2472 struct macb_queue *queue = &bp->queues[0];2473 2474 if (queue->rx_buffers) {2475 dma_free_coherent(&bp->pdev->dev,2476 bp->rx_ring_size * bp->rx_buffer_size,2477 queue->rx_buffers, queue->rx_buffers_dma);2478 queue->rx_buffers = NULL;2479 }2480}2481 2482static void macb_free_consistent(struct macb *bp)2483{2484 struct macb_queue *queue;2485 unsigned int q;2486 int size;2487 2488 if (bp->rx_ring_tieoff) {2489 dma_free_coherent(&bp->pdev->dev, macb_dma_desc_get_size(bp),2490 bp->rx_ring_tieoff, bp->rx_ring_tieoff_dma);2491 bp->rx_ring_tieoff = NULL;2492 }2493 2494 bp->macbgem_ops.mog_free_rx_buffers(bp);2495 2496 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {2497 kfree(queue->tx_skb);2498 queue->tx_skb = NULL;2499 if (queue->tx_ring) {2500 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch;2501 dma_free_coherent(&bp->pdev->dev, size,2502 queue->tx_ring, queue->tx_ring_dma);2503 queue->tx_ring = NULL;2504 }2505 if (queue->rx_ring) {2506 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch;2507 dma_free_coherent(&bp->pdev->dev, size,2508 queue->rx_ring, queue->rx_ring_dma);2509 queue->rx_ring = NULL;2510 }2511 }2512}2513 2514static int gem_alloc_rx_buffers(struct macb *bp)2515{2516 struct macb_queue *queue;2517 unsigned int q;2518 int size;2519 2520 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {2521 size = bp->rx_ring_size * sizeof(struct sk_buff *);2522 queue->rx_skbuff = kzalloc(size, GFP_KERNEL);2523 if (!queue->rx_skbuff)2524 return -ENOMEM;2525 else2526 netdev_dbg(bp->dev,2527 "Allocated %d RX struct sk_buff entries at %p\n",2528 bp->rx_ring_size, queue->rx_skbuff);2529 }2530 return 0;2531}2532 2533static int macb_alloc_rx_buffers(struct macb *bp)2534{2535 struct macb_queue *queue = &bp->queues[0];2536 int size;2537 2538 size = bp->rx_ring_size * bp->rx_buffer_size;2539 queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size,2540 &queue->rx_buffers_dma, GFP_KERNEL);2541 if (!queue->rx_buffers)2542 return -ENOMEM;2543 2544 netdev_dbg(bp->dev,2545 "Allocated RX buffers of %d bytes at %08lx (mapped %p)\n",2546 size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers);2547 return 0;2548}2549 2550static int macb_alloc_consistent(struct macb *bp)2551{2552 struct macb_queue *queue;2553 unsigned int q;2554 int size;2555 2556 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {2557 size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch;2558 queue->tx_ring = dma_alloc_coherent(&bp->pdev->dev, size,2559 &queue->tx_ring_dma,2560 GFP_KERNEL);2561 if (!queue->tx_ring)2562 goto out_err;2563 netdev_dbg(bp->dev,2564 "Allocated TX ring for queue %u of %d bytes at %08lx (mapped %p)\n",2565 q, size, (unsigned long)queue->tx_ring_dma,2566 queue->tx_ring);2567 2568 size = bp->tx_ring_size * sizeof(struct macb_tx_skb);2569 queue->tx_skb = kmalloc(size, GFP_KERNEL);2570 if (!queue->tx_skb)2571 goto out_err;2572 2573 size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch;2574 queue->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size,2575 &queue->rx_ring_dma, GFP_KERNEL);2576 if (!queue->rx_ring)2577 goto out_err;2578 netdev_dbg(bp->dev,2579 "Allocated RX ring of %d bytes at %08lx (mapped %p)\n",2580 size, (unsigned long)queue->rx_ring_dma, queue->rx_ring);2581 }2582 if (bp->macbgem_ops.mog_alloc_rx_buffers(bp))2583 goto out_err;2584 2585 /* Required for tie off descriptor for PM cases */2586 if (!(bp->caps & MACB_CAPS_QUEUE_DISABLE)) {2587 bp->rx_ring_tieoff = dma_alloc_coherent(&bp->pdev->dev,2588 macb_dma_desc_get_size(bp),2589 &bp->rx_ring_tieoff_dma,2590 GFP_KERNEL);2591 if (!bp->rx_ring_tieoff)2592 goto out_err;2593 }2594 2595 return 0;2596 2597out_err:2598 macb_free_consistent(bp);2599 return -ENOMEM;2600}2601 2602static void macb_init_tieoff(struct macb *bp)2603{2604 struct macb_dma_desc *desc = bp->rx_ring_tieoff;2605 2606 if (bp->caps & MACB_CAPS_QUEUE_DISABLE)2607 return;2608 /* Setup a wrapping descriptor with no free slots2609 * (WRAP and USED) to tie off/disable unused RX queues.2610 */2611 macb_set_addr(bp, desc, MACB_BIT(RX_WRAP) | MACB_BIT(RX_USED));2612 desc->ctrl = 0;2613}2614 2615static void gem_init_rings(struct macb *bp)2616{2617 struct macb_queue *queue;2618 struct macb_dma_desc *desc = NULL;2619 unsigned int q;2620 int i;2621 2622 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {2623 for (i = 0; i < bp->tx_ring_size; i++) {2624 desc = macb_tx_desc(queue, i);2625 macb_set_addr(bp, desc, 0);2626 desc->ctrl = MACB_BIT(TX_USED);2627 }2628 desc->ctrl |= MACB_BIT(TX_WRAP);2629 queue->tx_head = 0;2630 queue->tx_tail = 0;2631 2632 queue->rx_tail = 0;2633 queue->rx_prepared_head = 0;2634 2635 gem_rx_refill(queue);2636 }2637 2638 macb_init_tieoff(bp);2639}2640 2641static void macb_init_rings(struct macb *bp)2642{2643 int i;2644 struct macb_dma_desc *desc = NULL;2645 2646 macb_init_rx_ring(&bp->queues[0]);2647 2648 for (i = 0; i < bp->tx_ring_size; i++) {2649 desc = macb_tx_desc(&bp->queues[0], i);2650 macb_set_addr(bp, desc, 0);2651 desc->ctrl = MACB_BIT(TX_USED);2652 }2653 bp->queues[0].tx_head = 0;2654 bp->queues[0].tx_tail = 0;2655 desc->ctrl |= MACB_BIT(TX_WRAP);2656 2657 macb_init_tieoff(bp);2658}2659 2660static void macb_reset_hw(struct macb *bp)2661{2662 struct macb_queue *queue;2663 unsigned int q;2664 u32 ctrl = macb_readl(bp, NCR);2665 2666 /* Disable RX and TX (XXX: Should we halt the transmission2667 * more gracefully?)2668 */2669 ctrl &= ~(MACB_BIT(RE) | MACB_BIT(TE));2670 2671 /* Clear the stats registers (XXX: Update stats first?) */2672 ctrl |= MACB_BIT(CLRSTAT);2673 2674 macb_writel(bp, NCR, ctrl);2675 2676 /* Clear all status flags */2677 macb_writel(bp, TSR, -1);2678 macb_writel(bp, RSR, -1);2679 2680 /* Disable RX partial store and forward and reset watermark value */2681 gem_writel(bp, PBUFRXCUT, 0);2682 2683 /* Disable all interrupts */2684 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {2685 queue_writel(queue, IDR, -1);2686 queue_readl(queue, ISR);2687 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)2688 queue_writel(queue, ISR, -1);2689 }2690}2691 2692static u32 gem_mdc_clk_div(struct macb *bp)2693{2694 u32 config;2695 unsigned long pclk_hz = clk_get_rate(bp->pclk);2696 2697 if (pclk_hz <= 20000000)2698 config = GEM_BF(CLK, GEM_CLK_DIV8);2699 else if (pclk_hz <= 40000000)2700 config = GEM_BF(CLK, GEM_CLK_DIV16);2701 else if (pclk_hz <= 80000000)2702 config = GEM_BF(CLK, GEM_CLK_DIV32);2703 else if (pclk_hz <= 120000000)2704 config = GEM_BF(CLK, GEM_CLK_DIV48);2705 else if (pclk_hz <= 160000000)2706 config = GEM_BF(CLK, GEM_CLK_DIV64);2707 else if (pclk_hz <= 240000000)2708 config = GEM_BF(CLK, GEM_CLK_DIV96);2709 else if (pclk_hz <= 320000000)2710 config = GEM_BF(CLK, GEM_CLK_DIV128);2711 else2712 config = GEM_BF(CLK, GEM_CLK_DIV224);2713 2714 return config;2715}2716 2717static u32 macb_mdc_clk_div(struct macb *bp)2718{2719 u32 config;2720 unsigned long pclk_hz;2721 2722 if (macb_is_gem(bp))2723 return gem_mdc_clk_div(bp);2724 2725 pclk_hz = clk_get_rate(bp->pclk);2726 if (pclk_hz <= 20000000)2727 config = MACB_BF(CLK, MACB_CLK_DIV8);2728 else if (pclk_hz <= 40000000)2729 config = MACB_BF(CLK, MACB_CLK_DIV16);2730 else if (pclk_hz <= 80000000)2731 config = MACB_BF(CLK, MACB_CLK_DIV32);2732 else2733 config = MACB_BF(CLK, MACB_CLK_DIV64);2734 2735 return config;2736}2737 2738/* Get the DMA bus width field of the network configuration register that we2739 * should program. We find the width from decoding the design configuration2740 * register to find the maximum supported data bus width.2741 */2742static u32 macb_dbw(struct macb *bp)2743{2744 if (!macb_is_gem(bp))2745 return 0;2746 2747 switch (GEM_BFEXT(DBWDEF, gem_readl(bp, DCFG1))) {2748 case 4:2749 return GEM_BF(DBW, GEM_DBW128);2750 case 2:2751 return GEM_BF(DBW, GEM_DBW64);2752 case 1:2753 default:2754 return GEM_BF(DBW, GEM_DBW32);2755 }2756}2757 2758/* Configure the receive DMA engine2759 * - use the correct receive buffer size2760 * - set best burst length for DMA operations2761 * (if not supported by FIFO, it will fallback to default)2762 * - set both rx/tx packet buffers to full memory size2763 * These are configurable parameters for GEM.2764 */2765static void macb_configure_dma(struct macb *bp)2766{2767 struct macb_queue *queue;2768 u32 buffer_size;2769 unsigned int q;2770 u32 dmacfg;2771 2772 buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE;2773 if (macb_is_gem(bp)) {2774 dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L);2775 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {2776 if (q)2777 queue_writel(queue, RBQS, buffer_size);2778 else2779 dmacfg |= GEM_BF(RXBS, buffer_size);2780 }2781 if (bp->dma_burst_length)2782 dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg);2783 dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L);2784 dmacfg &= ~GEM_BIT(ENDIA_PKT);2785 2786 if (bp->native_io)2787 dmacfg &= ~GEM_BIT(ENDIA_DESC);2788 else2789 dmacfg |= GEM_BIT(ENDIA_DESC); /* CPU in big endian */2790 2791 if (bp->dev->features & NETIF_F_HW_CSUM)2792 dmacfg |= GEM_BIT(TXCOEN);2793 else2794 dmacfg &= ~GEM_BIT(TXCOEN);2795 2796 dmacfg &= ~GEM_BIT(ADDR64);2797#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT2798 if (bp->hw_dma_cap & HW_DMA_CAP_64B)2799 dmacfg |= GEM_BIT(ADDR64);2800#endif2801#ifdef CONFIG_MACB_USE_HWSTAMP2802 if (bp->hw_dma_cap & HW_DMA_CAP_PTP)2803 dmacfg |= GEM_BIT(RXEXT) | GEM_BIT(TXEXT);2804#endif2805 netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n",2806 dmacfg);2807 gem_writel(bp, DMACFG, dmacfg);2808 }2809}2810 2811static void macb_init_hw(struct macb *bp)2812{2813 u32 config;2814 2815 macb_reset_hw(bp);2816 macb_set_hwaddr(bp);2817 2818 config = macb_mdc_clk_div(bp);2819 config |= MACB_BF(RBOF, NET_IP_ALIGN); /* Make eth data aligned */2820 config |= MACB_BIT(DRFCS); /* Discard Rx FCS */2821 if (bp->caps & MACB_CAPS_JUMBO)2822 config |= MACB_BIT(JFRAME); /* Enable jumbo frames */2823 else2824 config |= MACB_BIT(BIG); /* Receive oversized frames */2825 if (bp->dev->flags & IFF_PROMISC)2826 config |= MACB_BIT(CAF); /* Copy All Frames */2827 else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM)2828 config |= GEM_BIT(RXCOEN);2829 if (!(bp->dev->flags & IFF_BROADCAST))2830 config |= MACB_BIT(NBC); /* No BroadCast */2831 config |= macb_dbw(bp);2832 macb_writel(bp, NCFGR, config);2833 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len)2834 gem_writel(bp, JML, bp->jumbo_max_len);2835 bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK;2836 if (bp->caps & MACB_CAPS_JUMBO)2837 bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK;2838 2839 macb_configure_dma(bp);2840 2841 /* Enable RX partial store and forward and set watermark */2842 if (bp->rx_watermark)2843 gem_writel(bp, PBUFRXCUT, (bp->rx_watermark | GEM_BIT(ENCUTTHRU)));2844}2845 2846/* The hash address register is 64 bits long and takes up two2847 * locations in the memory map. The least significant bits are stored2848 * in EMAC_HSL and the most significant bits in EMAC_HSH.2849 *2850 * The unicast hash enable and the multicast hash enable bits in the2851 * network configuration register enable the reception of hash matched2852 * frames. The destination address is reduced to a 6 bit index into2853 * the 64 bit hash register using the following hash function. The2854 * hash function is an exclusive or of every sixth bit of the2855 * destination address.2856 *2857 * hi[5] = da[5] ^ da[11] ^ da[17] ^ da[23] ^ da[29] ^ da[35] ^ da[41] ^ da[47]2858 * hi[4] = da[4] ^ da[10] ^ da[16] ^ da[22] ^ da[28] ^ da[34] ^ da[40] ^ da[46]2859 * hi[3] = da[3] ^ da[09] ^ da[15] ^ da[21] ^ da[27] ^ da[33] ^ da[39] ^ da[45]2860 * hi[2] = da[2] ^ da[08] ^ da[14] ^ da[20] ^ da[26] ^ da[32] ^ da[38] ^ da[44]2861 * hi[1] = da[1] ^ da[07] ^ da[13] ^ da[19] ^ da[25] ^ da[31] ^ da[37] ^ da[43]2862 * hi[0] = da[0] ^ da[06] ^ da[12] ^ da[18] ^ da[24] ^ da[30] ^ da[36] ^ da[42]2863 *2864 * da[0] represents the least significant bit of the first byte2865 * received, that is, the multicast/unicast indicator, and da[47]2866 * represents the most significant bit of the last byte received. If2867 * the hash index, hi[n], points to a bit that is set in the hash2868 * register then the frame will be matched according to whether the2869 * frame is multicast or unicast. A multicast match will be signalled2870 * if the multicast hash enable bit is set, da[0] is 1 and the hash2871 * index points to a bit set in the hash register. A unicast match2872 * will be signalled if the unicast hash enable bit is set, da[0] is 02873 * and the hash index points to a bit set in the hash register. To2874 * receive all multicast frames, the hash register should be set with2875 * all ones and the multicast hash enable bit should be set in the2876 * network configuration register.2877 */2878 2879static inline int hash_bit_value(int bitnr, __u8 *addr)2880{2881 if (addr[bitnr / 8] & (1 << (bitnr % 8)))2882 return 1;2883 return 0;2884}2885 2886/* Return the hash index value for the specified address. */2887static int hash_get_index(__u8 *addr)2888{2889 int i, j, bitval;2890 int hash_index = 0;2891 2892 for (j = 0; j < 6; j++) {2893 for (i = 0, bitval = 0; i < 8; i++)2894 bitval ^= hash_bit_value(i * 6 + j, addr);2895 2896 hash_index |= (bitval << j);2897 }2898 2899 return hash_index;2900}2901 2902/* Add multicast addresses to the internal multicast-hash table. */2903static void macb_sethashtable(struct net_device *dev)2904{2905 struct netdev_hw_addr *ha;2906 unsigned long mc_filter[2];2907 unsigned int bitnr;2908 struct macb *bp = netdev_priv(dev);2909 2910 mc_filter[0] = 0;2911 mc_filter[1] = 0;2912 2913 netdev_for_each_mc_addr(ha, dev) {2914 bitnr = hash_get_index(ha->addr);2915 mc_filter[bitnr >> 5] |= 1 << (bitnr & 31);2916 }2917 2918 macb_or_gem_writel(bp, HRB, mc_filter[0]);2919 macb_or_gem_writel(bp, HRT, mc_filter[1]);2920}2921 2922/* Enable/Disable promiscuous and multicast modes. */2923static void macb_set_rx_mode(struct net_device *dev)2924{2925 unsigned long cfg;2926 struct macb *bp = netdev_priv(dev);2927 2928 cfg = macb_readl(bp, NCFGR);2929 2930 if (dev->flags & IFF_PROMISC) {2931 /* Enable promiscuous mode */2932 cfg |= MACB_BIT(CAF);2933 2934 /* Disable RX checksum offload */2935 if (macb_is_gem(bp))2936 cfg &= ~GEM_BIT(RXCOEN);2937 } else {2938 /* Disable promiscuous mode */2939 cfg &= ~MACB_BIT(CAF);2940 2941 /* Enable RX checksum offload only if requested */2942 if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM)2943 cfg |= GEM_BIT(RXCOEN);2944 }2945 2946 if (dev->flags & IFF_ALLMULTI) {2947 /* Enable all multicast mode */2948 macb_or_gem_writel(bp, HRB, -1);2949 macb_or_gem_writel(bp, HRT, -1);2950 cfg |= MACB_BIT(NCFGR_MTI);2951 } else if (!netdev_mc_empty(dev)) {2952 /* Enable specific multicasts */2953 macb_sethashtable(dev);2954 cfg |= MACB_BIT(NCFGR_MTI);2955 } else if (dev->flags & (~IFF_ALLMULTI)) {2956 /* Disable all multicast mode */2957 macb_or_gem_writel(bp, HRB, 0);2958 macb_or_gem_writel(bp, HRT, 0);2959 cfg &= ~MACB_BIT(NCFGR_MTI);2960 }2961 2962 macb_writel(bp, NCFGR, cfg);2963}2964 2965static int macb_open(struct net_device *dev)2966{2967 size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN;2968 struct macb *bp = netdev_priv(dev);2969 struct macb_queue *queue;2970 unsigned int q;2971 int err;2972 2973 netdev_dbg(bp->dev, "open\n");2974 2975 err = pm_runtime_resume_and_get(&bp->pdev->dev);2976 if (err < 0)2977 return err;2978 2979 /* RX buffers initialization */2980 macb_init_rx_buffer_size(bp, bufsz);2981 2982 err = macb_alloc_consistent(bp);2983 if (err) {2984 netdev_err(dev, "Unable to allocate DMA memory (error %d)\n",2985 err);2986 goto pm_exit;2987 }2988 2989 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {2990 napi_enable(&queue->napi_rx);2991 napi_enable(&queue->napi_tx);2992 }2993 2994 macb_init_hw(bp);2995 2996 err = phy_power_on(bp->sgmii_phy);2997 if (err)2998 goto reset_hw;2999 3000 err = macb_phylink_connect(bp);3001 if (err)3002 goto phy_off;3003 3004 netif_tx_start_all_queues(dev);3005 3006 if (bp->ptp_info)3007 bp->ptp_info->ptp_init(dev);3008 3009 return 0;3010 3011phy_off:3012 phy_power_off(bp->sgmii_phy);3013 3014reset_hw:3015 macb_reset_hw(bp);3016 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {3017 napi_disable(&queue->napi_rx);3018 napi_disable(&queue->napi_tx);3019 }3020 macb_free_consistent(bp);3021pm_exit:3022 pm_runtime_put_sync(&bp->pdev->dev);3023 return err;3024}3025 3026static int macb_close(struct net_device *dev)3027{3028 struct macb *bp = netdev_priv(dev);3029 struct macb_queue *queue;3030 unsigned long flags;3031 unsigned int q;3032 3033 netif_tx_stop_all_queues(dev);3034 3035 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {3036 napi_disable(&queue->napi_rx);3037 napi_disable(&queue->napi_tx);3038 }3039 3040 phylink_stop(bp->phylink);3041 phylink_disconnect_phy(bp->phylink);3042 3043 phy_power_off(bp->sgmii_phy);3044 3045 spin_lock_irqsave(&bp->lock, flags);3046 macb_reset_hw(bp);3047 netif_carrier_off(dev);3048 spin_unlock_irqrestore(&bp->lock, flags);3049 3050 macb_free_consistent(bp);3051 3052 if (bp->ptp_info)3053 bp->ptp_info->ptp_remove(dev);3054 3055 pm_runtime_put(&bp->pdev->dev);3056 3057 return 0;3058}3059 3060static int macb_change_mtu(struct net_device *dev, int new_mtu)3061{3062 if (netif_running(dev))3063 return -EBUSY;3064 3065 WRITE_ONCE(dev->mtu, new_mtu);3066 3067 return 0;3068}3069 3070static int macb_set_mac_addr(struct net_device *dev, void *addr)3071{3072 int err;3073 3074 err = eth_mac_addr(dev, addr);3075 if (err < 0)3076 return err;3077 3078 macb_set_hwaddr(netdev_priv(dev));3079 return 0;3080}3081 3082static void gem_update_stats(struct macb *bp)3083{3084 struct macb_queue *queue;3085 unsigned int i, q, idx;3086 unsigned long *stat;3087 3088 u32 *p = &bp->hw_stats.gem.tx_octets_31_0;3089 3090 for (i = 0; i < GEM_STATS_LEN; ++i, ++p) {3091 u32 offset = gem_statistics[i].offset;3092 u64 val = bp->macb_reg_readl(bp, offset);3093 3094 bp->ethtool_stats[i] += val;3095 *p += val;3096 3097 if (offset == GEM_OCTTXL || offset == GEM_OCTRXL) {3098 /* Add GEM_OCTTXH, GEM_OCTRXH */3099 val = bp->macb_reg_readl(bp, offset + 4);3100 bp->ethtool_stats[i] += ((u64)val) << 32;3101 *(++p) += val;3102 }3103 }3104 3105 idx = GEM_STATS_LEN;3106 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)3107 for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat)3108 bp->ethtool_stats[idx++] = *stat;3109}3110 3111static struct net_device_stats *gem_get_stats(struct macb *bp)3112{3113 struct gem_stats *hwstat = &bp->hw_stats.gem;3114 struct net_device_stats *nstat = &bp->dev->stats;3115 3116 if (!netif_running(bp->dev))3117 return nstat;3118 3119 gem_update_stats(bp);3120 3121 nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors +3122 hwstat->rx_alignment_errors +3123 hwstat->rx_resource_errors +3124 hwstat->rx_overruns +3125 hwstat->rx_oversize_frames +3126 hwstat->rx_jabbers +3127 hwstat->rx_undersized_frames +3128 hwstat->rx_length_field_frame_errors);3129 nstat->tx_errors = (hwstat->tx_late_collisions +3130 hwstat->tx_excessive_collisions +3131 hwstat->tx_underrun +3132 hwstat->tx_carrier_sense_errors);3133 nstat->multicast = hwstat->rx_multicast_frames;3134 nstat->collisions = (hwstat->tx_single_collision_frames +3135 hwstat->tx_multiple_collision_frames +3136 hwstat->tx_excessive_collisions);3137 nstat->rx_length_errors = (hwstat->rx_oversize_frames +3138 hwstat->rx_jabbers +3139 hwstat->rx_undersized_frames +3140 hwstat->rx_length_field_frame_errors);3141 nstat->rx_over_errors = hwstat->rx_resource_errors;3142 nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors;3143 nstat->rx_frame_errors = hwstat->rx_alignment_errors;3144 nstat->rx_fifo_errors = hwstat->rx_overruns;3145 nstat->tx_aborted_errors = hwstat->tx_excessive_collisions;3146 nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors;3147 nstat->tx_fifo_errors = hwstat->tx_underrun;3148 3149 return nstat;3150}3151 3152static void gem_get_ethtool_stats(struct net_device *dev,3153 struct ethtool_stats *stats, u64 *data)3154{3155 struct macb *bp;3156 3157 bp = netdev_priv(dev);3158 gem_update_stats(bp);3159 memcpy(data, &bp->ethtool_stats, sizeof(u64)3160 * (GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES));3161}3162 3163static int gem_get_sset_count(struct net_device *dev, int sset)3164{3165 struct macb *bp = netdev_priv(dev);3166 3167 switch (sset) {3168 case ETH_SS_STATS:3169 return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN;3170 default:3171 return -EOPNOTSUPP;3172 }3173}3174 3175static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p)3176{3177 char stat_string[ETH_GSTRING_LEN];3178 struct macb *bp = netdev_priv(dev);3179 struct macb_queue *queue;3180 unsigned int i;3181 unsigned int q;3182 3183 switch (sset) {3184 case ETH_SS_STATS:3185 for (i = 0; i < GEM_STATS_LEN; i++, p += ETH_GSTRING_LEN)3186 memcpy(p, gem_statistics[i].stat_string,3187 ETH_GSTRING_LEN);3188 3189 for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {3190 for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) {3191 snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s",3192 q, queue_statistics[i].stat_string);3193 memcpy(p, stat_string, ETH_GSTRING_LEN);3194 }3195 }3196 break;3197 }3198}3199 3200static struct net_device_stats *macb_get_stats(struct net_device *dev)3201{3202 struct macb *bp = netdev_priv(dev);3203 struct net_device_stats *nstat = &bp->dev->stats;3204 struct macb_stats *hwstat = &bp->hw_stats.macb;3205 3206 if (macb_is_gem(bp))3207 return gem_get_stats(bp);3208 3209 /* read stats from hardware */3210 macb_update_stats(bp);3211 3212 /* Convert HW stats into netdevice stats */3213 nstat->rx_errors = (hwstat->rx_fcs_errors +3214 hwstat->rx_align_errors +3215 hwstat->rx_resource_errors +3216 hwstat->rx_overruns +3217 hwstat->rx_oversize_pkts +3218 hwstat->rx_jabbers +3219 hwstat->rx_undersize_pkts +3220 hwstat->rx_length_mismatch);3221 nstat->tx_errors = (hwstat->tx_late_cols +3222 hwstat->tx_excessive_cols +3223 hwstat->tx_underruns +3224 hwstat->tx_carrier_errors +3225 hwstat->sqe_test_errors);3226 nstat->collisions = (hwstat->tx_single_cols +3227 hwstat->tx_multiple_cols +3228 hwstat->tx_excessive_cols);3229 nstat->rx_length_errors = (hwstat->rx_oversize_pkts +3230 hwstat->rx_jabbers +3231 hwstat->rx_undersize_pkts +3232 hwstat->rx_length_mismatch);3233 nstat->rx_over_errors = hwstat->rx_resource_errors +3234 hwstat->rx_overruns;3235 nstat->rx_crc_errors = hwstat->rx_fcs_errors;3236 nstat->rx_frame_errors = hwstat->rx_align_errors;3237 nstat->rx_fifo_errors = hwstat->rx_overruns;3238 /* XXX: What does "missed" mean? */3239 nstat->tx_aborted_errors = hwstat->tx_excessive_cols;3240 nstat->tx_carrier_errors = hwstat->tx_carrier_errors;3241 nstat->tx_fifo_errors = hwstat->tx_underruns;3242 /* Don't know about heartbeat or window errors... */3243 3244 return nstat;3245}3246 3247static int macb_get_regs_len(struct net_device *netdev)3248{3249 return MACB_GREGS_NBR * sizeof(u32);3250}3251 3252static void macb_get_regs(struct net_device *dev, struct ethtool_regs *regs,3253 void *p)3254{3255 struct macb *bp = netdev_priv(dev);3256 unsigned int tail, head;3257 u32 *regs_buff = p;3258 3259 regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1))3260 | MACB_GREGS_VERSION;3261 3262 tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail);3263 head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head);3264 3265 regs_buff[0] = macb_readl(bp, NCR);3266 regs_buff[1] = macb_or_gem_readl(bp, NCFGR);3267 regs_buff[2] = macb_readl(bp, NSR);3268 regs_buff[3] = macb_readl(bp, TSR);3269 regs_buff[4] = macb_readl(bp, RBQP);3270 regs_buff[5] = macb_readl(bp, TBQP);3271 regs_buff[6] = macb_readl(bp, RSR);3272 regs_buff[7] = macb_readl(bp, IMR);3273 3274 regs_buff[8] = tail;3275 regs_buff[9] = head;3276 regs_buff[10] = macb_tx_dma(&bp->queues[0], tail);3277 regs_buff[11] = macb_tx_dma(&bp->queues[0], head);3278 3279 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))3280 regs_buff[12] = macb_or_gem_readl(bp, USRIO);3281 if (macb_is_gem(bp))3282 regs_buff[13] = gem_readl(bp, DMACFG);3283}3284 3285static void macb_get_wol(struct net_device *netdev, struct ethtool_wolinfo *wol)3286{3287 struct macb *bp = netdev_priv(netdev);3288 3289 phylink_ethtool_get_wol(bp->phylink, wol);3290 wol->supported |= (WAKE_MAGIC | WAKE_ARP);3291 3292 /* Add macb wolopts to phy wolopts */3293 wol->wolopts |= bp->wolopts;3294}3295 3296static int macb_set_wol(struct net_device *netdev, struct ethtool_wolinfo *wol)3297{3298 struct macb *bp = netdev_priv(netdev);3299 int ret;3300 3301 /* Pass the order to phylink layer */3302 ret = phylink_ethtool_set_wol(bp->phylink, wol);3303 /* Don't manage WoL on MAC, if PHY set_wol() fails */3304 if (ret && ret != -EOPNOTSUPP)3305 return ret;3306 3307 bp->wolopts = (wol->wolopts & WAKE_MAGIC) ? WAKE_MAGIC : 0;3308 bp->wolopts |= (wol->wolopts & WAKE_ARP) ? WAKE_ARP : 0;3309 bp->wol = (wol->wolopts) ? MACB_WOL_ENABLED : 0;3310 3311 device_set_wakeup_enable(&bp->pdev->dev, bp->wol);3312 3313 return 0;3314}3315 3316static int macb_get_link_ksettings(struct net_device *netdev,3317 struct ethtool_link_ksettings *kset)3318{3319 struct macb *bp = netdev_priv(netdev);3320 3321 return phylink_ethtool_ksettings_get(bp->phylink, kset);3322}3323 3324static int macb_set_link_ksettings(struct net_device *netdev,3325 const struct ethtool_link_ksettings *kset)3326{3327 struct macb *bp = netdev_priv(netdev);3328 3329 return phylink_ethtool_ksettings_set(bp->phylink, kset);3330}3331 3332static void macb_get_ringparam(struct net_device *netdev,3333 struct ethtool_ringparam *ring,3334 struct kernel_ethtool_ringparam *kernel_ring,3335 struct netlink_ext_ack *extack)3336{3337 struct macb *bp = netdev_priv(netdev);3338 3339 ring->rx_max_pending = MAX_RX_RING_SIZE;3340 ring->tx_max_pending = MAX_TX_RING_SIZE;3341 3342 ring->rx_pending = bp->rx_ring_size;3343 ring->tx_pending = bp->tx_ring_size;3344}3345 3346static int macb_set_ringparam(struct net_device *netdev,3347 struct ethtool_ringparam *ring,3348 struct kernel_ethtool_ringparam *kernel_ring,3349 struct netlink_ext_ack *extack)3350{3351 struct macb *bp = netdev_priv(netdev);3352 u32 new_rx_size, new_tx_size;3353 unsigned int reset = 0;3354 3355 if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending))3356 return -EINVAL;3357 3358 new_rx_size = clamp_t(u32, ring->rx_pending,3359 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE);3360 new_rx_size = roundup_pow_of_two(new_rx_size);3361 3362 new_tx_size = clamp_t(u32, ring->tx_pending,3363 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE);3364 new_tx_size = roundup_pow_of_two(new_tx_size);3365 3366 if ((new_tx_size == bp->tx_ring_size) &&3367 (new_rx_size == bp->rx_ring_size)) {3368 /* nothing to do */3369 return 0;3370 }3371 3372 if (netif_running(bp->dev)) {3373 reset = 1;3374 macb_close(bp->dev);3375 }3376 3377 bp->rx_ring_size = new_rx_size;3378 bp->tx_ring_size = new_tx_size;3379 3380 if (reset)3381 macb_open(bp->dev);3382 3383 return 0;3384}3385 3386#ifdef CONFIG_MACB_USE_HWSTAMP3387static unsigned int gem_get_tsu_rate(struct macb *bp)3388{3389 struct clk *tsu_clk;3390 unsigned int tsu_rate;3391 3392 tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk");3393 if (!IS_ERR(tsu_clk))3394 tsu_rate = clk_get_rate(tsu_clk);3395 /* try pclk instead */3396 else if (!IS_ERR(bp->pclk)) {3397 tsu_clk = bp->pclk;3398 tsu_rate = clk_get_rate(tsu_clk);3399 } else3400 return -ENOTSUPP;3401 return tsu_rate;3402}3403 3404static s32 gem_get_ptp_max_adj(void)3405{3406 return 64000000;3407}3408 3409static int gem_get_ts_info(struct net_device *dev,3410 struct kernel_ethtool_ts_info *info)3411{3412 struct macb *bp = netdev_priv(dev);3413 3414 if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) {3415 ethtool_op_get_ts_info(dev, info);3416 return 0;3417 }3418 3419 info->so_timestamping =3420 SOF_TIMESTAMPING_TX_SOFTWARE |3421 SOF_TIMESTAMPING_TX_HARDWARE |3422 SOF_TIMESTAMPING_RX_HARDWARE |3423 SOF_TIMESTAMPING_RAW_HARDWARE;3424 info->tx_types =3425 (1 << HWTSTAMP_TX_ONESTEP_SYNC) |3426 (1 << HWTSTAMP_TX_OFF) |3427 (1 << HWTSTAMP_TX_ON);3428 info->rx_filters =3429 (1 << HWTSTAMP_FILTER_NONE) |3430 (1 << HWTSTAMP_FILTER_ALL);3431 3432 if (bp->ptp_clock)3433 info->phc_index = ptp_clock_index(bp->ptp_clock);3434 3435 return 0;3436}3437 3438static struct macb_ptp_info gem_ptp_info = {3439 .ptp_init = gem_ptp_init,3440 .ptp_remove = gem_ptp_remove,3441 .get_ptp_max_adj = gem_get_ptp_max_adj,3442 .get_tsu_rate = gem_get_tsu_rate,3443 .get_ts_info = gem_get_ts_info,3444 .get_hwtst = gem_get_hwtst,3445 .set_hwtst = gem_set_hwtst,3446};3447#endif3448 3449static int macb_get_ts_info(struct net_device *netdev,3450 struct kernel_ethtool_ts_info *info)3451{3452 struct macb *bp = netdev_priv(netdev);3453 3454 if (bp->ptp_info)3455 return bp->ptp_info->get_ts_info(netdev, info);3456 3457 return ethtool_op_get_ts_info(netdev, info);3458}3459 3460static void gem_enable_flow_filters(struct macb *bp, bool enable)3461{3462 struct net_device *netdev = bp->dev;3463 struct ethtool_rx_fs_item *item;3464 u32 t2_scr;3465 int num_t2_scr;3466 3467 if (!(netdev->features & NETIF_F_NTUPLE))3468 return;3469 3470 num_t2_scr = GEM_BFEXT(T2SCR, gem_readl(bp, DCFG8));3471 3472 list_for_each_entry(item, &bp->rx_fs_list.list, list) {3473 struct ethtool_rx_flow_spec *fs = &item->fs;3474 struct ethtool_tcpip4_spec *tp4sp_m;3475 3476 if (fs->location >= num_t2_scr)3477 continue;3478 3479 t2_scr = gem_readl_n(bp, SCRT2, fs->location);3480 3481 /* enable/disable screener regs for the flow entry */3482 t2_scr = GEM_BFINS(ETHTEN, enable, t2_scr);3483 3484 /* only enable fields with no masking */3485 tp4sp_m = &(fs->m_u.tcp_ip4_spec);3486 3487 if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF))3488 t2_scr = GEM_BFINS(CMPAEN, 1, t2_scr);3489 else3490 t2_scr = GEM_BFINS(CMPAEN, 0, t2_scr);3491 3492 if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF))3493 t2_scr = GEM_BFINS(CMPBEN, 1, t2_scr);3494 else3495 t2_scr = GEM_BFINS(CMPBEN, 0, t2_scr);3496 3497 if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)))3498 t2_scr = GEM_BFINS(CMPCEN, 1, t2_scr);3499 else3500 t2_scr = GEM_BFINS(CMPCEN, 0, t2_scr);3501 3502 gem_writel_n(bp, SCRT2, fs->location, t2_scr);3503 }3504}3505 3506static void gem_prog_cmp_regs(struct macb *bp, struct ethtool_rx_flow_spec *fs)3507{3508 struct ethtool_tcpip4_spec *tp4sp_v, *tp4sp_m;3509 uint16_t index = fs->location;3510 u32 w0, w1, t2_scr;3511 bool cmp_a = false;3512 bool cmp_b = false;3513 bool cmp_c = false;3514 3515 if (!macb_is_gem(bp))3516 return;3517 3518 tp4sp_v = &(fs->h_u.tcp_ip4_spec);3519 tp4sp_m = &(fs->m_u.tcp_ip4_spec);3520 3521 /* ignore field if any masking set */3522 if (tp4sp_m->ip4src == 0xFFFFFFFF) {3523 /* 1st compare reg - IP source address */3524 w0 = 0;3525 w1 = 0;3526 w0 = tp4sp_v->ip4src;3527 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */3528 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1);3529 w1 = GEM_BFINS(T2OFST, ETYPE_SRCIP_OFFSET, w1);3530 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w0);3531 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4SRC_CMP(index)), w1);3532 cmp_a = true;3533 }3534 3535 /* ignore field if any masking set */3536 if (tp4sp_m->ip4dst == 0xFFFFFFFF) {3537 /* 2nd compare reg - IP destination address */3538 w0 = 0;3539 w1 = 0;3540 w0 = tp4sp_v->ip4dst;3541 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */3542 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_ETYPE, w1);3543 w1 = GEM_BFINS(T2OFST, ETYPE_DSTIP_OFFSET, w1);3544 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_IP4DST_CMP(index)), w0);3545 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_IP4DST_CMP(index)), w1);3546 cmp_b = true;3547 }3548 3549 /* ignore both port fields if masking set in both */3550 if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) {3551 /* 3rd compare reg - source port, destination port */3552 w0 = 0;3553 w1 = 0;3554 w1 = GEM_BFINS(T2CMPOFST, GEM_T2COMPOFST_IPHDR, w1);3555 if (tp4sp_m->psrc == tp4sp_m->pdst) {3556 w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0);3557 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0);3558 w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */3559 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1);3560 } else {3561 /* only one port definition */3562 w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */3563 w0 = GEM_BFINS(T2MASK, 0xFFFF, w0);3564 if (tp4sp_m->psrc == 0xFFFF) { /* src port */3565 w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0);3566 w1 = GEM_BFINS(T2OFST, IPHDR_SRCPORT_OFFSET, w1);3567 } else { /* dst port */3568 w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0);3569 w1 = GEM_BFINS(T2OFST, IPHDR_DSTPORT_OFFSET, w1);3570 }3571 }3572 gem_writel_n(bp, T2CMPW0, T2CMP_OFST(GEM_PORT_CMP(index)), w0);3573 gem_writel_n(bp, T2CMPW1, T2CMP_OFST(GEM_PORT_CMP(index)), w1);3574 cmp_c = true;3575 }3576 3577 t2_scr = 0;3578 t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr);3579 t2_scr = GEM_BFINS(ETHT2IDX, SCRT2_ETHT, t2_scr);3580 if (cmp_a)3581 t2_scr = GEM_BFINS(CMPA, GEM_IP4SRC_CMP(index), t2_scr);3582 if (cmp_b)3583 t2_scr = GEM_BFINS(CMPB, GEM_IP4DST_CMP(index), t2_scr);3584 if (cmp_c)3585 t2_scr = GEM_BFINS(CMPC, GEM_PORT_CMP(index), t2_scr);3586 gem_writel_n(bp, SCRT2, index, t2_scr);3587}3588 3589static int gem_add_flow_filter(struct net_device *netdev,3590 struct ethtool_rxnfc *cmd)3591{3592 struct macb *bp = netdev_priv(netdev);3593 struct ethtool_rx_flow_spec *fs = &cmd->fs;3594 struct ethtool_rx_fs_item *item, *newfs;3595 unsigned long flags;3596 int ret = -EINVAL;3597 bool added = false;3598 3599 newfs = kmalloc(sizeof(*newfs), GFP_KERNEL);3600 if (newfs == NULL)3601 return -ENOMEM;3602 memcpy(&newfs->fs, fs, sizeof(newfs->fs));3603 3604 netdev_dbg(netdev,3605 "Adding flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n",3606 fs->flow_type, (int)fs->ring_cookie, fs->location,3607 htonl(fs->h_u.tcp_ip4_spec.ip4src),3608 htonl(fs->h_u.tcp_ip4_spec.ip4dst),3609 be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc),3610 be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst));3611 3612 spin_lock_irqsave(&bp->rx_fs_lock, flags);3613 3614 /* find correct place to add in list */3615 list_for_each_entry(item, &bp->rx_fs_list.list, list) {3616 if (item->fs.location > newfs->fs.location) {3617 list_add_tail(&newfs->list, &item->list);3618 added = true;3619 break;3620 } else if (item->fs.location == fs->location) {3621 netdev_err(netdev, "Rule not added: location %d not free!\n",3622 fs->location);3623 ret = -EBUSY;3624 goto err;3625 }3626 }3627 if (!added)3628 list_add_tail(&newfs->list, &bp->rx_fs_list.list);3629 3630 gem_prog_cmp_regs(bp, fs);3631 bp->rx_fs_list.count++;3632 /* enable filtering if NTUPLE on */3633 gem_enable_flow_filters(bp, 1);3634 3635 spin_unlock_irqrestore(&bp->rx_fs_lock, flags);3636 return 0;3637 3638err:3639 spin_unlock_irqrestore(&bp->rx_fs_lock, flags);3640 kfree(newfs);3641 return ret;3642}3643 3644static int gem_del_flow_filter(struct net_device *netdev,3645 struct ethtool_rxnfc *cmd)3646{3647 struct macb *bp = netdev_priv(netdev);3648 struct ethtool_rx_fs_item *item;3649 struct ethtool_rx_flow_spec *fs;3650 unsigned long flags;3651 3652 spin_lock_irqsave(&bp->rx_fs_lock, flags);3653 3654 list_for_each_entry(item, &bp->rx_fs_list.list, list) {3655 if (item->fs.location == cmd->fs.location) {3656 /* disable screener regs for the flow entry */3657 fs = &(item->fs);3658 netdev_dbg(netdev,3659 "Deleting flow filter entry,type=%u,queue=%u,loc=%u,src=%08X,dst=%08X,ps=%u,pd=%u\n",3660 fs->flow_type, (int)fs->ring_cookie, fs->location,3661 htonl(fs->h_u.tcp_ip4_spec.ip4src),3662 htonl(fs->h_u.tcp_ip4_spec.ip4dst),3663 be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc),3664 be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst));3665 3666 gem_writel_n(bp, SCRT2, fs->location, 0);3667 3668 list_del(&item->list);3669 bp->rx_fs_list.count--;3670 spin_unlock_irqrestore(&bp->rx_fs_lock, flags);3671 kfree(item);3672 return 0;3673 }3674 }3675 3676 spin_unlock_irqrestore(&bp->rx_fs_lock, flags);3677 return -EINVAL;3678}3679 3680static int gem_get_flow_entry(struct net_device *netdev,3681 struct ethtool_rxnfc *cmd)3682{3683 struct macb *bp = netdev_priv(netdev);3684 struct ethtool_rx_fs_item *item;3685 3686 list_for_each_entry(item, &bp->rx_fs_list.list, list) {3687 if (item->fs.location == cmd->fs.location) {3688 memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs));3689 return 0;3690 }3691 }3692 return -EINVAL;3693}3694 3695static int gem_get_all_flow_entries(struct net_device *netdev,3696 struct ethtool_rxnfc *cmd, u32 *rule_locs)3697{3698 struct macb *bp = netdev_priv(netdev);3699 struct ethtool_rx_fs_item *item;3700 uint32_t cnt = 0;3701 3702 list_for_each_entry(item, &bp->rx_fs_list.list, list) {3703 if (cnt == cmd->rule_cnt)3704 return -EMSGSIZE;3705 rule_locs[cnt] = item->fs.location;3706 cnt++;3707 }3708 cmd->data = bp->max_tuples;3709 cmd->rule_cnt = cnt;3710 3711 return 0;3712}3713 3714static int gem_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd,3715 u32 *rule_locs)3716{3717 struct macb *bp = netdev_priv(netdev);3718 int ret = 0;3719 3720 switch (cmd->cmd) {3721 case ETHTOOL_GRXRINGS:3722 cmd->data = bp->num_queues;3723 break;3724 case ETHTOOL_GRXCLSRLCNT:3725 cmd->rule_cnt = bp->rx_fs_list.count;3726 break;3727 case ETHTOOL_GRXCLSRULE:3728 ret = gem_get_flow_entry(netdev, cmd);3729 break;3730 case ETHTOOL_GRXCLSRLALL:3731 ret = gem_get_all_flow_entries(netdev, cmd, rule_locs);3732 break;3733 default:3734 netdev_err(netdev,3735 "Command parameter %d is not supported\n", cmd->cmd);3736 ret = -EOPNOTSUPP;3737 }3738 3739 return ret;3740}3741 3742static int gem_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd)3743{3744 struct macb *bp = netdev_priv(netdev);3745 int ret;3746 3747 switch (cmd->cmd) {3748 case ETHTOOL_SRXCLSRLINS:3749 if ((cmd->fs.location >= bp->max_tuples)3750 || (cmd->fs.ring_cookie >= bp->num_queues)) {3751 ret = -EINVAL;3752 break;3753 }3754 ret = gem_add_flow_filter(netdev, cmd);3755 break;3756 case ETHTOOL_SRXCLSRLDEL:3757 ret = gem_del_flow_filter(netdev, cmd);3758 break;3759 default:3760 netdev_err(netdev,3761 "Command parameter %d is not supported\n", cmd->cmd);3762 ret = -EOPNOTSUPP;3763 }3764 3765 return ret;3766}3767 3768static const struct ethtool_ops macb_ethtool_ops = {3769 .get_regs_len = macb_get_regs_len,3770 .get_regs = macb_get_regs,3771 .get_link = ethtool_op_get_link,3772 .get_ts_info = ethtool_op_get_ts_info,3773 .get_wol = macb_get_wol,3774 .set_wol = macb_set_wol,3775 .get_link_ksettings = macb_get_link_ksettings,3776 .set_link_ksettings = macb_set_link_ksettings,3777 .get_ringparam = macb_get_ringparam,3778 .set_ringparam = macb_set_ringparam,3779};3780 3781static const struct ethtool_ops gem_ethtool_ops = {3782 .get_regs_len = macb_get_regs_len,3783 .get_regs = macb_get_regs,3784 .get_wol = macb_get_wol,3785 .set_wol = macb_set_wol,3786 .get_link = ethtool_op_get_link,3787 .get_ts_info = macb_get_ts_info,3788 .get_ethtool_stats = gem_get_ethtool_stats,3789 .get_strings = gem_get_ethtool_strings,3790 .get_sset_count = gem_get_sset_count,3791 .get_link_ksettings = macb_get_link_ksettings,3792 .set_link_ksettings = macb_set_link_ksettings,3793 .get_ringparam = macb_get_ringparam,3794 .set_ringparam = macb_set_ringparam,3795 .get_rxnfc = gem_get_rxnfc,3796 .set_rxnfc = gem_set_rxnfc,3797};3798 3799static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)3800{3801 struct macb *bp = netdev_priv(dev);3802 3803 if (!netif_running(dev))3804 return -EINVAL;3805 3806 return phylink_mii_ioctl(bp->phylink, rq, cmd);3807}3808 3809static int macb_hwtstamp_get(struct net_device *dev,3810 struct kernel_hwtstamp_config *cfg)3811{3812 struct macb *bp = netdev_priv(dev);3813 3814 if (!netif_running(dev))3815 return -EINVAL;3816 3817 if (!bp->ptp_info)3818 return -EOPNOTSUPP;3819 3820 return bp->ptp_info->get_hwtst(dev, cfg);3821}3822 3823static int macb_hwtstamp_set(struct net_device *dev,3824 struct kernel_hwtstamp_config *cfg,3825 struct netlink_ext_ack *extack)3826{3827 struct macb *bp = netdev_priv(dev);3828 3829 if (!netif_running(dev))3830 return -EINVAL;3831 3832 if (!bp->ptp_info)3833 return -EOPNOTSUPP;3834 3835 return bp->ptp_info->set_hwtst(dev, cfg, extack);3836}3837 3838static inline void macb_set_txcsum_feature(struct macb *bp,3839 netdev_features_t features)3840{3841 u32 val;3842 3843 if (!macb_is_gem(bp))3844 return;3845 3846 val = gem_readl(bp, DMACFG);3847 if (features & NETIF_F_HW_CSUM)3848 val |= GEM_BIT(TXCOEN);3849 else3850 val &= ~GEM_BIT(TXCOEN);3851 3852 gem_writel(bp, DMACFG, val);3853}3854 3855static inline void macb_set_rxcsum_feature(struct macb *bp,3856 netdev_features_t features)3857{3858 struct net_device *netdev = bp->dev;3859 u32 val;3860 3861 if (!macb_is_gem(bp))3862 return;3863 3864 val = gem_readl(bp, NCFGR);3865 if ((features & NETIF_F_RXCSUM) && !(netdev->flags & IFF_PROMISC))3866 val |= GEM_BIT(RXCOEN);3867 else3868 val &= ~GEM_BIT(RXCOEN);3869 3870 gem_writel(bp, NCFGR, val);3871}3872 3873static inline void macb_set_rxflow_feature(struct macb *bp,3874 netdev_features_t features)3875{3876 if (!macb_is_gem(bp))3877 return;3878 3879 gem_enable_flow_filters(bp, !!(features & NETIF_F_NTUPLE));3880}3881 3882static int macb_set_features(struct net_device *netdev,3883 netdev_features_t features)3884{3885 struct macb *bp = netdev_priv(netdev);3886 netdev_features_t changed = features ^ netdev->features;3887 3888 /* TX checksum offload */3889 if (changed & NETIF_F_HW_CSUM)3890 macb_set_txcsum_feature(bp, features);3891 3892 /* RX checksum offload */3893 if (changed & NETIF_F_RXCSUM)3894 macb_set_rxcsum_feature(bp, features);3895 3896 /* RX Flow Filters */3897 if (changed & NETIF_F_NTUPLE)3898 macb_set_rxflow_feature(bp, features);3899 3900 return 0;3901}3902 3903static void macb_restore_features(struct macb *bp)3904{3905 struct net_device *netdev = bp->dev;3906 netdev_features_t features = netdev->features;3907 struct ethtool_rx_fs_item *item;3908 3909 /* TX checksum offload */3910 macb_set_txcsum_feature(bp, features);3911 3912 /* RX checksum offload */3913 macb_set_rxcsum_feature(bp, features);3914 3915 /* RX Flow Filters */3916 list_for_each_entry(item, &bp->rx_fs_list.list, list)3917 gem_prog_cmp_regs(bp, &item->fs);3918 3919 macb_set_rxflow_feature(bp, features);3920}3921 3922static const struct net_device_ops macb_netdev_ops = {3923 .ndo_open = macb_open,3924 .ndo_stop = macb_close,3925 .ndo_start_xmit = macb_start_xmit,3926 .ndo_set_rx_mode = macb_set_rx_mode,3927 .ndo_get_stats = macb_get_stats,3928 .ndo_eth_ioctl = macb_ioctl,3929 .ndo_validate_addr = eth_validate_addr,3930 .ndo_change_mtu = macb_change_mtu,3931 .ndo_set_mac_address = macb_set_mac_addr,3932#ifdef CONFIG_NET_POLL_CONTROLLER3933 .ndo_poll_controller = macb_poll_controller,3934#endif3935 .ndo_set_features = macb_set_features,3936 .ndo_features_check = macb_features_check,3937 .ndo_hwtstamp_set = macb_hwtstamp_set,3938 .ndo_hwtstamp_get = macb_hwtstamp_get,3939};3940 3941/* Configure peripheral capabilities according to device tree3942 * and integration options used3943 */3944static void macb_configure_caps(struct macb *bp,3945 const struct macb_config *dt_conf)3946{3947 u32 dcfg;3948 3949 if (dt_conf)3950 bp->caps = dt_conf->caps;3951 3952 if (hw_is_gem(bp->regs, bp->native_io)) {3953 bp->caps |= MACB_CAPS_MACB_IS_GEM;3954 3955 dcfg = gem_readl(bp, DCFG1);3956 if (GEM_BFEXT(IRQCOR, dcfg) == 0)3957 bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE;3958 if (GEM_BFEXT(NO_PCS, dcfg) == 0)3959 bp->caps |= MACB_CAPS_PCS;3960 dcfg = gem_readl(bp, DCFG12);3961 if (GEM_BFEXT(HIGH_SPEED, dcfg) == 1)3962 bp->caps |= MACB_CAPS_HIGH_SPEED;3963 dcfg = gem_readl(bp, DCFG2);3964 if ((dcfg & (GEM_BIT(RX_PKT_BUFF) | GEM_BIT(TX_PKT_BUFF))) == 0)3965 bp->caps |= MACB_CAPS_FIFO_MODE;3966 if (gem_has_ptp(bp)) {3967 if (!GEM_BFEXT(TSU, gem_readl(bp, DCFG5)))3968 dev_err(&bp->pdev->dev,3969 "GEM doesn't support hardware ptp.\n");3970 else {3971#ifdef CONFIG_MACB_USE_HWSTAMP3972 bp->hw_dma_cap |= HW_DMA_CAP_PTP;3973 bp->ptp_info = &gem_ptp_info;3974#endif3975 }3976 }3977 }3978 3979 dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps);3980}3981 3982static void macb_probe_queues(void __iomem *mem,3983 bool native_io,3984 unsigned int *queue_mask,3985 unsigned int *num_queues)3986{3987 *queue_mask = 0x1;3988 *num_queues = 1;3989 3990 /* is it macb or gem ?3991 *3992 * We need to read directly from the hardware here because3993 * we are early in the probe process and don't have the3994 * MACB_CAPS_MACB_IS_GEM flag positioned3995 */3996 if (!hw_is_gem(mem, native_io))3997 return;3998 3999 /* bit 0 is never set but queue 0 always exists */4000 *queue_mask |= readl_relaxed(mem + GEM_DCFG6) & 0xff;4001 *num_queues = hweight32(*queue_mask);4002}4003 4004static void macb_clks_disable(struct clk *pclk, struct clk *hclk, struct clk *tx_clk,4005 struct clk *rx_clk, struct clk *tsu_clk)4006{4007 struct clk_bulk_data clks[] = {4008 { .clk = tsu_clk, },4009 { .clk = rx_clk, },4010 { .clk = pclk, },4011 { .clk = hclk, },4012 { .clk = tx_clk },4013 };4014 4015 clk_bulk_disable_unprepare(ARRAY_SIZE(clks), clks);4016}4017 4018static int macb_clk_init(struct platform_device *pdev, struct clk **pclk,4019 struct clk **hclk, struct clk **tx_clk,4020 struct clk **rx_clk, struct clk **tsu_clk)4021{4022 struct macb_platform_data *pdata;4023 int err;4024 4025 pdata = dev_get_platdata(&pdev->dev);4026 if (pdata) {4027 *pclk = pdata->pclk;4028 *hclk = pdata->hclk;4029 } else {4030 *pclk = devm_clk_get(&pdev->dev, "pclk");4031 *hclk = devm_clk_get(&pdev->dev, "hclk");4032 }4033 4034 if (IS_ERR_OR_NULL(*pclk))4035 return dev_err_probe(&pdev->dev,4036 IS_ERR(*pclk) ? PTR_ERR(*pclk) : -ENODEV,4037 "failed to get pclk\n");4038 4039 if (IS_ERR_OR_NULL(*hclk))4040 return dev_err_probe(&pdev->dev,4041 IS_ERR(*hclk) ? PTR_ERR(*hclk) : -ENODEV,4042 "failed to get hclk\n");4043 4044 *tx_clk = devm_clk_get_optional(&pdev->dev, "tx_clk");4045 if (IS_ERR(*tx_clk))4046 return PTR_ERR(*tx_clk);4047 4048 *rx_clk = devm_clk_get_optional(&pdev->dev, "rx_clk");4049 if (IS_ERR(*rx_clk))4050 return PTR_ERR(*rx_clk);4051 4052 *tsu_clk = devm_clk_get_optional(&pdev->dev, "tsu_clk");4053 if (IS_ERR(*tsu_clk))4054 return PTR_ERR(*tsu_clk);4055 4056 err = clk_prepare_enable(*pclk);4057 if (err) {4058 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err);4059 return err;4060 }4061 4062 err = clk_prepare_enable(*hclk);4063 if (err) {4064 dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err);4065 goto err_disable_pclk;4066 }4067 4068 err = clk_prepare_enable(*tx_clk);4069 if (err) {4070 dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err);4071 goto err_disable_hclk;4072 }4073 4074 err = clk_prepare_enable(*rx_clk);4075 if (err) {4076 dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err);4077 goto err_disable_txclk;4078 }4079 4080 err = clk_prepare_enable(*tsu_clk);4081 if (err) {4082 dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err);4083 goto err_disable_rxclk;4084 }4085 4086 return 0;4087 4088err_disable_rxclk:4089 clk_disable_unprepare(*rx_clk);4090 4091err_disable_txclk:4092 clk_disable_unprepare(*tx_clk);4093 4094err_disable_hclk:4095 clk_disable_unprepare(*hclk);4096 4097err_disable_pclk:4098 clk_disable_unprepare(*pclk);4099 4100 return err;4101}4102 4103static int macb_init(struct platform_device *pdev)4104{4105 struct net_device *dev = platform_get_drvdata(pdev);4106 unsigned int hw_q, q;4107 struct macb *bp = netdev_priv(dev);4108 struct macb_queue *queue;4109 int err;4110 u32 val, reg;4111 4112 bp->tx_ring_size = DEFAULT_TX_RING_SIZE;4113 bp->rx_ring_size = DEFAULT_RX_RING_SIZE;4114 4115 /* set the queue register mapping once for all: queue0 has a special4116 * register mapping but we don't want to test the queue index then4117 * compute the corresponding register offset at run time.4118 */4119 for (hw_q = 0, q = 0; hw_q < MACB_MAX_QUEUES; ++hw_q) {4120 if (!(bp->queue_mask & (1 << hw_q)))4121 continue;4122 4123 queue = &bp->queues[q];4124 queue->bp = bp;4125 spin_lock_init(&queue->tx_ptr_lock);4126 netif_napi_add(dev, &queue->napi_rx, macb_rx_poll);4127 netif_napi_add(dev, &queue->napi_tx, macb_tx_poll);4128 if (hw_q) {4129 queue->ISR = GEM_ISR(hw_q - 1);4130 queue->IER = GEM_IER(hw_q - 1);4131 queue->IDR = GEM_IDR(hw_q - 1);4132 queue->IMR = GEM_IMR(hw_q - 1);4133 queue->TBQP = GEM_TBQP(hw_q - 1);4134 queue->RBQP = GEM_RBQP(hw_q - 1);4135 queue->RBQS = GEM_RBQS(hw_q - 1);4136#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT4137 if (bp->hw_dma_cap & HW_DMA_CAP_64B) {4138 queue->TBQPH = GEM_TBQPH(hw_q - 1);4139 queue->RBQPH = GEM_RBQPH(hw_q - 1);4140 }4141#endif4142 } else {4143 /* queue0 uses legacy registers */4144 queue->ISR = MACB_ISR;4145 queue->IER = MACB_IER;4146 queue->IDR = MACB_IDR;4147 queue->IMR = MACB_IMR;4148 queue->TBQP = MACB_TBQP;4149 queue->RBQP = MACB_RBQP;4150#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT4151 if (bp->hw_dma_cap & HW_DMA_CAP_64B) {4152 queue->TBQPH = MACB_TBQPH;4153 queue->RBQPH = MACB_RBQPH;4154 }4155#endif4156 }4157 4158 /* get irq: here we use the linux queue index, not the hardware4159 * queue index. the queue irq definitions in the device tree4160 * must remove the optional gaps that could exist in the4161 * hardware queue mask.4162 */4163 queue->irq = platform_get_irq(pdev, q);4164 err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt,4165 IRQF_SHARED, dev->name, queue);4166 if (err) {4167 dev_err(&pdev->dev,4168 "Unable to request IRQ %d (error %d)\n",4169 queue->irq, err);4170 return err;4171 }4172 4173 INIT_WORK(&queue->tx_error_task, macb_tx_error_task);4174 q++;4175 }4176 4177 dev->netdev_ops = &macb_netdev_ops;4178 4179 /* setup appropriated routines according to adapter type */4180 if (macb_is_gem(bp)) {4181 bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers;4182 bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers;4183 bp->macbgem_ops.mog_init_rings = gem_init_rings;4184 bp->macbgem_ops.mog_rx = gem_rx;4185 dev->ethtool_ops = &gem_ethtool_ops;4186 } else {4187 bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers;4188 bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers;4189 bp->macbgem_ops.mog_init_rings = macb_init_rings;4190 bp->macbgem_ops.mog_rx = macb_rx;4191 dev->ethtool_ops = &macb_ethtool_ops;4192 }4193 4194 netdev_sw_irq_coalesce_default_on(dev);4195 4196 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;4197 4198 /* Set features */4199 dev->hw_features = NETIF_F_SG;4200 4201 /* Check LSO capability */4202 if (GEM_BFEXT(PBUF_LSO, gem_readl(bp, DCFG6)))4203 dev->hw_features |= MACB_NETIF_LSO;4204 4205 /* Checksum offload is only available on gem with packet buffer */4206 if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE))4207 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM;4208 if (bp->caps & MACB_CAPS_SG_DISABLED)4209 dev->hw_features &= ~NETIF_F_SG;4210 dev->features = dev->hw_features;4211 4212 /* Check RX Flow Filters support.4213 * Max Rx flows set by availability of screeners & compare regs:4214 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs4215 */4216 reg = gem_readl(bp, DCFG8);4217 bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3),4218 GEM_BFEXT(T2SCR, reg));4219 INIT_LIST_HEAD(&bp->rx_fs_list.list);4220 if (bp->max_tuples > 0) {4221 /* also needs one ethtype match to check IPv4 */4222 if (GEM_BFEXT(SCR2ETH, reg) > 0) {4223 /* program this reg now */4224 reg = 0;4225 reg = GEM_BFINS(ETHTCMP, (uint16_t)ETH_P_IP, reg);4226 gem_writel_n(bp, ETHT, SCRT2_ETHT, reg);4227 /* Filtering is supported in hw but don't enable it in kernel now */4228 dev->hw_features |= NETIF_F_NTUPLE;4229 /* init Rx flow definitions */4230 bp->rx_fs_list.count = 0;4231 spin_lock_init(&bp->rx_fs_lock);4232 } else4233 bp->max_tuples = 0;4234 }4235 4236 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) {4237 val = 0;4238 if (phy_interface_mode_is_rgmii(bp->phy_interface))4239 val = bp->usrio->rgmii;4240 else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII &&4241 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII))4242 val = bp->usrio->rmii;4243 else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII))4244 val = bp->usrio->mii;4245 4246 if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN)4247 val |= bp->usrio->refclk;4248 4249 macb_or_gem_writel(bp, USRIO, val);4250 }4251 4252 /* Set MII management clock divider */4253 val = macb_mdc_clk_div(bp);4254 val |= macb_dbw(bp);4255 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII)4256 val |= GEM_BIT(SGMIIEN) | GEM_BIT(PCSSEL);4257 macb_writel(bp, NCFGR, val);4258 4259 return 0;4260}4261 4262static const struct macb_usrio_config macb_default_usrio = {4263 .mii = MACB_BIT(MII),4264 .rmii = MACB_BIT(RMII),4265 .rgmii = GEM_BIT(RGMII),4266 .refclk = MACB_BIT(CLKEN),4267};4268 4269#if defined(CONFIG_OF)4270/* 1518 rounded up */4271#define AT91ETHER_MAX_RBUFF_SZ 0x6004272/* max number of receive buffers */4273#define AT91ETHER_MAX_RX_DESCR 94274 4275static struct sifive_fu540_macb_mgmt *mgmt;4276 4277static int at91ether_alloc_coherent(struct macb *lp)4278{4279 struct macb_queue *q = &lp->queues[0];4280 4281 q->rx_ring = dma_alloc_coherent(&lp->pdev->dev,4282 (AT91ETHER_MAX_RX_DESCR *4283 macb_dma_desc_get_size(lp)),4284 &q->rx_ring_dma, GFP_KERNEL);4285 if (!q->rx_ring)4286 return -ENOMEM;4287 4288 q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev,4289 AT91ETHER_MAX_RX_DESCR *4290 AT91ETHER_MAX_RBUFF_SZ,4291 &q->rx_buffers_dma, GFP_KERNEL);4292 if (!q->rx_buffers) {4293 dma_free_coherent(&lp->pdev->dev,4294 AT91ETHER_MAX_RX_DESCR *4295 macb_dma_desc_get_size(lp),4296 q->rx_ring, q->rx_ring_dma);4297 q->rx_ring = NULL;4298 return -ENOMEM;4299 }4300 4301 return 0;4302}4303 4304static void at91ether_free_coherent(struct macb *lp)4305{4306 struct macb_queue *q = &lp->queues[0];4307 4308 if (q->rx_ring) {4309 dma_free_coherent(&lp->pdev->dev,4310 AT91ETHER_MAX_RX_DESCR *4311 macb_dma_desc_get_size(lp),4312 q->rx_ring, q->rx_ring_dma);4313 q->rx_ring = NULL;4314 }4315 4316 if (q->rx_buffers) {4317 dma_free_coherent(&lp->pdev->dev,4318 AT91ETHER_MAX_RX_DESCR *4319 AT91ETHER_MAX_RBUFF_SZ,4320 q->rx_buffers, q->rx_buffers_dma);4321 q->rx_buffers = NULL;4322 }4323}4324 4325/* Initialize and start the Receiver and Transmit subsystems */4326static int at91ether_start(struct macb *lp)4327{4328 struct macb_queue *q = &lp->queues[0];4329 struct macb_dma_desc *desc;4330 dma_addr_t addr;4331 u32 ctl;4332 int i, ret;4333 4334 ret = at91ether_alloc_coherent(lp);4335 if (ret)4336 return ret;4337 4338 addr = q->rx_buffers_dma;4339 for (i = 0; i < AT91ETHER_MAX_RX_DESCR; i++) {4340 desc = macb_rx_desc(q, i);4341 macb_set_addr(lp, desc, addr);4342 desc->ctrl = 0;4343 addr += AT91ETHER_MAX_RBUFF_SZ;4344 }4345 4346 /* Set the Wrap bit on the last descriptor */4347 desc->addr |= MACB_BIT(RX_WRAP);4348 4349 /* Reset buffer index */4350 q->rx_tail = 0;4351 4352 /* Program address of descriptor list in Rx Buffer Queue register */4353 macb_writel(lp, RBQP, q->rx_ring_dma);4354 4355 /* Enable Receive and Transmit */4356 ctl = macb_readl(lp, NCR);4357 macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE));4358 4359 /* Enable MAC interrupts */4360 macb_writel(lp, IER, MACB_BIT(RCOMP) |4361 MACB_BIT(RXUBR) |4362 MACB_BIT(ISR_TUND) |4363 MACB_BIT(ISR_RLE) |4364 MACB_BIT(TCOMP) |4365 MACB_BIT(ISR_ROVR) |4366 MACB_BIT(HRESP));4367 4368 return 0;4369}4370 4371static void at91ether_stop(struct macb *lp)4372{4373 u32 ctl;4374 4375 /* Disable MAC interrupts */4376 macb_writel(lp, IDR, MACB_BIT(RCOMP) |4377 MACB_BIT(RXUBR) |4378 MACB_BIT(ISR_TUND) |4379 MACB_BIT(ISR_RLE) |4380 MACB_BIT(TCOMP) |4381 MACB_BIT(ISR_ROVR) |4382 MACB_BIT(HRESP));4383 4384 /* Disable Receiver and Transmitter */4385 ctl = macb_readl(lp, NCR);4386 macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE)));4387 4388 /* Free resources. */4389 at91ether_free_coherent(lp);4390}4391 4392/* Open the ethernet interface */4393static int at91ether_open(struct net_device *dev)4394{4395 struct macb *lp = netdev_priv(dev);4396 u32 ctl;4397 int ret;4398 4399 ret = pm_runtime_resume_and_get(&lp->pdev->dev);4400 if (ret < 0)4401 return ret;4402 4403 /* Clear internal statistics */4404 ctl = macb_readl(lp, NCR);4405 macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT));4406 4407 macb_set_hwaddr(lp);4408 4409 ret = at91ether_start(lp);4410 if (ret)4411 goto pm_exit;4412 4413 ret = macb_phylink_connect(lp);4414 if (ret)4415 goto stop;4416 4417 netif_start_queue(dev);4418 4419 return 0;4420 4421stop:4422 at91ether_stop(lp);4423pm_exit:4424 pm_runtime_put_sync(&lp->pdev->dev);4425 return ret;4426}4427 4428/* Close the interface */4429static int at91ether_close(struct net_device *dev)4430{4431 struct macb *lp = netdev_priv(dev);4432 4433 netif_stop_queue(dev);4434 4435 phylink_stop(lp->phylink);4436 phylink_disconnect_phy(lp->phylink);4437 4438 at91ether_stop(lp);4439 4440 return pm_runtime_put(&lp->pdev->dev);4441}4442 4443/* Transmit packet */4444static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb,4445 struct net_device *dev)4446{4447 struct macb *lp = netdev_priv(dev);4448 4449 if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) {4450 int desc = 0;4451 4452 netif_stop_queue(dev);4453 4454 /* Store packet information (to free when Tx completed) */4455 lp->rm9200_txq[desc].skb = skb;4456 lp->rm9200_txq[desc].size = skb->len;4457 lp->rm9200_txq[desc].mapping = dma_map_single(&lp->pdev->dev, skb->data,4458 skb->len, DMA_TO_DEVICE);4459 if (dma_mapping_error(&lp->pdev->dev, lp->rm9200_txq[desc].mapping)) {4460 dev_kfree_skb_any(skb);4461 dev->stats.tx_dropped++;4462 netdev_err(dev, "%s: DMA mapping error\n", __func__);4463 return NETDEV_TX_OK;4464 }4465 4466 /* Set address of the data in the Transmit Address register */4467 macb_writel(lp, TAR, lp->rm9200_txq[desc].mapping);4468 /* Set length of the packet in the Transmit Control register */4469 macb_writel(lp, TCR, skb->len);4470 4471 } else {4472 netdev_err(dev, "%s called, but device is busy!\n", __func__);4473 return NETDEV_TX_BUSY;4474 }4475 4476 return NETDEV_TX_OK;4477}4478 4479/* Extract received frame from buffer descriptors and sent to upper layers.4480 * (Called from interrupt context)4481 */4482static void at91ether_rx(struct net_device *dev)4483{4484 struct macb *lp = netdev_priv(dev);4485 struct macb_queue *q = &lp->queues[0];4486 struct macb_dma_desc *desc;4487 unsigned char *p_recv;4488 struct sk_buff *skb;4489 unsigned int pktlen;4490 4491 desc = macb_rx_desc(q, q->rx_tail);4492 while (desc->addr & MACB_BIT(RX_USED)) {4493 p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ;4494 pktlen = MACB_BF(RX_FRMLEN, desc->ctrl);4495 skb = netdev_alloc_skb(dev, pktlen + 2);4496 if (skb) {4497 skb_reserve(skb, 2);4498 skb_put_data(skb, p_recv, pktlen);4499 4500 skb->protocol = eth_type_trans(skb, dev);4501 dev->stats.rx_packets++;4502 dev->stats.rx_bytes += pktlen;4503 netif_rx(skb);4504 } else {4505 dev->stats.rx_dropped++;4506 }4507 4508 if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH))4509 dev->stats.multicast++;4510 4511 /* reset ownership bit */4512 desc->addr &= ~MACB_BIT(RX_USED);4513 4514 /* wrap after last buffer */4515 if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1)4516 q->rx_tail = 0;4517 else4518 q->rx_tail++;4519 4520 desc = macb_rx_desc(q, q->rx_tail);4521 }4522}4523 4524/* MAC interrupt handler */4525static irqreturn_t at91ether_interrupt(int irq, void *dev_id)4526{4527 struct net_device *dev = dev_id;4528 struct macb *lp = netdev_priv(dev);4529 u32 intstatus, ctl;4530 unsigned int desc;4531 4532 /* MAC Interrupt Status register indicates what interrupts are pending.4533 * It is automatically cleared once read.4534 */4535 intstatus = macb_readl(lp, ISR);4536 4537 /* Receive complete */4538 if (intstatus & MACB_BIT(RCOMP))4539 at91ether_rx(dev);4540 4541 /* Transmit complete */4542 if (intstatus & MACB_BIT(TCOMP)) {4543 /* The TCOM bit is set even if the transmission failed */4544 if (intstatus & (MACB_BIT(ISR_TUND) | MACB_BIT(ISR_RLE)))4545 dev->stats.tx_errors++;4546 4547 desc = 0;4548 if (lp->rm9200_txq[desc].skb) {4549 dev_consume_skb_irq(lp->rm9200_txq[desc].skb);4550 lp->rm9200_txq[desc].skb = NULL;4551 dma_unmap_single(&lp->pdev->dev, lp->rm9200_txq[desc].mapping,4552 lp->rm9200_txq[desc].size, DMA_TO_DEVICE);4553 dev->stats.tx_packets++;4554 dev->stats.tx_bytes += lp->rm9200_txq[desc].size;4555 }4556 netif_wake_queue(dev);4557 }4558 4559 /* Work-around for EMAC Errata section 41.3.1 */4560 if (intstatus & MACB_BIT(RXUBR)) {4561 ctl = macb_readl(lp, NCR);4562 macb_writel(lp, NCR, ctl & ~MACB_BIT(RE));4563 wmb();4564 macb_writel(lp, NCR, ctl | MACB_BIT(RE));4565 }4566 4567 if (intstatus & MACB_BIT(ISR_ROVR))4568 netdev_err(dev, "ROVR error\n");4569 4570 return IRQ_HANDLED;4571}4572 4573#ifdef CONFIG_NET_POLL_CONTROLLER4574static void at91ether_poll_controller(struct net_device *dev)4575{4576 unsigned long flags;4577 4578 local_irq_save(flags);4579 at91ether_interrupt(dev->irq, dev);4580 local_irq_restore(flags);4581}4582#endif4583 4584static const struct net_device_ops at91ether_netdev_ops = {4585 .ndo_open = at91ether_open,4586 .ndo_stop = at91ether_close,4587 .ndo_start_xmit = at91ether_start_xmit,4588 .ndo_get_stats = macb_get_stats,4589 .ndo_set_rx_mode = macb_set_rx_mode,4590 .ndo_set_mac_address = eth_mac_addr,4591 .ndo_eth_ioctl = macb_ioctl,4592 .ndo_validate_addr = eth_validate_addr,4593#ifdef CONFIG_NET_POLL_CONTROLLER4594 .ndo_poll_controller = at91ether_poll_controller,4595#endif4596 .ndo_hwtstamp_set = macb_hwtstamp_set,4597 .ndo_hwtstamp_get = macb_hwtstamp_get,4598};4599 4600static int at91ether_clk_init(struct platform_device *pdev, struct clk **pclk,4601 struct clk **hclk, struct clk **tx_clk,4602 struct clk **rx_clk, struct clk **tsu_clk)4603{4604 int err;4605 4606 *hclk = NULL;4607 *tx_clk = NULL;4608 *rx_clk = NULL;4609 *tsu_clk = NULL;4610 4611 *pclk = devm_clk_get(&pdev->dev, "ether_clk");4612 if (IS_ERR(*pclk))4613 return PTR_ERR(*pclk);4614 4615 err = clk_prepare_enable(*pclk);4616 if (err) {4617 dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err);4618 return err;4619 }4620 4621 return 0;4622}4623 4624static int at91ether_init(struct platform_device *pdev)4625{4626 struct net_device *dev = platform_get_drvdata(pdev);4627 struct macb *bp = netdev_priv(dev);4628 int err;4629 4630 bp->queues[0].bp = bp;4631 4632 dev->netdev_ops = &at91ether_netdev_ops;4633 dev->ethtool_ops = &macb_ethtool_ops;4634 4635 err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt,4636 0, dev->name, dev);4637 if (err)4638 return err;4639 4640 macb_writel(bp, NCR, 0);4641 4642 macb_writel(bp, NCFGR, MACB_BF(CLK, MACB_CLK_DIV32) | MACB_BIT(BIG));4643 4644 return 0;4645}4646 4647static unsigned long fu540_macb_tx_recalc_rate(struct clk_hw *hw,4648 unsigned long parent_rate)4649{4650 return mgmt->rate;4651}4652 4653static long fu540_macb_tx_round_rate(struct clk_hw *hw, unsigned long rate,4654 unsigned long *parent_rate)4655{4656 if (WARN_ON(rate < 2500000))4657 return 2500000;4658 else if (rate == 2500000)4659 return 2500000;4660 else if (WARN_ON(rate < 13750000))4661 return 2500000;4662 else if (WARN_ON(rate < 25000000))4663 return 25000000;4664 else if (rate == 25000000)4665 return 25000000;4666 else if (WARN_ON(rate < 75000000))4667 return 25000000;4668 else if (WARN_ON(rate < 125000000))4669 return 125000000;4670 else if (rate == 125000000)4671 return 125000000;4672 4673 WARN_ON(rate > 125000000);4674 4675 return 125000000;4676}4677 4678static int fu540_macb_tx_set_rate(struct clk_hw *hw, unsigned long rate,4679 unsigned long parent_rate)4680{4681 rate = fu540_macb_tx_round_rate(hw, rate, &parent_rate);4682 if (rate != 125000000)4683 iowrite32(1, mgmt->reg);4684 else4685 iowrite32(0, mgmt->reg);4686 mgmt->rate = rate;4687 4688 return 0;4689}4690 4691static const struct clk_ops fu540_c000_ops = {4692 .recalc_rate = fu540_macb_tx_recalc_rate,4693 .round_rate = fu540_macb_tx_round_rate,4694 .set_rate = fu540_macb_tx_set_rate,4695};4696 4697static int fu540_c000_clk_init(struct platform_device *pdev, struct clk **pclk,4698 struct clk **hclk, struct clk **tx_clk,4699 struct clk **rx_clk, struct clk **tsu_clk)4700{4701 struct clk_init_data init;4702 int err = 0;4703 4704 err = macb_clk_init(pdev, pclk, hclk, tx_clk, rx_clk, tsu_clk);4705 if (err)4706 return err;4707 4708 mgmt = devm_kzalloc(&pdev->dev, sizeof(*mgmt), GFP_KERNEL);4709 if (!mgmt) {4710 err = -ENOMEM;4711 goto err_disable_clks;4712 }4713 4714 init.name = "sifive-gemgxl-mgmt";4715 init.ops = &fu540_c000_ops;4716 init.flags = 0;4717 init.num_parents = 0;4718 4719 mgmt->rate = 0;4720 mgmt->hw.init = &init;4721 4722 *tx_clk = devm_clk_register(&pdev->dev, &mgmt->hw);4723 if (IS_ERR(*tx_clk)) {4724 err = PTR_ERR(*tx_clk);4725 goto err_disable_clks;4726 }4727 4728 err = clk_prepare_enable(*tx_clk);4729 if (err) {4730 dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err);4731 *tx_clk = NULL;4732 goto err_disable_clks;4733 } else {4734 dev_info(&pdev->dev, "Registered clk switch '%s'\n", init.name);4735 }4736 4737 return 0;4738 4739err_disable_clks:4740 macb_clks_disable(*pclk, *hclk, *tx_clk, *rx_clk, *tsu_clk);4741 4742 return err;4743}4744 4745static int fu540_c000_init(struct platform_device *pdev)4746{4747 mgmt->reg = devm_platform_ioremap_resource(pdev, 1);4748 if (IS_ERR(mgmt->reg))4749 return PTR_ERR(mgmt->reg);4750 4751 return macb_init(pdev);4752}4753 4754static int init_reset_optional(struct platform_device *pdev)4755{4756 struct net_device *dev = platform_get_drvdata(pdev);4757 struct macb *bp = netdev_priv(dev);4758 int ret;4759 4760 if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) {4761 /* Ensure PHY device used in SGMII mode is ready */4762 bp->sgmii_phy = devm_phy_optional_get(&pdev->dev, NULL);4763 4764 if (IS_ERR(bp->sgmii_phy))4765 return dev_err_probe(&pdev->dev, PTR_ERR(bp->sgmii_phy),4766 "failed to get SGMII PHY\n");4767 4768 ret = phy_init(bp->sgmii_phy);4769 if (ret)4770 return dev_err_probe(&pdev->dev, ret,4771 "failed to init SGMII PHY\n");4772 4773 ret = zynqmp_pm_is_function_supported(PM_IOCTL, IOCTL_SET_GEM_CONFIG);4774 if (!ret) {4775 u32 pm_info[2];4776 4777 ret = of_property_read_u32_array(pdev->dev.of_node, "power-domains",4778 pm_info, ARRAY_SIZE(pm_info));4779 if (ret) {4780 dev_err(&pdev->dev, "Failed to read power management information\n");4781 goto err_out_phy_exit;4782 }4783 ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_FIXED, 0);4784 if (ret)4785 goto err_out_phy_exit;4786 4787 ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_SGMII_MODE, 1);4788 if (ret)4789 goto err_out_phy_exit;4790 }4791 4792 }4793 4794 /* Fully reset controller at hardware level if mapped in device tree */4795 ret = device_reset_optional(&pdev->dev);4796 if (ret) {4797 phy_exit(bp->sgmii_phy);4798 return dev_err_probe(&pdev->dev, ret, "failed to reset controller");4799 }4800 4801 ret = macb_init(pdev);4802 4803err_out_phy_exit:4804 if (ret)4805 phy_exit(bp->sgmii_phy);4806 4807 return ret;4808}4809 4810static const struct macb_usrio_config sama7g5_usrio = {4811 .mii = 0,4812 .rmii = 1,4813 .rgmii = 2,4814 .refclk = BIT(2),4815 .hdfctlen = BIT(6),4816};4817 4818static const struct macb_config fu540_c000_config = {4819 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO |4820 MACB_CAPS_GEM_HAS_PTP,4821 .dma_burst_length = 16,4822 .clk_init = fu540_c000_clk_init,4823 .init = fu540_c000_init,4824 .jumbo_max_len = 10240,4825 .usrio = &macb_default_usrio,4826};4827 4828static const struct macb_config at91sam9260_config = {4829 .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII,4830 .clk_init = macb_clk_init,4831 .init = macb_init,4832 .usrio = &macb_default_usrio,4833};4834 4835static const struct macb_config sama5d3macb_config = {4836 .caps = MACB_CAPS_SG_DISABLED |4837 MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII,4838 .clk_init = macb_clk_init,4839 .init = macb_init,4840 .usrio = &macb_default_usrio,4841};4842 4843static const struct macb_config pc302gem_config = {4844 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE,4845 .dma_burst_length = 16,4846 .clk_init = macb_clk_init,4847 .init = macb_init,4848 .usrio = &macb_default_usrio,4849};4850 4851static const struct macb_config sama5d2_config = {4852 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII,4853 .dma_burst_length = 16,4854 .clk_init = macb_clk_init,4855 .init = macb_init,4856 .usrio = &macb_default_usrio,4857};4858 4859static const struct macb_config sama5d29_config = {4860 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_GEM_HAS_PTP,4861 .dma_burst_length = 16,4862 .clk_init = macb_clk_init,4863 .init = macb_init,4864 .usrio = &macb_default_usrio,4865};4866 4867static const struct macb_config sama5d3_config = {4868 .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE |4869 MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO,4870 .dma_burst_length = 16,4871 .clk_init = macb_clk_init,4872 .init = macb_init,4873 .jumbo_max_len = 10240,4874 .usrio = &macb_default_usrio,4875};4876 4877static const struct macb_config sama5d4_config = {4878 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII,4879 .dma_burst_length = 4,4880 .clk_init = macb_clk_init,4881 .init = macb_init,4882 .usrio = &macb_default_usrio,4883};4884 4885static const struct macb_config emac_config = {4886 .caps = MACB_CAPS_NEEDS_RSTONUBR | MACB_CAPS_MACB_IS_EMAC,4887 .clk_init = at91ether_clk_init,4888 .init = at91ether_init,4889 .usrio = &macb_default_usrio,4890};4891 4892static const struct macb_config np4_config = {4893 .caps = MACB_CAPS_USRIO_DISABLED,4894 .clk_init = macb_clk_init,4895 .init = macb_init,4896 .usrio = &macb_default_usrio,4897};4898 4899static const struct macb_config zynqmp_config = {4900 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE |4901 MACB_CAPS_JUMBO |4902 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH,4903 .dma_burst_length = 16,4904 .clk_init = macb_clk_init,4905 .init = init_reset_optional,4906 .jumbo_max_len = 10240,4907 .usrio = &macb_default_usrio,4908};4909 4910static const struct macb_config zynq_config = {4911 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF |4912 MACB_CAPS_NEEDS_RSTONUBR,4913 .dma_burst_length = 16,4914 .clk_init = macb_clk_init,4915 .init = macb_init,4916 .usrio = &macb_default_usrio,4917};4918 4919static const struct macb_config mpfs_config = {4920 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE |4921 MACB_CAPS_JUMBO |4922 MACB_CAPS_GEM_HAS_PTP,4923 .dma_burst_length = 16,4924 .clk_init = macb_clk_init,4925 .init = init_reset_optional,4926 .usrio = &macb_default_usrio,4927 .max_tx_length = 4040, /* Cadence Erratum 1686 */4928 .jumbo_max_len = 4040,4929};4930 4931static const struct macb_config sama7g5_gem_config = {4932 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_CLK_HW_CHG |4933 MACB_CAPS_MIIONRGMII | MACB_CAPS_GEM_HAS_PTP,4934 .dma_burst_length = 16,4935 .clk_init = macb_clk_init,4936 .init = macb_init,4937 .usrio = &sama7g5_usrio,4938};4939 4940static const struct macb_config sama7g5_emac_config = {4941 .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII |4942 MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_MIIONRGMII |4943 MACB_CAPS_GEM_HAS_PTP,4944 .dma_burst_length = 16,4945 .clk_init = macb_clk_init,4946 .init = macb_init,4947 .usrio = &sama7g5_usrio,4948};4949 4950static const struct macb_config versal_config = {4951 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO |4952 MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH | MACB_CAPS_NEED_TSUCLK |4953 MACB_CAPS_QUEUE_DISABLE,4954 .dma_burst_length = 16,4955 .clk_init = macb_clk_init,4956 .init = init_reset_optional,4957 .jumbo_max_len = 10240,4958 .usrio = &macb_default_usrio,4959};4960 4961static const struct of_device_id macb_dt_ids[] = {4962 { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config },4963 { .compatible = "cdns,macb" },4964 { .compatible = "cdns,np4-macb", .data = &np4_config },4965 { .compatible = "cdns,pc302-gem", .data = &pc302gem_config },4966 { .compatible = "cdns,gem", .data = &pc302gem_config },4967 { .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config },4968 { .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config },4969 { .compatible = "atmel,sama5d29-gem", .data = &sama5d29_config },4970 { .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config },4971 { .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config },4972 { .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config },4973 { .compatible = "cdns,at91rm9200-emac", .data = &emac_config },4974 { .compatible = "cdns,emac", .data = &emac_config },4975 { .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, /* deprecated */4976 { .compatible = "cdns,zynq-gem", .data = &zynq_config }, /* deprecated */4977 { .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config },4978 { .compatible = "microchip,mpfs-macb", .data = &mpfs_config },4979 { .compatible = "microchip,sama7g5-gem", .data = &sama7g5_gem_config },4980 { .compatible = "microchip,sama7g5-emac", .data = &sama7g5_emac_config },4981 { .compatible = "xlnx,zynqmp-gem", .data = &zynqmp_config},4982 { .compatible = "xlnx,zynq-gem", .data = &zynq_config },4983 { .compatible = "xlnx,versal-gem", .data = &versal_config},4984 { /* sentinel */ }4985};4986MODULE_DEVICE_TABLE(of, macb_dt_ids);4987#endif /* CONFIG_OF */4988 4989static const struct macb_config default_gem_config = {4990 .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE |4991 MACB_CAPS_JUMBO |4992 MACB_CAPS_GEM_HAS_PTP,4993 .dma_burst_length = 16,4994 .clk_init = macb_clk_init,4995 .init = macb_init,4996 .usrio = &macb_default_usrio,4997 .jumbo_max_len = 10240,4998};4999 5000static int macb_probe(struct platform_device *pdev)5001{5002 const struct macb_config *macb_config = &default_gem_config;5003 int (*clk_init)(struct platform_device *, struct clk **,5004 struct clk **, struct clk **, struct clk **,5005 struct clk **) = macb_config->clk_init;5006 int (*init)(struct platform_device *) = macb_config->init;5007 struct device_node *np = pdev->dev.of_node;5008 struct clk *pclk, *hclk = NULL, *tx_clk = NULL, *rx_clk = NULL;5009 struct clk *tsu_clk = NULL;5010 unsigned int queue_mask, num_queues;5011 bool native_io;5012 phy_interface_t interface;5013 struct net_device *dev;5014 struct resource *regs;5015 u32 wtrmrk_rst_val;5016 void __iomem *mem;5017 struct macb *bp;5018 int err, val;5019 5020 mem = devm_platform_get_and_ioremap_resource(pdev, 0, ®s);5021 if (IS_ERR(mem))5022 return PTR_ERR(mem);5023 5024 if (np) {5025 const struct of_device_id *match;5026 5027 match = of_match_node(macb_dt_ids, np);5028 if (match && match->data) {5029 macb_config = match->data;5030 clk_init = macb_config->clk_init;5031 init = macb_config->init;5032 }5033 }5034 5035 err = clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk);5036 if (err)5037 return err;5038 5039 pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT);5040 pm_runtime_use_autosuspend(&pdev->dev);5041 pm_runtime_get_noresume(&pdev->dev);5042 pm_runtime_set_active(&pdev->dev);5043 pm_runtime_enable(&pdev->dev);5044 native_io = hw_is_native_io(mem);5045 5046 macb_probe_queues(mem, native_io, &queue_mask, &num_queues);5047 dev = alloc_etherdev_mq(sizeof(*bp), num_queues);5048 if (!dev) {5049 err = -ENOMEM;5050 goto err_disable_clocks;5051 }5052 5053 dev->base_addr = regs->start;5054 5055 SET_NETDEV_DEV(dev, &pdev->dev);5056 5057 bp = netdev_priv(dev);5058 bp->pdev = pdev;5059 bp->dev = dev;5060 bp->regs = mem;5061 bp->native_io = native_io;5062 if (native_io) {5063 bp->macb_reg_readl = hw_readl_native;5064 bp->macb_reg_writel = hw_writel_native;5065 } else {5066 bp->macb_reg_readl = hw_readl;5067 bp->macb_reg_writel = hw_writel;5068 }5069 bp->num_queues = num_queues;5070 bp->queue_mask = queue_mask;5071 if (macb_config)5072 bp->dma_burst_length = macb_config->dma_burst_length;5073 bp->pclk = pclk;5074 bp->hclk = hclk;5075 bp->tx_clk = tx_clk;5076 bp->rx_clk = rx_clk;5077 bp->tsu_clk = tsu_clk;5078 if (macb_config)5079 bp->jumbo_max_len = macb_config->jumbo_max_len;5080 5081 if (!hw_is_gem(bp->regs, bp->native_io))5082 bp->max_tx_length = MACB_MAX_TX_LEN;5083 else if (macb_config->max_tx_length)5084 bp->max_tx_length = macb_config->max_tx_length;5085 else5086 bp->max_tx_length = GEM_MAX_TX_LEN;5087 5088 bp->wol = 0;5089 device_set_wakeup_capable(&pdev->dev, 1);5090 5091 bp->usrio = macb_config->usrio;5092 5093 /* By default we set to partial store and forward mode for zynqmp.5094 * Disable if not set in devicetree.5095 */5096 if (GEM_BFEXT(PBUF_CUTTHRU, gem_readl(bp, DCFG6))) {5097 err = of_property_read_u32(bp->pdev->dev.of_node,5098 "cdns,rx-watermark",5099 &bp->rx_watermark);5100 5101 if (!err) {5102 /* Disable partial store and forward in case of error or5103 * invalid watermark value5104 */5105 wtrmrk_rst_val = (1 << (GEM_BFEXT(RX_PBUF_ADDR, gem_readl(bp, DCFG2)))) - 1;5106 if (bp->rx_watermark > wtrmrk_rst_val || !bp->rx_watermark) {5107 dev_info(&bp->pdev->dev, "Invalid watermark value\n");5108 bp->rx_watermark = 0;5109 }5110 }5111 }5112 spin_lock_init(&bp->lock);5113 5114 /* setup capabilities */5115 macb_configure_caps(bp, macb_config);5116 5117#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT5118 if (GEM_BFEXT(DAW64, gem_readl(bp, DCFG6))) {5119 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44));5120 bp->hw_dma_cap |= HW_DMA_CAP_64B;5121 }5122#endif5123 platform_set_drvdata(pdev, dev);5124 5125 dev->irq = platform_get_irq(pdev, 0);5126 if (dev->irq < 0) {5127 err = dev->irq;5128 goto err_out_free_netdev;5129 }5130 5131 /* MTU range: 68 - 1518 or 10240 */5132 dev->min_mtu = GEM_MTU_MIN_SIZE;5133 if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len)5134 dev->max_mtu = bp->jumbo_max_len - ETH_HLEN - ETH_FCS_LEN;5135 else5136 dev->max_mtu = 1536 - ETH_HLEN - ETH_FCS_LEN;5137 5138 if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) {5139 val = GEM_BFEXT(RXBD_RDBUFF, gem_readl(bp, DCFG10));5140 if (val)5141 bp->rx_bd_rd_prefetch = (2 << (val - 1)) *5142 macb_dma_desc_get_size(bp);5143 5144 val = GEM_BFEXT(TXBD_RDBUFF, gem_readl(bp, DCFG10));5145 if (val)5146 bp->tx_bd_rd_prefetch = (2 << (val - 1)) *5147 macb_dma_desc_get_size(bp);5148 }5149 5150 bp->rx_intr_mask = MACB_RX_INT_FLAGS;5151 if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR)5152 bp->rx_intr_mask |= MACB_BIT(RXUBR);5153 5154 err = of_get_ethdev_address(np, bp->dev);5155 if (err == -EPROBE_DEFER)5156 goto err_out_free_netdev;5157 else if (err)5158 macb_get_hwaddr(bp);5159 5160 err = of_get_phy_mode(np, &interface);5161 if (err)5162 /* not found in DT, MII by default */5163 bp->phy_interface = PHY_INTERFACE_MODE_MII;5164 else5165 bp->phy_interface = interface;5166 5167 /* IP specific init */5168 err = init(pdev);5169 if (err)5170 goto err_out_free_netdev;5171 5172 err = macb_mii_init(bp);5173 if (err)5174 goto err_out_phy_exit;5175 5176 netif_carrier_off(dev);5177 5178 err = register_netdev(dev);5179 if (err) {5180 dev_err(&pdev->dev, "Cannot register net device, aborting.\n");5181 goto err_out_unregister_mdio;5182 }5183 5184 INIT_WORK(&bp->hresp_err_bh_work, macb_hresp_error_task);5185 5186 netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n",5187 macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID),5188 dev->base_addr, dev->irq, dev->dev_addr);5189 5190 pm_runtime_mark_last_busy(&bp->pdev->dev);5191 pm_runtime_put_autosuspend(&bp->pdev->dev);5192 5193 return 0;5194 5195err_out_unregister_mdio:5196 mdiobus_unregister(bp->mii_bus);5197 mdiobus_free(bp->mii_bus);5198 5199err_out_phy_exit:5200 phy_exit(bp->sgmii_phy);5201 5202err_out_free_netdev:5203 free_netdev(dev);5204 5205err_disable_clocks:5206 macb_clks_disable(pclk, hclk, tx_clk, rx_clk, tsu_clk);5207 pm_runtime_disable(&pdev->dev);5208 pm_runtime_set_suspended(&pdev->dev);5209 pm_runtime_dont_use_autosuspend(&pdev->dev);5210 5211 return err;5212}5213 5214static void macb_remove(struct platform_device *pdev)5215{5216 struct net_device *dev;5217 struct macb *bp;5218 5219 dev = platform_get_drvdata(pdev);5220 5221 if (dev) {5222 bp = netdev_priv(dev);5223 phy_exit(bp->sgmii_phy);5224 mdiobus_unregister(bp->mii_bus);5225 mdiobus_free(bp->mii_bus);5226 5227 unregister_netdev(dev);5228 cancel_work_sync(&bp->hresp_err_bh_work);5229 pm_runtime_disable(&pdev->dev);5230 pm_runtime_dont_use_autosuspend(&pdev->dev);5231 if (!pm_runtime_suspended(&pdev->dev)) {5232 macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk,5233 bp->rx_clk, bp->tsu_clk);5234 pm_runtime_set_suspended(&pdev->dev);5235 }5236 phylink_destroy(bp->phylink);5237 free_netdev(dev);5238 }5239}5240 5241static int __maybe_unused macb_suspend(struct device *dev)5242{5243 struct net_device *netdev = dev_get_drvdata(dev);5244 struct macb *bp = netdev_priv(netdev);5245 struct in_ifaddr *ifa = NULL;5246 struct macb_queue *queue;5247 struct in_device *idev;5248 unsigned long flags;5249 unsigned int q;5250 int err;5251 u32 tmp;5252 5253 if (!device_may_wakeup(&bp->dev->dev))5254 phy_exit(bp->sgmii_phy);5255 5256 if (!netif_running(netdev))5257 return 0;5258 5259 if (bp->wol & MACB_WOL_ENABLED) {5260 /* Check for IP address in WOL ARP mode */5261 idev = __in_dev_get_rcu(bp->dev);5262 if (idev)5263 ifa = rcu_dereference(idev->ifa_list);5264 if ((bp->wolopts & WAKE_ARP) && !ifa) {5265 netdev_err(netdev, "IP address not assigned as required by WoL walk ARP\n");5266 return -EOPNOTSUPP;5267 }5268 spin_lock_irqsave(&bp->lock, flags);5269 5270 /* Disable Tx and Rx engines before disabling the queues,5271 * this is mandatory as per the IP spec sheet5272 */5273 tmp = macb_readl(bp, NCR);5274 macb_writel(bp, NCR, tmp & ~(MACB_BIT(TE) | MACB_BIT(RE)));5275 for (q = 0, queue = bp->queues; q < bp->num_queues;5276 ++q, ++queue) {5277 /* Disable RX queues */5278 if (bp->caps & MACB_CAPS_QUEUE_DISABLE) {5279 queue_writel(queue, RBQP, MACB_BIT(QUEUE_DISABLE));5280 } else {5281 /* Tie off RX queues */5282 queue_writel(queue, RBQP,5283 lower_32_bits(bp->rx_ring_tieoff_dma));5284#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT5285 queue_writel(queue, RBQPH,5286 upper_32_bits(bp->rx_ring_tieoff_dma));5287#endif5288 }5289 /* Disable all interrupts */5290 queue_writel(queue, IDR, -1);5291 queue_readl(queue, ISR);5292 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)5293 queue_writel(queue, ISR, -1);5294 }5295 /* Enable Receive engine */5296 macb_writel(bp, NCR, tmp | MACB_BIT(RE));5297 /* Flush all status bits */5298 macb_writel(bp, TSR, -1);5299 macb_writel(bp, RSR, -1);5300 5301 tmp = (bp->wolopts & WAKE_MAGIC) ? MACB_BIT(MAG) : 0;5302 if (bp->wolopts & WAKE_ARP) {5303 tmp |= MACB_BIT(ARP);5304 /* write IP address into register */5305 tmp |= MACB_BFEXT(IP, be32_to_cpu(ifa->ifa_local));5306 }5307 5308 /* Change interrupt handler and5309 * Enable WoL IRQ on queue 05310 */5311 devm_free_irq(dev, bp->queues[0].irq, bp->queues);5312 if (macb_is_gem(bp)) {5313 err = devm_request_irq(dev, bp->queues[0].irq, gem_wol_interrupt,5314 IRQF_SHARED, netdev->name, bp->queues);5315 if (err) {5316 dev_err(dev,5317 "Unable to request IRQ %d (error %d)\n",5318 bp->queues[0].irq, err);5319 spin_unlock_irqrestore(&bp->lock, flags);5320 return err;5321 }5322 queue_writel(bp->queues, IER, GEM_BIT(WOL));5323 gem_writel(bp, WOL, tmp);5324 } else {5325 err = devm_request_irq(dev, bp->queues[0].irq, macb_wol_interrupt,5326 IRQF_SHARED, netdev->name, bp->queues);5327 if (err) {5328 dev_err(dev,5329 "Unable to request IRQ %d (error %d)\n",5330 bp->queues[0].irq, err);5331 spin_unlock_irqrestore(&bp->lock, flags);5332 return err;5333 }5334 queue_writel(bp->queues, IER, MACB_BIT(WOL));5335 macb_writel(bp, WOL, tmp);5336 }5337 spin_unlock_irqrestore(&bp->lock, flags);5338 5339 enable_irq_wake(bp->queues[0].irq);5340 }5341 5342 netif_device_detach(netdev);5343 for (q = 0, queue = bp->queues; q < bp->num_queues;5344 ++q, ++queue) {5345 napi_disable(&queue->napi_rx);5346 napi_disable(&queue->napi_tx);5347 }5348 5349 if (!(bp->wol & MACB_WOL_ENABLED)) {5350 rtnl_lock();5351 phylink_stop(bp->phylink);5352 rtnl_unlock();5353 spin_lock_irqsave(&bp->lock, flags);5354 macb_reset_hw(bp);5355 spin_unlock_irqrestore(&bp->lock, flags);5356 }5357 5358 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))5359 bp->pm_data.usrio = macb_or_gem_readl(bp, USRIO);5360 5361 if (netdev->hw_features & NETIF_F_NTUPLE)5362 bp->pm_data.scrt2 = gem_readl_n(bp, ETHT, SCRT2_ETHT);5363 5364 if (bp->ptp_info)5365 bp->ptp_info->ptp_remove(netdev);5366 if (!device_may_wakeup(dev))5367 pm_runtime_force_suspend(dev);5368 5369 return 0;5370}5371 5372static int __maybe_unused macb_resume(struct device *dev)5373{5374 struct net_device *netdev = dev_get_drvdata(dev);5375 struct macb *bp = netdev_priv(netdev);5376 struct macb_queue *queue;5377 unsigned long flags;5378 unsigned int q;5379 int err;5380 5381 if (!device_may_wakeup(&bp->dev->dev))5382 phy_init(bp->sgmii_phy);5383 5384 if (!netif_running(netdev))5385 return 0;5386 5387 if (!device_may_wakeup(dev))5388 pm_runtime_force_resume(dev);5389 5390 if (bp->wol & MACB_WOL_ENABLED) {5391 spin_lock_irqsave(&bp->lock, flags);5392 /* Disable WoL */5393 if (macb_is_gem(bp)) {5394 queue_writel(bp->queues, IDR, GEM_BIT(WOL));5395 gem_writel(bp, WOL, 0);5396 } else {5397 queue_writel(bp->queues, IDR, MACB_BIT(WOL));5398 macb_writel(bp, WOL, 0);5399 }5400 /* Clear ISR on queue 0 */5401 queue_readl(bp->queues, ISR);5402 if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)5403 queue_writel(bp->queues, ISR, -1);5404 /* Replace interrupt handler on queue 0 */5405 devm_free_irq(dev, bp->queues[0].irq, bp->queues);5406 err = devm_request_irq(dev, bp->queues[0].irq, macb_interrupt,5407 IRQF_SHARED, netdev->name, bp->queues);5408 if (err) {5409 dev_err(dev,5410 "Unable to request IRQ %d (error %d)\n",5411 bp->queues[0].irq, err);5412 spin_unlock_irqrestore(&bp->lock, flags);5413 return err;5414 }5415 spin_unlock_irqrestore(&bp->lock, flags);5416 5417 disable_irq_wake(bp->queues[0].irq);5418 5419 /* Now make sure we disable phy before moving5420 * to common restore path5421 */5422 rtnl_lock();5423 phylink_stop(bp->phylink);5424 rtnl_unlock();5425 }5426 5427 for (q = 0, queue = bp->queues; q < bp->num_queues;5428 ++q, ++queue) {5429 napi_enable(&queue->napi_rx);5430 napi_enable(&queue->napi_tx);5431 }5432 5433 if (netdev->hw_features & NETIF_F_NTUPLE)5434 gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2);5435 5436 if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))5437 macb_or_gem_writel(bp, USRIO, bp->pm_data.usrio);5438 5439 macb_writel(bp, NCR, MACB_BIT(MPE));5440 macb_init_hw(bp);5441 macb_set_rx_mode(netdev);5442 macb_restore_features(bp);5443 rtnl_lock();5444 5445 phylink_start(bp->phylink);5446 rtnl_unlock();5447 5448 netif_device_attach(netdev);5449 if (bp->ptp_info)5450 bp->ptp_info->ptp_init(netdev);5451 5452 return 0;5453}5454 5455static int __maybe_unused macb_runtime_suspend(struct device *dev)5456{5457 struct net_device *netdev = dev_get_drvdata(dev);5458 struct macb *bp = netdev_priv(netdev);5459 5460 if (!(device_may_wakeup(dev)))5461 macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk, bp->rx_clk, bp->tsu_clk);5462 else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK))5463 macb_clks_disable(NULL, NULL, NULL, NULL, bp->tsu_clk);5464 5465 return 0;5466}5467 5468static int __maybe_unused macb_runtime_resume(struct device *dev)5469{5470 struct net_device *netdev = dev_get_drvdata(dev);5471 struct macb *bp = netdev_priv(netdev);5472 5473 if (!(device_may_wakeup(dev))) {5474 clk_prepare_enable(bp->pclk);5475 clk_prepare_enable(bp->hclk);5476 clk_prepare_enable(bp->tx_clk);5477 clk_prepare_enable(bp->rx_clk);5478 clk_prepare_enable(bp->tsu_clk);5479 } else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) {5480 clk_prepare_enable(bp->tsu_clk);5481 }5482 5483 return 0;5484}5485 5486static const struct dev_pm_ops macb_pm_ops = {5487 SET_SYSTEM_SLEEP_PM_OPS(macb_suspend, macb_resume)5488 SET_RUNTIME_PM_OPS(macb_runtime_suspend, macb_runtime_resume, NULL)5489};5490 5491static struct platform_driver macb_driver = {5492 .probe = macb_probe,5493 .remove_new = macb_remove,5494 .driver = {5495 .name = "macb",5496 .of_match_table = of_match_ptr(macb_dt_ids),5497 .pm = &macb_pm_ops,5498 },5499};5500 5501module_platform_driver(macb_driver);5502 5503MODULE_LICENSE("GPL");5504MODULE_DESCRIPTION("Cadence MACB/GEM Ethernet driver");5505MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");5506MODULE_ALIAS("platform:macb");5507