1484 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.3 */4 5/* Qualcomm Technologies, Inc. EMAC Ethernet Controller MAC layer support6 */7 8#include <linux/tcp.h>9#include <linux/ip.h>10#include <linux/ipv6.h>11#include <linux/crc32.h>12#include <linux/if_vlan.h>13#include <linux/jiffies.h>14#include <linux/phy.h>15#include <linux/of.h>16#include <net/ip6_checksum.h>17#include "emac.h"18#include "emac-sgmii.h"19 20/* EMAC_MAC_CTRL */21#define SINGLE_PAUSE_MODE 0x1000000022#define DEBUG_MODE 0x0800000023#define BROAD_EN 0x0400000024#define MULTI_ALL 0x0200000025#define RX_CHKSUM_EN 0x0100000026#define HUGE 0x0080000027#define SPEED(x) (((x) & 0x3) << 20)28#define SPEED_MASK SPEED(0x3)29#define SIMR 0x0008000030#define TPAUSE 0x0001000031#define PROM_MODE 0x0000800032#define VLAN_STRIP 0x0000400033#define PRLEN_BMSK 0x00003c0034#define PRLEN_SHFT 1035#define HUGEN 0x0000020036#define FLCHK 0x0000010037#define PCRCE 0x0000008038#define CRCE 0x0000004039#define FULLD 0x0000002040#define MAC_LP_EN 0x0000001041#define RXFC 0x0000000842#define TXFC 0x0000000443#define RXEN 0x0000000244#define TXEN 0x0000000145 46/* EMAC_DESC_CTRL_3 */47#define RFD_RING_SIZE_BMSK 0xfff48 49/* EMAC_DESC_CTRL_4 */50#define RX_BUFFER_SIZE_BMSK 0xffff51 52/* EMAC_DESC_CTRL_6 */53#define RRD_RING_SIZE_BMSK 0xfff54 55/* EMAC_DESC_CTRL_9 */56#define TPD_RING_SIZE_BMSK 0xffff57 58/* EMAC_TXQ_CTRL_0 */59#define NUM_TXF_BURST_PREF_BMSK 0xffff000060#define NUM_TXF_BURST_PREF_SHFT 1661#define LS_8023_SP 0x8062#define TXQ_MODE 0x4063#define TXQ_EN 0x2064#define IP_OP_SP 0x1065#define NUM_TPD_BURST_PREF_BMSK 0xf66#define NUM_TPD_BURST_PREF_SHFT 067 68/* EMAC_TXQ_CTRL_1 */69#define JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK 0x7ff70 71/* EMAC_TXQ_CTRL_2 */72#define TXF_HWM_BMSK 0xfff000073#define TXF_LWM_BMSK 0xfff74 75/* EMAC_RXQ_CTRL_0 */76#define RXQ_EN BIT(31)77#define CUT_THRU_EN BIT(30)78#define RSS_HASH_EN BIT(29)79#define NUM_RFD_BURST_PREF_BMSK 0x3f0000080#define NUM_RFD_BURST_PREF_SHFT 2081#define IDT_TABLE_SIZE_BMSK 0x1ff0082#define IDT_TABLE_SIZE_SHFT 883#define SP_IPV6 0x8084 85/* EMAC_RXQ_CTRL_1 */86#define JUMBO_1KAH_BMSK 0xf00087#define JUMBO_1KAH_SHFT 1288#define RFD_PREF_LOW_TH 0x1089#define RFD_PREF_LOW_THRESHOLD_BMSK 0xfc090#define RFD_PREF_LOW_THRESHOLD_SHFT 691#define RFD_PREF_UP_TH 0x1092#define RFD_PREF_UP_THRESHOLD_BMSK 0x3f93#define RFD_PREF_UP_THRESHOLD_SHFT 094 95/* EMAC_RXQ_CTRL_2 */96#define RXF_DOF_THRESFHOLD 0x1a097#define RXF_DOF_THRESHOLD_BMSK 0xfff000098#define RXF_DOF_THRESHOLD_SHFT 1699#define RXF_UOF_THRESFHOLD 0xbe100#define RXF_UOF_THRESHOLD_BMSK 0xfff101#define RXF_UOF_THRESHOLD_SHFT 0102 103/* EMAC_RXQ_CTRL_3 */104#define RXD_TIMER_BMSK 0xffff0000105#define RXD_THRESHOLD_BMSK 0xfff106#define RXD_THRESHOLD_SHFT 0107 108/* EMAC_DMA_CTRL */109#define DMAW_DLY_CNT_BMSK 0xf0000110#define DMAW_DLY_CNT_SHFT 16111#define DMAR_DLY_CNT_BMSK 0xf800112#define DMAR_DLY_CNT_SHFT 11113#define DMAR_REQ_PRI 0x400114#define REGWRBLEN_BMSK 0x380115#define REGWRBLEN_SHFT 7116#define REGRDBLEN_BMSK 0x70117#define REGRDBLEN_SHFT 4118#define OUT_ORDER_MODE 0x4119#define ENH_ORDER_MODE 0x2120#define IN_ORDER_MODE 0x1121 122/* EMAC_MAILBOX_13 */123#define RFD3_PROC_IDX_BMSK 0xfff0000124#define RFD3_PROC_IDX_SHFT 16125#define RFD3_PROD_IDX_BMSK 0xfff126#define RFD3_PROD_IDX_SHFT 0127 128/* EMAC_MAILBOX_2 */129#define NTPD_CONS_IDX_BMSK 0xffff0000130#define NTPD_CONS_IDX_SHFT 16131 132/* EMAC_MAILBOX_3 */133#define RFD0_CONS_IDX_BMSK 0xfff134#define RFD0_CONS_IDX_SHFT 0135 136/* EMAC_MAILBOX_11 */137#define H3TPD_PROD_IDX_BMSK 0xffff0000138#define H3TPD_PROD_IDX_SHFT 16139 140/* EMAC_AXI_MAST_CTRL */141#define DATA_BYTE_SWAP 0x8142#define MAX_BOUND 0x2143#define MAX_BTYPE 0x1144 145/* EMAC_MAILBOX_12 */146#define H3TPD_CONS_IDX_BMSK 0xffff0000147#define H3TPD_CONS_IDX_SHFT 16148 149/* EMAC_MAILBOX_9 */150#define H2TPD_PROD_IDX_BMSK 0xffff151#define H2TPD_PROD_IDX_SHFT 0152 153/* EMAC_MAILBOX_10 */154#define H1TPD_CONS_IDX_BMSK 0xffff0000155#define H1TPD_CONS_IDX_SHFT 16156#define H2TPD_CONS_IDX_BMSK 0xffff157#define H2TPD_CONS_IDX_SHFT 0158 159/* EMAC_ATHR_HEADER_CTRL */160#define HEADER_CNT_EN 0x2161#define HEADER_ENABLE 0x1162 163/* EMAC_MAILBOX_0 */164#define RFD0_PROC_IDX_BMSK 0xfff0000165#define RFD0_PROC_IDX_SHFT 16166#define RFD0_PROD_IDX_BMSK 0xfff167#define RFD0_PROD_IDX_SHFT 0168 169/* EMAC_MAILBOX_5 */170#define RFD1_PROC_IDX_BMSK 0xfff0000171#define RFD1_PROC_IDX_SHFT 16172#define RFD1_PROD_IDX_BMSK 0xfff173#define RFD1_PROD_IDX_SHFT 0174 175/* EMAC_MISC_CTRL */176#define RX_UNCPL_INT_EN 0x1177 178/* EMAC_MAILBOX_7 */179#define RFD2_CONS_IDX_BMSK 0xfff0000180#define RFD2_CONS_IDX_SHFT 16181#define RFD1_CONS_IDX_BMSK 0xfff182#define RFD1_CONS_IDX_SHFT 0183 184/* EMAC_MAILBOX_8 */185#define RFD3_CONS_IDX_BMSK 0xfff186#define RFD3_CONS_IDX_SHFT 0187 188/* EMAC_MAILBOX_15 */189#define NTPD_PROD_IDX_BMSK 0xffff190#define NTPD_PROD_IDX_SHFT 0191 192/* EMAC_MAILBOX_16 */193#define H1TPD_PROD_IDX_BMSK 0xffff194#define H1TPD_PROD_IDX_SHFT 0195 196#define RXQ0_RSS_HSTYP_IPV6_TCP_EN 0x20197#define RXQ0_RSS_HSTYP_IPV6_EN 0x10198#define RXQ0_RSS_HSTYP_IPV4_TCP_EN 0x8199#define RXQ0_RSS_HSTYP_IPV4_EN 0x4200 201/* EMAC_EMAC_WRAPPER_TX_TS_INX */202#define EMAC_WRAPPER_TX_TS_EMPTY BIT(31)203#define EMAC_WRAPPER_TX_TS_INX_BMSK 0xffff204 205struct emac_skb_cb {206 u32 tpd_idx;207 unsigned long jiffies;208};209 210#define EMAC_SKB_CB(skb) ((struct emac_skb_cb *)(skb)->cb)211#define EMAC_RSS_IDT_SIZE 256212#define JUMBO_1KAH 0x4213#define RXD_TH 0x100214#define EMAC_TPD_LAST_FRAGMENT 0x80000000215#define EMAC_TPD_TSTAMP_SAVE 0x80000000216 217/* EMAC Errors in emac_rrd.word[3] */218#define EMAC_RRD_L4F BIT(14)219#define EMAC_RRD_IPF BIT(15)220#define EMAC_RRD_CRC BIT(21)221#define EMAC_RRD_FAE BIT(22)222#define EMAC_RRD_TRN BIT(23)223#define EMAC_RRD_RNT BIT(24)224#define EMAC_RRD_INC BIT(25)225#define EMAC_RRD_FOV BIT(29)226#define EMAC_RRD_LEN BIT(30)227 228/* Error bits that will result in a received frame being discarded */229#define EMAC_RRD_ERROR (EMAC_RRD_IPF | EMAC_RRD_CRC | EMAC_RRD_FAE | \230 EMAC_RRD_TRN | EMAC_RRD_RNT | EMAC_RRD_INC | \231 EMAC_RRD_FOV | EMAC_RRD_LEN)232#define EMAC_RRD_STATS_DW_IDX 3233 234#define EMAC_RRD(RXQ, SIZE, IDX) ((RXQ)->rrd.v_addr + (SIZE * (IDX)))235#define EMAC_RFD(RXQ, SIZE, IDX) ((RXQ)->rfd.v_addr + (SIZE * (IDX)))236#define EMAC_TPD(TXQ, SIZE, IDX) ((TXQ)->tpd.v_addr + (SIZE * (IDX)))237 238#define GET_RFD_BUFFER(RXQ, IDX) (&((RXQ)->rfd.rfbuff[(IDX)]))239#define GET_TPD_BUFFER(RTQ, IDX) (&((RTQ)->tpd.tpbuff[(IDX)]))240 241#define EMAC_TX_POLL_HWTXTSTAMP_THRESHOLD 8242 243#define ISR_RX_PKT (\244 RX_PKT_INT0 |\245 RX_PKT_INT1 |\246 RX_PKT_INT2 |\247 RX_PKT_INT3)248 249void emac_mac_multicast_addr_set(struct emac_adapter *adpt, u8 *addr)250{251 u32 crc32, bit, reg, mta;252 253 /* Calculate the CRC of the MAC address */254 crc32 = ether_crc(ETH_ALEN, addr);255 256 /* The HASH Table is an array of 2 32-bit registers. It is257 * treated like an array of 64 bits (BitArray[hash_value]).258 * Use the upper 6 bits of the above CRC as the hash value.259 */260 reg = (crc32 >> 31) & 0x1;261 bit = (crc32 >> 26) & 0x1F;262 263 mta = readl(adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2));264 mta |= BIT(bit);265 writel(mta, adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2));266}267 268void emac_mac_multicast_addr_clear(struct emac_adapter *adpt)269{270 writel(0, adpt->base + EMAC_HASH_TAB_REG0);271 writel(0, adpt->base + EMAC_HASH_TAB_REG1);272}273 274/* definitions for RSS */275#define EMAC_RSS_KEY(_i, _type) \276 (EMAC_RSS_KEY0 + ((_i) * sizeof(_type)))277#define EMAC_RSS_TBL(_i, _type) \278 (EMAC_IDT_TABLE0 + ((_i) * sizeof(_type)))279 280/* Config MAC modes */281void emac_mac_mode_config(struct emac_adapter *adpt)282{283 struct net_device *netdev = adpt->netdev;284 u32 mac;285 286 mac = readl(adpt->base + EMAC_MAC_CTRL);287 mac &= ~(VLAN_STRIP | PROM_MODE | MULTI_ALL | MAC_LP_EN);288 289 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX)290 mac |= VLAN_STRIP;291 292 if (netdev->flags & IFF_PROMISC)293 mac |= PROM_MODE;294 295 if (netdev->flags & IFF_ALLMULTI)296 mac |= MULTI_ALL;297 298 writel(mac, adpt->base + EMAC_MAC_CTRL);299}300 301/* Config descriptor rings */302static void emac_mac_dma_rings_config(struct emac_adapter *adpt)303{304 /* TPD (Transmit Packet Descriptor) */305 writel(upper_32_bits(adpt->tx_q.tpd.dma_addr),306 adpt->base + EMAC_DESC_CTRL_1);307 308 writel(lower_32_bits(adpt->tx_q.tpd.dma_addr),309 adpt->base + EMAC_DESC_CTRL_8);310 311 writel(adpt->tx_q.tpd.count & TPD_RING_SIZE_BMSK,312 adpt->base + EMAC_DESC_CTRL_9);313 314 /* RFD (Receive Free Descriptor) & RRD (Receive Return Descriptor) */315 writel(upper_32_bits(adpt->rx_q.rfd.dma_addr),316 adpt->base + EMAC_DESC_CTRL_0);317 318 writel(lower_32_bits(adpt->rx_q.rfd.dma_addr),319 adpt->base + EMAC_DESC_CTRL_2);320 writel(lower_32_bits(adpt->rx_q.rrd.dma_addr),321 adpt->base + EMAC_DESC_CTRL_5);322 323 writel(adpt->rx_q.rfd.count & RFD_RING_SIZE_BMSK,324 adpt->base + EMAC_DESC_CTRL_3);325 writel(adpt->rx_q.rrd.count & RRD_RING_SIZE_BMSK,326 adpt->base + EMAC_DESC_CTRL_6);327 328 writel(adpt->rxbuf_size & RX_BUFFER_SIZE_BMSK,329 adpt->base + EMAC_DESC_CTRL_4);330 331 writel(0, adpt->base + EMAC_DESC_CTRL_11);332 333 /* Load all of the base addresses above and ensure that triggering HW to334 * read ring pointers is flushed335 */336 writel(1, adpt->base + EMAC_INTER_SRAM_PART9);337}338 339/* Config transmit parameters */340static void emac_mac_tx_config(struct emac_adapter *adpt)341{342 u32 val;343 344 writel((EMAC_MAX_TX_OFFLOAD_THRESH >> 3) &345 JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK, adpt->base + EMAC_TXQ_CTRL_1);346 347 val = (adpt->tpd_burst << NUM_TPD_BURST_PREF_SHFT) &348 NUM_TPD_BURST_PREF_BMSK;349 350 val |= TXQ_MODE | LS_8023_SP;351 val |= (0x0100 << NUM_TXF_BURST_PREF_SHFT) &352 NUM_TXF_BURST_PREF_BMSK;353 354 writel(val, adpt->base + EMAC_TXQ_CTRL_0);355 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_2,356 (TXF_HWM_BMSK | TXF_LWM_BMSK), 0);357}358 359/* Config receive parameters */360static void emac_mac_rx_config(struct emac_adapter *adpt)361{362 u32 val;363 364 val = (adpt->rfd_burst << NUM_RFD_BURST_PREF_SHFT) &365 NUM_RFD_BURST_PREF_BMSK;366 val |= (SP_IPV6 | CUT_THRU_EN);367 368 writel(val, adpt->base + EMAC_RXQ_CTRL_0);369 370 val = readl(adpt->base + EMAC_RXQ_CTRL_1);371 val &= ~(JUMBO_1KAH_BMSK | RFD_PREF_LOW_THRESHOLD_BMSK |372 RFD_PREF_UP_THRESHOLD_BMSK);373 val |= (JUMBO_1KAH << JUMBO_1KAH_SHFT) |374 (RFD_PREF_LOW_TH << RFD_PREF_LOW_THRESHOLD_SHFT) |375 (RFD_PREF_UP_TH << RFD_PREF_UP_THRESHOLD_SHFT);376 writel(val, adpt->base + EMAC_RXQ_CTRL_1);377 378 val = readl(adpt->base + EMAC_RXQ_CTRL_2);379 val &= ~(RXF_DOF_THRESHOLD_BMSK | RXF_UOF_THRESHOLD_BMSK);380 val |= (RXF_DOF_THRESFHOLD << RXF_DOF_THRESHOLD_SHFT) |381 (RXF_UOF_THRESFHOLD << RXF_UOF_THRESHOLD_SHFT);382 writel(val, adpt->base + EMAC_RXQ_CTRL_2);383 384 val = readl(adpt->base + EMAC_RXQ_CTRL_3);385 val &= ~(RXD_TIMER_BMSK | RXD_THRESHOLD_BMSK);386 val |= RXD_TH << RXD_THRESHOLD_SHFT;387 writel(val, adpt->base + EMAC_RXQ_CTRL_3);388}389 390/* Config dma */391static void emac_mac_dma_config(struct emac_adapter *adpt)392{393 u32 dma_ctrl = DMAR_REQ_PRI;394 395 switch (adpt->dma_order) {396 case emac_dma_ord_in:397 dma_ctrl |= IN_ORDER_MODE;398 break;399 case emac_dma_ord_enh:400 dma_ctrl |= ENH_ORDER_MODE;401 break;402 case emac_dma_ord_out:403 dma_ctrl |= OUT_ORDER_MODE;404 break;405 default:406 break;407 }408 409 dma_ctrl |= (((u32)adpt->dmar_block) << REGRDBLEN_SHFT) &410 REGRDBLEN_BMSK;411 dma_ctrl |= (((u32)adpt->dmaw_block) << REGWRBLEN_SHFT) &412 REGWRBLEN_BMSK;413 dma_ctrl |= (((u32)adpt->dmar_dly_cnt) << DMAR_DLY_CNT_SHFT) &414 DMAR_DLY_CNT_BMSK;415 dma_ctrl |= (((u32)adpt->dmaw_dly_cnt) << DMAW_DLY_CNT_SHFT) &416 DMAW_DLY_CNT_BMSK;417 418 /* config DMA and ensure that configuration is flushed to HW */419 writel(dma_ctrl, adpt->base + EMAC_DMA_CTRL);420}421 422/* set MAC address */423static void emac_set_mac_address(struct emac_adapter *adpt, const u8 *addr)424{425 u32 sta;426 427 /* for example: 00-A0-C6-11-22-33428 * 0<-->C6112233, 1<-->00A0.429 */430 431 /* low 32bit word */432 sta = (((u32)addr[2]) << 24) | (((u32)addr[3]) << 16) |433 (((u32)addr[4]) << 8) | (((u32)addr[5]));434 writel(sta, adpt->base + EMAC_MAC_STA_ADDR0);435 436 /* hight 32bit word */437 sta = (((u32)addr[0]) << 8) | (u32)addr[1];438 writel(sta, adpt->base + EMAC_MAC_STA_ADDR1);439}440 441static void emac_mac_config(struct emac_adapter *adpt)442{443 struct net_device *netdev = adpt->netdev;444 unsigned int max_frame;445 u32 val;446 447 emac_set_mac_address(adpt, netdev->dev_addr);448 449 max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;450 adpt->rxbuf_size = netdev->mtu > EMAC_DEF_RX_BUF_SIZE ?451 ALIGN(max_frame, 8) : EMAC_DEF_RX_BUF_SIZE;452 453 emac_mac_dma_rings_config(adpt);454 455 writel(netdev->mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN,456 adpt->base + EMAC_MAX_FRAM_LEN_CTRL);457 458 emac_mac_tx_config(adpt);459 emac_mac_rx_config(adpt);460 emac_mac_dma_config(adpt);461 462 val = readl(adpt->base + EMAC_AXI_MAST_CTRL);463 val &= ~(DATA_BYTE_SWAP | MAX_BOUND);464 val |= MAX_BTYPE;465 writel(val, adpt->base + EMAC_AXI_MAST_CTRL);466 writel(0, adpt->base + EMAC_CLK_GATE_CTRL);467 writel(RX_UNCPL_INT_EN, adpt->base + EMAC_MISC_CTRL);468}469 470void emac_mac_reset(struct emac_adapter *adpt)471{472 emac_mac_stop(adpt);473 474 emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, SOFT_RST);475 usleep_range(100, 150); /* reset may take up to 100usec */476 477 /* interrupt clear-on-read */478 emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, INT_RD_CLR_EN);479}480 481static void emac_mac_start(struct emac_adapter *adpt)482{483 struct phy_device *phydev = adpt->phydev;484 u32 mac, csr1;485 486 /* enable tx queue */487 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, 0, TXQ_EN);488 489 /* enable rx queue */490 emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, 0, RXQ_EN);491 492 /* enable mac control */493 mac = readl(adpt->base + EMAC_MAC_CTRL);494 csr1 = readl(adpt->csr + EMAC_EMAC_WRAPPER_CSR1);495 496 mac |= TXEN | RXEN; /* enable RX/TX */497 498 /* Configure MAC flow control. If set to automatic, then match499 * whatever the PHY does. Otherwise, enable or disable it, depending500 * on what the user configured via ethtool.501 */502 mac &= ~(RXFC | TXFC);503 504 if (adpt->automatic) {505 /* If it's set to automatic, then update our local values */506 adpt->rx_flow_control = phydev->pause;507 adpt->tx_flow_control = phydev->pause != phydev->asym_pause;508 }509 mac |= adpt->rx_flow_control ? RXFC : 0;510 mac |= adpt->tx_flow_control ? TXFC : 0;511 512 /* setup link speed */513 mac &= ~SPEED_MASK;514 if (phydev->speed == SPEED_1000) {515 mac |= SPEED(2);516 csr1 |= FREQ_MODE;517 } else {518 mac |= SPEED(1);519 csr1 &= ~FREQ_MODE;520 }521 522 if (phydev->duplex == DUPLEX_FULL)523 mac |= FULLD;524 else525 mac &= ~FULLD;526 527 /* other parameters */528 mac |= (CRCE | PCRCE);529 mac |= ((adpt->preamble << PRLEN_SHFT) & PRLEN_BMSK);530 mac |= BROAD_EN;531 mac |= FLCHK;532 mac &= ~RX_CHKSUM_EN;533 mac &= ~(HUGEN | VLAN_STRIP | TPAUSE | SIMR | HUGE | MULTI_ALL |534 DEBUG_MODE | SINGLE_PAUSE_MODE);535 536 /* Enable single-pause-frame mode if requested.537 *538 * If enabled, the EMAC will send a single pause frame when the RX539 * queue is full. This normally leads to packet loss because540 * the pause frame disables the remote MAC only for 33ms (the quanta),541 * and then the remote MAC continues sending packets even though542 * the RX queue is still full.543 *544 * If disabled, the EMAC sends a pause frame every 31ms until the RX545 * queue is no longer full. Normally, this is the preferred546 * method of operation. However, when the system is hung (e.g.547 * cores are halted), the EMAC interrupt handler is never called548 * and so the RX queue fills up quickly and stays full. The resuling549 * non-stop "flood" of pause frames sometimes has the effect of550 * disabling nearby switches. In some cases, other nearby switches551 * are also affected, shutting down the entire network.552 *553 * The user can enable or disable single-pause-frame mode554 * via ethtool.555 */556 mac |= adpt->single_pause_mode ? SINGLE_PAUSE_MODE : 0;557 558 writel_relaxed(csr1, adpt->csr + EMAC_EMAC_WRAPPER_CSR1);559 560 writel_relaxed(mac, adpt->base + EMAC_MAC_CTRL);561 562 /* enable interrupt read clear, low power sleep mode and563 * the irq moderators564 */565 566 writel_relaxed(adpt->irq_mod, adpt->base + EMAC_IRQ_MOD_TIM_INIT);567 writel_relaxed(INT_RD_CLR_EN | LPW_MODE | IRQ_MODERATOR_EN |568 IRQ_MODERATOR2_EN, adpt->base + EMAC_DMA_MAS_CTRL);569 570 emac_mac_mode_config(adpt);571 572 emac_reg_update32(adpt->base + EMAC_ATHR_HEADER_CTRL,573 (HEADER_ENABLE | HEADER_CNT_EN), 0);574}575 576void emac_mac_stop(struct emac_adapter *adpt)577{578 emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, RXQ_EN, 0);579 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, TXQ_EN, 0);580 emac_reg_update32(adpt->base + EMAC_MAC_CTRL, TXEN | RXEN, 0);581 usleep_range(1000, 1050); /* stopping mac may take upto 1msec */582}583 584/* Free all descriptors of given transmit queue */585static void emac_tx_q_descs_free(struct emac_adapter *adpt)586{587 struct emac_tx_queue *tx_q = &adpt->tx_q;588 unsigned int i;589 size_t size;590 591 /* ring already cleared, nothing to do */592 if (!tx_q->tpd.tpbuff)593 return;594 595 for (i = 0; i < tx_q->tpd.count; i++) {596 struct emac_buffer *tpbuf = GET_TPD_BUFFER(tx_q, i);597 598 if (tpbuf->dma_addr) {599 dma_unmap_single(adpt->netdev->dev.parent,600 tpbuf->dma_addr, tpbuf->length,601 DMA_TO_DEVICE);602 tpbuf->dma_addr = 0;603 }604 if (tpbuf->skb) {605 dev_kfree_skb_any(tpbuf->skb);606 tpbuf->skb = NULL;607 }608 }609 610 size = sizeof(struct emac_buffer) * tx_q->tpd.count;611 memset(tx_q->tpd.tpbuff, 0, size);612 613 /* clear the descriptor ring */614 memset(tx_q->tpd.v_addr, 0, tx_q->tpd.size);615 616 tx_q->tpd.consume_idx = 0;617 tx_q->tpd.produce_idx = 0;618}619 620/* Free all descriptors of given receive queue */621static void emac_rx_q_free_descs(struct emac_adapter *adpt)622{623 struct device *dev = adpt->netdev->dev.parent;624 struct emac_rx_queue *rx_q = &adpt->rx_q;625 unsigned int i;626 size_t size;627 628 /* ring already cleared, nothing to do */629 if (!rx_q->rfd.rfbuff)630 return;631 632 for (i = 0; i < rx_q->rfd.count; i++) {633 struct emac_buffer *rfbuf = GET_RFD_BUFFER(rx_q, i);634 635 if (rfbuf->dma_addr) {636 dma_unmap_single(dev, rfbuf->dma_addr, rfbuf->length,637 DMA_FROM_DEVICE);638 rfbuf->dma_addr = 0;639 }640 if (rfbuf->skb) {641 dev_kfree_skb(rfbuf->skb);642 rfbuf->skb = NULL;643 }644 }645 646 size = sizeof(struct emac_buffer) * rx_q->rfd.count;647 memset(rx_q->rfd.rfbuff, 0, size);648 649 /* clear the descriptor rings */650 memset(rx_q->rrd.v_addr, 0, rx_q->rrd.size);651 rx_q->rrd.produce_idx = 0;652 rx_q->rrd.consume_idx = 0;653 654 memset(rx_q->rfd.v_addr, 0, rx_q->rfd.size);655 rx_q->rfd.produce_idx = 0;656 rx_q->rfd.consume_idx = 0;657}658 659/* Free all buffers associated with given transmit queue */660static void emac_tx_q_bufs_free(struct emac_adapter *adpt)661{662 struct emac_tx_queue *tx_q = &adpt->tx_q;663 664 emac_tx_q_descs_free(adpt);665 666 kfree(tx_q->tpd.tpbuff);667 tx_q->tpd.tpbuff = NULL;668 tx_q->tpd.v_addr = NULL;669 tx_q->tpd.dma_addr = 0;670 tx_q->tpd.size = 0;671}672 673/* Allocate TX descriptor ring for the given transmit queue */674static int emac_tx_q_desc_alloc(struct emac_adapter *adpt,675 struct emac_tx_queue *tx_q)676{677 struct emac_ring_header *ring_header = &adpt->ring_header;678 int node = dev_to_node(adpt->netdev->dev.parent);679 size_t size;680 681 size = sizeof(struct emac_buffer) * tx_q->tpd.count;682 tx_q->tpd.tpbuff = kzalloc_node(size, GFP_KERNEL, node);683 if (!tx_q->tpd.tpbuff)684 return -ENOMEM;685 686 tx_q->tpd.size = tx_q->tpd.count * (adpt->tpd_size * 4);687 tx_q->tpd.dma_addr = ring_header->dma_addr + ring_header->used;688 tx_q->tpd.v_addr = ring_header->v_addr + ring_header->used;689 ring_header->used += ALIGN(tx_q->tpd.size, 8);690 tx_q->tpd.produce_idx = 0;691 tx_q->tpd.consume_idx = 0;692 693 return 0;694}695 696/* Free all buffers associated with given transmit queue */697static void emac_rx_q_bufs_free(struct emac_adapter *adpt)698{699 struct emac_rx_queue *rx_q = &adpt->rx_q;700 701 emac_rx_q_free_descs(adpt);702 703 kfree(rx_q->rfd.rfbuff);704 rx_q->rfd.rfbuff = NULL;705 706 rx_q->rfd.v_addr = NULL;707 rx_q->rfd.dma_addr = 0;708 rx_q->rfd.size = 0;709 710 rx_q->rrd.v_addr = NULL;711 rx_q->rrd.dma_addr = 0;712 rx_q->rrd.size = 0;713}714 715/* Allocate RX descriptor rings for the given receive queue */716static int emac_rx_descs_alloc(struct emac_adapter *adpt)717{718 struct emac_ring_header *ring_header = &adpt->ring_header;719 int node = dev_to_node(adpt->netdev->dev.parent);720 struct emac_rx_queue *rx_q = &adpt->rx_q;721 size_t size;722 723 size = sizeof(struct emac_buffer) * rx_q->rfd.count;724 rx_q->rfd.rfbuff = kzalloc_node(size, GFP_KERNEL, node);725 if (!rx_q->rfd.rfbuff)726 return -ENOMEM;727 728 rx_q->rrd.size = rx_q->rrd.count * (adpt->rrd_size * 4);729 rx_q->rfd.size = rx_q->rfd.count * (adpt->rfd_size * 4);730 731 rx_q->rrd.dma_addr = ring_header->dma_addr + ring_header->used;732 rx_q->rrd.v_addr = ring_header->v_addr + ring_header->used;733 ring_header->used += ALIGN(rx_q->rrd.size, 8);734 735 rx_q->rfd.dma_addr = ring_header->dma_addr + ring_header->used;736 rx_q->rfd.v_addr = ring_header->v_addr + ring_header->used;737 ring_header->used += ALIGN(rx_q->rfd.size, 8);738 739 rx_q->rrd.produce_idx = 0;740 rx_q->rrd.consume_idx = 0;741 742 rx_q->rfd.produce_idx = 0;743 rx_q->rfd.consume_idx = 0;744 745 return 0;746}747 748/* Allocate all TX and RX descriptor rings */749int emac_mac_rx_tx_rings_alloc_all(struct emac_adapter *adpt)750{751 struct emac_ring_header *ring_header = &adpt->ring_header;752 struct device *dev = adpt->netdev->dev.parent;753 unsigned int num_tx_descs = adpt->tx_desc_cnt;754 unsigned int num_rx_descs = adpt->rx_desc_cnt;755 int ret;756 757 adpt->tx_q.tpd.count = adpt->tx_desc_cnt;758 759 adpt->rx_q.rrd.count = adpt->rx_desc_cnt;760 adpt->rx_q.rfd.count = adpt->rx_desc_cnt;761 762 /* Ring DMA buffer. Each ring may need up to 8 bytes for alignment,763 * hence the additional padding bytes are allocated.764 */765 ring_header->size = num_tx_descs * (adpt->tpd_size * 4) +766 num_rx_descs * (adpt->rfd_size * 4) +767 num_rx_descs * (adpt->rrd_size * 4) +768 8 + 2 * 8; /* 8 byte per one Tx and two Rx rings */769 770 ring_header->used = 0;771 ring_header->v_addr = dma_alloc_coherent(dev, ring_header->size,772 &ring_header->dma_addr,773 GFP_KERNEL);774 if (!ring_header->v_addr)775 return -ENOMEM;776 777 ring_header->used = ALIGN(ring_header->dma_addr, 8) -778 ring_header->dma_addr;779 780 ret = emac_tx_q_desc_alloc(adpt, &adpt->tx_q);781 if (ret) {782 netdev_err(adpt->netdev, "error: Tx Queue alloc failed\n");783 goto err_alloc_tx;784 }785 786 ret = emac_rx_descs_alloc(adpt);787 if (ret) {788 netdev_err(adpt->netdev, "error: Rx Queue alloc failed\n");789 goto err_alloc_rx;790 }791 792 return 0;793 794err_alloc_rx:795 emac_tx_q_bufs_free(adpt);796err_alloc_tx:797 dma_free_coherent(dev, ring_header->size,798 ring_header->v_addr, ring_header->dma_addr);799 800 ring_header->v_addr = NULL;801 ring_header->dma_addr = 0;802 ring_header->size = 0;803 ring_header->used = 0;804 805 return ret;806}807 808/* Free all TX and RX descriptor rings */809void emac_mac_rx_tx_rings_free_all(struct emac_adapter *adpt)810{811 struct emac_ring_header *ring_header = &adpt->ring_header;812 struct device *dev = adpt->netdev->dev.parent;813 814 emac_tx_q_bufs_free(adpt);815 emac_rx_q_bufs_free(adpt);816 817 dma_free_coherent(dev, ring_header->size,818 ring_header->v_addr, ring_header->dma_addr);819 820 ring_header->v_addr = NULL;821 ring_header->dma_addr = 0;822 ring_header->size = 0;823 ring_header->used = 0;824}825 826/* Initialize descriptor rings */827static void emac_mac_rx_tx_ring_reset_all(struct emac_adapter *adpt)828{829 unsigned int i;830 831 adpt->tx_q.tpd.produce_idx = 0;832 adpt->tx_q.tpd.consume_idx = 0;833 for (i = 0; i < adpt->tx_q.tpd.count; i++)834 adpt->tx_q.tpd.tpbuff[i].dma_addr = 0;835 836 adpt->rx_q.rrd.produce_idx = 0;837 adpt->rx_q.rrd.consume_idx = 0;838 adpt->rx_q.rfd.produce_idx = 0;839 adpt->rx_q.rfd.consume_idx = 0;840 for (i = 0; i < adpt->rx_q.rfd.count; i++)841 adpt->rx_q.rfd.rfbuff[i].dma_addr = 0;842}843 844/* Produce new receive free descriptor */845static void emac_mac_rx_rfd_create(struct emac_adapter *adpt,846 struct emac_rx_queue *rx_q,847 dma_addr_t addr)848{849 u32 *hw_rfd = EMAC_RFD(rx_q, adpt->rfd_size, rx_q->rfd.produce_idx);850 851 *(hw_rfd++) = lower_32_bits(addr);852 *hw_rfd = upper_32_bits(addr);853 854 if (++rx_q->rfd.produce_idx == rx_q->rfd.count)855 rx_q->rfd.produce_idx = 0;856}857 858/* Fill up receive queue's RFD with preallocated receive buffers */859static void emac_mac_rx_descs_refill(struct emac_adapter *adpt,860 struct emac_rx_queue *rx_q)861{862 struct emac_buffer *curr_rxbuf;863 struct emac_buffer *next_rxbuf;864 unsigned int count = 0;865 u32 next_produce_idx;866 867 next_produce_idx = rx_q->rfd.produce_idx + 1;868 if (next_produce_idx == rx_q->rfd.count)869 next_produce_idx = 0;870 871 curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx);872 next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx);873 874 /* this always has a blank rx_buffer*/875 while (!next_rxbuf->dma_addr) {876 struct sk_buff *skb;877 int ret;878 879 skb = netdev_alloc_skb_ip_align(adpt->netdev, adpt->rxbuf_size);880 if (!skb)881 break;882 883 curr_rxbuf->dma_addr =884 dma_map_single(adpt->netdev->dev.parent, skb->data,885 adpt->rxbuf_size, DMA_FROM_DEVICE);886 887 ret = dma_mapping_error(adpt->netdev->dev.parent,888 curr_rxbuf->dma_addr);889 if (ret) {890 dev_kfree_skb(skb);891 break;892 }893 curr_rxbuf->skb = skb;894 curr_rxbuf->length = adpt->rxbuf_size;895 896 emac_mac_rx_rfd_create(adpt, rx_q, curr_rxbuf->dma_addr);897 next_produce_idx = rx_q->rfd.produce_idx + 1;898 if (next_produce_idx == rx_q->rfd.count)899 next_produce_idx = 0;900 901 curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx);902 next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx);903 count++;904 }905 906 if (count) {907 u32 prod_idx = (rx_q->rfd.produce_idx << rx_q->produce_shift) &908 rx_q->produce_mask;909 emac_reg_update32(adpt->base + rx_q->produce_reg,910 rx_q->produce_mask, prod_idx);911 }912}913 914static void emac_adjust_link(struct net_device *netdev)915{916 struct emac_adapter *adpt = netdev_priv(netdev);917 struct phy_device *phydev = netdev->phydev;918 919 if (phydev->link) {920 emac_mac_start(adpt);921 emac_sgmii_link_change(adpt, true);922 } else {923 emac_sgmii_link_change(adpt, false);924 emac_mac_stop(adpt);925 }926 927 phy_print_status(phydev);928}929 930/* Bringup the interface/HW */931int emac_mac_up(struct emac_adapter *adpt)932{933 struct net_device *netdev = adpt->netdev;934 int ret;935 936 emac_mac_rx_tx_ring_reset_all(adpt);937 emac_mac_config(adpt);938 emac_mac_rx_descs_refill(adpt, &adpt->rx_q);939 940 adpt->phydev->irq = PHY_POLL;941 ret = phy_connect_direct(netdev, adpt->phydev, emac_adjust_link,942 PHY_INTERFACE_MODE_SGMII);943 if (ret) {944 netdev_err(adpt->netdev, "could not connect phy\n");945 return ret;946 }947 948 phy_attached_print(adpt->phydev, NULL);949 950 /* enable mac irq */951 writel((u32)~DIS_INT, adpt->base + EMAC_INT_STATUS);952 writel(adpt->irq.mask, adpt->base + EMAC_INT_MASK);953 954 phy_start(adpt->phydev);955 956 napi_enable(&adpt->rx_q.napi);957 netif_start_queue(netdev);958 959 return 0;960}961 962/* Bring down the interface/HW */963void emac_mac_down(struct emac_adapter *adpt)964{965 struct net_device *netdev = adpt->netdev;966 967 netif_stop_queue(netdev);968 napi_disable(&adpt->rx_q.napi);969 970 phy_stop(adpt->phydev);971 972 /* Interrupts must be disabled before the PHY is disconnected, to973 * avoid a race condition where adjust_link is null when we get974 * an interrupt.975 */976 writel(DIS_INT, adpt->base + EMAC_INT_STATUS);977 writel(0, adpt->base + EMAC_INT_MASK);978 synchronize_irq(adpt->irq.irq);979 980 phy_disconnect(adpt->phydev);981 982 emac_mac_reset(adpt);983 984 emac_tx_q_descs_free(adpt);985 netdev_reset_queue(adpt->netdev);986 emac_rx_q_free_descs(adpt);987}988 989/* Consume next received packet descriptor */990static bool emac_rx_process_rrd(struct emac_adapter *adpt,991 struct emac_rx_queue *rx_q,992 struct emac_rrd *rrd)993{994 u32 *hw_rrd = EMAC_RRD(rx_q, adpt->rrd_size, rx_q->rrd.consume_idx);995 996 rrd->word[3] = *(hw_rrd + 3);997 998 if (!RRD_UPDT(rrd))999 return false;1000 1001 rrd->word[4] = 0;1002 rrd->word[5] = 0;1003 1004 rrd->word[0] = *(hw_rrd++);1005 rrd->word[1] = *(hw_rrd++);1006 rrd->word[2] = *(hw_rrd++);1007 1008 if (unlikely(RRD_NOR(rrd) != 1)) {1009 netdev_err(adpt->netdev,1010 "error: multi-RFD not support yet! nor:%lu\n",1011 RRD_NOR(rrd));1012 }1013 1014 /* mark rrd as processed */1015 RRD_UPDT_SET(rrd, 0);1016 *hw_rrd = rrd->word[3];1017 1018 if (++rx_q->rrd.consume_idx == rx_q->rrd.count)1019 rx_q->rrd.consume_idx = 0;1020 1021 return true;1022}1023 1024/* Produce new transmit descriptor */1025static void emac_tx_tpd_create(struct emac_adapter *adpt,1026 struct emac_tx_queue *tx_q, struct emac_tpd *tpd)1027{1028 u32 *hw_tpd;1029 1030 tx_q->tpd.last_produce_idx = tx_q->tpd.produce_idx;1031 hw_tpd = EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.produce_idx);1032 1033 if (++tx_q->tpd.produce_idx == tx_q->tpd.count)1034 tx_q->tpd.produce_idx = 0;1035 1036 *(hw_tpd++) = tpd->word[0];1037 *(hw_tpd++) = tpd->word[1];1038 *(hw_tpd++) = tpd->word[2];1039 *hw_tpd = tpd->word[3];1040}1041 1042/* Mark the last transmit descriptor as such (for the transmit packet) */1043static void emac_tx_tpd_mark_last(struct emac_adapter *adpt,1044 struct emac_tx_queue *tx_q)1045{1046 u32 *hw_tpd =1047 EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.last_produce_idx);1048 u32 tmp_tpd;1049 1050 tmp_tpd = *(hw_tpd + 1);1051 tmp_tpd |= EMAC_TPD_LAST_FRAGMENT;1052 *(hw_tpd + 1) = tmp_tpd;1053}1054 1055static void emac_rx_rfd_clean(struct emac_rx_queue *rx_q, struct emac_rrd *rrd)1056{1057 struct emac_buffer *rfbuf = rx_q->rfd.rfbuff;1058 u32 consume_idx = RRD_SI(rrd);1059 unsigned int i;1060 1061 for (i = 0; i < RRD_NOR(rrd); i++) {1062 rfbuf[consume_idx].skb = NULL;1063 if (++consume_idx == rx_q->rfd.count)1064 consume_idx = 0;1065 }1066 1067 rx_q->rfd.consume_idx = consume_idx;1068 rx_q->rfd.process_idx = consume_idx;1069}1070 1071/* Push the received skb to upper layers */1072static void emac_receive_skb(struct emac_rx_queue *rx_q,1073 struct sk_buff *skb,1074 u16 vlan_tag, bool vlan_flag)1075{1076 if (vlan_flag) {1077 u16 vlan;1078 1079 EMAC_TAG_TO_VLAN(vlan_tag, vlan);1080 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan);1081 }1082 1083 napi_gro_receive(&rx_q->napi, skb);1084}1085 1086/* Process receive event */1087void emac_mac_rx_process(struct emac_adapter *adpt, struct emac_rx_queue *rx_q,1088 int *num_pkts, int max_pkts)1089{1090 u32 proc_idx, hw_consume_idx, num_consume_pkts;1091 struct net_device *netdev = adpt->netdev;1092 struct emac_buffer *rfbuf;1093 unsigned int count = 0;1094 struct emac_rrd rrd;1095 struct sk_buff *skb;1096 u32 reg;1097 1098 reg = readl_relaxed(adpt->base + rx_q->consume_reg);1099 1100 hw_consume_idx = (reg & rx_q->consume_mask) >> rx_q->consume_shift;1101 num_consume_pkts = (hw_consume_idx >= rx_q->rrd.consume_idx) ?1102 (hw_consume_idx - rx_q->rrd.consume_idx) :1103 (hw_consume_idx + rx_q->rrd.count - rx_q->rrd.consume_idx);1104 1105 do {1106 if (!num_consume_pkts)1107 break;1108 1109 if (!emac_rx_process_rrd(adpt, rx_q, &rrd))1110 break;1111 1112 if (likely(RRD_NOR(&rrd) == 1)) {1113 /* good receive */1114 rfbuf = GET_RFD_BUFFER(rx_q, RRD_SI(&rrd));1115 dma_unmap_single(adpt->netdev->dev.parent,1116 rfbuf->dma_addr, rfbuf->length,1117 DMA_FROM_DEVICE);1118 rfbuf->dma_addr = 0;1119 skb = rfbuf->skb;1120 } else {1121 netdev_err(adpt->netdev,1122 "error: multi-RFD not support yet!\n");1123 break;1124 }1125 emac_rx_rfd_clean(rx_q, &rrd);1126 num_consume_pkts--;1127 count++;1128 1129 /* Due to a HW issue in L4 check sum detection (UDP/TCP frags1130 * with DF set are marked as error), drop packets based on the1131 * error mask rather than the summary bit (ignoring L4F errors)1132 */1133 if (rrd.word[EMAC_RRD_STATS_DW_IDX] & EMAC_RRD_ERROR) {1134 netif_dbg(adpt, rx_status, adpt->netdev,1135 "Drop error packet[RRD: 0x%x:0x%x:0x%x:0x%x]\n",1136 rrd.word[0], rrd.word[1],1137 rrd.word[2], rrd.word[3]);1138 1139 dev_kfree_skb(skb);1140 continue;1141 }1142 1143 skb_put(skb, RRD_PKT_SIZE(&rrd) - ETH_FCS_LEN);1144 skb->dev = netdev;1145 skb->protocol = eth_type_trans(skb, skb->dev);1146 if (netdev->features & NETIF_F_RXCSUM)1147 skb->ip_summed = RRD_L4F(&rrd) ?1148 CHECKSUM_NONE : CHECKSUM_UNNECESSARY;1149 else1150 skb_checksum_none_assert(skb);1151 1152 emac_receive_skb(rx_q, skb, (u16)RRD_CVALN_TAG(&rrd),1153 (bool)RRD_CVTAG(&rrd));1154 1155 (*num_pkts)++;1156 } while (*num_pkts < max_pkts);1157 1158 if (count) {1159 proc_idx = (rx_q->rfd.process_idx << rx_q->process_shft) &1160 rx_q->process_mask;1161 emac_reg_update32(adpt->base + rx_q->process_reg,1162 rx_q->process_mask, proc_idx);1163 emac_mac_rx_descs_refill(adpt, rx_q);1164 }1165}1166 1167/* get the number of free transmit descriptors */1168static unsigned int emac_tpd_num_free_descs(struct emac_tx_queue *tx_q)1169{1170 u32 produce_idx = tx_q->tpd.produce_idx;1171 u32 consume_idx = tx_q->tpd.consume_idx;1172 1173 return (consume_idx > produce_idx) ?1174 (consume_idx - produce_idx - 1) :1175 (tx_q->tpd.count + consume_idx - produce_idx - 1);1176}1177 1178/* Process transmit event */1179void emac_mac_tx_process(struct emac_adapter *adpt, struct emac_tx_queue *tx_q)1180{1181 u32 reg = readl_relaxed(adpt->base + tx_q->consume_reg);1182 u32 hw_consume_idx, pkts_compl = 0, bytes_compl = 0;1183 struct emac_buffer *tpbuf;1184 1185 hw_consume_idx = (reg & tx_q->consume_mask) >> tx_q->consume_shift;1186 1187 while (tx_q->tpd.consume_idx != hw_consume_idx) {1188 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.consume_idx);1189 if (tpbuf->dma_addr) {1190 dma_unmap_page(adpt->netdev->dev.parent,1191 tpbuf->dma_addr, tpbuf->length,1192 DMA_TO_DEVICE);1193 tpbuf->dma_addr = 0;1194 }1195 1196 if (tpbuf->skb) {1197 pkts_compl++;1198 bytes_compl += tpbuf->skb->len;1199 dev_consume_skb_irq(tpbuf->skb);1200 tpbuf->skb = NULL;1201 }1202 1203 if (++tx_q->tpd.consume_idx == tx_q->tpd.count)1204 tx_q->tpd.consume_idx = 0;1205 }1206 1207 netdev_completed_queue(adpt->netdev, pkts_compl, bytes_compl);1208 1209 if (netif_queue_stopped(adpt->netdev))1210 if (emac_tpd_num_free_descs(tx_q) > (MAX_SKB_FRAGS + 1))1211 netif_wake_queue(adpt->netdev);1212}1213 1214/* Initialize all queue data structures */1215void emac_mac_rx_tx_ring_init_all(struct platform_device *pdev,1216 struct emac_adapter *adpt)1217{1218 adpt->rx_q.netdev = adpt->netdev;1219 1220 adpt->rx_q.produce_reg = EMAC_MAILBOX_0;1221 adpt->rx_q.produce_mask = RFD0_PROD_IDX_BMSK;1222 adpt->rx_q.produce_shift = RFD0_PROD_IDX_SHFT;1223 1224 adpt->rx_q.process_reg = EMAC_MAILBOX_0;1225 adpt->rx_q.process_mask = RFD0_PROC_IDX_BMSK;1226 adpt->rx_q.process_shft = RFD0_PROC_IDX_SHFT;1227 1228 adpt->rx_q.consume_reg = EMAC_MAILBOX_3;1229 adpt->rx_q.consume_mask = RFD0_CONS_IDX_BMSK;1230 adpt->rx_q.consume_shift = RFD0_CONS_IDX_SHFT;1231 1232 adpt->rx_q.irq = &adpt->irq;1233 adpt->rx_q.intr = adpt->irq.mask & ISR_RX_PKT;1234 1235 adpt->tx_q.produce_reg = EMAC_MAILBOX_15;1236 adpt->tx_q.produce_mask = NTPD_PROD_IDX_BMSK;1237 adpt->tx_q.produce_shift = NTPD_PROD_IDX_SHFT;1238 1239 adpt->tx_q.consume_reg = EMAC_MAILBOX_2;1240 adpt->tx_q.consume_mask = NTPD_CONS_IDX_BMSK;1241 adpt->tx_q.consume_shift = NTPD_CONS_IDX_SHFT;1242}1243 1244/* Fill up transmit descriptors with TSO and Checksum offload information */1245static int emac_tso_csum(struct emac_adapter *adpt,1246 struct emac_tx_queue *tx_q,1247 struct sk_buff *skb,1248 struct emac_tpd *tpd)1249{1250 unsigned int hdr_len;1251 int ret;1252 1253 if (skb_is_gso(skb)) {1254 if (skb_header_cloned(skb)) {1255 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);1256 if (unlikely(ret))1257 return ret;1258 }1259 1260 if (skb->protocol == htons(ETH_P_IP)) {1261 u32 pkt_len = ((unsigned char *)ip_hdr(skb) - skb->data)1262 + ntohs(ip_hdr(skb)->tot_len);1263 if (skb->len > pkt_len) {1264 ret = pskb_trim(skb, pkt_len);1265 if (unlikely(ret))1266 return ret;1267 }1268 }1269 1270 hdr_len = skb_tcp_all_headers(skb);1271 if (unlikely(skb->len == hdr_len)) {1272 /* we only need to do csum */1273 netif_warn(adpt, tx_err, adpt->netdev,1274 "tso not needed for packet with 0 data\n");1275 goto do_csum;1276 }1277 1278 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {1279 ip_hdr(skb)->check = 0;1280 tcp_hdr(skb)->check =1281 ~csum_tcpudp_magic(ip_hdr(skb)->saddr,1282 ip_hdr(skb)->daddr,1283 0, IPPROTO_TCP, 0);1284 TPD_IPV4_SET(tpd, 1);1285 }1286 1287 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) {1288 /* ipv6 tso need an extra tpd */1289 struct emac_tpd extra_tpd;1290 1291 memset(tpd, 0, sizeof(*tpd));1292 memset(&extra_tpd, 0, sizeof(extra_tpd));1293 1294 tcp_v6_gso_csum_prep(skb);1295 1296 TPD_PKT_LEN_SET(&extra_tpd, skb->len);1297 TPD_LSO_SET(&extra_tpd, 1);1298 TPD_LSOV_SET(&extra_tpd, 1);1299 emac_tx_tpd_create(adpt, tx_q, &extra_tpd);1300 TPD_LSOV_SET(tpd, 1);1301 }1302 1303 TPD_LSO_SET(tpd, 1);1304 TPD_TCPHDR_OFFSET_SET(tpd, skb_transport_offset(skb));1305 TPD_MSS_SET(tpd, skb_shinfo(skb)->gso_size);1306 return 0;1307 }1308 1309do_csum:1310 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {1311 unsigned int css, cso;1312 1313 cso = skb_transport_offset(skb);1314 if (unlikely(cso & 0x1)) {1315 netdev_err(adpt->netdev,1316 "error: payload offset should be even\n");1317 return -EINVAL;1318 }1319 css = cso + skb->csum_offset;1320 1321 TPD_PAYLOAD_OFFSET_SET(tpd, cso >> 1);1322 TPD_CXSUM_OFFSET_SET(tpd, css >> 1);1323 TPD_CSX_SET(tpd, 1);1324 }1325 1326 return 0;1327}1328 1329/* Fill up transmit descriptors */1330static void emac_tx_fill_tpd(struct emac_adapter *adpt,1331 struct emac_tx_queue *tx_q, struct sk_buff *skb,1332 struct emac_tpd *tpd)1333{1334 unsigned int nr_frags = skb_shinfo(skb)->nr_frags;1335 unsigned int first = tx_q->tpd.produce_idx;1336 unsigned int len = skb_headlen(skb);1337 struct emac_buffer *tpbuf = NULL;1338 unsigned int mapped_len = 0;1339 unsigned int i;1340 int count = 0;1341 int ret;1342 1343 /* if Large Segment Offload is (in TCP Segmentation Offload struct) */1344 if (TPD_LSO(tpd)) {1345 mapped_len = skb_tcp_all_headers(skb);1346 1347 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);1348 tpbuf->length = mapped_len;1349 tpbuf->dma_addr = dma_map_page(adpt->netdev->dev.parent,1350 virt_to_page(skb->data),1351 offset_in_page(skb->data),1352 tpbuf->length,1353 DMA_TO_DEVICE);1354 ret = dma_mapping_error(adpt->netdev->dev.parent,1355 tpbuf->dma_addr);1356 if (ret)1357 goto error;1358 1359 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));1360 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));1361 TPD_BUF_LEN_SET(tpd, tpbuf->length);1362 emac_tx_tpd_create(adpt, tx_q, tpd);1363 count++;1364 }1365 1366 if (mapped_len < len) {1367 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);1368 tpbuf->length = len - mapped_len;1369 tpbuf->dma_addr = dma_map_page(adpt->netdev->dev.parent,1370 virt_to_page(skb->data +1371 mapped_len),1372 offset_in_page(skb->data +1373 mapped_len),1374 tpbuf->length, DMA_TO_DEVICE);1375 ret = dma_mapping_error(adpt->netdev->dev.parent,1376 tpbuf->dma_addr);1377 if (ret)1378 goto error;1379 1380 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));1381 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));1382 TPD_BUF_LEN_SET(tpd, tpbuf->length);1383 emac_tx_tpd_create(adpt, tx_q, tpd);1384 count++;1385 }1386 1387 for (i = 0; i < nr_frags; i++) {1388 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];1389 1390 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);1391 tpbuf->length = skb_frag_size(frag);1392 tpbuf->dma_addr = skb_frag_dma_map(adpt->netdev->dev.parent,1393 frag, 0, tpbuf->length,1394 DMA_TO_DEVICE);1395 ret = dma_mapping_error(adpt->netdev->dev.parent,1396 tpbuf->dma_addr);1397 if (ret)1398 goto error;1399 1400 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));1401 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));1402 TPD_BUF_LEN_SET(tpd, tpbuf->length);1403 emac_tx_tpd_create(adpt, tx_q, tpd);1404 count++;1405 }1406 1407 /* The last tpd */1408 wmb();1409 emac_tx_tpd_mark_last(adpt, tx_q);1410 1411 /* The last buffer info contain the skb address,1412 * so it will be freed after unmap1413 */1414 tpbuf->skb = skb;1415 1416 return;1417 1418error:1419 /* One of the memory mappings failed, so undo everything */1420 tx_q->tpd.produce_idx = first;1421 1422 while (count--) {1423 tpbuf = GET_TPD_BUFFER(tx_q, first);1424 dma_unmap_page(adpt->netdev->dev.parent, tpbuf->dma_addr,1425 tpbuf->length, DMA_TO_DEVICE);1426 tpbuf->dma_addr = 0;1427 tpbuf->length = 0;1428 1429 if (++first == tx_q->tpd.count)1430 first = 0;1431 }1432 1433 dev_kfree_skb(skb);1434}1435 1436/* Transmit the packet using specified transmit queue */1437netdev_tx_t emac_mac_tx_buf_send(struct emac_adapter *adpt,1438 struct emac_tx_queue *tx_q,1439 struct sk_buff *skb)1440{1441 struct emac_tpd tpd;1442 u32 prod_idx;1443 int len;1444 1445 memset(&tpd, 0, sizeof(tpd));1446 1447 if (emac_tso_csum(adpt, tx_q, skb, &tpd) != 0) {1448 dev_kfree_skb_any(skb);1449 return NETDEV_TX_OK;1450 }1451 1452 if (skb_vlan_tag_present(skb)) {1453 u16 tag;1454 1455 EMAC_VLAN_TO_TAG(skb_vlan_tag_get(skb), tag);1456 TPD_CVLAN_TAG_SET(&tpd, tag);1457 TPD_INSTC_SET(&tpd, 1);1458 }1459 1460 if (skb_network_offset(skb) != ETH_HLEN)1461 TPD_TYP_SET(&tpd, 1);1462 1463 len = skb->len;1464 emac_tx_fill_tpd(adpt, tx_q, skb, &tpd);1465 1466 netdev_sent_queue(adpt->netdev, len);1467 1468 /* Make sure the are enough free descriptors to hold one1469 * maximum-sized SKB. We need one desc for each fragment,1470 * one for the checksum (emac_tso_csum), one for TSO, and1471 * one for the SKB header.1472 */1473 if (emac_tpd_num_free_descs(tx_q) < (MAX_SKB_FRAGS + 3))1474 netif_stop_queue(adpt->netdev);1475 1476 /* update produce idx */1477 prod_idx = (tx_q->tpd.produce_idx << tx_q->produce_shift) &1478 tx_q->produce_mask;1479 emac_reg_update32(adpt->base + tx_q->produce_reg,1480 tx_q->produce_mask, prod_idx);1481 1482 return NETDEV_TX_OK;1483}1484