2289 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause2/*3 * rtase is the Linux device driver released for Realtek Automotive Switch4 * controllers with PCI-Express interface.5 *6 * Copyright(c) 2024 Realtek Semiconductor Corp.7 *8 * Below is a simplified block diagram of the chip and its relevant interfaces.9 *10 * *************************11 * * *12 * * CPU network device *13 * * *14 * * +-------------+ *15 * * | PCIE Host | *16 * ***********++************17 * ||18 * PCIE19 * ||20 * ********************++**********************21 * * | PCIE Endpoint | *22 * * +---------------+ *23 * * | GMAC | *24 * * +--++--+ Realtek *25 * * || RTL90xx Series *26 * * || *27 * * +-------------++----------------+ *28 * * | | MAC | | *29 * * | +-----+ | *30 * * | | *31 * * | Ethernet Switch Core | *32 * * | | *33 * * | +-----+ +-----+ | *34 * * | | MAC |...........| MAC | | *35 * * +---+-----+-----------+-----+---+ *36 * * | PHY |...........| PHY | *37 * * +--++-+ +--++-+ *38 * *************||****************||***********39 *40 * The block of the Realtek RTL90xx series is our entire chip architecture,41 * the GMAC is connected to the switch core, and there is no PHY in between.42 * In addition, this driver is mainly used to control GMAC, but does not43 * control the switch core, so it is not the same as DSA. Linux only plays44 * the role of a normal leaf node in this model.45 */46 47#include <linux/crc32.h>48#include <linux/dma-mapping.h>49#include <linux/etherdevice.h>50#include <linux/if_vlan.h>51#include <linux/in.h>52#include <linux/init.h>53#include <linux/interrupt.h>54#include <linux/io.h>55#include <linux/iopoll.h>56#include <linux/ip.h>57#include <linux/ipv6.h>58#include <linux/mdio.h>59#include <linux/module.h>60#include <linux/netdevice.h>61#include <linux/pci.h>62#include <linux/pm_runtime.h>63#include <linux/prefetch.h>64#include <linux/rtnetlink.h>65#include <linux/tcp.h>66#include <asm/irq.h>67#include <net/ip6_checksum.h>68#include <net/netdev_queues.h>69#include <net/page_pool/helpers.h>70#include <net/pkt_cls.h>71 72#include "rtase.h"73 74#define RTK_OPTS1_DEBUG_VALUE 0x0BADBEEF75#define RTK_MAGIC_NUMBER 0x0BADBADBADBADBAD76 77static const struct pci_device_id rtase_pci_tbl[] = {78 {PCI_VDEVICE(REALTEK, 0x906A)},79 {}80};81 82MODULE_DEVICE_TABLE(pci, rtase_pci_tbl);83 84MODULE_AUTHOR("Realtek ARD Software Team");85MODULE_DESCRIPTION("Network Driver for the PCIe interface of Realtek Automotive Ethernet Switch");86MODULE_LICENSE("Dual BSD/GPL");87 88struct rtase_counters {89 __le64 tx_packets;90 __le64 rx_packets;91 __le64 tx_errors;92 __le32 rx_errors;93 __le16 rx_missed;94 __le16 align_errors;95 __le32 tx_one_collision;96 __le32 tx_multi_collision;97 __le64 rx_unicast;98 __le64 rx_broadcast;99 __le32 rx_multicast;100 __le16 tx_aborted;101 __le16 tx_underrun;102} __packed;103 104static void rtase_w8(const struct rtase_private *tp, u16 reg, u8 val8)105{106 writeb(val8, tp->mmio_addr + reg);107}108 109static void rtase_w16(const struct rtase_private *tp, u16 reg, u16 val16)110{111 writew(val16, tp->mmio_addr + reg);112}113 114static void rtase_w32(const struct rtase_private *tp, u16 reg, u32 val32)115{116 writel(val32, tp->mmio_addr + reg);117}118 119static u8 rtase_r8(const struct rtase_private *tp, u16 reg)120{121 return readb(tp->mmio_addr + reg);122}123 124static u16 rtase_r16(const struct rtase_private *tp, u16 reg)125{126 return readw(tp->mmio_addr + reg);127}128 129static u32 rtase_r32(const struct rtase_private *tp, u16 reg)130{131 return readl(tp->mmio_addr + reg);132}133 134static void rtase_free_desc(struct rtase_private *tp)135{136 struct pci_dev *pdev = tp->pdev;137 u32 i;138 139 for (i = 0; i < tp->func_tx_queue_num; i++) {140 if (!tp->tx_ring[i].desc)141 continue;142 143 dma_free_coherent(&pdev->dev, RTASE_TX_RING_DESC_SIZE,144 tp->tx_ring[i].desc,145 tp->tx_ring[i].phy_addr);146 tp->tx_ring[i].desc = NULL;147 }148 149 for (i = 0; i < tp->func_rx_queue_num; i++) {150 if (!tp->rx_ring[i].desc)151 continue;152 153 dma_free_coherent(&pdev->dev, RTASE_RX_RING_DESC_SIZE,154 tp->rx_ring[i].desc,155 tp->rx_ring[i].phy_addr);156 tp->rx_ring[i].desc = NULL;157 }158}159 160static int rtase_alloc_desc(struct rtase_private *tp)161{162 struct pci_dev *pdev = tp->pdev;163 u32 i;164 165 /* rx and tx descriptors needs 256 bytes alignment.166 * dma_alloc_coherent provides more.167 */168 for (i = 0; i < tp->func_tx_queue_num; i++) {169 tp->tx_ring[i].desc =170 dma_alloc_coherent(&pdev->dev,171 RTASE_TX_RING_DESC_SIZE,172 &tp->tx_ring[i].phy_addr,173 GFP_KERNEL);174 if (!tp->tx_ring[i].desc)175 goto err_out;176 }177 178 for (i = 0; i < tp->func_rx_queue_num; i++) {179 tp->rx_ring[i].desc =180 dma_alloc_coherent(&pdev->dev,181 RTASE_RX_RING_DESC_SIZE,182 &tp->rx_ring[i].phy_addr,183 GFP_KERNEL);184 if (!tp->rx_ring[i].desc)185 goto err_out;186 }187 188 return 0;189 190err_out:191 rtase_free_desc(tp);192 return -ENOMEM;193}194 195static void rtase_unmap_tx_skb(struct pci_dev *pdev, u32 len,196 struct rtase_tx_desc *desc)197{198 dma_unmap_single(&pdev->dev, le64_to_cpu(desc->addr), len,199 DMA_TO_DEVICE);200 desc->opts1 = cpu_to_le32(RTK_OPTS1_DEBUG_VALUE);201 desc->opts2 = 0x00;202 desc->addr = cpu_to_le64(RTK_MAGIC_NUMBER);203}204 205static void rtase_tx_clear_range(struct rtase_ring *ring, u32 start, u32 n)206{207 struct rtase_tx_desc *desc_base = ring->desc;208 struct rtase_private *tp = ring->ivec->tp;209 u32 i;210 211 for (i = 0; i < n; i++) {212 u32 entry = (start + i) % RTASE_NUM_DESC;213 struct rtase_tx_desc *desc = desc_base + entry;214 u32 len = ring->mis.len[entry];215 struct sk_buff *skb;216 217 if (len == 0)218 continue;219 220 rtase_unmap_tx_skb(tp->pdev, len, desc);221 ring->mis.len[entry] = 0;222 skb = ring->skbuff[entry];223 if (!skb)224 continue;225 226 tp->stats.tx_dropped++;227 dev_kfree_skb_any(skb);228 ring->skbuff[entry] = NULL;229 }230}231 232static void rtase_tx_clear(struct rtase_private *tp)233{234 struct rtase_ring *ring;235 u16 i;236 237 for (i = 0; i < tp->func_tx_queue_num; i++) {238 ring = &tp->tx_ring[i];239 rtase_tx_clear_range(ring, ring->dirty_idx, RTASE_NUM_DESC);240 ring->cur_idx = 0;241 ring->dirty_idx = 0;242 }243}244 245static void rtase_mark_to_asic(union rtase_rx_desc *desc, u32 rx_buf_sz)246{247 u32 eor = le32_to_cpu(desc->desc_cmd.opts1) & RTASE_RING_END;248 249 desc->desc_status.opts2 = 0;250 /* force memory writes to complete before releasing descriptor */251 dma_wmb();252 WRITE_ONCE(desc->desc_cmd.opts1,253 cpu_to_le32(RTASE_DESC_OWN | eor | rx_buf_sz));254}255 256static u32 rtase_tx_avail(struct rtase_ring *ring)257{258 return READ_ONCE(ring->dirty_idx) + RTASE_NUM_DESC -259 READ_ONCE(ring->cur_idx);260}261 262static int tx_handler(struct rtase_ring *ring, int budget)263{264 const struct rtase_private *tp = ring->ivec->tp;265 struct net_device *dev = tp->dev;266 u32 dirty_tx, tx_left;267 u32 bytes_compl = 0;268 u32 pkts_compl = 0;269 int workdone = 0;270 271 dirty_tx = ring->dirty_idx;272 tx_left = READ_ONCE(ring->cur_idx) - dirty_tx;273 274 while (tx_left > 0) {275 u32 entry = dirty_tx % RTASE_NUM_DESC;276 struct rtase_tx_desc *desc = ring->desc +277 sizeof(struct rtase_tx_desc) * entry;278 u32 status;279 280 status = le32_to_cpu(desc->opts1);281 282 if (status & RTASE_DESC_OWN)283 break;284 285 rtase_unmap_tx_skb(tp->pdev, ring->mis.len[entry], desc);286 ring->mis.len[entry] = 0;287 if (ring->skbuff[entry]) {288 pkts_compl++;289 bytes_compl += ring->skbuff[entry]->len;290 napi_consume_skb(ring->skbuff[entry], budget);291 ring->skbuff[entry] = NULL;292 }293 294 dirty_tx++;295 tx_left--;296 workdone++;297 298 if (workdone == RTASE_TX_BUDGET_DEFAULT)299 break;300 }301 302 if (ring->dirty_idx != dirty_tx) {303 dev_sw_netstats_tx_add(dev, pkts_compl, bytes_compl);304 WRITE_ONCE(ring->dirty_idx, dirty_tx);305 306 netif_subqueue_completed_wake(dev, ring->index, pkts_compl,307 bytes_compl,308 rtase_tx_avail(ring),309 RTASE_TX_START_THRS);310 311 if (ring->cur_idx != dirty_tx)312 rtase_w8(tp, RTASE_TPPOLL, BIT(ring->index));313 }314 315 return 0;316}317 318static void rtase_tx_desc_init(struct rtase_private *tp, u16 idx)319{320 struct rtase_ring *ring = &tp->tx_ring[idx];321 struct rtase_tx_desc *desc;322 u32 i;323 324 memset(ring->desc, 0x0, RTASE_TX_RING_DESC_SIZE);325 memset(ring->skbuff, 0x0, sizeof(ring->skbuff));326 ring->cur_idx = 0;327 ring->dirty_idx = 0;328 ring->index = idx;329 ring->alloc_fail = 0;330 331 for (i = 0; i < RTASE_NUM_DESC; i++) {332 ring->mis.len[i] = 0;333 if ((RTASE_NUM_DESC - 1) == i) {334 desc = ring->desc + sizeof(struct rtase_tx_desc) * i;335 desc->opts1 = cpu_to_le32(RTASE_RING_END);336 }337 }338 339 ring->ring_handler = tx_handler;340 if (idx < 4) {341 ring->ivec = &tp->int_vector[idx];342 list_add_tail(&ring->ring_entry,343 &tp->int_vector[idx].ring_list);344 } else {345 ring->ivec = &tp->int_vector[0];346 list_add_tail(&ring->ring_entry, &tp->int_vector[0].ring_list);347 }348}349 350static void rtase_map_to_asic(union rtase_rx_desc *desc, dma_addr_t mapping,351 u32 rx_buf_sz)352{353 desc->desc_cmd.addr = cpu_to_le64(mapping);354 355 rtase_mark_to_asic(desc, rx_buf_sz);356}357 358static void rtase_make_unusable_by_asic(union rtase_rx_desc *desc)359{360 desc->desc_cmd.addr = cpu_to_le64(RTK_MAGIC_NUMBER);361 desc->desc_cmd.opts1 &= ~cpu_to_le32(RTASE_DESC_OWN | RSVD_MASK);362}363 364static int rtase_alloc_rx_data_buf(struct rtase_ring *ring,365 void **p_data_buf,366 union rtase_rx_desc *desc,367 dma_addr_t *rx_phy_addr)368{369 struct rtase_int_vector *ivec = ring->ivec;370 const struct rtase_private *tp = ivec->tp;371 dma_addr_t mapping;372 struct page *page;373 374 page = page_pool_dev_alloc_pages(tp->page_pool);375 if (!page) {376 ring->alloc_fail++;377 goto err_out;378 }379 380 *p_data_buf = page_address(page);381 mapping = page_pool_get_dma_addr(page);382 *rx_phy_addr = mapping;383 rtase_map_to_asic(desc, mapping, tp->rx_buf_sz);384 385 return 0;386 387err_out:388 rtase_make_unusable_by_asic(desc);389 390 return -ENOMEM;391}392 393static u32 rtase_rx_ring_fill(struct rtase_ring *ring, u32 ring_start,394 u32 ring_end)395{396 union rtase_rx_desc *desc_base = ring->desc;397 u32 cur;398 399 for (cur = ring_start; ring_end - cur > 0; cur++) {400 u32 i = cur % RTASE_NUM_DESC;401 union rtase_rx_desc *desc = desc_base + i;402 int ret;403 404 if (ring->data_buf[i])405 continue;406 407 ret = rtase_alloc_rx_data_buf(ring, &ring->data_buf[i], desc,408 &ring->mis.data_phy_addr[i]);409 if (ret)410 break;411 }412 413 return cur - ring_start;414}415 416static void rtase_mark_as_last_descriptor(union rtase_rx_desc *desc)417{418 desc->desc_cmd.opts1 |= cpu_to_le32(RTASE_RING_END);419}420 421static void rtase_rx_ring_clear(struct page_pool *page_pool,422 struct rtase_ring *ring)423{424 union rtase_rx_desc *desc;425 struct page *page;426 u32 i;427 428 for (i = 0; i < RTASE_NUM_DESC; i++) {429 desc = ring->desc + sizeof(union rtase_rx_desc) * i;430 page = virt_to_head_page(ring->data_buf[i]);431 432 if (ring->data_buf[i])433 page_pool_put_full_page(page_pool, page, true);434 435 rtase_make_unusable_by_asic(desc);436 }437}438 439static int rtase_fragmented_frame(u32 status)440{441 return (status & (RTASE_RX_FIRST_FRAG | RTASE_RX_LAST_FRAG)) !=442 (RTASE_RX_FIRST_FRAG | RTASE_RX_LAST_FRAG);443}444 445static void rtase_rx_csum(const struct rtase_private *tp, struct sk_buff *skb,446 const union rtase_rx_desc *desc)447{448 u32 opts2 = le32_to_cpu(desc->desc_status.opts2);449 450 /* rx csum offload */451 if (((opts2 & RTASE_RX_V4F) && !(opts2 & RTASE_RX_IPF)) ||452 (opts2 & RTASE_RX_V6F)) {453 if (((opts2 & RTASE_RX_TCPT) && !(opts2 & RTASE_RX_TCPF)) ||454 ((opts2 & RTASE_RX_UDPT) && !(opts2 & RTASE_RX_UDPF)))455 skb->ip_summed = CHECKSUM_UNNECESSARY;456 else457 skb->ip_summed = CHECKSUM_NONE;458 } else {459 skb->ip_summed = CHECKSUM_NONE;460 }461}462 463static void rtase_rx_vlan_skb(union rtase_rx_desc *desc, struct sk_buff *skb)464{465 u32 opts2 = le32_to_cpu(desc->desc_status.opts2);466 467 if (!(opts2 & RTASE_RX_VLAN_TAG))468 return;469 470 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),471 swab16(opts2 & RTASE_VLAN_TAG_MASK));472}473 474static void rtase_rx_skb(const struct rtase_ring *ring, struct sk_buff *skb)475{476 struct rtase_int_vector *ivec = ring->ivec;477 478 napi_gro_receive(&ivec->napi, skb);479}480 481static int rx_handler(struct rtase_ring *ring, int budget)482{483 union rtase_rx_desc *desc_base = ring->desc;484 u32 pkt_size, cur_rx, delta, entry, status;485 struct rtase_private *tp = ring->ivec->tp;486 struct net_device *dev = tp->dev;487 union rtase_rx_desc *desc;488 struct sk_buff *skb;489 int workdone = 0;490 491 cur_rx = ring->cur_idx;492 entry = cur_rx % RTASE_NUM_DESC;493 desc = &desc_base[entry];494 495 while (workdone < budget) {496 status = le32_to_cpu(desc->desc_status.opts1);497 498 if (status & RTASE_DESC_OWN)499 break;500 501 /* This barrier is needed to keep us from reading502 * any other fields out of the rx descriptor until503 * we know the status of RTASE_DESC_OWN504 */505 dma_rmb();506 507 if (unlikely(status & RTASE_RX_RES)) {508 if (net_ratelimit())509 netdev_warn(dev, "Rx ERROR. status = %08x\n",510 status);511 512 tp->stats.rx_errors++;513 514 if (status & (RTASE_RX_RWT | RTASE_RX_RUNT))515 tp->stats.rx_length_errors++;516 517 if (status & RTASE_RX_CRC)518 tp->stats.rx_crc_errors++;519 520 if (dev->features & NETIF_F_RXALL)521 goto process_pkt;522 523 rtase_mark_to_asic(desc, tp->rx_buf_sz);524 goto skip_process_pkt;525 }526 527process_pkt:528 pkt_size = status & RTASE_RX_PKT_SIZE_MASK;529 if (likely(!(dev->features & NETIF_F_RXFCS)))530 pkt_size -= ETH_FCS_LEN;531 532 /* The driver does not support incoming fragmented frames.533 * They are seen as a symptom of over-mtu sized frames.534 */535 if (unlikely(rtase_fragmented_frame(status))) {536 tp->stats.rx_dropped++;537 tp->stats.rx_length_errors++;538 rtase_mark_to_asic(desc, tp->rx_buf_sz);539 goto skip_process_pkt;540 }541 542 dma_sync_single_for_cpu(&tp->pdev->dev,543 ring->mis.data_phy_addr[entry],544 tp->rx_buf_sz, DMA_FROM_DEVICE);545 546 skb = build_skb(ring->data_buf[entry], PAGE_SIZE);547 if (!skb) {548 tp->stats.rx_dropped++;549 rtase_mark_to_asic(desc, tp->rx_buf_sz);550 goto skip_process_pkt;551 }552 ring->data_buf[entry] = NULL;553 554 if (dev->features & NETIF_F_RXCSUM)555 rtase_rx_csum(tp, skb, desc);556 557 skb_put(skb, pkt_size);558 skb_mark_for_recycle(skb);559 skb->protocol = eth_type_trans(skb, dev);560 561 if (skb->pkt_type == PACKET_MULTICAST)562 tp->stats.multicast++;563 564 rtase_rx_vlan_skb(desc, skb);565 rtase_rx_skb(ring, skb);566 567 dev_sw_netstats_rx_add(dev, pkt_size);568 569skip_process_pkt:570 workdone++;571 cur_rx++;572 entry = cur_rx % RTASE_NUM_DESC;573 desc = ring->desc + sizeof(union rtase_rx_desc) * entry;574 }575 576 ring->cur_idx = cur_rx;577 delta = rtase_rx_ring_fill(ring, ring->dirty_idx, ring->cur_idx);578 ring->dirty_idx += delta;579 580 return workdone;581}582 583static void rtase_rx_desc_init(struct rtase_private *tp, u16 idx)584{585 struct rtase_ring *ring = &tp->rx_ring[idx];586 u16 i;587 588 memset(ring->desc, 0x0, RTASE_RX_RING_DESC_SIZE);589 memset(ring->data_buf, 0x0, sizeof(ring->data_buf));590 ring->cur_idx = 0;591 ring->dirty_idx = 0;592 ring->index = idx;593 ring->alloc_fail = 0;594 595 for (i = 0; i < RTASE_NUM_DESC; i++)596 ring->mis.data_phy_addr[i] = 0;597 598 ring->ring_handler = rx_handler;599 ring->ivec = &tp->int_vector[idx];600 list_add_tail(&ring->ring_entry, &tp->int_vector[idx].ring_list);601}602 603static void rtase_rx_clear(struct rtase_private *tp)604{605 u32 i;606 607 for (i = 0; i < tp->func_rx_queue_num; i++)608 rtase_rx_ring_clear(tp->page_pool, &tp->rx_ring[i]);609 610 page_pool_destroy(tp->page_pool);611 tp->page_pool = NULL;612}613 614static int rtase_init_ring(const struct net_device *dev)615{616 struct rtase_private *tp = netdev_priv(dev);617 struct page_pool_params pp_params = { 0 };618 struct page_pool *page_pool;619 u32 num;620 u16 i;621 622 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;623 pp_params.order = 0;624 pp_params.pool_size = RTASE_NUM_DESC * tp->func_rx_queue_num;625 pp_params.nid = dev_to_node(&tp->pdev->dev);626 pp_params.dev = &tp->pdev->dev;627 pp_params.dma_dir = DMA_FROM_DEVICE;628 pp_params.max_len = PAGE_SIZE;629 pp_params.offset = 0;630 631 page_pool = page_pool_create(&pp_params);632 if (IS_ERR(page_pool)) {633 netdev_err(tp->dev, "failed to create page pool\n");634 return -ENOMEM;635 }636 637 tp->page_pool = page_pool;638 639 for (i = 0; i < tp->func_tx_queue_num; i++)640 rtase_tx_desc_init(tp, i);641 642 for (i = 0; i < tp->func_rx_queue_num; i++) {643 rtase_rx_desc_init(tp, i);644 645 num = rtase_rx_ring_fill(&tp->rx_ring[i], 0, RTASE_NUM_DESC);646 if (num != RTASE_NUM_DESC)647 goto err_out;648 649 rtase_mark_as_last_descriptor(tp->rx_ring[i].desc +650 sizeof(union rtase_rx_desc) *651 (RTASE_NUM_DESC - 1));652 }653 654 return 0;655 656err_out:657 rtase_rx_clear(tp);658 return -ENOMEM;659}660 661static void rtase_interrupt_mitigation(const struct rtase_private *tp)662{663 u32 i;664 665 for (i = 0; i < tp->func_tx_queue_num; i++)666 rtase_w16(tp, RTASE_INT_MITI_TX + i * 2, tp->tx_int_mit);667 668 for (i = 0; i < tp->func_rx_queue_num; i++)669 rtase_w16(tp, RTASE_INT_MITI_RX + i * 2, tp->rx_int_mit);670}671 672static void rtase_tally_counter_addr_fill(const struct rtase_private *tp)673{674 rtase_w32(tp, RTASE_DTCCR4, upper_32_bits(tp->tally_paddr));675 rtase_w32(tp, RTASE_DTCCR0, lower_32_bits(tp->tally_paddr));676}677 678static void rtase_tally_counter_clear(const struct rtase_private *tp)679{680 u32 cmd = lower_32_bits(tp->tally_paddr);681 682 rtase_w32(tp, RTASE_DTCCR4, upper_32_bits(tp->tally_paddr));683 rtase_w32(tp, RTASE_DTCCR0, cmd | RTASE_COUNTER_RESET);684}685 686static void rtase_desc_addr_fill(const struct rtase_private *tp)687{688 const struct rtase_ring *ring;689 u16 i, cmd, val;690 int err;691 692 for (i = 0; i < tp->func_tx_queue_num; i++) {693 ring = &tp->tx_ring[i];694 695 rtase_w32(tp, RTASE_TX_DESC_ADDR0,696 lower_32_bits(ring->phy_addr));697 rtase_w32(tp, RTASE_TX_DESC_ADDR4,698 upper_32_bits(ring->phy_addr));699 700 cmd = i | RTASE_TX_DESC_CMD_WE | RTASE_TX_DESC_CMD_CS;701 rtase_w16(tp, RTASE_TX_DESC_COMMAND, cmd);702 703 err = read_poll_timeout(rtase_r16, val,704 !(val & RTASE_TX_DESC_CMD_CS), 10,705 1000, false, tp,706 RTASE_TX_DESC_COMMAND);707 708 if (err == -ETIMEDOUT)709 netdev_err(tp->dev,710 "error occurred in fill tx descriptor\n");711 }712 713 for (i = 0; i < tp->func_rx_queue_num; i++) {714 ring = &tp->rx_ring[i];715 716 if (i == 0) {717 rtase_w32(tp, RTASE_Q0_RX_DESC_ADDR0,718 lower_32_bits(ring->phy_addr));719 rtase_w32(tp, RTASE_Q0_RX_DESC_ADDR4,720 upper_32_bits(ring->phy_addr));721 } else {722 rtase_w32(tp, (RTASE_Q1_RX_DESC_ADDR0 + ((i - 1) * 8)),723 lower_32_bits(ring->phy_addr));724 rtase_w32(tp, (RTASE_Q1_RX_DESC_ADDR4 + ((i - 1) * 8)),725 upper_32_bits(ring->phy_addr));726 }727 }728}729 730static void rtase_hw_set_features(const struct net_device *dev,731 netdev_features_t features)732{733 const struct rtase_private *tp = netdev_priv(dev);734 u16 rx_config, val;735 736 rx_config = rtase_r16(tp, RTASE_RX_CONFIG_0);737 if (features & NETIF_F_RXALL)738 rx_config |= (RTASE_ACCEPT_ERR | RTASE_ACCEPT_RUNT);739 else740 rx_config &= ~(RTASE_ACCEPT_ERR | RTASE_ACCEPT_RUNT);741 742 rtase_w16(tp, RTASE_RX_CONFIG_0, rx_config);743 744 val = rtase_r16(tp, RTASE_CPLUS_CMD);745 if (features & NETIF_F_RXCSUM)746 rtase_w16(tp, RTASE_CPLUS_CMD, val | RTASE_RX_CHKSUM);747 else748 rtase_w16(tp, RTASE_CPLUS_CMD, val & ~RTASE_RX_CHKSUM);749 750 rx_config = rtase_r16(tp, RTASE_RX_CONFIG_1);751 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX)752 rx_config |= (RTASE_INNER_VLAN_DETAG_EN |753 RTASE_OUTER_VLAN_DETAG_EN);754 else755 rx_config &= ~(RTASE_INNER_VLAN_DETAG_EN |756 RTASE_OUTER_VLAN_DETAG_EN);757 758 rtase_w16(tp, RTASE_RX_CONFIG_1, rx_config);759}760 761static void rtase_hw_set_rx_packet_filter(struct net_device *dev)762{763 u32 mc_filter[2] = { 0xFFFFFFFF, 0xFFFFFFFF };764 struct rtase_private *tp = netdev_priv(dev);765 u16 rx_mode;766 767 rx_mode = rtase_r16(tp, RTASE_RX_CONFIG_0) & ~RTASE_ACCEPT_MASK;768 rx_mode |= RTASE_ACCEPT_BROADCAST | RTASE_ACCEPT_MYPHYS;769 770 if (dev->flags & IFF_PROMISC) {771 rx_mode |= RTASE_ACCEPT_MULTICAST | RTASE_ACCEPT_ALLPHYS;772 } else if (dev->flags & IFF_ALLMULTI) {773 rx_mode |= RTASE_ACCEPT_MULTICAST;774 } else {775 struct netdev_hw_addr *hw_addr;776 777 mc_filter[0] = 0;778 mc_filter[1] = 0;779 780 netdev_for_each_mc_addr(hw_addr, dev) {781 u32 bit_nr = eth_hw_addr_crc(hw_addr);782 u32 idx = u32_get_bits(bit_nr, BIT(31));783 u32 bit = u32_get_bits(bit_nr,784 RTASE_MULTICAST_FILTER_MASK);785 786 mc_filter[idx] |= BIT(bit);787 rx_mode |= RTASE_ACCEPT_MULTICAST;788 }789 }790 791 if (dev->features & NETIF_F_RXALL)792 rx_mode |= RTASE_ACCEPT_ERR | RTASE_ACCEPT_RUNT;793 794 rtase_w32(tp, RTASE_MAR0, swab32(mc_filter[1]));795 rtase_w32(tp, RTASE_MAR1, swab32(mc_filter[0]));796 rtase_w16(tp, RTASE_RX_CONFIG_0, rx_mode);797}798 799static void rtase_irq_dis_and_clear(const struct rtase_private *tp)800{801 const struct rtase_int_vector *ivec = &tp->int_vector[0];802 u32 val1;803 u16 val2;804 u8 i;805 806 rtase_w32(tp, ivec->imr_addr, 0);807 val1 = rtase_r32(tp, ivec->isr_addr);808 rtase_w32(tp, ivec->isr_addr, val1);809 810 for (i = 1; i < tp->int_nums; i++) {811 ivec = &tp->int_vector[i];812 rtase_w16(tp, ivec->imr_addr, 0);813 val2 = rtase_r16(tp, ivec->isr_addr);814 rtase_w16(tp, ivec->isr_addr, val2);815 }816}817 818static void rtase_poll_timeout(const struct rtase_private *tp, u32 cond,819 u32 sleep_us, u64 timeout_us, u16 reg)820{821 int err;822 u8 val;823 824 err = read_poll_timeout(rtase_r8, val, val & cond, sleep_us,825 timeout_us, false, tp, reg);826 827 if (err == -ETIMEDOUT)828 netdev_err(tp->dev, "poll reg 0x00%x timeout\n", reg);829}830 831static void rtase_nic_reset(const struct net_device *dev)832{833 const struct rtase_private *tp = netdev_priv(dev);834 u16 rx_config;835 u8 val;836 837 rx_config = rtase_r16(tp, RTASE_RX_CONFIG_0);838 rtase_w16(tp, RTASE_RX_CONFIG_0, rx_config & ~RTASE_ACCEPT_MASK);839 840 val = rtase_r8(tp, RTASE_MISC);841 rtase_w8(tp, RTASE_MISC, val | RTASE_RX_DV_GATE_EN);842 843 val = rtase_r8(tp, RTASE_CHIP_CMD);844 rtase_w8(tp, RTASE_CHIP_CMD, val | RTASE_STOP_REQ);845 mdelay(2);846 847 rtase_poll_timeout(tp, RTASE_STOP_REQ_DONE, 100, 150000,848 RTASE_CHIP_CMD);849 850 rtase_poll_timeout(tp, RTASE_TX_FIFO_EMPTY, 100, 100000,851 RTASE_FIFOR);852 853 rtase_poll_timeout(tp, RTASE_RX_FIFO_EMPTY, 100, 100000,854 RTASE_FIFOR);855 856 val = rtase_r8(tp, RTASE_CHIP_CMD);857 rtase_w8(tp, RTASE_CHIP_CMD, val & ~(RTASE_TE | RTASE_RE));858 val = rtase_r8(tp, RTASE_CHIP_CMD);859 rtase_w8(tp, RTASE_CHIP_CMD, val & ~RTASE_STOP_REQ);860 861 rtase_w16(tp, RTASE_RX_CONFIG_0, rx_config);862}863 864static void rtase_hw_reset(const struct net_device *dev)865{866 const struct rtase_private *tp = netdev_priv(dev);867 868 rtase_irq_dis_and_clear(tp);869 870 rtase_nic_reset(dev);871}872 873static void rtase_set_rx_queue(const struct rtase_private *tp)874{875 u16 reg_data;876 877 reg_data = rtase_r16(tp, RTASE_FCR);878 switch (tp->func_rx_queue_num) {879 case 1:880 u16p_replace_bits(®_data, 0x1, RTASE_FCR_RXQ_MASK);881 break;882 case 2:883 u16p_replace_bits(®_data, 0x2, RTASE_FCR_RXQ_MASK);884 break;885 case 4:886 u16p_replace_bits(®_data, 0x3, RTASE_FCR_RXQ_MASK);887 break;888 }889 rtase_w16(tp, RTASE_FCR, reg_data);890}891 892static void rtase_set_tx_queue(const struct rtase_private *tp)893{894 u16 reg_data;895 896 reg_data = rtase_r16(tp, RTASE_TX_CONFIG_1);897 switch (tp->tx_queue_ctrl) {898 case 1:899 u16p_replace_bits(®_data, 0x0, RTASE_TC_MODE_MASK);900 break;901 case 2:902 u16p_replace_bits(®_data, 0x1, RTASE_TC_MODE_MASK);903 break;904 case 3:905 case 4:906 u16p_replace_bits(®_data, 0x2, RTASE_TC_MODE_MASK);907 break;908 default:909 u16p_replace_bits(®_data, 0x3, RTASE_TC_MODE_MASK);910 break;911 }912 rtase_w16(tp, RTASE_TX_CONFIG_1, reg_data);913}914 915static void rtase_hw_config(struct net_device *dev)916{917 const struct rtase_private *tp = netdev_priv(dev);918 u32 reg_data32;919 u16 reg_data16;920 921 rtase_hw_reset(dev);922 923 /* set rx dma burst */924 reg_data16 = rtase_r16(tp, RTASE_RX_CONFIG_0);925 reg_data16 &= ~(RTASE_RX_SINGLE_TAG | RTASE_RX_SINGLE_FETCH);926 u16p_replace_bits(®_data16, RTASE_RX_DMA_BURST_256,927 RTASE_RX_MX_DMA_MASK);928 rtase_w16(tp, RTASE_RX_CONFIG_0, reg_data16);929 930 /* new rx descritpor */931 reg_data16 = rtase_r16(tp, RTASE_RX_CONFIG_1);932 reg_data16 |= RTASE_RX_NEW_DESC_FORMAT_EN | RTASE_PCIE_NEW_FLOW;933 u16p_replace_bits(®_data16, 0xF, RTASE_RX_MAX_FETCH_DESC_MASK);934 rtase_w16(tp, RTASE_RX_CONFIG_1, reg_data16);935 936 rtase_set_rx_queue(tp);937 938 rtase_interrupt_mitigation(tp);939 940 /* set tx dma burst size and interframe gap time */941 reg_data32 = rtase_r32(tp, RTASE_TX_CONFIG_0);942 u32p_replace_bits(®_data32, RTASE_TX_DMA_BURST_UNLIMITED,943 RTASE_TX_DMA_MASK);944 u32p_replace_bits(®_data32, RTASE_INTERFRAMEGAP,945 RTASE_TX_INTER_FRAME_GAP_MASK);946 rtase_w32(tp, RTASE_TX_CONFIG_0, reg_data32);947 948 /* new tx descriptor */949 reg_data16 = rtase_r16(tp, RTASE_TFUN_CTRL);950 rtase_w16(tp, RTASE_TFUN_CTRL, reg_data16 |951 RTASE_TX_NEW_DESC_FORMAT_EN);952 953 /* tx fetch desc number */954 rtase_w8(tp, RTASE_TDFNR, 0x10);955 956 /* tag num select */957 reg_data16 = rtase_r16(tp, RTASE_MTPS);958 u16p_replace_bits(®_data16, 0x4, RTASE_TAG_NUM_SEL_MASK);959 rtase_w16(tp, RTASE_MTPS, reg_data16);960 961 rtase_set_tx_queue(tp);962 963 rtase_w16(tp, RTASE_TOKSEL, 0x5555);964 965 rtase_tally_counter_addr_fill(tp);966 rtase_desc_addr_fill(tp);967 rtase_hw_set_features(dev, dev->features);968 969 /* enable flow control */970 reg_data16 = rtase_r16(tp, RTASE_CPLUS_CMD);971 reg_data16 |= (RTASE_FORCE_TXFLOW_EN | RTASE_FORCE_RXFLOW_EN);972 rtase_w16(tp, RTASE_CPLUS_CMD, reg_data16);973 /* set near fifo threshold - rx missed issue. */974 rtase_w16(tp, RTASE_RFIFONFULL, 0x190);975 976 rtase_w16(tp, RTASE_RMS, tp->rx_buf_sz);977 978 rtase_hw_set_rx_packet_filter(dev);979}980 981static void rtase_nic_enable(const struct net_device *dev)982{983 const struct rtase_private *tp = netdev_priv(dev);984 u16 rcr = rtase_r16(tp, RTASE_RX_CONFIG_1);985 u8 val;986 987 rtase_w16(tp, RTASE_RX_CONFIG_1, rcr & ~RTASE_PCIE_RELOAD_EN);988 rtase_w16(tp, RTASE_RX_CONFIG_1, rcr | RTASE_PCIE_RELOAD_EN);989 990 val = rtase_r8(tp, RTASE_CHIP_CMD);991 rtase_w8(tp, RTASE_CHIP_CMD, val | RTASE_TE | RTASE_RE);992 993 val = rtase_r8(tp, RTASE_MISC);994 rtase_w8(tp, RTASE_MISC, val & ~RTASE_RX_DV_GATE_EN);995}996 997static void rtase_enable_hw_interrupt(const struct rtase_private *tp)998{999 const struct rtase_int_vector *ivec = &tp->int_vector[0];1000 u32 i;1001 1002 rtase_w32(tp, ivec->imr_addr, ivec->imr);1003 1004 for (i = 1; i < tp->int_nums; i++) {1005 ivec = &tp->int_vector[i];1006 rtase_w16(tp, ivec->imr_addr, ivec->imr);1007 }1008}1009 1010static void rtase_hw_start(const struct net_device *dev)1011{1012 const struct rtase_private *tp = netdev_priv(dev);1013 1014 rtase_nic_enable(dev);1015 rtase_enable_hw_interrupt(tp);1016}1017 1018/* the interrupt handler does RXQ0 and TXQ0, TXQ4~7 interrutp status1019 */1020static irqreturn_t rtase_interrupt(int irq, void *dev_instance)1021{1022 const struct rtase_private *tp;1023 struct rtase_int_vector *ivec;1024 u32 status;1025 1026 ivec = dev_instance;1027 tp = ivec->tp;1028 status = rtase_r32(tp, ivec->isr_addr);1029 1030 rtase_w32(tp, ivec->imr_addr, 0x0);1031 rtase_w32(tp, ivec->isr_addr, status & ~RTASE_FOVW);1032 1033 if (napi_schedule_prep(&ivec->napi))1034 __napi_schedule(&ivec->napi);1035 1036 return IRQ_HANDLED;1037}1038 1039/* the interrupt handler does RXQ1&TXQ1 or RXQ2&TXQ2 or RXQ3&TXQ3 interrupt1040 * status according to interrupt vector1041 */1042static irqreturn_t rtase_q_interrupt(int irq, void *dev_instance)1043{1044 const struct rtase_private *tp;1045 struct rtase_int_vector *ivec;1046 u16 status;1047 1048 ivec = dev_instance;1049 tp = ivec->tp;1050 status = rtase_r16(tp, ivec->isr_addr);1051 1052 rtase_w16(tp, ivec->imr_addr, 0x0);1053 rtase_w16(tp, ivec->isr_addr, status);1054 1055 if (napi_schedule_prep(&ivec->napi))1056 __napi_schedule(&ivec->napi);1057 1058 return IRQ_HANDLED;1059}1060 1061static int rtase_poll(struct napi_struct *napi, int budget)1062{1063 const struct rtase_int_vector *ivec;1064 const struct rtase_private *tp;1065 struct rtase_ring *ring;1066 int total_workdone = 0;1067 1068 ivec = container_of(napi, struct rtase_int_vector, napi);1069 tp = ivec->tp;1070 1071 list_for_each_entry(ring, &ivec->ring_list, ring_entry)1072 total_workdone += ring->ring_handler(ring, budget);1073 1074 if (total_workdone >= budget)1075 return budget;1076 1077 if (napi_complete_done(napi, total_workdone)) {1078 if (!ivec->index)1079 rtase_w32(tp, ivec->imr_addr, ivec->imr);1080 else1081 rtase_w16(tp, ivec->imr_addr, ivec->imr);1082 }1083 1084 return total_workdone;1085}1086 1087static int rtase_open(struct net_device *dev)1088{1089 struct rtase_private *tp = netdev_priv(dev);1090 const struct pci_dev *pdev = tp->pdev;1091 struct rtase_int_vector *ivec;1092 u16 i = 0, j;1093 int ret;1094 1095 ivec = &tp->int_vector[0];1096 tp->rx_buf_sz = RTASE_RX_BUF_SIZE;1097 1098 ret = rtase_alloc_desc(tp);1099 if (ret)1100 return ret;1101 1102 ret = rtase_init_ring(dev);1103 if (ret)1104 goto err_free_all_allocated_mem;1105 1106 rtase_hw_config(dev);1107 1108 if (tp->sw_flag & RTASE_SWF_MSIX_ENABLED) {1109 ret = request_irq(ivec->irq, rtase_interrupt, 0,1110 dev->name, ivec);1111 if (ret)1112 goto err_free_all_allocated_irq;1113 1114 /* request other interrupts to handle multiqueue */1115 for (i = 1; i < tp->int_nums; i++) {1116 ivec = &tp->int_vector[i];1117 snprintf(ivec->name, sizeof(ivec->name), "%s_int%i",1118 tp->dev->name, i);1119 ret = request_irq(ivec->irq, rtase_q_interrupt, 0,1120 ivec->name, ivec);1121 if (ret)1122 goto err_free_all_allocated_irq;1123 }1124 } else {1125 ret = request_irq(pdev->irq, rtase_interrupt, 0, dev->name,1126 ivec);1127 if (ret)1128 goto err_free_all_allocated_mem;1129 }1130 1131 rtase_hw_start(dev);1132 1133 for (i = 0; i < tp->int_nums; i++) {1134 ivec = &tp->int_vector[i];1135 napi_enable(&ivec->napi);1136 }1137 1138 netif_carrier_on(dev);1139 netif_wake_queue(dev);1140 1141 return 0;1142 1143err_free_all_allocated_irq:1144 for (j = 0; j < i; j++)1145 free_irq(tp->int_vector[j].irq, &tp->int_vector[j]);1146 1147err_free_all_allocated_mem:1148 rtase_free_desc(tp);1149 1150 return ret;1151}1152 1153static void rtase_down(struct net_device *dev)1154{1155 struct rtase_private *tp = netdev_priv(dev);1156 struct rtase_int_vector *ivec;1157 struct rtase_ring *ring, *tmp;1158 u32 i;1159 1160 for (i = 0; i < tp->int_nums; i++) {1161 ivec = &tp->int_vector[i];1162 napi_disable(&ivec->napi);1163 list_for_each_entry_safe(ring, tmp, &ivec->ring_list,1164 ring_entry)1165 list_del(&ring->ring_entry);1166 }1167 1168 netif_tx_disable(dev);1169 1170 netif_carrier_off(dev);1171 1172 rtase_hw_reset(dev);1173 1174 rtase_tx_clear(tp);1175 1176 rtase_rx_clear(tp);1177}1178 1179static int rtase_close(struct net_device *dev)1180{1181 struct rtase_private *tp = netdev_priv(dev);1182 const struct pci_dev *pdev = tp->pdev;1183 u32 i;1184 1185 rtase_down(dev);1186 1187 if (tp->sw_flag & RTASE_SWF_MSIX_ENABLED) {1188 for (i = 0; i < tp->int_nums; i++)1189 free_irq(tp->int_vector[i].irq, &tp->int_vector[i]);1190 1191 } else {1192 free_irq(pdev->irq, &tp->int_vector[0]);1193 }1194 1195 rtase_free_desc(tp);1196 1197 return 0;1198}1199 1200static u32 rtase_tx_vlan_tag(const struct rtase_private *tp,1201 const struct sk_buff *skb)1202{1203 return (skb_vlan_tag_present(skb)) ?1204 (RTASE_TX_VLAN_TAG | swab16(skb_vlan_tag_get(skb))) : 0x00;1205}1206 1207static u32 rtase_tx_csum(struct sk_buff *skb, const struct net_device *dev)1208{1209 u32 csum_cmd = 0;1210 u8 ip_protocol;1211 1212 switch (vlan_get_protocol(skb)) {1213 case htons(ETH_P_IP):1214 csum_cmd = RTASE_TX_IPCS_C;1215 ip_protocol = ip_hdr(skb)->protocol;1216 break;1217 1218 case htons(ETH_P_IPV6):1219 csum_cmd = RTASE_TX_IPV6F_C;1220 ip_protocol = ipv6_hdr(skb)->nexthdr;1221 break;1222 1223 default:1224 ip_protocol = IPPROTO_RAW;1225 break;1226 }1227 1228 if (ip_protocol == IPPROTO_TCP)1229 csum_cmd |= RTASE_TX_TCPCS_C;1230 else if (ip_protocol == IPPROTO_UDP)1231 csum_cmd |= RTASE_TX_UDPCS_C;1232 1233 csum_cmd |= u32_encode_bits(skb_transport_offset(skb),1234 RTASE_TCPHO_MASK);1235 1236 return csum_cmd;1237}1238 1239static int rtase_xmit_frags(struct rtase_ring *ring, struct sk_buff *skb,1240 u32 opts1, u32 opts2)1241{1242 const struct skb_shared_info *info = skb_shinfo(skb);1243 const struct rtase_private *tp = ring->ivec->tp;1244 const u8 nr_frags = info->nr_frags;1245 struct rtase_tx_desc *txd = NULL;1246 u32 cur_frag, entry;1247 1248 entry = ring->cur_idx;1249 for (cur_frag = 0; cur_frag < nr_frags; cur_frag++) {1250 const skb_frag_t *frag = &info->frags[cur_frag];1251 dma_addr_t mapping;1252 u32 status, len;1253 void *addr;1254 1255 entry = (entry + 1) % RTASE_NUM_DESC;1256 1257 txd = ring->desc + sizeof(struct rtase_tx_desc) * entry;1258 len = skb_frag_size(frag);1259 addr = skb_frag_address(frag);1260 mapping = dma_map_single(&tp->pdev->dev, addr, len,1261 DMA_TO_DEVICE);1262 1263 if (unlikely(dma_mapping_error(&tp->pdev->dev, mapping))) {1264 if (unlikely(net_ratelimit()))1265 netdev_err(tp->dev,1266 "Failed to map TX fragments DMA!\n");1267 1268 goto err_out;1269 }1270 1271 if (((entry + 1) % RTASE_NUM_DESC) == 0)1272 status = (opts1 | len | RTASE_RING_END);1273 else1274 status = opts1 | len;1275 1276 if (cur_frag == (nr_frags - 1)) {1277 ring->skbuff[entry] = skb;1278 status |= RTASE_TX_LAST_FRAG;1279 }1280 1281 ring->mis.len[entry] = len;1282 txd->addr = cpu_to_le64(mapping);1283 txd->opts2 = cpu_to_le32(opts2);1284 1285 /* make sure the operating fields have been updated */1286 dma_wmb();1287 txd->opts1 = cpu_to_le32(status);1288 }1289 1290 return cur_frag;1291 1292err_out:1293 rtase_tx_clear_range(ring, ring->cur_idx + 1, cur_frag);1294 return -EIO;1295}1296 1297static netdev_tx_t rtase_start_xmit(struct sk_buff *skb,1298 struct net_device *dev)1299{1300 struct skb_shared_info *shinfo = skb_shinfo(skb);1301 struct rtase_private *tp = netdev_priv(dev);1302 u32 q_idx, entry, len, opts1, opts2;1303 struct netdev_queue *tx_queue;1304 bool stop_queue, door_bell;1305 u32 mss = shinfo->gso_size;1306 struct rtase_tx_desc *txd;1307 struct rtase_ring *ring;1308 dma_addr_t mapping;1309 int frags;1310 1311 /* multiqueues */1312 q_idx = skb_get_queue_mapping(skb);1313 ring = &tp->tx_ring[q_idx];1314 tx_queue = netdev_get_tx_queue(dev, q_idx);1315 1316 if (unlikely(!rtase_tx_avail(ring))) {1317 if (net_ratelimit())1318 netdev_err(dev,1319 "BUG! Tx Ring full when queue awake!\n");1320 1321 netif_stop_queue(dev);1322 return NETDEV_TX_BUSY;1323 }1324 1325 entry = ring->cur_idx % RTASE_NUM_DESC;1326 txd = ring->desc + sizeof(struct rtase_tx_desc) * entry;1327 1328 opts1 = RTASE_DESC_OWN;1329 opts2 = rtase_tx_vlan_tag(tp, skb);1330 1331 /* tcp segmentation offload (or tcp large send) */1332 if (mss) {1333 if (shinfo->gso_type & SKB_GSO_TCPV4) {1334 opts1 |= RTASE_GIANT_SEND_V4;1335 } else if (shinfo->gso_type & SKB_GSO_TCPV6) {1336 if (skb_cow_head(skb, 0))1337 goto err_dma_0;1338 1339 tcp_v6_gso_csum_prep(skb);1340 opts1 |= RTASE_GIANT_SEND_V6;1341 } else {1342 WARN_ON_ONCE(1);1343 }1344 1345 opts1 |= u32_encode_bits(skb_transport_offset(skb),1346 RTASE_TCPHO_MASK);1347 opts2 |= u32_encode_bits(mss, RTASE_MSS_MASK);1348 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {1349 opts2 |= rtase_tx_csum(skb, dev);1350 }1351 1352 frags = rtase_xmit_frags(ring, skb, opts1, opts2);1353 if (unlikely(frags < 0))1354 goto err_dma_0;1355 1356 if (frags) {1357 len = skb_headlen(skb);1358 opts1 |= RTASE_TX_FIRST_FRAG;1359 } else {1360 len = skb->len;1361 ring->skbuff[entry] = skb;1362 opts1 |= RTASE_TX_FIRST_FRAG | RTASE_TX_LAST_FRAG;1363 }1364 1365 if (((entry + 1) % RTASE_NUM_DESC) == 0)1366 opts1 |= (len | RTASE_RING_END);1367 else1368 opts1 |= len;1369 1370 mapping = dma_map_single(&tp->pdev->dev, skb->data, len,1371 DMA_TO_DEVICE);1372 1373 if (unlikely(dma_mapping_error(&tp->pdev->dev, mapping))) {1374 if (unlikely(net_ratelimit()))1375 netdev_err(dev, "Failed to map TX DMA!\n");1376 1377 goto err_dma_1;1378 }1379 1380 ring->mis.len[entry] = len;1381 txd->addr = cpu_to_le64(mapping);1382 txd->opts2 = cpu_to_le32(opts2);1383 txd->opts1 = cpu_to_le32(opts1 & ~RTASE_DESC_OWN);1384 1385 /* make sure the operating fields have been updated */1386 dma_wmb();1387 1388 door_bell = __netdev_tx_sent_queue(tx_queue, skb->len,1389 netdev_xmit_more());1390 1391 txd->opts1 = cpu_to_le32(opts1);1392 1393 skb_tx_timestamp(skb);1394 1395 /* tx needs to see descriptor changes before updated cur_idx */1396 smp_wmb();1397 1398 WRITE_ONCE(ring->cur_idx, ring->cur_idx + frags + 1);1399 1400 stop_queue = !netif_subqueue_maybe_stop(dev, ring->index,1401 rtase_tx_avail(ring),1402 RTASE_TX_STOP_THRS,1403 RTASE_TX_START_THRS);1404 1405 if (door_bell || stop_queue)1406 rtase_w8(tp, RTASE_TPPOLL, BIT(ring->index));1407 1408 return NETDEV_TX_OK;1409 1410err_dma_1:1411 ring->skbuff[entry] = NULL;1412 rtase_tx_clear_range(ring, ring->cur_idx + 1, frags);1413 1414err_dma_0:1415 tp->stats.tx_dropped++;1416 dev_kfree_skb_any(skb);1417 return NETDEV_TX_OK;1418}1419 1420static void rtase_set_rx_mode(struct net_device *dev)1421{1422 rtase_hw_set_rx_packet_filter(dev);1423}1424 1425static void rtase_enable_eem_write(const struct rtase_private *tp)1426{1427 u8 val;1428 1429 val = rtase_r8(tp, RTASE_EEM);1430 rtase_w8(tp, RTASE_EEM, val | RTASE_EEM_UNLOCK);1431}1432 1433static void rtase_disable_eem_write(const struct rtase_private *tp)1434{1435 u8 val;1436 1437 val = rtase_r8(tp, RTASE_EEM);1438 rtase_w8(tp, RTASE_EEM, val & ~RTASE_EEM_UNLOCK);1439}1440 1441static void rtase_rar_set(const struct rtase_private *tp, const u8 *addr)1442{1443 u32 rar_low, rar_high;1444 1445 rar_low = (u32)addr[0] | ((u32)addr[1] << 8) |1446 ((u32)addr[2] << 16) | ((u32)addr[3] << 24);1447 1448 rar_high = (u32)addr[4] | ((u32)addr[5] << 8);1449 1450 rtase_enable_eem_write(tp);1451 rtase_w32(tp, RTASE_MAC0, rar_low);1452 rtase_w32(tp, RTASE_MAC4, rar_high);1453 rtase_disable_eem_write(tp);1454 rtase_w16(tp, RTASE_LBK_CTRL, RTASE_LBK_ATLD | RTASE_LBK_CLR);1455}1456 1457static int rtase_set_mac_address(struct net_device *dev, void *p)1458{1459 struct rtase_private *tp = netdev_priv(dev);1460 int ret;1461 1462 ret = eth_mac_addr(dev, p);1463 if (ret)1464 return ret;1465 1466 rtase_rar_set(tp, dev->dev_addr);1467 1468 return 0;1469}1470 1471static int rtase_change_mtu(struct net_device *dev, int new_mtu)1472{1473 dev->mtu = new_mtu;1474 1475 netdev_update_features(dev);1476 1477 return 0;1478}1479 1480static void rtase_wait_for_quiescence(const struct net_device *dev)1481{1482 struct rtase_private *tp = netdev_priv(dev);1483 struct rtase_int_vector *ivec;1484 u32 i;1485 1486 for (i = 0; i < tp->int_nums; i++) {1487 ivec = &tp->int_vector[i];1488 synchronize_irq(ivec->irq);1489 /* wait for any pending NAPI task to complete */1490 napi_disable(&ivec->napi);1491 }1492 1493 rtase_irq_dis_and_clear(tp);1494 1495 for (i = 0; i < tp->int_nums; i++) {1496 ivec = &tp->int_vector[i];1497 napi_enable(&ivec->napi);1498 }1499}1500 1501static void rtase_sw_reset(struct net_device *dev)1502{1503 struct rtase_private *tp = netdev_priv(dev);1504 int ret;1505 1506 netif_stop_queue(dev);1507 netif_carrier_off(dev);1508 rtase_hw_reset(dev);1509 1510 /* let's wait a bit while any (async) irq lands on */1511 rtase_wait_for_quiescence(dev);1512 rtase_tx_clear(tp);1513 rtase_rx_clear(tp);1514 1515 ret = rtase_init_ring(dev);1516 if (ret) {1517 netdev_err(dev, "unable to init ring\n");1518 rtase_free_desc(tp);1519 return;1520 }1521 1522 rtase_hw_config(dev);1523 /* always link, so start to transmit & receive */1524 rtase_hw_start(dev);1525 1526 netif_carrier_on(dev);1527 netif_wake_queue(dev);1528}1529 1530static void rtase_dump_tally_counter(const struct rtase_private *tp)1531{1532 dma_addr_t paddr = tp->tally_paddr;1533 u32 cmd = lower_32_bits(paddr);1534 u32 val;1535 int err;1536 1537 rtase_w32(tp, RTASE_DTCCR4, upper_32_bits(paddr));1538 rtase_w32(tp, RTASE_DTCCR0, cmd);1539 rtase_w32(tp, RTASE_DTCCR0, cmd | RTASE_COUNTER_DUMP);1540 1541 err = read_poll_timeout(rtase_r32, val, !(val & RTASE_COUNTER_DUMP),1542 10, 250, false, tp, RTASE_DTCCR0);1543 1544 if (err == -ETIMEDOUT)1545 netdev_err(tp->dev, "error occurred in dump tally counter\n");1546}1547 1548static void rtase_dump_state(const struct net_device *dev)1549{1550 const struct rtase_private *tp = netdev_priv(dev);1551 int max_reg_size = RTASE_PCI_REGS_SIZE;1552 const struct rtase_counters *counters;1553 const struct rtase_ring *ring;1554 u32 dword_rd;1555 int n = 0;1556 1557 ring = &tp->tx_ring[0];1558 netdev_err(dev, "Tx descriptor info:\n");1559 netdev_err(dev, "Tx curIdx = 0x%x\n", ring->cur_idx);1560 netdev_err(dev, "Tx dirtyIdx = 0x%x\n", ring->dirty_idx);1561 netdev_err(dev, "Tx phyAddr = %pad\n", &ring->phy_addr);1562 1563 ring = &tp->rx_ring[0];1564 netdev_err(dev, "Rx descriptor info:\n");1565 netdev_err(dev, "Rx curIdx = 0x%x\n", ring->cur_idx);1566 netdev_err(dev, "Rx dirtyIdx = 0x%x\n", ring->dirty_idx);1567 netdev_err(dev, "Rx phyAddr = %pad\n", &ring->phy_addr);1568 1569 netdev_err(dev, "Device Registers:\n");1570 netdev_err(dev, "Chip Command = 0x%02x\n",1571 rtase_r8(tp, RTASE_CHIP_CMD));1572 netdev_err(dev, "IMR = %08x\n", rtase_r32(tp, RTASE_IMR0));1573 netdev_err(dev, "ISR = %08x\n", rtase_r32(tp, RTASE_ISR0));1574 netdev_err(dev, "Boot Ctrl Reg(0xE004) = %04x\n",1575 rtase_r16(tp, RTASE_BOOT_CTL));1576 netdev_err(dev, "EPHY ISR(0xE014) = %04x\n",1577 rtase_r16(tp, RTASE_EPHY_ISR));1578 netdev_err(dev, "EPHY IMR(0xE016) = %04x\n",1579 rtase_r16(tp, RTASE_EPHY_IMR));1580 netdev_err(dev, "CLKSW SET REG(0xE018) = %04x\n",1581 rtase_r16(tp, RTASE_CLKSW_SET));1582 1583 netdev_err(dev, "Dump PCI Registers:\n");1584 1585 while (n < max_reg_size) {1586 if ((n % RTASE_DWORD_MOD) == 0)1587 netdev_err(tp->dev, "0x%03x:\n", n);1588 1589 pci_read_config_dword(tp->pdev, n, &dword_rd);1590 netdev_err(tp->dev, "%08x\n", dword_rd);1591 n += 4;1592 }1593 1594 netdev_err(dev, "Dump tally counter:\n");1595 counters = tp->tally_vaddr;1596 rtase_dump_tally_counter(tp);1597 1598 netdev_err(dev, "tx_packets %lld\n",1599 le64_to_cpu(counters->tx_packets));1600 netdev_err(dev, "rx_packets %lld\n",1601 le64_to_cpu(counters->rx_packets));1602 netdev_err(dev, "tx_errors %lld\n",1603 le64_to_cpu(counters->tx_errors));1604 netdev_err(dev, "rx_errors %d\n",1605 le32_to_cpu(counters->rx_errors));1606 netdev_err(dev, "rx_missed %d\n",1607 le16_to_cpu(counters->rx_missed));1608 netdev_err(dev, "align_errors %d\n",1609 le16_to_cpu(counters->align_errors));1610 netdev_err(dev, "tx_one_collision %d\n",1611 le32_to_cpu(counters->tx_one_collision));1612 netdev_err(dev, "tx_multi_collision %d\n",1613 le32_to_cpu(counters->tx_multi_collision));1614 netdev_err(dev, "rx_unicast %lld\n",1615 le64_to_cpu(counters->rx_unicast));1616 netdev_err(dev, "rx_broadcast %lld\n",1617 le64_to_cpu(counters->rx_broadcast));1618 netdev_err(dev, "rx_multicast %d\n",1619 le32_to_cpu(counters->rx_multicast));1620 netdev_err(dev, "tx_aborted %d\n",1621 le16_to_cpu(counters->tx_aborted));1622 netdev_err(dev, "tx_underrun %d\n",1623 le16_to_cpu(counters->tx_underrun));1624}1625 1626static void rtase_tx_timeout(struct net_device *dev, unsigned int txqueue)1627{1628 rtase_dump_state(dev);1629 rtase_sw_reset(dev);1630}1631 1632static void rtase_get_stats64(struct net_device *dev,1633 struct rtnl_link_stats64 *stats)1634{1635 const struct rtase_private *tp = netdev_priv(dev);1636 const struct rtase_counters *counters;1637 1638 counters = tp->tally_vaddr;1639 1640 dev_fetch_sw_netstats(stats, dev->tstats);1641 1642 /* fetch additional counter values missing in stats collected by driver1643 * from tally counter1644 */1645 rtase_dump_tally_counter(tp);1646 stats->rx_errors = tp->stats.rx_errors;1647 stats->tx_errors = le64_to_cpu(counters->tx_errors);1648 stats->rx_dropped = tp->stats.rx_dropped;1649 stats->tx_dropped = tp->stats.tx_dropped;1650 stats->multicast = tp->stats.multicast;1651 stats->rx_length_errors = tp->stats.rx_length_errors;1652}1653 1654static netdev_features_t rtase_fix_features(struct net_device *dev,1655 netdev_features_t features)1656{1657 netdev_features_t features_fix = features;1658 1659 /* not support TSO for jumbo frames */1660 if (dev->mtu > ETH_DATA_LEN)1661 features_fix &= ~NETIF_F_ALL_TSO;1662 1663 return features_fix;1664}1665 1666static int rtase_set_features(struct net_device *dev,1667 netdev_features_t features)1668{1669 netdev_features_t features_set = features;1670 1671 features_set &= NETIF_F_RXALL | NETIF_F_RXCSUM |1672 NETIF_F_HW_VLAN_CTAG_RX;1673 1674 if (features_set ^ dev->features)1675 rtase_hw_set_features(dev, features_set);1676 1677 return 0;1678}1679 1680static const struct net_device_ops rtase_netdev_ops = {1681 .ndo_open = rtase_open,1682 .ndo_stop = rtase_close,1683 .ndo_start_xmit = rtase_start_xmit,1684 .ndo_set_rx_mode = rtase_set_rx_mode,1685 .ndo_set_mac_address = rtase_set_mac_address,1686 .ndo_change_mtu = rtase_change_mtu,1687 .ndo_tx_timeout = rtase_tx_timeout,1688 .ndo_get_stats64 = rtase_get_stats64,1689 .ndo_fix_features = rtase_fix_features,1690 .ndo_set_features = rtase_set_features,1691};1692 1693static void rtase_get_mac_address(struct net_device *dev)1694{1695 struct rtase_private *tp = netdev_priv(dev);1696 u8 mac_addr[ETH_ALEN] __aligned(2) = {};1697 u32 i;1698 1699 for (i = 0; i < ETH_ALEN; i++)1700 mac_addr[i] = rtase_r8(tp, RTASE_MAC0 + i);1701 1702 if (!is_valid_ether_addr(mac_addr)) {1703 eth_hw_addr_random(dev);1704 netdev_warn(dev, "Random ether addr %pM\n", dev->dev_addr);1705 } else {1706 eth_hw_addr_set(dev, mac_addr);1707 ether_addr_copy(dev->perm_addr, dev->dev_addr);1708 }1709 1710 rtase_rar_set(tp, dev->dev_addr);1711}1712 1713static int rtase_get_settings(struct net_device *dev,1714 struct ethtool_link_ksettings *cmd)1715{1716 u32 supported = SUPPORTED_MII | SUPPORTED_Pause | SUPPORTED_Asym_Pause;1717 1718 ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported,1719 supported);1720 cmd->base.speed = SPEED_5000;1721 cmd->base.duplex = DUPLEX_FULL;1722 cmd->base.port = PORT_MII;1723 cmd->base.autoneg = AUTONEG_DISABLE;1724 1725 return 0;1726}1727 1728static void rtase_get_pauseparam(struct net_device *dev,1729 struct ethtool_pauseparam *pause)1730{1731 const struct rtase_private *tp = netdev_priv(dev);1732 u16 value = rtase_r16(tp, RTASE_CPLUS_CMD);1733 1734 pause->autoneg = AUTONEG_DISABLE;1735 pause->tx_pause = !!(value & RTASE_FORCE_TXFLOW_EN);1736 pause->rx_pause = !!(value & RTASE_FORCE_RXFLOW_EN);1737}1738 1739static int rtase_set_pauseparam(struct net_device *dev,1740 struct ethtool_pauseparam *pause)1741{1742 const struct rtase_private *tp = netdev_priv(dev);1743 u16 value = rtase_r16(tp, RTASE_CPLUS_CMD);1744 1745 if (pause->autoneg)1746 return -EOPNOTSUPP;1747 1748 value &= ~(RTASE_FORCE_TXFLOW_EN | RTASE_FORCE_RXFLOW_EN);1749 1750 if (pause->tx_pause)1751 value |= RTASE_FORCE_TXFLOW_EN;1752 1753 if (pause->rx_pause)1754 value |= RTASE_FORCE_RXFLOW_EN;1755 1756 rtase_w16(tp, RTASE_CPLUS_CMD, value);1757 return 0;1758}1759 1760static void rtase_get_eth_mac_stats(struct net_device *dev,1761 struct ethtool_eth_mac_stats *stats)1762{1763 struct rtase_private *tp = netdev_priv(dev);1764 const struct rtase_counters *counters;1765 1766 counters = tp->tally_vaddr;1767 1768 rtase_dump_tally_counter(tp);1769 1770 stats->FramesTransmittedOK = le64_to_cpu(counters->tx_packets);1771 stats->FramesReceivedOK = le64_to_cpu(counters->rx_packets);1772 stats->FramesLostDueToIntMACXmitError =1773 le64_to_cpu(counters->tx_errors);1774 stats->BroadcastFramesReceivedOK = le64_to_cpu(counters->rx_broadcast);1775}1776 1777static const struct ethtool_ops rtase_ethtool_ops = {1778 .get_link = ethtool_op_get_link,1779 .get_link_ksettings = rtase_get_settings,1780 .get_pauseparam = rtase_get_pauseparam,1781 .set_pauseparam = rtase_set_pauseparam,1782 .get_eth_mac_stats = rtase_get_eth_mac_stats,1783 .get_ts_info = ethtool_op_get_ts_info,1784};1785 1786static void rtase_init_netdev_ops(struct net_device *dev)1787{1788 dev->netdev_ops = &rtase_netdev_ops;1789 dev->ethtool_ops = &rtase_ethtool_ops;1790}1791 1792static void rtase_reset_interrupt(struct pci_dev *pdev,1793 const struct rtase_private *tp)1794{1795 if (tp->sw_flag & RTASE_SWF_MSIX_ENABLED)1796 pci_disable_msix(pdev);1797 else1798 pci_disable_msi(pdev);1799}1800 1801static int rtase_alloc_msix(struct pci_dev *pdev, struct rtase_private *tp)1802{1803 int ret, irq;1804 u16 i;1805 1806 memset(tp->msix_entry, 0x0, RTASE_NUM_MSIX *1807 sizeof(struct msix_entry));1808 1809 for (i = 0; i < RTASE_NUM_MSIX; i++)1810 tp->msix_entry[i].entry = i;1811 1812 ret = pci_enable_msix_exact(pdev, tp->msix_entry, tp->int_nums);1813 1814 if (ret)1815 return ret;1816 1817 for (i = 0; i < tp->int_nums; i++) {1818 irq = pci_irq_vector(pdev, i);1819 if (!irq) {1820 pci_disable_msix(pdev);1821 return irq;1822 }1823 1824 tp->int_vector[i].irq = irq;1825 }1826 1827 return 0;1828}1829 1830static int rtase_alloc_interrupt(struct pci_dev *pdev,1831 struct rtase_private *tp)1832{1833 int ret;1834 1835 ret = rtase_alloc_msix(pdev, tp);1836 if (ret) {1837 ret = pci_enable_msi(pdev);1838 if (ret) {1839 dev_err(&pdev->dev,1840 "unable to alloc interrupt.(MSI)\n");1841 return ret;1842 }1843 1844 tp->sw_flag |= RTASE_SWF_MSI_ENABLED;1845 } else {1846 tp->sw_flag |= RTASE_SWF_MSIX_ENABLED;1847 }1848 1849 return 0;1850}1851 1852static void rtase_init_hardware(const struct rtase_private *tp)1853{1854 u16 i;1855 1856 for (i = 0; i < RTASE_VLAN_FILTER_ENTRY_NUM; i++)1857 rtase_w32(tp, RTASE_VLAN_ENTRY_0 + i * 4, 0);1858}1859 1860static void rtase_init_int_vector(struct rtase_private *tp)1861{1862 u16 i;1863 1864 /* interrupt vector 0 */1865 tp->int_vector[0].tp = tp;1866 tp->int_vector[0].index = 0;1867 tp->int_vector[0].imr_addr = RTASE_IMR0;1868 tp->int_vector[0].isr_addr = RTASE_ISR0;1869 tp->int_vector[0].imr = RTASE_ROK | RTASE_RDU | RTASE_TOK |1870 RTASE_TOK4 | RTASE_TOK5 | RTASE_TOK6 |1871 RTASE_TOK7;1872 tp->int_vector[0].poll = rtase_poll;1873 1874 memset(tp->int_vector[0].name, 0x0, sizeof(tp->int_vector[0].name));1875 INIT_LIST_HEAD(&tp->int_vector[0].ring_list);1876 1877 netif_napi_add(tp->dev, &tp->int_vector[0].napi,1878 tp->int_vector[0].poll);1879 1880 /* interrupt vector 1 ~ 3 */1881 for (i = 1; i < tp->int_nums; i++) {1882 tp->int_vector[i].tp = tp;1883 tp->int_vector[i].index = i;1884 tp->int_vector[i].imr_addr = RTASE_IMR1 + (i - 1) * 4;1885 tp->int_vector[i].isr_addr = RTASE_ISR1 + (i - 1) * 4;1886 tp->int_vector[i].imr = RTASE_Q_ROK | RTASE_Q_RDU |1887 RTASE_Q_TOK;1888 tp->int_vector[i].poll = rtase_poll;1889 1890 memset(tp->int_vector[i].name, 0x0,1891 sizeof(tp->int_vector[0].name));1892 INIT_LIST_HEAD(&tp->int_vector[i].ring_list);1893 1894 netif_napi_add(tp->dev, &tp->int_vector[i].napi,1895 tp->int_vector[i].poll);1896 }1897}1898 1899static u16 rtase_calc_time_mitigation(u32 time_us)1900{1901 u8 msb, time_count, time_unit;1902 u16 int_miti;1903 1904 time_us = min_t(int, time_us, RTASE_MITI_MAX_TIME);1905 1906 msb = fls(time_us);1907 if (msb >= RTASE_MITI_COUNT_BIT_NUM) {1908 time_unit = msb - RTASE_MITI_COUNT_BIT_NUM;1909 time_count = time_us >> (msb - RTASE_MITI_COUNT_BIT_NUM);1910 } else {1911 time_unit = 0;1912 time_count = time_us;1913 }1914 1915 int_miti = u16_encode_bits(time_count, RTASE_MITI_TIME_COUNT_MASK) |1916 u16_encode_bits(time_unit, RTASE_MITI_TIME_UNIT_MASK);1917 1918 return int_miti;1919}1920 1921static u16 rtase_calc_packet_num_mitigation(u16 pkt_num)1922{1923 u8 msb, pkt_num_count, pkt_num_unit;1924 u16 int_miti;1925 1926 pkt_num = min_t(int, pkt_num, RTASE_MITI_MAX_PKT_NUM);1927 1928 if (pkt_num > 60) {1929 pkt_num_unit = RTASE_MITI_MAX_PKT_NUM_IDX;1930 pkt_num_count = pkt_num / RTASE_MITI_MAX_PKT_NUM_UNIT;1931 } else {1932 msb = fls(pkt_num);1933 if (msb >= RTASE_MITI_COUNT_BIT_NUM) {1934 pkt_num_unit = msb - RTASE_MITI_COUNT_BIT_NUM;1935 pkt_num_count = pkt_num >> (msb -1936 RTASE_MITI_COUNT_BIT_NUM);1937 } else {1938 pkt_num_unit = 0;1939 pkt_num_count = pkt_num;1940 }1941 }1942 1943 int_miti = u16_encode_bits(pkt_num_count,1944 RTASE_MITI_PKT_NUM_COUNT_MASK) |1945 u16_encode_bits(pkt_num_unit,1946 RTASE_MITI_PKT_NUM_UNIT_MASK);1947 1948 return int_miti;1949}1950 1951static void rtase_init_software_variable(struct pci_dev *pdev,1952 struct rtase_private *tp)1953{1954 u16 int_miti;1955 1956 tp->tx_queue_ctrl = RTASE_TXQ_CTRL;1957 tp->func_tx_queue_num = RTASE_FUNC_TXQ_NUM;1958 tp->func_rx_queue_num = RTASE_FUNC_RXQ_NUM;1959 tp->int_nums = RTASE_INTERRUPT_NUM;1960 1961 int_miti = rtase_calc_time_mitigation(RTASE_MITI_DEFAULT_TIME) |1962 rtase_calc_packet_num_mitigation(RTASE_MITI_DEFAULT_PKT_NUM);1963 tp->tx_int_mit = int_miti;1964 tp->rx_int_mit = int_miti;1965 1966 tp->sw_flag = 0;1967 1968 rtase_init_int_vector(tp);1969 1970 /* MTU range: 60 - hw-specific max */1971 tp->dev->min_mtu = ETH_ZLEN;1972 tp->dev->max_mtu = RTASE_MAX_JUMBO_SIZE;1973}1974 1975static bool rtase_check_mac_version_valid(struct rtase_private *tp)1976{1977 u32 hw_ver = rtase_r32(tp, RTASE_TX_CONFIG_0) & RTASE_HW_VER_MASK;1978 bool known_ver = false;1979 1980 switch (hw_ver) {1981 case 0x00800000:1982 case 0x04000000:1983 case 0x04800000:1984 known_ver = true;1985 break;1986 }1987 1988 return known_ver;1989}1990 1991static int rtase_init_board(struct pci_dev *pdev, struct net_device **dev_out,1992 void __iomem **ioaddr_out)1993{1994 struct net_device *dev;1995 void __iomem *ioaddr;1996 int ret = -ENOMEM;1997 1998 /* dev zeroed in alloc_etherdev */1999 dev = alloc_etherdev_mq(sizeof(struct rtase_private),2000 RTASE_FUNC_TXQ_NUM);2001 if (!dev)2002 goto err_out;2003 2004 SET_NETDEV_DEV(dev, &pdev->dev);2005 2006 ret = pci_enable_device(pdev);2007 if (ret < 0)2008 goto err_out_free_dev;2009 2010 /* make sure PCI base addr 1 is MMIO */2011 if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) {2012 ret = -ENODEV;2013 goto err_out_disable;2014 }2015 2016 /* check for weird/broken PCI region reporting */2017 if (pci_resource_len(pdev, 2) < RTASE_REGS_SIZE) {2018 ret = -ENODEV;2019 goto err_out_disable;2020 }2021 2022 ret = pci_request_regions(pdev, KBUILD_MODNAME);2023 if (ret < 0)2024 goto err_out_disable;2025 2026 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));2027 if (ret) {2028 dev_err(&pdev->dev, "no usable dma addressing method\n");2029 goto err_out_free_res;2030 }2031 2032 pci_set_master(pdev);2033 2034 /* ioremap MMIO region */2035 ioaddr = ioremap(pci_resource_start(pdev, 2),2036 pci_resource_len(pdev, 2));2037 if (!ioaddr) {2038 ret = -EIO;2039 goto err_out_free_res;2040 }2041 2042 *ioaddr_out = ioaddr;2043 *dev_out = dev;2044 2045 return ret;2046 2047err_out_free_res:2048 pci_release_regions(pdev);2049 2050err_out_disable:2051 pci_disable_device(pdev);2052 2053err_out_free_dev:2054 free_netdev(dev);2055 2056err_out:2057 *ioaddr_out = NULL;2058 *dev_out = NULL;2059 2060 return ret;2061}2062 2063static void rtase_release_board(struct pci_dev *pdev, struct net_device *dev,2064 void __iomem *ioaddr)2065{2066 const struct rtase_private *tp = netdev_priv(dev);2067 2068 rtase_rar_set(tp, tp->dev->perm_addr);2069 iounmap(ioaddr);2070 2071 if (tp->sw_flag & RTASE_SWF_MSIX_ENABLED)2072 pci_disable_msix(pdev);2073 else2074 pci_disable_msi(pdev);2075 2076 pci_release_regions(pdev);2077 pci_disable_device(pdev);2078 free_netdev(dev);2079}2080 2081static int rtase_init_one(struct pci_dev *pdev,2082 const struct pci_device_id *ent)2083{2084 struct net_device *dev = NULL;2085 struct rtase_int_vector *ivec;2086 void __iomem *ioaddr = NULL;2087 struct rtase_private *tp;2088 int ret, i;2089 2090 if (!pdev->is_physfn && pdev->is_virtfn) {2091 dev_err(&pdev->dev,2092 "This module does not support a virtual function.");2093 return -EINVAL;2094 }2095 2096 dev_dbg(&pdev->dev, "Automotive Switch Ethernet driver loaded\n");2097 2098 ret = rtase_init_board(pdev, &dev, &ioaddr);2099 if (ret != 0)2100 return ret;2101 2102 tp = netdev_priv(dev);2103 tp->mmio_addr = ioaddr;2104 tp->dev = dev;2105 tp->pdev = pdev;2106 2107 /* identify chip attached to board */2108 if (!rtase_check_mac_version_valid(tp))2109 return dev_err_probe(&pdev->dev, -ENODEV,2110 "unknown chip version, contact rtase maintainers (see MAINTAINERS file)\n");2111 2112 rtase_init_software_variable(pdev, tp);2113 rtase_init_hardware(tp);2114 2115 ret = rtase_alloc_interrupt(pdev, tp);2116 if (ret < 0) {2117 dev_err(&pdev->dev, "unable to alloc MSIX/MSI\n");2118 goto err_out_1;2119 }2120 2121 rtase_init_netdev_ops(dev);2122 2123 dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;2124 2125 dev->features |= NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |2126 NETIF_F_IP_CSUM | NETIF_F_HIGHDMA |2127 NETIF_F_RXCSUM | NETIF_F_SG |2128 NETIF_F_TSO | NETIF_F_IPV6_CSUM |2129 NETIF_F_TSO6;2130 2131 dev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM |2132 NETIF_F_TSO | NETIF_F_RXCSUM |2133 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |2134 NETIF_F_RXALL | NETIF_F_RXFCS |2135 NETIF_F_IPV6_CSUM | NETIF_F_TSO6;2136 2137 dev->vlan_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO |2138 NETIF_F_HIGHDMA;2139 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;2140 netif_set_tso_max_size(dev, RTASE_LSO_64K);2141 netif_set_tso_max_segs(dev, RTASE_NIC_MAX_PHYS_BUF_COUNT_LSO2);2142 2143 rtase_get_mac_address(dev);2144 2145 tp->tally_vaddr = dma_alloc_coherent(&pdev->dev,2146 sizeof(*tp->tally_vaddr),2147 &tp->tally_paddr,2148 GFP_KERNEL);2149 if (!tp->tally_vaddr) {2150 ret = -ENOMEM;2151 goto err_out;2152 }2153 2154 rtase_tally_counter_clear(tp);2155 2156 pci_set_drvdata(pdev, dev);2157 2158 netif_carrier_off(dev);2159 2160 ret = register_netdev(dev);2161 if (ret != 0)2162 goto err_out;2163 2164 netdev_dbg(dev, "%pM, IRQ %d\n", dev->dev_addr, dev->irq);2165 2166 return 0;2167 2168err_out:2169 if (tp->tally_vaddr) {2170 dma_free_coherent(&pdev->dev,2171 sizeof(*tp->tally_vaddr),2172 tp->tally_vaddr,2173 tp->tally_paddr);2174 2175 tp->tally_vaddr = NULL;2176 }2177 2178err_out_1:2179 for (i = 0; i < tp->int_nums; i++) {2180 ivec = &tp->int_vector[i];2181 netif_napi_del(&ivec->napi);2182 }2183 2184 rtase_release_board(pdev, dev, ioaddr);2185 2186 return ret;2187}2188 2189static void rtase_remove_one(struct pci_dev *pdev)2190{2191 struct net_device *dev = pci_get_drvdata(pdev);2192 struct rtase_private *tp = netdev_priv(dev);2193 struct rtase_int_vector *ivec;2194 u32 i;2195 2196 unregister_netdev(dev);2197 2198 for (i = 0; i < tp->int_nums; i++) {2199 ivec = &tp->int_vector[i];2200 netif_napi_del(&ivec->napi);2201 }2202 2203 rtase_reset_interrupt(pdev, tp);2204 if (tp->tally_vaddr) {2205 dma_free_coherent(&pdev->dev,2206 sizeof(*tp->tally_vaddr),2207 tp->tally_vaddr,2208 tp->tally_paddr);2209 tp->tally_vaddr = NULL;2210 }2211 2212 rtase_release_board(pdev, dev, tp->mmio_addr);2213 pci_set_drvdata(pdev, NULL);2214}2215 2216static void rtase_shutdown(struct pci_dev *pdev)2217{2218 struct net_device *dev = pci_get_drvdata(pdev);2219 const struct rtase_private *tp;2220 2221 tp = netdev_priv(dev);2222 2223 if (netif_running(dev))2224 rtase_close(dev);2225 2226 rtase_reset_interrupt(pdev, tp);2227}2228 2229static int rtase_suspend(struct device *device)2230{2231 struct net_device *dev = dev_get_drvdata(device);2232 2233 if (netif_running(dev)) {2234 netif_device_detach(dev);2235 rtase_hw_reset(dev);2236 }2237 2238 return 0;2239}2240 2241static int rtase_resume(struct device *device)2242{2243 struct net_device *dev = dev_get_drvdata(device);2244 struct rtase_private *tp = netdev_priv(dev);2245 int ret;2246 2247 /* restore last modified mac address */2248 rtase_rar_set(tp, dev->dev_addr);2249 2250 if (!netif_running(dev))2251 goto out;2252 2253 rtase_wait_for_quiescence(dev);2254 2255 rtase_tx_clear(tp);2256 rtase_rx_clear(tp);2257 2258 ret = rtase_init_ring(dev);2259 if (ret) {2260 netdev_err(dev, "unable to init ring\n");2261 rtase_free_desc(tp);2262 return -ENOMEM;2263 }2264 2265 rtase_hw_config(dev);2266 /* always link, so start to transmit & receive */2267 rtase_hw_start(dev);2268 2269 netif_device_attach(dev);2270out:2271 2272 return 0;2273}2274 2275static const struct dev_pm_ops rtase_pm_ops = {2276 SYSTEM_SLEEP_PM_OPS(rtase_suspend, rtase_resume)2277};2278 2279static struct pci_driver rtase_pci_driver = {2280 .name = KBUILD_MODNAME,2281 .id_table = rtase_pci_tbl,2282 .probe = rtase_init_one,2283 .remove = rtase_remove_one,2284 .shutdown = rtase_shutdown,2285 .driver.pm = pm_ptr(&rtase_pm_ops),2286};2287 2288module_pci_driver(rtase_pci_driver);2289