2699 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright (C) 2021 Gerhard Engleder <gerhard@engleder-embedded.com> */3 4/* TSN endpoint Ethernet MAC driver5 *6 * The TSN endpoint Ethernet MAC is a FPGA based network device for real-time7 * communication. It is designed for endpoints within TSN (Time Sensitive8 * Networking) networks; e.g., for PLCs in the industrial automation case.9 *10 * It supports multiple TX/RX queue pairs. The first TX/RX queue pair is used11 * by the driver.12 *13 * More information can be found here:14 * - www.embedded-experts.at/tsn15 * - www.engleder-embedded.com16 */17 18#include "tsnep.h"19#include "tsnep_hw.h"20 21#include <linux/module.h>22#include <linux/of.h>23#include <linux/of_net.h>24#include <linux/of_mdio.h>25#include <linux/interrupt.h>26#include <linux/etherdevice.h>27#include <linux/phy.h>28#include <linux/iopoll.h>29#include <linux/bpf.h>30#include <linux/bpf_trace.h>31#include <net/page_pool/helpers.h>32#include <net/xdp_sock_drv.h>33 34#define TSNEP_RX_OFFSET (max(NET_SKB_PAD, XDP_PACKET_HEADROOM) + NET_IP_ALIGN)35#define TSNEP_HEADROOM ALIGN(TSNEP_RX_OFFSET, 4)36#define TSNEP_MAX_RX_BUF_SIZE (PAGE_SIZE - TSNEP_HEADROOM - \37 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))38/* XSK buffer shall store at least Q-in-Q frame */39#define TSNEP_XSK_RX_BUF_SIZE (ALIGN(TSNEP_RX_INLINE_METADATA_SIZE + \40 ETH_FRAME_LEN + ETH_FCS_LEN + \41 VLAN_HLEN * 2, 4))42 43#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT44#define DMA_ADDR_HIGH(dma_addr) ((u32)(((dma_addr) >> 32) & 0xFFFFFFFF))45#else46#define DMA_ADDR_HIGH(dma_addr) ((u32)(0))47#endif48#define DMA_ADDR_LOW(dma_addr) ((u32)((dma_addr) & 0xFFFFFFFF))49 50#define TSNEP_COALESCE_USECS_DEFAULT 6451#define TSNEP_COALESCE_USECS_MAX ((ECM_INT_DELAY_MASK >> ECM_INT_DELAY_SHIFT) * \52 ECM_INT_DELAY_BASE_US + ECM_INT_DELAY_BASE_US - 1)53 54/* mapping type */55#define TSNEP_TX_TYPE_MAP BIT(0)56#define TSNEP_TX_TYPE_MAP_PAGE BIT(1)57#define TSNEP_TX_TYPE_INLINE BIT(2)58/* buffer type */59#define TSNEP_TX_TYPE_SKB BIT(8)60#define TSNEP_TX_TYPE_SKB_MAP (TSNEP_TX_TYPE_SKB | TSNEP_TX_TYPE_MAP)61#define TSNEP_TX_TYPE_SKB_INLINE (TSNEP_TX_TYPE_SKB | TSNEP_TX_TYPE_INLINE)62#define TSNEP_TX_TYPE_SKB_FRAG BIT(9)63#define TSNEP_TX_TYPE_SKB_FRAG_MAP_PAGE (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_MAP_PAGE)64#define TSNEP_TX_TYPE_SKB_FRAG_INLINE (TSNEP_TX_TYPE_SKB_FRAG | TSNEP_TX_TYPE_INLINE)65#define TSNEP_TX_TYPE_XDP_TX BIT(10)66#define TSNEP_TX_TYPE_XDP_NDO BIT(11)67#define TSNEP_TX_TYPE_XDP_NDO_MAP_PAGE (TSNEP_TX_TYPE_XDP_NDO | TSNEP_TX_TYPE_MAP_PAGE)68#define TSNEP_TX_TYPE_XDP (TSNEP_TX_TYPE_XDP_TX | TSNEP_TX_TYPE_XDP_NDO)69#define TSNEP_TX_TYPE_XSK BIT(12)70 71#define TSNEP_XDP_TX BIT(0)72#define TSNEP_XDP_REDIRECT BIT(1)73 74static void tsnep_enable_irq(struct tsnep_adapter *adapter, u32 mask)75{76 iowrite32(mask, adapter->addr + ECM_INT_ENABLE);77}78 79static void tsnep_disable_irq(struct tsnep_adapter *adapter, u32 mask)80{81 mask |= ECM_INT_DISABLE;82 iowrite32(mask, adapter->addr + ECM_INT_ENABLE);83}84 85static irqreturn_t tsnep_irq(int irq, void *arg)86{87 struct tsnep_adapter *adapter = arg;88 u32 active = ioread32(adapter->addr + ECM_INT_ACTIVE);89 90 /* acknowledge interrupt */91 if (active != 0)92 iowrite32(active, adapter->addr + ECM_INT_ACKNOWLEDGE);93 94 /* handle link interrupt */95 if ((active & ECM_INT_LINK) != 0)96 phy_mac_interrupt(adapter->netdev->phydev);97 98 /* handle TX/RX queue 0 interrupt */99 if ((active & adapter->queue[0].irq_mask) != 0) {100 if (napi_schedule_prep(&adapter->queue[0].napi)) {101 tsnep_disable_irq(adapter, adapter->queue[0].irq_mask);102 /* schedule after masking to avoid races */103 __napi_schedule(&adapter->queue[0].napi);104 }105 }106 107 return IRQ_HANDLED;108}109 110static irqreturn_t tsnep_irq_txrx(int irq, void *arg)111{112 struct tsnep_queue *queue = arg;113 114 /* handle TX/RX queue interrupt */115 if (napi_schedule_prep(&queue->napi)) {116 tsnep_disable_irq(queue->adapter, queue->irq_mask);117 /* schedule after masking to avoid races */118 __napi_schedule(&queue->napi);119 }120 121 return IRQ_HANDLED;122}123 124int tsnep_set_irq_coalesce(struct tsnep_queue *queue, u32 usecs)125{126 if (usecs > TSNEP_COALESCE_USECS_MAX)127 return -ERANGE;128 129 usecs /= ECM_INT_DELAY_BASE_US;130 usecs <<= ECM_INT_DELAY_SHIFT;131 usecs &= ECM_INT_DELAY_MASK;132 133 queue->irq_delay &= ~ECM_INT_DELAY_MASK;134 queue->irq_delay |= usecs;135 iowrite8(queue->irq_delay, queue->irq_delay_addr);136 137 return 0;138}139 140u32 tsnep_get_irq_coalesce(struct tsnep_queue *queue)141{142 u32 usecs;143 144 usecs = (queue->irq_delay & ECM_INT_DELAY_MASK);145 usecs >>= ECM_INT_DELAY_SHIFT;146 usecs *= ECM_INT_DELAY_BASE_US;147 148 return usecs;149}150 151static int tsnep_mdiobus_read(struct mii_bus *bus, int addr, int regnum)152{153 struct tsnep_adapter *adapter = bus->priv;154 u32 md;155 int retval;156 157 md = ECM_MD_READ;158 if (!adapter->suppress_preamble)159 md |= ECM_MD_PREAMBLE;160 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK;161 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK;162 iowrite32(md, adapter->addr + ECM_MD_CONTROL);163 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md,164 !(md & ECM_MD_BUSY), 16, 1000);165 if (retval != 0)166 return retval;167 168 return (md & ECM_MD_DATA_MASK) >> ECM_MD_DATA_SHIFT;169}170 171static int tsnep_mdiobus_write(struct mii_bus *bus, int addr, int regnum,172 u16 val)173{174 struct tsnep_adapter *adapter = bus->priv;175 u32 md;176 int retval;177 178 md = ECM_MD_WRITE;179 if (!adapter->suppress_preamble)180 md |= ECM_MD_PREAMBLE;181 md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK;182 md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK;183 md |= ((u32)val << ECM_MD_DATA_SHIFT) & ECM_MD_DATA_MASK;184 iowrite32(md, adapter->addr + ECM_MD_CONTROL);185 retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md,186 !(md & ECM_MD_BUSY), 16, 1000);187 if (retval != 0)188 return retval;189 190 return 0;191}192 193static void tsnep_set_link_mode(struct tsnep_adapter *adapter)194{195 u32 mode;196 197 switch (adapter->phydev->speed) {198 case SPEED_100:199 mode = ECM_LINK_MODE_100;200 break;201 case SPEED_1000:202 mode = ECM_LINK_MODE_1000;203 break;204 default:205 mode = ECM_LINK_MODE_OFF;206 break;207 }208 iowrite32(mode, adapter->addr + ECM_STATUS);209}210 211static void tsnep_phy_link_status_change(struct net_device *netdev)212{213 struct tsnep_adapter *adapter = netdev_priv(netdev);214 struct phy_device *phydev = netdev->phydev;215 216 if (phydev->link)217 tsnep_set_link_mode(adapter);218 219 phy_print_status(netdev->phydev);220}221 222static int tsnep_phy_loopback(struct tsnep_adapter *adapter, bool enable)223{224 int retval;225 226 retval = phy_loopback(adapter->phydev, enable);227 228 /* PHY link state change is not signaled if loopback is enabled, it229 * would delay a working loopback anyway, let's ensure that loopback230 * is working immediately by setting link mode directly231 */232 if (!retval && enable) {233 netif_carrier_on(adapter->netdev);234 tsnep_set_link_mode(adapter);235 }236 237 return retval;238}239 240static int tsnep_phy_open(struct tsnep_adapter *adapter)241{242 struct phy_device *phydev;243 struct ethtool_keee ethtool_keee;244 int retval;245 246 retval = phy_connect_direct(adapter->netdev, adapter->phydev,247 tsnep_phy_link_status_change,248 adapter->phy_mode);249 if (retval)250 return retval;251 phydev = adapter->netdev->phydev;252 253 /* MAC supports only 100Mbps|1000Mbps full duplex254 * SPE (Single Pair Ethernet) is also an option but not implemented yet255 */256 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Half_BIT);257 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Full_BIT);258 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_100baseT_Half_BIT);259 phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_1000baseT_Half_BIT);260 261 /* disable EEE autoneg, EEE not supported by TSNEP */262 memset(ðtool_keee, 0, sizeof(ethtool_keee));263 phy_ethtool_set_eee(adapter->phydev, ðtool_keee);264 265 adapter->phydev->irq = PHY_MAC_INTERRUPT;266 phy_start(adapter->phydev);267 268 return 0;269}270 271static void tsnep_phy_close(struct tsnep_adapter *adapter)272{273 phy_stop(adapter->netdev->phydev);274 phy_disconnect(adapter->netdev->phydev);275}276 277static void tsnep_tx_ring_cleanup(struct tsnep_tx *tx)278{279 struct device *dmadev = tx->adapter->dmadev;280 int i;281 282 memset(tx->entry, 0, sizeof(tx->entry));283 284 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {285 if (tx->page[i]) {286 dma_free_coherent(dmadev, PAGE_SIZE, tx->page[i],287 tx->page_dma[i]);288 tx->page[i] = NULL;289 tx->page_dma[i] = 0;290 }291 }292}293 294static int tsnep_tx_ring_create(struct tsnep_tx *tx)295{296 struct device *dmadev = tx->adapter->dmadev;297 struct tsnep_tx_entry *entry;298 struct tsnep_tx_entry *next_entry;299 int i, j;300 int retval;301 302 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {303 tx->page[i] =304 dma_alloc_coherent(dmadev, PAGE_SIZE, &tx->page_dma[i],305 GFP_KERNEL);306 if (!tx->page[i]) {307 retval = -ENOMEM;308 goto alloc_failed;309 }310 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) {311 entry = &tx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j];312 entry->desc_wb = (struct tsnep_tx_desc_wb *)313 (((u8 *)tx->page[i]) + TSNEP_DESC_SIZE * j);314 entry->desc = (struct tsnep_tx_desc *)315 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET);316 entry->desc_dma = tx->page_dma[i] + TSNEP_DESC_SIZE * j;317 entry->owner_user_flag = false;318 }319 }320 for (i = 0; i < TSNEP_RING_SIZE; i++) {321 entry = &tx->entry[i];322 next_entry = &tx->entry[(i + 1) & TSNEP_RING_MASK];323 entry->desc->next = __cpu_to_le64(next_entry->desc_dma);324 }325 326 return 0;327 328alloc_failed:329 tsnep_tx_ring_cleanup(tx);330 return retval;331}332 333static void tsnep_tx_init(struct tsnep_tx *tx)334{335 dma_addr_t dma;336 337 dma = tx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER;338 iowrite32(DMA_ADDR_LOW(dma), tx->addr + TSNEP_TX_DESC_ADDR_LOW);339 iowrite32(DMA_ADDR_HIGH(dma), tx->addr + TSNEP_TX_DESC_ADDR_HIGH);340 tx->write = 0;341 tx->read = 0;342 tx->owner_counter = 1;343 tx->increment_owner_counter = TSNEP_RING_SIZE - 1;344}345 346static void tsnep_tx_enable(struct tsnep_tx *tx)347{348 struct netdev_queue *nq;349 350 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);351 352 __netif_tx_lock_bh(nq);353 netif_tx_wake_queue(nq);354 __netif_tx_unlock_bh(nq);355}356 357static void tsnep_tx_disable(struct tsnep_tx *tx, struct napi_struct *napi)358{359 struct netdev_queue *nq;360 u32 val;361 362 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);363 364 __netif_tx_lock_bh(nq);365 netif_tx_stop_queue(nq);366 __netif_tx_unlock_bh(nq);367 368 /* wait until TX is done in hardware */369 readx_poll_timeout(ioread32, tx->addr + TSNEP_CONTROL, val,370 ((val & TSNEP_CONTROL_TX_ENABLE) == 0), 10000,371 1000000);372 373 /* wait until TX is also done in software */374 while (READ_ONCE(tx->read) != tx->write) {375 napi_schedule(napi);376 napi_synchronize(napi);377 }378}379 380static void tsnep_tx_activate(struct tsnep_tx *tx, int index, int length,381 bool last)382{383 struct tsnep_tx_entry *entry = &tx->entry[index];384 385 entry->properties = 0;386 /* xdpf and zc are union with skb */387 if (entry->skb) {388 entry->properties = length & TSNEP_DESC_LENGTH_MASK;389 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG;390 if ((entry->type & TSNEP_TX_TYPE_SKB) &&391 (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS))392 entry->properties |= TSNEP_DESC_EXTENDED_WRITEBACK_FLAG;393 394 /* toggle user flag to prevent false acknowledge395 *396 * Only the first fragment is acknowledged. For all other397 * fragments no acknowledge is done and the last written owner398 * counter stays in the writeback descriptor. Therefore, it is399 * possible that the last written owner counter is identical to400 * the new incremented owner counter and a false acknowledge is401 * detected before the real acknowledge has been done by402 * hardware.403 *404 * The user flag is used to prevent this situation. The user405 * flag is copied to the writeback descriptor by the hardware406 * and is used as additional acknowledge data. By toggeling the407 * user flag only for the first fragment (which is408 * acknowledged), it is guaranteed that the last acknowledge409 * done for this descriptor has used a different user flag and410 * cannot be detected as false acknowledge.411 */412 entry->owner_user_flag = !entry->owner_user_flag;413 }414 if (last)415 entry->properties |= TSNEP_TX_DESC_LAST_FRAGMENT_FLAG;416 if (index == tx->increment_owner_counter) {417 tx->owner_counter++;418 if (tx->owner_counter == 4)419 tx->owner_counter = 1;420 tx->increment_owner_counter--;421 if (tx->increment_owner_counter < 0)422 tx->increment_owner_counter = TSNEP_RING_SIZE - 1;423 }424 entry->properties |=425 (tx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) &426 TSNEP_DESC_OWNER_COUNTER_MASK;427 if (entry->owner_user_flag)428 entry->properties |= TSNEP_TX_DESC_OWNER_USER_FLAG;429 entry->desc->more_properties =430 __cpu_to_le32(entry->len & TSNEP_DESC_LENGTH_MASK);431 if (entry->type & TSNEP_TX_TYPE_INLINE)432 entry->properties |= TSNEP_TX_DESC_DATA_AFTER_DESC_FLAG;433 434 /* descriptor properties shall be written last, because valid data is435 * signaled there436 */437 dma_wmb();438 439 entry->desc->properties = __cpu_to_le32(entry->properties);440}441 442static int tsnep_tx_desc_available(struct tsnep_tx *tx)443{444 if (tx->read <= tx->write)445 return TSNEP_RING_SIZE - tx->write + tx->read - 1;446 else447 return tx->read - tx->write - 1;448}449 450static int tsnep_tx_map_frag(skb_frag_t *frag, struct tsnep_tx_entry *entry,451 struct device *dmadev, dma_addr_t *dma)452{453 unsigned int len;454 int mapped;455 456 len = skb_frag_size(frag);457 if (likely(len > TSNEP_DESC_SIZE_DATA_AFTER_INLINE)) {458 *dma = skb_frag_dma_map(dmadev, frag, 0, len, DMA_TO_DEVICE);459 if (dma_mapping_error(dmadev, *dma))460 return -ENOMEM;461 entry->type = TSNEP_TX_TYPE_SKB_FRAG_MAP_PAGE;462 mapped = 1;463 } else {464 void *fragdata = skb_frag_address_safe(frag);465 466 if (likely(fragdata)) {467 memcpy(&entry->desc->tx, fragdata, len);468 } else {469 struct page *page = skb_frag_page(frag);470 471 fragdata = kmap_local_page(page);472 memcpy(&entry->desc->tx, fragdata + skb_frag_off(frag),473 len);474 kunmap_local(fragdata);475 }476 entry->type = TSNEP_TX_TYPE_SKB_FRAG_INLINE;477 mapped = 0;478 }479 480 return mapped;481}482 483static int tsnep_tx_map(struct sk_buff *skb, struct tsnep_tx *tx, int count)484{485 struct device *dmadev = tx->adapter->dmadev;486 struct tsnep_tx_entry *entry;487 unsigned int len;488 int map_len = 0;489 dma_addr_t dma;490 int i, mapped;491 492 for (i = 0; i < count; i++) {493 entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK];494 495 if (!i) {496 len = skb_headlen(skb);497 if (likely(len > TSNEP_DESC_SIZE_DATA_AFTER_INLINE)) {498 dma = dma_map_single(dmadev, skb->data, len,499 DMA_TO_DEVICE);500 if (dma_mapping_error(dmadev, dma))501 return -ENOMEM;502 entry->type = TSNEP_TX_TYPE_SKB_MAP;503 mapped = 1;504 } else {505 memcpy(&entry->desc->tx, skb->data, len);506 entry->type = TSNEP_TX_TYPE_SKB_INLINE;507 mapped = 0;508 }509 } else {510 skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];511 512 len = skb_frag_size(frag);513 mapped = tsnep_tx_map_frag(frag, entry, dmadev, &dma);514 if (mapped < 0)515 return mapped;516 }517 518 entry->len = len;519 if (likely(mapped)) {520 dma_unmap_addr_set(entry, dma, dma);521 entry->desc->tx = __cpu_to_le64(dma);522 }523 524 map_len += len;525 }526 527 return map_len;528}529 530static int tsnep_tx_unmap(struct tsnep_tx *tx, int index, int count)531{532 struct device *dmadev = tx->adapter->dmadev;533 struct tsnep_tx_entry *entry;534 int map_len = 0;535 int i;536 537 for (i = 0; i < count; i++) {538 entry = &tx->entry[(index + i) & TSNEP_RING_MASK];539 540 if (entry->len) {541 if (entry->type & TSNEP_TX_TYPE_MAP)542 dma_unmap_single(dmadev,543 dma_unmap_addr(entry, dma),544 dma_unmap_len(entry, len),545 DMA_TO_DEVICE);546 else if (entry->type & TSNEP_TX_TYPE_MAP_PAGE)547 dma_unmap_page(dmadev,548 dma_unmap_addr(entry, dma),549 dma_unmap_len(entry, len),550 DMA_TO_DEVICE);551 map_len += entry->len;552 entry->len = 0;553 }554 }555 556 return map_len;557}558 559static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb,560 struct tsnep_tx *tx)561{562 int count = 1;563 struct tsnep_tx_entry *entry;564 int length;565 int i;566 int retval;567 568 if (skb_shinfo(skb)->nr_frags > 0)569 count += skb_shinfo(skb)->nr_frags;570 571 if (tsnep_tx_desc_available(tx) < count) {572 /* ring full, shall not happen because queue is stopped if full573 * below574 */575 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index);576 577 return NETDEV_TX_BUSY;578 }579 580 entry = &tx->entry[tx->write];581 entry->skb = skb;582 583 retval = tsnep_tx_map(skb, tx, count);584 if (retval < 0) {585 tsnep_tx_unmap(tx, tx->write, count);586 dev_kfree_skb_any(entry->skb);587 entry->skb = NULL;588 589 tx->dropped++;590 591 return NETDEV_TX_OK;592 }593 length = retval;594 595 if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)596 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;597 598 for (i = 0; i < count; i++)599 tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length,600 i == count - 1);601 tx->write = (tx->write + count) & TSNEP_RING_MASK;602 603 skb_tx_timestamp(skb);604 605 /* descriptor properties shall be valid before hardware is notified */606 dma_wmb();607 608 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL);609 610 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1)) {611 /* ring can get full with next frame */612 netif_stop_subqueue(tx->adapter->netdev, tx->queue_index);613 }614 615 return NETDEV_TX_OK;616}617 618static int tsnep_xdp_tx_map(struct xdp_frame *xdpf, struct tsnep_tx *tx,619 struct skb_shared_info *shinfo, int count, u32 type)620{621 struct device *dmadev = tx->adapter->dmadev;622 struct tsnep_tx_entry *entry;623 struct page *page;624 skb_frag_t *frag;625 unsigned int len;626 int map_len = 0;627 dma_addr_t dma;628 void *data;629 int i;630 631 frag = NULL;632 len = xdpf->len;633 for (i = 0; i < count; i++) {634 entry = &tx->entry[(tx->write + i) & TSNEP_RING_MASK];635 if (type & TSNEP_TX_TYPE_XDP_NDO) {636 data = unlikely(frag) ? skb_frag_address(frag) :637 xdpf->data;638 dma = dma_map_single(dmadev, data, len, DMA_TO_DEVICE);639 if (dma_mapping_error(dmadev, dma))640 return -ENOMEM;641 642 entry->type = TSNEP_TX_TYPE_XDP_NDO_MAP_PAGE;643 } else {644 page = unlikely(frag) ? skb_frag_page(frag) :645 virt_to_page(xdpf->data);646 dma = page_pool_get_dma_addr(page);647 if (unlikely(frag))648 dma += skb_frag_off(frag);649 else650 dma += sizeof(*xdpf) + xdpf->headroom;651 dma_sync_single_for_device(dmadev, dma, len,652 DMA_BIDIRECTIONAL);653 654 entry->type = TSNEP_TX_TYPE_XDP_TX;655 }656 657 entry->len = len;658 dma_unmap_addr_set(entry, dma, dma);659 660 entry->desc->tx = __cpu_to_le64(dma);661 662 map_len += len;663 664 if (i + 1 < count) {665 frag = &shinfo->frags[i];666 len = skb_frag_size(frag);667 }668 }669 670 return map_len;671}672 673/* This function requires __netif_tx_lock is held by the caller. */674static bool tsnep_xdp_xmit_frame_ring(struct xdp_frame *xdpf,675 struct tsnep_tx *tx, u32 type)676{677 struct skb_shared_info *shinfo = xdp_get_shared_info_from_frame(xdpf);678 struct tsnep_tx_entry *entry;679 int count, length, retval, i;680 681 count = 1;682 if (unlikely(xdp_frame_has_frags(xdpf)))683 count += shinfo->nr_frags;684 685 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS686 * will be available for normal TX path and queue is stopped there if687 * necessary688 */689 if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1 + count))690 return false;691 692 entry = &tx->entry[tx->write];693 entry->xdpf = xdpf;694 695 retval = tsnep_xdp_tx_map(xdpf, tx, shinfo, count, type);696 if (retval < 0) {697 tsnep_tx_unmap(tx, tx->write, count);698 entry->xdpf = NULL;699 700 tx->dropped++;701 702 return false;703 }704 length = retval;705 706 for (i = 0; i < count; i++)707 tsnep_tx_activate(tx, (tx->write + i) & TSNEP_RING_MASK, length,708 i == count - 1);709 tx->write = (tx->write + count) & TSNEP_RING_MASK;710 711 /* descriptor properties shall be valid before hardware is notified */712 dma_wmb();713 714 return true;715}716 717static void tsnep_xdp_xmit_flush(struct tsnep_tx *tx)718{719 iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL);720}721 722static bool tsnep_xdp_xmit_back(struct tsnep_adapter *adapter,723 struct xdp_buff *xdp,724 struct netdev_queue *tx_nq, struct tsnep_tx *tx,725 bool zc)726{727 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);728 bool xmit;729 u32 type;730 731 if (unlikely(!xdpf))732 return false;733 734 /* no page pool for zero copy */735 if (zc)736 type = TSNEP_TX_TYPE_XDP_NDO;737 else738 type = TSNEP_TX_TYPE_XDP_TX;739 740 __netif_tx_lock(tx_nq, smp_processor_id());741 742 xmit = tsnep_xdp_xmit_frame_ring(xdpf, tx, type);743 744 /* Avoid transmit queue timeout since we share it with the slow path */745 if (xmit)746 txq_trans_cond_update(tx_nq);747 748 __netif_tx_unlock(tx_nq);749 750 return xmit;751}752 753static int tsnep_xdp_tx_map_zc(struct xdp_desc *xdpd, struct tsnep_tx *tx)754{755 struct tsnep_tx_entry *entry;756 dma_addr_t dma;757 758 entry = &tx->entry[tx->write];759 entry->zc = true;760 761 dma = xsk_buff_raw_get_dma(tx->xsk_pool, xdpd->addr);762 xsk_buff_raw_dma_sync_for_device(tx->xsk_pool, dma, xdpd->len);763 764 entry->type = TSNEP_TX_TYPE_XSK;765 entry->len = xdpd->len;766 767 entry->desc->tx = __cpu_to_le64(dma);768 769 return xdpd->len;770}771 772static void tsnep_xdp_xmit_frame_ring_zc(struct xdp_desc *xdpd,773 struct tsnep_tx *tx)774{775 int length;776 777 length = tsnep_xdp_tx_map_zc(xdpd, tx);778 779 tsnep_tx_activate(tx, tx->write, length, true);780 tx->write = (tx->write + 1) & TSNEP_RING_MASK;781}782 783static void tsnep_xdp_xmit_zc(struct tsnep_tx *tx)784{785 int desc_available = tsnep_tx_desc_available(tx);786 struct xdp_desc *descs = tx->xsk_pool->tx_descs;787 int batch, i;788 789 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS790 * will be available for normal TX path and queue is stopped there if791 * necessary792 */793 if (desc_available <= (MAX_SKB_FRAGS + 1))794 return;795 desc_available -= MAX_SKB_FRAGS + 1;796 797 batch = xsk_tx_peek_release_desc_batch(tx->xsk_pool, desc_available);798 for (i = 0; i < batch; i++)799 tsnep_xdp_xmit_frame_ring_zc(&descs[i], tx);800 801 if (batch) {802 /* descriptor properties shall be valid before hardware is803 * notified804 */805 dma_wmb();806 807 tsnep_xdp_xmit_flush(tx);808 }809}810 811static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget)812{813 struct tsnep_tx_entry *entry;814 struct netdev_queue *nq;815 int xsk_frames = 0;816 int budget = 128;817 int length;818 int count;819 820 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);821 __netif_tx_lock(nq, smp_processor_id());822 823 do {824 if (tx->read == tx->write)825 break;826 827 entry = &tx->entry[tx->read];828 if ((__le32_to_cpu(entry->desc_wb->properties) &829 TSNEP_TX_DESC_OWNER_MASK) !=830 (entry->properties & TSNEP_TX_DESC_OWNER_MASK))831 break;832 833 /* descriptor properties shall be read first, because valid data834 * is signaled there835 */836 dma_rmb();837 838 count = 1;839 if ((entry->type & TSNEP_TX_TYPE_SKB) &&840 skb_shinfo(entry->skb)->nr_frags > 0)841 count += skb_shinfo(entry->skb)->nr_frags;842 else if ((entry->type & TSNEP_TX_TYPE_XDP) &&843 xdp_frame_has_frags(entry->xdpf))844 count += xdp_get_shared_info_from_frame(entry->xdpf)->nr_frags;845 846 length = tsnep_tx_unmap(tx, tx->read, count);847 848 if ((entry->type & TSNEP_TX_TYPE_SKB) &&849 (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS) &&850 (__le32_to_cpu(entry->desc_wb->properties) &851 TSNEP_DESC_EXTENDED_WRITEBACK_FLAG)) {852 struct skb_shared_hwtstamps hwtstamps;853 u64 timestamp;854 855 if (skb_shinfo(entry->skb)->tx_flags &856 SKBTX_HW_TSTAMP_USE_CYCLES)857 timestamp =858 __le64_to_cpu(entry->desc_wb->counter);859 else860 timestamp =861 __le64_to_cpu(entry->desc_wb->timestamp);862 863 memset(&hwtstamps, 0, sizeof(hwtstamps));864 hwtstamps.hwtstamp = ns_to_ktime(timestamp);865 866 skb_tstamp_tx(entry->skb, &hwtstamps);867 }868 869 if (entry->type & TSNEP_TX_TYPE_SKB)870 napi_consume_skb(entry->skb, napi_budget);871 else if (entry->type & TSNEP_TX_TYPE_XDP)872 xdp_return_frame_rx_napi(entry->xdpf);873 else874 xsk_frames++;875 /* xdpf and zc are union with skb */876 entry->skb = NULL;877 878 tx->read = (tx->read + count) & TSNEP_RING_MASK;879 880 tx->packets++;881 tx->bytes += length + ETH_FCS_LEN;882 883 budget--;884 } while (likely(budget));885 886 if (tx->xsk_pool) {887 if (xsk_frames)888 xsk_tx_completed(tx->xsk_pool, xsk_frames);889 if (xsk_uses_need_wakeup(tx->xsk_pool))890 xsk_set_tx_need_wakeup(tx->xsk_pool);891 tsnep_xdp_xmit_zc(tx);892 }893 894 if ((tsnep_tx_desc_available(tx) >= ((MAX_SKB_FRAGS + 1) * 2)) &&895 netif_tx_queue_stopped(nq)) {896 netif_tx_wake_queue(nq);897 }898 899 __netif_tx_unlock(nq);900 901 return budget != 0;902}903 904static bool tsnep_tx_pending(struct tsnep_tx *tx)905{906 struct tsnep_tx_entry *entry;907 struct netdev_queue *nq;908 bool pending = false;909 910 nq = netdev_get_tx_queue(tx->adapter->netdev, tx->queue_index);911 __netif_tx_lock(nq, smp_processor_id());912 913 if (tx->read != tx->write) {914 entry = &tx->entry[tx->read];915 if ((__le32_to_cpu(entry->desc_wb->properties) &916 TSNEP_TX_DESC_OWNER_MASK) ==917 (entry->properties & TSNEP_TX_DESC_OWNER_MASK))918 pending = true;919 }920 921 __netif_tx_unlock(nq);922 923 return pending;924}925 926static int tsnep_tx_open(struct tsnep_tx *tx)927{928 int retval;929 930 retval = tsnep_tx_ring_create(tx);931 if (retval)932 return retval;933 934 tsnep_tx_init(tx);935 936 return 0;937}938 939static void tsnep_tx_close(struct tsnep_tx *tx)940{941 tsnep_tx_ring_cleanup(tx);942}943 944static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx)945{946 struct device *dmadev = rx->adapter->dmadev;947 struct tsnep_rx_entry *entry;948 int i;949 950 for (i = 0; i < TSNEP_RING_SIZE; i++) {951 entry = &rx->entry[i];952 if (!rx->xsk_pool && entry->page)953 page_pool_put_full_page(rx->page_pool, entry->page,954 false);955 if (rx->xsk_pool && entry->xdp)956 xsk_buff_free(entry->xdp);957 /* xdp is union with page */958 entry->page = NULL;959 }960 961 if (rx->page_pool)962 page_pool_destroy(rx->page_pool);963 964 memset(rx->entry, 0, sizeof(rx->entry));965 966 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {967 if (rx->page[i]) {968 dma_free_coherent(dmadev, PAGE_SIZE, rx->page[i],969 rx->page_dma[i]);970 rx->page[i] = NULL;971 rx->page_dma[i] = 0;972 }973 }974}975 976static int tsnep_rx_ring_create(struct tsnep_rx *rx)977{978 struct device *dmadev = rx->adapter->dmadev;979 struct tsnep_rx_entry *entry;980 struct page_pool_params pp_params = { 0 };981 struct tsnep_rx_entry *next_entry;982 int i, j;983 int retval;984 985 for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {986 rx->page[i] =987 dma_alloc_coherent(dmadev, PAGE_SIZE, &rx->page_dma[i],988 GFP_KERNEL);989 if (!rx->page[i]) {990 retval = -ENOMEM;991 goto failed;992 }993 for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) {994 entry = &rx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j];995 entry->desc_wb = (struct tsnep_rx_desc_wb *)996 (((u8 *)rx->page[i]) + TSNEP_DESC_SIZE * j);997 entry->desc = (struct tsnep_rx_desc *)998 (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET);999 entry->desc_dma = rx->page_dma[i] + TSNEP_DESC_SIZE * j;1000 }1001 }1002 1003 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;1004 pp_params.order = 0;1005 pp_params.pool_size = TSNEP_RING_SIZE;1006 pp_params.nid = dev_to_node(dmadev);1007 pp_params.dev = dmadev;1008 pp_params.dma_dir = DMA_BIDIRECTIONAL;1009 pp_params.max_len = TSNEP_MAX_RX_BUF_SIZE;1010 pp_params.offset = TSNEP_RX_OFFSET;1011 rx->page_pool = page_pool_create(&pp_params);1012 if (IS_ERR(rx->page_pool)) {1013 retval = PTR_ERR(rx->page_pool);1014 rx->page_pool = NULL;1015 goto failed;1016 }1017 1018 for (i = 0; i < TSNEP_RING_SIZE; i++) {1019 entry = &rx->entry[i];1020 next_entry = &rx->entry[(i + 1) & TSNEP_RING_MASK];1021 entry->desc->next = __cpu_to_le64(next_entry->desc_dma);1022 }1023 1024 return 0;1025 1026failed:1027 tsnep_rx_ring_cleanup(rx);1028 return retval;1029}1030 1031static void tsnep_rx_init(struct tsnep_rx *rx)1032{1033 dma_addr_t dma;1034 1035 dma = rx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER;1036 iowrite32(DMA_ADDR_LOW(dma), rx->addr + TSNEP_RX_DESC_ADDR_LOW);1037 iowrite32(DMA_ADDR_HIGH(dma), rx->addr + TSNEP_RX_DESC_ADDR_HIGH);1038 rx->write = 0;1039 rx->read = 0;1040 rx->owner_counter = 1;1041 rx->increment_owner_counter = TSNEP_RING_SIZE - 1;1042}1043 1044static void tsnep_rx_enable(struct tsnep_rx *rx)1045{1046 /* descriptor properties shall be valid before hardware is notified */1047 dma_wmb();1048 1049 iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL);1050}1051 1052static void tsnep_rx_disable(struct tsnep_rx *rx)1053{1054 u32 val;1055 1056 iowrite32(TSNEP_CONTROL_RX_DISABLE, rx->addr + TSNEP_CONTROL);1057 readx_poll_timeout(ioread32, rx->addr + TSNEP_CONTROL, val,1058 ((val & TSNEP_CONTROL_RX_ENABLE) == 0), 10000,1059 1000000);1060}1061 1062static int tsnep_rx_desc_available(struct tsnep_rx *rx)1063{1064 if (rx->read <= rx->write)1065 return TSNEP_RING_SIZE - rx->write + rx->read - 1;1066 else1067 return rx->read - rx->write - 1;1068}1069 1070static void tsnep_rx_free_page_buffer(struct tsnep_rx *rx)1071{1072 struct page **page;1073 1074 /* last entry of page_buffer is always zero, because ring cannot be1075 * filled completely1076 */1077 page = rx->page_buffer;1078 while (*page) {1079 page_pool_put_full_page(rx->page_pool, *page, false);1080 *page = NULL;1081 page++;1082 }1083}1084 1085static int tsnep_rx_alloc_page_buffer(struct tsnep_rx *rx)1086{1087 int i;1088 1089 /* alloc for all ring entries except the last one, because ring cannot1090 * be filled completely1091 */1092 for (i = 0; i < TSNEP_RING_SIZE - 1; i++) {1093 rx->page_buffer[i] = page_pool_dev_alloc_pages(rx->page_pool);1094 if (!rx->page_buffer[i]) {1095 tsnep_rx_free_page_buffer(rx);1096 1097 return -ENOMEM;1098 }1099 }1100 1101 return 0;1102}1103 1104static void tsnep_rx_set_page(struct tsnep_rx *rx, struct tsnep_rx_entry *entry,1105 struct page *page)1106{1107 entry->page = page;1108 entry->len = TSNEP_MAX_RX_BUF_SIZE;1109 entry->dma = page_pool_get_dma_addr(entry->page);1110 entry->desc->rx = __cpu_to_le64(entry->dma + TSNEP_RX_OFFSET);1111}1112 1113static int tsnep_rx_alloc_buffer(struct tsnep_rx *rx, int index)1114{1115 struct tsnep_rx_entry *entry = &rx->entry[index];1116 struct page *page;1117 1118 page = page_pool_dev_alloc_pages(rx->page_pool);1119 if (unlikely(!page))1120 return -ENOMEM;1121 tsnep_rx_set_page(rx, entry, page);1122 1123 return 0;1124}1125 1126static void tsnep_rx_reuse_buffer(struct tsnep_rx *rx, int index)1127{1128 struct tsnep_rx_entry *entry = &rx->entry[index];1129 struct tsnep_rx_entry *read = &rx->entry[rx->read];1130 1131 tsnep_rx_set_page(rx, entry, read->page);1132 read->page = NULL;1133}1134 1135static void tsnep_rx_activate(struct tsnep_rx *rx, int index)1136{1137 struct tsnep_rx_entry *entry = &rx->entry[index];1138 1139 /* TSNEP_MAX_RX_BUF_SIZE and TSNEP_XSK_RX_BUF_SIZE are multiple of 4 */1140 entry->properties = entry->len & TSNEP_DESC_LENGTH_MASK;1141 entry->properties |= TSNEP_DESC_INTERRUPT_FLAG;1142 if (index == rx->increment_owner_counter) {1143 rx->owner_counter++;1144 if (rx->owner_counter == 4)1145 rx->owner_counter = 1;1146 rx->increment_owner_counter--;1147 if (rx->increment_owner_counter < 0)1148 rx->increment_owner_counter = TSNEP_RING_SIZE - 1;1149 }1150 entry->properties |=1151 (rx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) &1152 TSNEP_DESC_OWNER_COUNTER_MASK;1153 1154 /* descriptor properties shall be written last, because valid data is1155 * signaled there1156 */1157 dma_wmb();1158 1159 entry->desc->properties = __cpu_to_le32(entry->properties);1160}1161 1162static int tsnep_rx_alloc(struct tsnep_rx *rx, int count, bool reuse)1163{1164 bool alloc_failed = false;1165 int i, index;1166 1167 for (i = 0; i < count && !alloc_failed; i++) {1168 index = (rx->write + i) & TSNEP_RING_MASK;1169 1170 if (unlikely(tsnep_rx_alloc_buffer(rx, index))) {1171 rx->alloc_failed++;1172 alloc_failed = true;1173 1174 /* reuse only if no other allocation was successful */1175 if (i == 0 && reuse)1176 tsnep_rx_reuse_buffer(rx, index);1177 else1178 break;1179 }1180 1181 tsnep_rx_activate(rx, index);1182 }1183 1184 if (i)1185 rx->write = (rx->write + i) & TSNEP_RING_MASK;1186 1187 return i;1188}1189 1190static int tsnep_rx_refill(struct tsnep_rx *rx, int count, bool reuse)1191{1192 int desc_refilled;1193 1194 desc_refilled = tsnep_rx_alloc(rx, count, reuse);1195 if (desc_refilled)1196 tsnep_rx_enable(rx);1197 1198 return desc_refilled;1199}1200 1201static void tsnep_rx_set_xdp(struct tsnep_rx *rx, struct tsnep_rx_entry *entry,1202 struct xdp_buff *xdp)1203{1204 entry->xdp = xdp;1205 entry->len = TSNEP_XSK_RX_BUF_SIZE;1206 entry->dma = xsk_buff_xdp_get_dma(entry->xdp);1207 entry->desc->rx = __cpu_to_le64(entry->dma);1208}1209 1210static void tsnep_rx_reuse_buffer_zc(struct tsnep_rx *rx, int index)1211{1212 struct tsnep_rx_entry *entry = &rx->entry[index];1213 struct tsnep_rx_entry *read = &rx->entry[rx->read];1214 1215 tsnep_rx_set_xdp(rx, entry, read->xdp);1216 read->xdp = NULL;1217}1218 1219static int tsnep_rx_alloc_zc(struct tsnep_rx *rx, int count, bool reuse)1220{1221 u32 allocated;1222 int i;1223 1224 allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch, count);1225 for (i = 0; i < allocated; i++) {1226 int index = (rx->write + i) & TSNEP_RING_MASK;1227 struct tsnep_rx_entry *entry = &rx->entry[index];1228 1229 tsnep_rx_set_xdp(rx, entry, rx->xdp_batch[i]);1230 tsnep_rx_activate(rx, index);1231 }1232 if (i == 0) {1233 rx->alloc_failed++;1234 1235 if (reuse) {1236 tsnep_rx_reuse_buffer_zc(rx, rx->write);1237 tsnep_rx_activate(rx, rx->write);1238 }1239 }1240 1241 if (i)1242 rx->write = (rx->write + i) & TSNEP_RING_MASK;1243 1244 return i;1245}1246 1247static void tsnep_rx_free_zc(struct tsnep_rx *rx)1248{1249 int i;1250 1251 for (i = 0; i < TSNEP_RING_SIZE; i++) {1252 struct tsnep_rx_entry *entry = &rx->entry[i];1253 1254 if (entry->xdp)1255 xsk_buff_free(entry->xdp);1256 entry->xdp = NULL;1257 }1258}1259 1260static int tsnep_rx_refill_zc(struct tsnep_rx *rx, int count, bool reuse)1261{1262 int desc_refilled;1263 1264 desc_refilled = tsnep_rx_alloc_zc(rx, count, reuse);1265 if (desc_refilled)1266 tsnep_rx_enable(rx);1267 1268 return desc_refilled;1269}1270 1271static void tsnep_xsk_rx_need_wakeup(struct tsnep_rx *rx, int desc_available)1272{1273 if (desc_available)1274 xsk_set_rx_need_wakeup(rx->xsk_pool);1275 else1276 xsk_clear_rx_need_wakeup(rx->xsk_pool);1277}1278 1279static bool tsnep_xdp_run_prog(struct tsnep_rx *rx, struct bpf_prog *prog,1280 struct xdp_buff *xdp, int *status,1281 struct netdev_queue *tx_nq, struct tsnep_tx *tx)1282{1283 unsigned int length;1284 unsigned int sync;1285 u32 act;1286 1287 length = xdp->data_end - xdp->data_hard_start - XDP_PACKET_HEADROOM;1288 1289 act = bpf_prog_run_xdp(prog, xdp);1290 switch (act) {1291 case XDP_PASS:1292 return false;1293 case XDP_TX:1294 if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx, false))1295 goto out_failure;1296 *status |= TSNEP_XDP_TX;1297 return true;1298 case XDP_REDIRECT:1299 if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0)1300 goto out_failure;1301 *status |= TSNEP_XDP_REDIRECT;1302 return true;1303 default:1304 bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act);1305 fallthrough;1306 case XDP_ABORTED:1307out_failure:1308 trace_xdp_exception(rx->adapter->netdev, prog, act);1309 fallthrough;1310 case XDP_DROP:1311 /* Due xdp_adjust_tail: DMA sync for_device cover max len CPU1312 * touch1313 */1314 sync = xdp->data_end - xdp->data_hard_start -1315 XDP_PACKET_HEADROOM;1316 sync = max(sync, length);1317 page_pool_put_page(rx->page_pool, virt_to_head_page(xdp->data),1318 sync, true);1319 return true;1320 }1321}1322 1323static bool tsnep_xdp_run_prog_zc(struct tsnep_rx *rx, struct bpf_prog *prog,1324 struct xdp_buff *xdp, int *status,1325 struct netdev_queue *tx_nq,1326 struct tsnep_tx *tx)1327{1328 u32 act;1329 1330 act = bpf_prog_run_xdp(prog, xdp);1331 1332 /* XDP_REDIRECT is the main action for zero-copy */1333 if (likely(act == XDP_REDIRECT)) {1334 if (xdp_do_redirect(rx->adapter->netdev, xdp, prog) < 0)1335 goto out_failure;1336 *status |= TSNEP_XDP_REDIRECT;1337 return true;1338 }1339 1340 switch (act) {1341 case XDP_PASS:1342 return false;1343 case XDP_TX:1344 if (!tsnep_xdp_xmit_back(rx->adapter, xdp, tx_nq, tx, true))1345 goto out_failure;1346 *status |= TSNEP_XDP_TX;1347 return true;1348 default:1349 bpf_warn_invalid_xdp_action(rx->adapter->netdev, prog, act);1350 fallthrough;1351 case XDP_ABORTED:1352out_failure:1353 trace_xdp_exception(rx->adapter->netdev, prog, act);1354 fallthrough;1355 case XDP_DROP:1356 xsk_buff_free(xdp);1357 return true;1358 }1359}1360 1361static void tsnep_finalize_xdp(struct tsnep_adapter *adapter, int status,1362 struct netdev_queue *tx_nq, struct tsnep_tx *tx)1363{1364 if (status & TSNEP_XDP_TX) {1365 __netif_tx_lock(tx_nq, smp_processor_id());1366 tsnep_xdp_xmit_flush(tx);1367 __netif_tx_unlock(tx_nq);1368 }1369 1370 if (status & TSNEP_XDP_REDIRECT)1371 xdp_do_flush();1372}1373 1374static struct sk_buff *tsnep_build_skb(struct tsnep_rx *rx, struct page *page,1375 int length)1376{1377 struct sk_buff *skb;1378 1379 skb = napi_build_skb(page_address(page), PAGE_SIZE);1380 if (unlikely(!skb))1381 return NULL;1382 1383 /* update pointers within the skb to store the data */1384 skb_reserve(skb, TSNEP_RX_OFFSET + TSNEP_RX_INLINE_METADATA_SIZE);1385 __skb_put(skb, length - ETH_FCS_LEN);1386 1387 if (rx->adapter->hwtstamp_config.rx_filter == HWTSTAMP_FILTER_ALL) {1388 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb);1389 struct tsnep_rx_inline *rx_inline =1390 (struct tsnep_rx_inline *)(page_address(page) +1391 TSNEP_RX_OFFSET);1392 1393 skb_shinfo(skb)->tx_flags |=1394 SKBTX_HW_TSTAMP_NETDEV;1395 memset(hwtstamps, 0, sizeof(*hwtstamps));1396 hwtstamps->netdev_data = rx_inline;1397 }1398 1399 skb_record_rx_queue(skb, rx->queue_index);1400 skb->protocol = eth_type_trans(skb, rx->adapter->netdev);1401 1402 return skb;1403}1404 1405static void tsnep_rx_page(struct tsnep_rx *rx, struct napi_struct *napi,1406 struct page *page, int length)1407{1408 struct sk_buff *skb;1409 1410 skb = tsnep_build_skb(rx, page, length);1411 if (skb) {1412 skb_mark_for_recycle(skb);1413 1414 rx->packets++;1415 rx->bytes += length;1416 if (skb->pkt_type == PACKET_MULTICAST)1417 rx->multicast++;1418 1419 napi_gro_receive(napi, skb);1420 } else {1421 page_pool_recycle_direct(rx->page_pool, page);1422 1423 rx->dropped++;1424 }1425}1426 1427static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi,1428 int budget)1429{1430 struct device *dmadev = rx->adapter->dmadev;1431 enum dma_data_direction dma_dir;1432 struct tsnep_rx_entry *entry;1433 struct netdev_queue *tx_nq;1434 struct bpf_prog *prog;1435 struct xdp_buff xdp;1436 struct tsnep_tx *tx;1437 int desc_available;1438 int xdp_status = 0;1439 int done = 0;1440 int length;1441 1442 desc_available = tsnep_rx_desc_available(rx);1443 dma_dir = page_pool_get_dma_dir(rx->page_pool);1444 prog = READ_ONCE(rx->adapter->xdp_prog);1445 if (prog) {1446 tx_nq = netdev_get_tx_queue(rx->adapter->netdev,1447 rx->tx_queue_index);1448 tx = &rx->adapter->tx[rx->tx_queue_index];1449 1450 xdp_init_buff(&xdp, PAGE_SIZE, &rx->xdp_rxq);1451 }1452 1453 while (likely(done < budget) && (rx->read != rx->write)) {1454 entry = &rx->entry[rx->read];1455 if ((__le32_to_cpu(entry->desc_wb->properties) &1456 TSNEP_DESC_OWNER_COUNTER_MASK) !=1457 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))1458 break;1459 done++;1460 1461 if (desc_available >= TSNEP_RING_RX_REFILL) {1462 bool reuse = desc_available >= TSNEP_RING_RX_REUSE;1463 1464 desc_available -= tsnep_rx_refill(rx, desc_available,1465 reuse);1466 if (!entry->page) {1467 /* buffer has been reused for refill to prevent1468 * empty RX ring, thus buffer cannot be used for1469 * RX processing1470 */1471 rx->read = (rx->read + 1) & TSNEP_RING_MASK;1472 desc_available++;1473 1474 rx->dropped++;1475 1476 continue;1477 }1478 }1479 1480 /* descriptor properties shall be read first, because valid data1481 * is signaled there1482 */1483 dma_rmb();1484 1485 prefetch(page_address(entry->page) + TSNEP_RX_OFFSET);1486 length = __le32_to_cpu(entry->desc_wb->properties) &1487 TSNEP_DESC_LENGTH_MASK;1488 dma_sync_single_range_for_cpu(dmadev, entry->dma,1489 TSNEP_RX_OFFSET, length, dma_dir);1490 1491 /* RX metadata with timestamps is in front of actual data,1492 * subtract metadata size to get length of actual data and1493 * consider metadata size as offset of actual data during RX1494 * processing1495 */1496 length -= TSNEP_RX_INLINE_METADATA_SIZE;1497 1498 rx->read = (rx->read + 1) & TSNEP_RING_MASK;1499 desc_available++;1500 1501 if (prog) {1502 bool consume;1503 1504 xdp_prepare_buff(&xdp, page_address(entry->page),1505 XDP_PACKET_HEADROOM + TSNEP_RX_INLINE_METADATA_SIZE,1506 length - ETH_FCS_LEN, false);1507 1508 consume = tsnep_xdp_run_prog(rx, prog, &xdp,1509 &xdp_status, tx_nq, tx);1510 if (consume) {1511 rx->packets++;1512 rx->bytes += length;1513 1514 entry->page = NULL;1515 1516 continue;1517 }1518 }1519 1520 tsnep_rx_page(rx, napi, entry->page, length);1521 entry->page = NULL;1522 }1523 1524 if (xdp_status)1525 tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx);1526 1527 if (desc_available)1528 tsnep_rx_refill(rx, desc_available, false);1529 1530 return done;1531}1532 1533static int tsnep_rx_poll_zc(struct tsnep_rx *rx, struct napi_struct *napi,1534 int budget)1535{1536 struct tsnep_rx_entry *entry;1537 struct netdev_queue *tx_nq;1538 struct bpf_prog *prog;1539 struct tsnep_tx *tx;1540 int desc_available;1541 int xdp_status = 0;1542 struct page *page;1543 int done = 0;1544 int length;1545 1546 desc_available = tsnep_rx_desc_available(rx);1547 prog = READ_ONCE(rx->adapter->xdp_prog);1548 if (prog) {1549 tx_nq = netdev_get_tx_queue(rx->adapter->netdev,1550 rx->tx_queue_index);1551 tx = &rx->adapter->tx[rx->tx_queue_index];1552 }1553 1554 while (likely(done < budget) && (rx->read != rx->write)) {1555 entry = &rx->entry[rx->read];1556 if ((__le32_to_cpu(entry->desc_wb->properties) &1557 TSNEP_DESC_OWNER_COUNTER_MASK) !=1558 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))1559 break;1560 done++;1561 1562 if (desc_available >= TSNEP_RING_RX_REFILL) {1563 bool reuse = desc_available >= TSNEP_RING_RX_REUSE;1564 1565 desc_available -= tsnep_rx_refill_zc(rx, desc_available,1566 reuse);1567 if (!entry->xdp) {1568 /* buffer has been reused for refill to prevent1569 * empty RX ring, thus buffer cannot be used for1570 * RX processing1571 */1572 rx->read = (rx->read + 1) & TSNEP_RING_MASK;1573 desc_available++;1574 1575 rx->dropped++;1576 1577 continue;1578 }1579 }1580 1581 /* descriptor properties shall be read first, because valid data1582 * is signaled there1583 */1584 dma_rmb();1585 1586 prefetch(entry->xdp->data);1587 length = __le32_to_cpu(entry->desc_wb->properties) &1588 TSNEP_DESC_LENGTH_MASK;1589 xsk_buff_set_size(entry->xdp, length - ETH_FCS_LEN);1590 xsk_buff_dma_sync_for_cpu(entry->xdp);1591 1592 /* RX metadata with timestamps is in front of actual data,1593 * subtract metadata size to get length of actual data and1594 * consider metadata size as offset of actual data during RX1595 * processing1596 */1597 length -= TSNEP_RX_INLINE_METADATA_SIZE;1598 1599 rx->read = (rx->read + 1) & TSNEP_RING_MASK;1600 desc_available++;1601 1602 if (prog) {1603 bool consume;1604 1605 entry->xdp->data += TSNEP_RX_INLINE_METADATA_SIZE;1606 entry->xdp->data_meta += TSNEP_RX_INLINE_METADATA_SIZE;1607 1608 consume = tsnep_xdp_run_prog_zc(rx, prog, entry->xdp,1609 &xdp_status, tx_nq, tx);1610 if (consume) {1611 rx->packets++;1612 rx->bytes += length;1613 1614 entry->xdp = NULL;1615 1616 continue;1617 }1618 }1619 1620 page = page_pool_dev_alloc_pages(rx->page_pool);1621 if (page) {1622 memcpy(page_address(page) + TSNEP_RX_OFFSET,1623 entry->xdp->data - TSNEP_RX_INLINE_METADATA_SIZE,1624 length + TSNEP_RX_INLINE_METADATA_SIZE);1625 tsnep_rx_page(rx, napi, page, length);1626 } else {1627 rx->dropped++;1628 }1629 xsk_buff_free(entry->xdp);1630 entry->xdp = NULL;1631 }1632 1633 if (xdp_status)1634 tsnep_finalize_xdp(rx->adapter, xdp_status, tx_nq, tx);1635 1636 if (desc_available)1637 desc_available -= tsnep_rx_refill_zc(rx, desc_available, false);1638 1639 if (xsk_uses_need_wakeup(rx->xsk_pool)) {1640 tsnep_xsk_rx_need_wakeup(rx, desc_available);1641 1642 return done;1643 }1644 1645 return desc_available ? budget : done;1646}1647 1648static bool tsnep_rx_pending(struct tsnep_rx *rx)1649{1650 struct tsnep_rx_entry *entry;1651 1652 if (rx->read != rx->write) {1653 entry = &rx->entry[rx->read];1654 if ((__le32_to_cpu(entry->desc_wb->properties) &1655 TSNEP_DESC_OWNER_COUNTER_MASK) ==1656 (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK))1657 return true;1658 }1659 1660 return false;1661}1662 1663static int tsnep_rx_open(struct tsnep_rx *rx)1664{1665 int desc_available;1666 int retval;1667 1668 retval = tsnep_rx_ring_create(rx);1669 if (retval)1670 return retval;1671 1672 tsnep_rx_init(rx);1673 1674 desc_available = tsnep_rx_desc_available(rx);1675 if (rx->xsk_pool)1676 retval = tsnep_rx_alloc_zc(rx, desc_available, false);1677 else1678 retval = tsnep_rx_alloc(rx, desc_available, false);1679 if (retval != desc_available) {1680 retval = -ENOMEM;1681 1682 goto alloc_failed;1683 }1684 1685 /* prealloc pages to prevent allocation failures when XSK pool is1686 * disabled at runtime1687 */1688 if (rx->xsk_pool) {1689 retval = tsnep_rx_alloc_page_buffer(rx);1690 if (retval)1691 goto alloc_failed;1692 }1693 1694 return 0;1695 1696alloc_failed:1697 tsnep_rx_ring_cleanup(rx);1698 return retval;1699}1700 1701static void tsnep_rx_close(struct tsnep_rx *rx)1702{1703 if (rx->xsk_pool)1704 tsnep_rx_free_page_buffer(rx);1705 1706 tsnep_rx_ring_cleanup(rx);1707}1708 1709static void tsnep_rx_reopen(struct tsnep_rx *rx)1710{1711 struct page **page = rx->page_buffer;1712 int i;1713 1714 tsnep_rx_init(rx);1715 1716 for (i = 0; i < TSNEP_RING_SIZE; i++) {1717 struct tsnep_rx_entry *entry = &rx->entry[i];1718 1719 /* defined initial values for properties are required for1720 * correct owner counter checking1721 */1722 entry->desc->properties = 0;1723 entry->desc_wb->properties = 0;1724 1725 /* prevent allocation failures by reusing kept pages */1726 if (*page) {1727 tsnep_rx_set_page(rx, entry, *page);1728 tsnep_rx_activate(rx, rx->write);1729 rx->write++;1730 1731 *page = NULL;1732 page++;1733 }1734 }1735}1736 1737static void tsnep_rx_reopen_xsk(struct tsnep_rx *rx)1738{1739 struct page **page = rx->page_buffer;1740 u32 allocated;1741 int i;1742 1743 tsnep_rx_init(rx);1744 1745 /* alloc all ring entries except the last one, because ring cannot be1746 * filled completely, as many buffers as possible is enough as wakeup is1747 * done if new buffers are available1748 */1749 allocated = xsk_buff_alloc_batch(rx->xsk_pool, rx->xdp_batch,1750 TSNEP_RING_SIZE - 1);1751 1752 for (i = 0; i < TSNEP_RING_SIZE; i++) {1753 struct tsnep_rx_entry *entry = &rx->entry[i];1754 1755 /* keep pages to prevent allocation failures when xsk is1756 * disabled1757 */1758 if (entry->page) {1759 *page = entry->page;1760 entry->page = NULL;1761 1762 page++;1763 }1764 1765 /* defined initial values for properties are required for1766 * correct owner counter checking1767 */1768 entry->desc->properties = 0;1769 entry->desc_wb->properties = 0;1770 1771 if (allocated) {1772 tsnep_rx_set_xdp(rx, entry,1773 rx->xdp_batch[allocated - 1]);1774 tsnep_rx_activate(rx, rx->write);1775 rx->write++;1776 1777 allocated--;1778 }1779 }1780 1781 /* set need wakeup flag immediately if ring is not filled completely,1782 * first polling would be too late as need wakeup signalisation would1783 * be delayed for an indefinite time1784 */1785 if (xsk_uses_need_wakeup(rx->xsk_pool))1786 tsnep_xsk_rx_need_wakeup(rx, tsnep_rx_desc_available(rx));1787}1788 1789static bool tsnep_pending(struct tsnep_queue *queue)1790{1791 if (queue->tx && tsnep_tx_pending(queue->tx))1792 return true;1793 1794 if (queue->rx && tsnep_rx_pending(queue->rx))1795 return true;1796 1797 return false;1798}1799 1800static int tsnep_poll(struct napi_struct *napi, int budget)1801{1802 struct tsnep_queue *queue = container_of(napi, struct tsnep_queue,1803 napi);1804 bool complete = true;1805 int done = 0;1806 1807 if (queue->tx)1808 complete = tsnep_tx_poll(queue->tx, budget);1809 1810 /* handle case where we are called by netpoll with a budget of 0 */1811 if (unlikely(budget <= 0))1812 return budget;1813 1814 if (queue->rx) {1815 done = queue->rx->xsk_pool ?1816 tsnep_rx_poll_zc(queue->rx, napi, budget) :1817 tsnep_rx_poll(queue->rx, napi, budget);1818 if (done >= budget)1819 complete = false;1820 }1821 1822 /* if all work not completed, return budget and keep polling */1823 if (!complete)1824 return budget;1825 1826 if (likely(napi_complete_done(napi, done))) {1827 tsnep_enable_irq(queue->adapter, queue->irq_mask);1828 1829 /* reschedule if work is already pending, prevent rotten packets1830 * which are transmitted or received after polling but before1831 * interrupt enable1832 */1833 if (tsnep_pending(queue)) {1834 tsnep_disable_irq(queue->adapter, queue->irq_mask);1835 napi_schedule(napi);1836 }1837 }1838 1839 return min(done, budget - 1);1840}1841 1842static int tsnep_request_irq(struct tsnep_queue *queue, bool first)1843{1844 const char *name = netdev_name(queue->adapter->netdev);1845 irq_handler_t handler;1846 void *dev;1847 int retval;1848 1849 if (first) {1850 sprintf(queue->name, "%s-mac", name);1851 handler = tsnep_irq;1852 dev = queue->adapter;1853 } else {1854 if (queue->tx && queue->rx)1855 snprintf(queue->name, sizeof(queue->name), "%s-txrx-%d",1856 name, queue->rx->queue_index);1857 else if (queue->tx)1858 snprintf(queue->name, sizeof(queue->name), "%s-tx-%d",1859 name, queue->tx->queue_index);1860 else1861 snprintf(queue->name, sizeof(queue->name), "%s-rx-%d",1862 name, queue->rx->queue_index);1863 handler = tsnep_irq_txrx;1864 dev = queue;1865 }1866 1867 retval = request_irq(queue->irq, handler, 0, queue->name, dev);1868 if (retval) {1869 /* if name is empty, then interrupt won't be freed */1870 memset(queue->name, 0, sizeof(queue->name));1871 }1872 1873 return retval;1874}1875 1876static void tsnep_free_irq(struct tsnep_queue *queue, bool first)1877{1878 void *dev;1879 1880 if (!strlen(queue->name))1881 return;1882 1883 if (first)1884 dev = queue->adapter;1885 else1886 dev = queue;1887 1888 free_irq(queue->irq, dev);1889 memset(queue->name, 0, sizeof(queue->name));1890}1891 1892static void tsnep_queue_close(struct tsnep_queue *queue, bool first)1893{1894 struct tsnep_rx *rx = queue->rx;1895 1896 tsnep_free_irq(queue, first);1897 1898 if (rx) {1899 if (xdp_rxq_info_is_reg(&rx->xdp_rxq))1900 xdp_rxq_info_unreg(&rx->xdp_rxq);1901 if (xdp_rxq_info_is_reg(&rx->xdp_rxq_zc))1902 xdp_rxq_info_unreg(&rx->xdp_rxq_zc);1903 }1904 1905 netif_napi_del(&queue->napi);1906}1907 1908static int tsnep_queue_open(struct tsnep_adapter *adapter,1909 struct tsnep_queue *queue, bool first)1910{1911 struct tsnep_rx *rx = queue->rx;1912 struct tsnep_tx *tx = queue->tx;1913 int retval;1914 1915 netif_napi_add(adapter->netdev, &queue->napi, tsnep_poll);1916 1917 if (rx) {1918 /* choose TX queue for XDP_TX */1919 if (tx)1920 rx->tx_queue_index = tx->queue_index;1921 else if (rx->queue_index < adapter->num_tx_queues)1922 rx->tx_queue_index = rx->queue_index;1923 else1924 rx->tx_queue_index = 0;1925 1926 /* prepare both memory models to eliminate possible registration1927 * errors when memory model is switched between page pool and1928 * XSK pool during runtime1929 */1930 retval = xdp_rxq_info_reg(&rx->xdp_rxq, adapter->netdev,1931 rx->queue_index, queue->napi.napi_id);1932 if (retval)1933 goto failed;1934 retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq,1935 MEM_TYPE_PAGE_POOL,1936 rx->page_pool);1937 if (retval)1938 goto failed;1939 retval = xdp_rxq_info_reg(&rx->xdp_rxq_zc, adapter->netdev,1940 rx->queue_index, queue->napi.napi_id);1941 if (retval)1942 goto failed;1943 retval = xdp_rxq_info_reg_mem_model(&rx->xdp_rxq_zc,1944 MEM_TYPE_XSK_BUFF_POOL,1945 NULL);1946 if (retval)1947 goto failed;1948 if (rx->xsk_pool)1949 xsk_pool_set_rxq_info(rx->xsk_pool, &rx->xdp_rxq_zc);1950 }1951 1952 retval = tsnep_request_irq(queue, first);1953 if (retval) {1954 netif_err(adapter, drv, adapter->netdev,1955 "can't get assigned irq %d.\n", queue->irq);1956 goto failed;1957 }1958 1959 return 0;1960 1961failed:1962 tsnep_queue_close(queue, first);1963 1964 return retval;1965}1966 1967static void tsnep_queue_enable(struct tsnep_queue *queue)1968{1969 napi_enable(&queue->napi);1970 tsnep_enable_irq(queue->adapter, queue->irq_mask);1971 1972 if (queue->tx)1973 tsnep_tx_enable(queue->tx);1974 1975 if (queue->rx)1976 tsnep_rx_enable(queue->rx);1977}1978 1979static void tsnep_queue_disable(struct tsnep_queue *queue)1980{1981 if (queue->tx)1982 tsnep_tx_disable(queue->tx, &queue->napi);1983 1984 napi_disable(&queue->napi);1985 tsnep_disable_irq(queue->adapter, queue->irq_mask);1986 1987 /* disable RX after NAPI polling has been disabled, because RX can be1988 * enabled during NAPI polling1989 */1990 if (queue->rx)1991 tsnep_rx_disable(queue->rx);1992}1993 1994static int tsnep_netdev_open(struct net_device *netdev)1995{1996 struct tsnep_adapter *adapter = netdev_priv(netdev);1997 int i, retval;1998 1999 for (i = 0; i < adapter->num_queues; i++) {2000 if (adapter->queue[i].tx) {2001 retval = tsnep_tx_open(adapter->queue[i].tx);2002 if (retval)2003 goto failed;2004 }2005 if (adapter->queue[i].rx) {2006 retval = tsnep_rx_open(adapter->queue[i].rx);2007 if (retval)2008 goto failed;2009 }2010 2011 retval = tsnep_queue_open(adapter, &adapter->queue[i], i == 0);2012 if (retval)2013 goto failed;2014 }2015 2016 retval = netif_set_real_num_tx_queues(adapter->netdev,2017 adapter->num_tx_queues);2018 if (retval)2019 goto failed;2020 retval = netif_set_real_num_rx_queues(adapter->netdev,2021 adapter->num_rx_queues);2022 if (retval)2023 goto failed;2024 2025 tsnep_enable_irq(adapter, ECM_INT_LINK);2026 retval = tsnep_phy_open(adapter);2027 if (retval)2028 goto phy_failed;2029 2030 for (i = 0; i < adapter->num_queues; i++)2031 tsnep_queue_enable(&adapter->queue[i]);2032 2033 return 0;2034 2035phy_failed:2036 tsnep_disable_irq(adapter, ECM_INT_LINK);2037failed:2038 for (i = 0; i < adapter->num_queues; i++) {2039 tsnep_queue_close(&adapter->queue[i], i == 0);2040 2041 if (adapter->queue[i].rx)2042 tsnep_rx_close(adapter->queue[i].rx);2043 if (adapter->queue[i].tx)2044 tsnep_tx_close(adapter->queue[i].tx);2045 }2046 return retval;2047}2048 2049static int tsnep_netdev_close(struct net_device *netdev)2050{2051 struct tsnep_adapter *adapter = netdev_priv(netdev);2052 int i;2053 2054 tsnep_disable_irq(adapter, ECM_INT_LINK);2055 tsnep_phy_close(adapter);2056 2057 for (i = 0; i < adapter->num_queues; i++) {2058 tsnep_queue_disable(&adapter->queue[i]);2059 2060 tsnep_queue_close(&adapter->queue[i], i == 0);2061 2062 if (adapter->queue[i].rx)2063 tsnep_rx_close(adapter->queue[i].rx);2064 if (adapter->queue[i].tx)2065 tsnep_tx_close(adapter->queue[i].tx);2066 }2067 2068 return 0;2069}2070 2071int tsnep_enable_xsk(struct tsnep_queue *queue, struct xsk_buff_pool *pool)2072{2073 bool running = netif_running(queue->adapter->netdev);2074 u32 frame_size;2075 2076 frame_size = xsk_pool_get_rx_frame_size(pool);2077 if (frame_size < TSNEP_XSK_RX_BUF_SIZE)2078 return -EOPNOTSUPP;2079 2080 queue->rx->page_buffer = kcalloc(TSNEP_RING_SIZE,2081 sizeof(*queue->rx->page_buffer),2082 GFP_KERNEL);2083 if (!queue->rx->page_buffer)2084 return -ENOMEM;2085 queue->rx->xdp_batch = kcalloc(TSNEP_RING_SIZE,2086 sizeof(*queue->rx->xdp_batch),2087 GFP_KERNEL);2088 if (!queue->rx->xdp_batch) {2089 kfree(queue->rx->page_buffer);2090 queue->rx->page_buffer = NULL;2091 2092 return -ENOMEM;2093 }2094 2095 xsk_pool_set_rxq_info(pool, &queue->rx->xdp_rxq_zc);2096 2097 if (running)2098 tsnep_queue_disable(queue);2099 2100 queue->tx->xsk_pool = pool;2101 queue->rx->xsk_pool = pool;2102 2103 if (running) {2104 tsnep_rx_reopen_xsk(queue->rx);2105 tsnep_queue_enable(queue);2106 }2107 2108 return 0;2109}2110 2111void tsnep_disable_xsk(struct tsnep_queue *queue)2112{2113 bool running = netif_running(queue->adapter->netdev);2114 2115 if (running)2116 tsnep_queue_disable(queue);2117 2118 tsnep_rx_free_zc(queue->rx);2119 2120 queue->rx->xsk_pool = NULL;2121 queue->tx->xsk_pool = NULL;2122 2123 if (running) {2124 tsnep_rx_reopen(queue->rx);2125 tsnep_queue_enable(queue);2126 }2127 2128 kfree(queue->rx->xdp_batch);2129 queue->rx->xdp_batch = NULL;2130 kfree(queue->rx->page_buffer);2131 queue->rx->page_buffer = NULL;2132}2133 2134static netdev_tx_t tsnep_netdev_xmit_frame(struct sk_buff *skb,2135 struct net_device *netdev)2136{2137 struct tsnep_adapter *adapter = netdev_priv(netdev);2138 u16 queue_mapping = skb_get_queue_mapping(skb);2139 2140 if (queue_mapping >= adapter->num_tx_queues)2141 queue_mapping = 0;2142 2143 return tsnep_xmit_frame_ring(skb, &adapter->tx[queue_mapping]);2144}2145 2146static int tsnep_netdev_ioctl(struct net_device *netdev, struct ifreq *ifr,2147 int cmd)2148{2149 if (!netif_running(netdev))2150 return -EINVAL;2151 if (cmd == SIOCSHWTSTAMP || cmd == SIOCGHWTSTAMP)2152 return tsnep_ptp_ioctl(netdev, ifr, cmd);2153 return phy_mii_ioctl(netdev->phydev, ifr, cmd);2154}2155 2156static void tsnep_netdev_set_multicast(struct net_device *netdev)2157{2158 struct tsnep_adapter *adapter = netdev_priv(netdev);2159 2160 u16 rx_filter = 0;2161 2162 /* configured MAC address and broadcasts are never filtered */2163 if (netdev->flags & IFF_PROMISC) {2164 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS;2165 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_UNICASTS;2166 } else if (!netdev_mc_empty(netdev) || (netdev->flags & IFF_ALLMULTI)) {2167 rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS;2168 }2169 iowrite16(rx_filter, adapter->addr + TSNEP_RX_FILTER);2170}2171 2172static void tsnep_netdev_get_stats64(struct net_device *netdev,2173 struct rtnl_link_stats64 *stats)2174{2175 struct tsnep_adapter *adapter = netdev_priv(netdev);2176 u32 reg;2177 u32 val;2178 int i;2179 2180 for (i = 0; i < adapter->num_tx_queues; i++) {2181 stats->tx_packets += adapter->tx[i].packets;2182 stats->tx_bytes += adapter->tx[i].bytes;2183 stats->tx_dropped += adapter->tx[i].dropped;2184 }2185 for (i = 0; i < adapter->num_rx_queues; i++) {2186 stats->rx_packets += adapter->rx[i].packets;2187 stats->rx_bytes += adapter->rx[i].bytes;2188 stats->rx_dropped += adapter->rx[i].dropped;2189 stats->multicast += adapter->rx[i].multicast;2190 2191 reg = ioread32(adapter->addr + TSNEP_QUEUE(i) +2192 TSNEP_RX_STATISTIC);2193 val = (reg & TSNEP_RX_STATISTIC_NO_DESC_MASK) >>2194 TSNEP_RX_STATISTIC_NO_DESC_SHIFT;2195 stats->rx_dropped += val;2196 val = (reg & TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_MASK) >>2197 TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_SHIFT;2198 stats->rx_dropped += val;2199 val = (reg & TSNEP_RX_STATISTIC_FIFO_OVERFLOW_MASK) >>2200 TSNEP_RX_STATISTIC_FIFO_OVERFLOW_SHIFT;2201 stats->rx_errors += val;2202 stats->rx_fifo_errors += val;2203 val = (reg & TSNEP_RX_STATISTIC_INVALID_FRAME_MASK) >>2204 TSNEP_RX_STATISTIC_INVALID_FRAME_SHIFT;2205 stats->rx_errors += val;2206 stats->rx_frame_errors += val;2207 }2208 2209 reg = ioread32(adapter->addr + ECM_STAT);2210 val = (reg & ECM_STAT_RX_ERR_MASK) >> ECM_STAT_RX_ERR_SHIFT;2211 stats->rx_errors += val;2212 val = (reg & ECM_STAT_INV_FRM_MASK) >> ECM_STAT_INV_FRM_SHIFT;2213 stats->rx_errors += val;2214 stats->rx_crc_errors += val;2215 val = (reg & ECM_STAT_FWD_RX_ERR_MASK) >> ECM_STAT_FWD_RX_ERR_SHIFT;2216 stats->rx_errors += val;2217}2218 2219static void tsnep_mac_set_address(struct tsnep_adapter *adapter, u8 *addr)2220{2221 iowrite32(*(u32 *)addr, adapter->addr + TSNEP_MAC_ADDRESS_LOW);2222 iowrite16(*(u16 *)(addr + sizeof(u32)),2223 adapter->addr + TSNEP_MAC_ADDRESS_HIGH);2224 2225 ether_addr_copy(adapter->mac_address, addr);2226 netif_info(adapter, drv, adapter->netdev, "MAC address set to %pM\n",2227 addr);2228}2229 2230static int tsnep_netdev_set_mac_address(struct net_device *netdev, void *addr)2231{2232 struct tsnep_adapter *adapter = netdev_priv(netdev);2233 struct sockaddr *sock_addr = addr;2234 int retval;2235 2236 retval = eth_prepare_mac_addr_change(netdev, sock_addr);2237 if (retval)2238 return retval;2239 eth_hw_addr_set(netdev, sock_addr->sa_data);2240 tsnep_mac_set_address(adapter, sock_addr->sa_data);2241 2242 return 0;2243}2244 2245static int tsnep_netdev_set_features(struct net_device *netdev,2246 netdev_features_t features)2247{2248 struct tsnep_adapter *adapter = netdev_priv(netdev);2249 netdev_features_t changed = netdev->features ^ features;2250 bool enable;2251 int retval = 0;2252 2253 if (changed & NETIF_F_LOOPBACK) {2254 enable = !!(features & NETIF_F_LOOPBACK);2255 retval = tsnep_phy_loopback(adapter, enable);2256 }2257 2258 return retval;2259}2260 2261static ktime_t tsnep_netdev_get_tstamp(struct net_device *netdev,2262 const struct skb_shared_hwtstamps *hwtstamps,2263 bool cycles)2264{2265 struct tsnep_rx_inline *rx_inline = hwtstamps->netdev_data;2266 u64 timestamp;2267 2268 if (cycles)2269 timestamp = __le64_to_cpu(rx_inline->counter);2270 else2271 timestamp = __le64_to_cpu(rx_inline->timestamp);2272 2273 return ns_to_ktime(timestamp);2274}2275 2276static int tsnep_netdev_bpf(struct net_device *dev, struct netdev_bpf *bpf)2277{2278 struct tsnep_adapter *adapter = netdev_priv(dev);2279 2280 switch (bpf->command) {2281 case XDP_SETUP_PROG:2282 return tsnep_xdp_setup_prog(adapter, bpf->prog, bpf->extack);2283 case XDP_SETUP_XSK_POOL:2284 return tsnep_xdp_setup_pool(adapter, bpf->xsk.pool,2285 bpf->xsk.queue_id);2286 default:2287 return -EOPNOTSUPP;2288 }2289}2290 2291static struct tsnep_tx *tsnep_xdp_get_tx(struct tsnep_adapter *adapter, u32 cpu)2292{2293 if (cpu >= TSNEP_MAX_QUEUES)2294 cpu &= TSNEP_MAX_QUEUES - 1;2295 2296 while (cpu >= adapter->num_tx_queues)2297 cpu -= adapter->num_tx_queues;2298 2299 return &adapter->tx[cpu];2300}2301 2302static int tsnep_netdev_xdp_xmit(struct net_device *dev, int n,2303 struct xdp_frame **xdp, u32 flags)2304{2305 struct tsnep_adapter *adapter = netdev_priv(dev);2306 u32 cpu = smp_processor_id();2307 struct netdev_queue *nq;2308 struct tsnep_tx *tx;2309 int nxmit;2310 bool xmit;2311 2312 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))2313 return -EINVAL;2314 2315 tx = tsnep_xdp_get_tx(adapter, cpu);2316 nq = netdev_get_tx_queue(adapter->netdev, tx->queue_index);2317 2318 __netif_tx_lock(nq, cpu);2319 2320 for (nxmit = 0; nxmit < n; nxmit++) {2321 xmit = tsnep_xdp_xmit_frame_ring(xdp[nxmit], tx,2322 TSNEP_TX_TYPE_XDP_NDO);2323 if (!xmit)2324 break;2325 2326 /* avoid transmit queue timeout since we share it with the slow2327 * path2328 */2329 txq_trans_cond_update(nq);2330 }2331 2332 if (flags & XDP_XMIT_FLUSH)2333 tsnep_xdp_xmit_flush(tx);2334 2335 __netif_tx_unlock(nq);2336 2337 return nxmit;2338}2339 2340static int tsnep_netdev_xsk_wakeup(struct net_device *dev, u32 queue_id,2341 u32 flags)2342{2343 struct tsnep_adapter *adapter = netdev_priv(dev);2344 struct tsnep_queue *queue;2345 2346 if (queue_id >= adapter->num_rx_queues ||2347 queue_id >= adapter->num_tx_queues)2348 return -EINVAL;2349 2350 queue = &adapter->queue[queue_id];2351 2352 if (!napi_if_scheduled_mark_missed(&queue->napi))2353 napi_schedule(&queue->napi);2354 2355 return 0;2356}2357 2358static const struct net_device_ops tsnep_netdev_ops = {2359 .ndo_open = tsnep_netdev_open,2360 .ndo_stop = tsnep_netdev_close,2361 .ndo_start_xmit = tsnep_netdev_xmit_frame,2362 .ndo_eth_ioctl = tsnep_netdev_ioctl,2363 .ndo_set_rx_mode = tsnep_netdev_set_multicast,2364 .ndo_get_stats64 = tsnep_netdev_get_stats64,2365 .ndo_set_mac_address = tsnep_netdev_set_mac_address,2366 .ndo_set_features = tsnep_netdev_set_features,2367 .ndo_get_tstamp = tsnep_netdev_get_tstamp,2368 .ndo_setup_tc = tsnep_tc_setup,2369 .ndo_bpf = tsnep_netdev_bpf,2370 .ndo_xdp_xmit = tsnep_netdev_xdp_xmit,2371 .ndo_xsk_wakeup = tsnep_netdev_xsk_wakeup,2372};2373 2374static int tsnep_mac_init(struct tsnep_adapter *adapter)2375{2376 int retval;2377 2378 /* initialize RX filtering, at least configured MAC address and2379 * broadcast are not filtered2380 */2381 iowrite16(0, adapter->addr + TSNEP_RX_FILTER);2382 2383 /* try to get MAC address in the following order:2384 * - device tree2385 * - valid MAC address already set2386 * - MAC address register if valid2387 * - random MAC address2388 */2389 retval = of_get_mac_address(adapter->pdev->dev.of_node,2390 adapter->mac_address);2391 if (retval == -EPROBE_DEFER)2392 return retval;2393 if (retval && !is_valid_ether_addr(adapter->mac_address)) {2394 *(u32 *)adapter->mac_address =2395 ioread32(adapter->addr + TSNEP_MAC_ADDRESS_LOW);2396 *(u16 *)(adapter->mac_address + sizeof(u32)) =2397 ioread16(adapter->addr + TSNEP_MAC_ADDRESS_HIGH);2398 if (!is_valid_ether_addr(adapter->mac_address))2399 eth_random_addr(adapter->mac_address);2400 }2401 2402 tsnep_mac_set_address(adapter, adapter->mac_address);2403 eth_hw_addr_set(adapter->netdev, adapter->mac_address);2404 2405 return 0;2406}2407 2408static int tsnep_mdio_init(struct tsnep_adapter *adapter)2409{2410 struct device_node *np = adapter->pdev->dev.of_node;2411 int retval;2412 2413 if (np) {2414 np = of_get_child_by_name(np, "mdio");2415 if (!np)2416 return 0;2417 2418 adapter->suppress_preamble =2419 of_property_read_bool(np, "suppress-preamble");2420 }2421 2422 adapter->mdiobus = devm_mdiobus_alloc(&adapter->pdev->dev);2423 if (!adapter->mdiobus) {2424 retval = -ENOMEM;2425 2426 goto out;2427 }2428 2429 adapter->mdiobus->priv = (void *)adapter;2430 adapter->mdiobus->parent = &adapter->pdev->dev;2431 adapter->mdiobus->read = tsnep_mdiobus_read;2432 adapter->mdiobus->write = tsnep_mdiobus_write;2433 adapter->mdiobus->name = TSNEP "-mdiobus";2434 snprintf(adapter->mdiobus->id, MII_BUS_ID_SIZE, "%s",2435 adapter->pdev->name);2436 2437 /* do not scan broadcast address */2438 adapter->mdiobus->phy_mask = 0x0000001;2439 2440 retval = of_mdiobus_register(adapter->mdiobus, np);2441 2442out:2443 of_node_put(np);2444 2445 return retval;2446}2447 2448static int tsnep_phy_init(struct tsnep_adapter *adapter)2449{2450 struct device_node *phy_node;2451 int retval;2452 2453 retval = of_get_phy_mode(adapter->pdev->dev.of_node,2454 &adapter->phy_mode);2455 if (retval)2456 adapter->phy_mode = PHY_INTERFACE_MODE_GMII;2457 2458 phy_node = of_parse_phandle(adapter->pdev->dev.of_node, "phy-handle",2459 0);2460 adapter->phydev = of_phy_find_device(phy_node);2461 of_node_put(phy_node);2462 if (!adapter->phydev && adapter->mdiobus)2463 adapter->phydev = phy_find_first(adapter->mdiobus);2464 if (!adapter->phydev)2465 return -EIO;2466 2467 return 0;2468}2469 2470static int tsnep_queue_init(struct tsnep_adapter *adapter, int queue_count)2471{2472 u32 irq_mask = ECM_INT_TX_0 | ECM_INT_RX_0;2473 char name[8];2474 int i;2475 int retval;2476 2477 /* one TX/RX queue pair for netdev is mandatory */2478 if (platform_irq_count(adapter->pdev) == 1)2479 retval = platform_get_irq(adapter->pdev, 0);2480 else2481 retval = platform_get_irq_byname(adapter->pdev, "mac");2482 if (retval < 0)2483 return retval;2484 adapter->num_tx_queues = 1;2485 adapter->num_rx_queues = 1;2486 adapter->num_queues = 1;2487 adapter->queue[0].adapter = adapter;2488 adapter->queue[0].irq = retval;2489 adapter->queue[0].tx = &adapter->tx[0];2490 adapter->queue[0].tx->adapter = adapter;2491 adapter->queue[0].tx->addr = adapter->addr + TSNEP_QUEUE(0);2492 adapter->queue[0].tx->queue_index = 0;2493 adapter->queue[0].rx = &adapter->rx[0];2494 adapter->queue[0].rx->adapter = adapter;2495 adapter->queue[0].rx->addr = adapter->addr + TSNEP_QUEUE(0);2496 adapter->queue[0].rx->queue_index = 0;2497 adapter->queue[0].irq_mask = irq_mask;2498 adapter->queue[0].irq_delay_addr = adapter->addr + ECM_INT_DELAY;2499 retval = tsnep_set_irq_coalesce(&adapter->queue[0],2500 TSNEP_COALESCE_USECS_DEFAULT);2501 if (retval < 0)2502 return retval;2503 2504 adapter->netdev->irq = adapter->queue[0].irq;2505 2506 /* add additional TX/RX queue pairs only if dedicated interrupt is2507 * available2508 */2509 for (i = 1; i < queue_count; i++) {2510 sprintf(name, "txrx-%d", i);2511 retval = platform_get_irq_byname_optional(adapter->pdev, name);2512 if (retval < 0)2513 break;2514 2515 adapter->num_tx_queues++;2516 adapter->num_rx_queues++;2517 adapter->num_queues++;2518 adapter->queue[i].adapter = adapter;2519 adapter->queue[i].irq = retval;2520 adapter->queue[i].tx = &adapter->tx[i];2521 adapter->queue[i].tx->adapter = adapter;2522 adapter->queue[i].tx->addr = adapter->addr + TSNEP_QUEUE(i);2523 adapter->queue[i].tx->queue_index = i;2524 adapter->queue[i].rx = &adapter->rx[i];2525 adapter->queue[i].rx->adapter = adapter;2526 adapter->queue[i].rx->addr = adapter->addr + TSNEP_QUEUE(i);2527 adapter->queue[i].rx->queue_index = i;2528 adapter->queue[i].irq_mask =2529 irq_mask << (ECM_INT_TXRX_SHIFT * i);2530 adapter->queue[i].irq_delay_addr =2531 adapter->addr + ECM_INT_DELAY + ECM_INT_DELAY_OFFSET * i;2532 retval = tsnep_set_irq_coalesce(&adapter->queue[i],2533 TSNEP_COALESCE_USECS_DEFAULT);2534 if (retval < 0)2535 return retval;2536 }2537 2538 return 0;2539}2540 2541static int tsnep_probe(struct platform_device *pdev)2542{2543 struct tsnep_adapter *adapter;2544 struct net_device *netdev;2545 struct resource *io;2546 u32 type;2547 int revision;2548 int version;2549 int queue_count;2550 int retval;2551 2552 netdev = devm_alloc_etherdev_mqs(&pdev->dev,2553 sizeof(struct tsnep_adapter),2554 TSNEP_MAX_QUEUES, TSNEP_MAX_QUEUES);2555 if (!netdev)2556 return -ENODEV;2557 SET_NETDEV_DEV(netdev, &pdev->dev);2558 adapter = netdev_priv(netdev);2559 platform_set_drvdata(pdev, adapter);2560 adapter->pdev = pdev;2561 adapter->dmadev = &pdev->dev;2562 adapter->netdev = netdev;2563 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE |2564 NETIF_MSG_LINK | NETIF_MSG_IFUP |2565 NETIF_MSG_IFDOWN | NETIF_MSG_TX_QUEUED;2566 2567 netdev->min_mtu = ETH_MIN_MTU;2568 netdev->max_mtu = TSNEP_MAX_FRAME_SIZE;2569 2570 mutex_init(&adapter->gate_control_lock);2571 mutex_init(&adapter->rxnfc_lock);2572 INIT_LIST_HEAD(&adapter->rxnfc_rules);2573 2574 adapter->addr = devm_platform_get_and_ioremap_resource(pdev, 0, &io);2575 if (IS_ERR(adapter->addr))2576 return PTR_ERR(adapter->addr);2577 netdev->mem_start = io->start;2578 netdev->mem_end = io->end;2579 2580 type = ioread32(adapter->addr + ECM_TYPE);2581 revision = (type & ECM_REVISION_MASK) >> ECM_REVISION_SHIFT;2582 version = (type & ECM_VERSION_MASK) >> ECM_VERSION_SHIFT;2583 queue_count = (type & ECM_QUEUE_COUNT_MASK) >> ECM_QUEUE_COUNT_SHIFT;2584 adapter->gate_control = type & ECM_GATE_CONTROL;2585 adapter->rxnfc_max = TSNEP_RX_ASSIGN_ETHER_TYPE_COUNT;2586 2587 tsnep_disable_irq(adapter, ECM_INT_ALL);2588 2589 retval = tsnep_queue_init(adapter, queue_count);2590 if (retval)2591 return retval;2592 2593 retval = dma_set_mask_and_coherent(&adapter->pdev->dev,2594 DMA_BIT_MASK(64));2595 if (retval) {2596 dev_err(&adapter->pdev->dev, "no usable DMA configuration.\n");2597 return retval;2598 }2599 2600 retval = tsnep_mac_init(adapter);2601 if (retval)2602 return retval;2603 2604 retval = tsnep_mdio_init(adapter);2605 if (retval)2606 goto mdio_init_failed;2607 2608 retval = tsnep_phy_init(adapter);2609 if (retval)2610 goto phy_init_failed;2611 2612 retval = tsnep_ptp_init(adapter);2613 if (retval)2614 goto ptp_init_failed;2615 2616 retval = tsnep_tc_init(adapter);2617 if (retval)2618 goto tc_init_failed;2619 2620 retval = tsnep_rxnfc_init(adapter);2621 if (retval)2622 goto rxnfc_init_failed;2623 2624 netdev->netdev_ops = &tsnep_netdev_ops;2625 netdev->ethtool_ops = &tsnep_ethtool_ops;2626 netdev->features = NETIF_F_SG;2627 netdev->hw_features = netdev->features | NETIF_F_LOOPBACK;2628 2629 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |2630 NETDEV_XDP_ACT_NDO_XMIT |2631 NETDEV_XDP_ACT_NDO_XMIT_SG |2632 NETDEV_XDP_ACT_XSK_ZEROCOPY;2633 2634 /* carrier off reporting is important to ethtool even BEFORE open */2635 netif_carrier_off(netdev);2636 2637 retval = register_netdev(netdev);2638 if (retval)2639 goto register_failed;2640 2641 dev_info(&adapter->pdev->dev, "device version %d.%02d\n", version,2642 revision);2643 if (adapter->gate_control)2644 dev_info(&adapter->pdev->dev, "gate control detected\n");2645 2646 return 0;2647 2648register_failed:2649 tsnep_rxnfc_cleanup(adapter);2650rxnfc_init_failed:2651 tsnep_tc_cleanup(adapter);2652tc_init_failed:2653 tsnep_ptp_cleanup(adapter);2654ptp_init_failed:2655phy_init_failed:2656 if (adapter->mdiobus)2657 mdiobus_unregister(adapter->mdiobus);2658mdio_init_failed:2659 return retval;2660}2661 2662static void tsnep_remove(struct platform_device *pdev)2663{2664 struct tsnep_adapter *adapter = platform_get_drvdata(pdev);2665 2666 unregister_netdev(adapter->netdev);2667 2668 tsnep_rxnfc_cleanup(adapter);2669 2670 tsnep_tc_cleanup(adapter);2671 2672 tsnep_ptp_cleanup(adapter);2673 2674 if (adapter->mdiobus)2675 mdiobus_unregister(adapter->mdiobus);2676 2677 tsnep_disable_irq(adapter, ECM_INT_ALL);2678}2679 2680static const struct of_device_id tsnep_of_match[] = {2681 { .compatible = "engleder,tsnep", },2682{ },2683};2684MODULE_DEVICE_TABLE(of, tsnep_of_match);2685 2686static struct platform_driver tsnep_driver = {2687 .driver = {2688 .name = TSNEP,2689 .of_match_table = tsnep_of_match,2690 },2691 .probe = tsnep_probe,2692 .remove_new = tsnep_remove,2693};2694module_platform_driver(tsnep_driver);2695 2696MODULE_AUTHOR("Gerhard Engleder <gerhard@engleder-embedded.com>");2697MODULE_DESCRIPTION("TSN endpoint Ethernet MAC driver");2698MODULE_LICENSE("GPL");2699