547 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Marvell Octeon EP (EndPoint) Ethernet Driver3 *4 * Copyright (C) 2020 Marvell.5 *6 */7 8#include <linux/pci.h>9#include <linux/etherdevice.h>10#include <linux/vmalloc.h>11 12#include "octep_config.h"13#include "octep_main.h"14 15static void octep_oq_reset_indices(struct octep_oq *oq)16{17 oq->host_read_idx = 0;18 oq->host_refill_idx = 0;19 oq->refill_count = 0;20 oq->last_pkt_count = 0;21 oq->pkts_pending = 0;22}23 24/**25 * octep_oq_fill_ring_buffers() - fill initial receive buffers for Rx ring.26 *27 * @oq: Octeon Rx queue data structure.28 *29 * Return: 0, if successfully filled receive buffers for all descriptors.30 * -1, if failed to allocate a buffer or failed to map for DMA.31 */32static int octep_oq_fill_ring_buffers(struct octep_oq *oq)33{34 struct octep_oq_desc_hw *desc_ring = oq->desc_ring;35 struct page *page;36 u32 i;37 38 for (i = 0; i < oq->max_count; i++) {39 page = dev_alloc_page();40 if (unlikely(!page)) {41 dev_err(oq->dev, "Rx buffer alloc failed\n");42 goto rx_buf_alloc_err;43 }44 desc_ring[i].buffer_ptr = dma_map_page(oq->dev, page, 0,45 PAGE_SIZE,46 DMA_FROM_DEVICE);47 if (dma_mapping_error(oq->dev, desc_ring[i].buffer_ptr)) {48 dev_err(oq->dev,49 "OQ-%d buffer alloc: DMA mapping error!\n",50 oq->q_no);51 put_page(page);52 goto dma_map_err;53 }54 oq->buff_info[i].page = page;55 }56 57 return 0;58 59dma_map_err:60rx_buf_alloc_err:61 while (i) {62 i--;63 dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr, PAGE_SIZE, DMA_FROM_DEVICE);64 put_page(oq->buff_info[i].page);65 oq->buff_info[i].page = NULL;66 }67 68 return -1;69}70 71/**72 * octep_oq_refill() - refill buffers for used Rx ring descriptors.73 *74 * @oct: Octeon device private data structure.75 * @oq: Octeon Rx queue data structure.76 *77 * Return: number of descriptors successfully refilled with receive buffers.78 */79static int octep_oq_refill(struct octep_device *oct, struct octep_oq *oq)80{81 struct octep_oq_desc_hw *desc_ring = oq->desc_ring;82 struct page *page;83 u32 refill_idx, i;84 85 refill_idx = oq->host_refill_idx;86 for (i = 0; i < oq->refill_count; i++) {87 page = dev_alloc_page();88 if (unlikely(!page)) {89 dev_err(oq->dev, "refill: rx buffer alloc failed\n");90 oq->stats.alloc_failures++;91 break;92 }93 94 desc_ring[refill_idx].buffer_ptr = dma_map_page(oq->dev, page, 0,95 PAGE_SIZE, DMA_FROM_DEVICE);96 if (dma_mapping_error(oq->dev, desc_ring[refill_idx].buffer_ptr)) {97 dev_err(oq->dev,98 "OQ-%d buffer refill: DMA mapping error!\n",99 oq->q_no);100 put_page(page);101 oq->stats.alloc_failures++;102 break;103 }104 oq->buff_info[refill_idx].page = page;105 refill_idx++;106 if (refill_idx == oq->max_count)107 refill_idx = 0;108 }109 oq->host_refill_idx = refill_idx;110 oq->refill_count -= i;111 112 return i;113}114 115/**116 * octep_setup_oq() - Setup a Rx queue.117 *118 * @oct: Octeon device private data structure.119 * @q_no: Rx queue number to be setup.120 *121 * Allocate resources for a Rx queue.122 */123static int octep_setup_oq(struct octep_device *oct, int q_no)124{125 struct octep_oq *oq;126 u32 desc_ring_size;127 128 oq = vzalloc(sizeof(*oq));129 if (!oq)130 goto create_oq_fail;131 oct->oq[q_no] = oq;132 133 oq->octep_dev = oct;134 oq->netdev = oct->netdev;135 oq->dev = &oct->pdev->dev;136 oq->q_no = q_no;137 oq->max_count = CFG_GET_OQ_NUM_DESC(oct->conf);138 oq->ring_size_mask = oq->max_count - 1;139 oq->buffer_size = CFG_GET_OQ_BUF_SIZE(oct->conf);140 oq->max_single_buffer_size = oq->buffer_size - OCTEP_OQ_RESP_HW_SIZE;141 142 /* When the hardware/firmware supports additional capabilities,143 * additional header is filled-in by Octeon after length field in144 * Rx packets. this header contains additional packet information.145 */146 if (oct->conf->fw_info.rx_ol_flags)147 oq->max_single_buffer_size -= OCTEP_OQ_RESP_HW_EXT_SIZE;148 149 oq->refill_threshold = CFG_GET_OQ_REFILL_THRESHOLD(oct->conf);150 151 desc_ring_size = oq->max_count * OCTEP_OQ_DESC_SIZE;152 oq->desc_ring = dma_alloc_coherent(oq->dev, desc_ring_size,153 &oq->desc_ring_dma, GFP_KERNEL);154 155 if (unlikely(!oq->desc_ring)) {156 dev_err(oq->dev,157 "Failed to allocate DMA memory for OQ-%d !!\n", q_no);158 goto desc_dma_alloc_err;159 }160 161 oq->buff_info = vcalloc(oq->max_count, OCTEP_OQ_RECVBUF_SIZE);162 if (unlikely(!oq->buff_info)) {163 dev_err(&oct->pdev->dev,164 "Failed to allocate buffer info for OQ-%d\n", q_no);165 goto buf_list_err;166 }167 168 if (octep_oq_fill_ring_buffers(oq))169 goto oq_fill_buff_err;170 171 octep_oq_reset_indices(oq);172 oct->hw_ops.setup_oq_regs(oct, q_no);173 oct->num_oqs++;174 175 return 0;176 177oq_fill_buff_err:178 vfree(oq->buff_info);179 oq->buff_info = NULL;180buf_list_err:181 dma_free_coherent(oq->dev, desc_ring_size,182 oq->desc_ring, oq->desc_ring_dma);183 oq->desc_ring = NULL;184desc_dma_alloc_err:185 vfree(oq);186 oct->oq[q_no] = NULL;187create_oq_fail:188 return -1;189}190 191/**192 * octep_oq_free_ring_buffers() - Free ring buffers.193 *194 * @oq: Octeon Rx queue data structure.195 *196 * Free receive buffers in unused Rx queue descriptors.197 */198static void octep_oq_free_ring_buffers(struct octep_oq *oq)199{200 struct octep_oq_desc_hw *desc_ring = oq->desc_ring;201 int i;202 203 if (!oq->desc_ring || !oq->buff_info)204 return;205 206 for (i = 0; i < oq->max_count; i++) {207 if (oq->buff_info[i].page) {208 dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr,209 PAGE_SIZE, DMA_FROM_DEVICE);210 put_page(oq->buff_info[i].page);211 oq->buff_info[i].page = NULL;212 desc_ring[i].buffer_ptr = 0;213 }214 }215 octep_oq_reset_indices(oq);216}217 218/**219 * octep_free_oq() - Free Rx queue resources.220 *221 * @oq: Octeon Rx queue data structure.222 *223 * Free all resources of a Rx queue.224 */225static int octep_free_oq(struct octep_oq *oq)226{227 struct octep_device *oct = oq->octep_dev;228 int q_no = oq->q_no;229 230 octep_oq_free_ring_buffers(oq);231 232 vfree(oq->buff_info);233 234 if (oq->desc_ring)235 dma_free_coherent(oq->dev,236 oq->max_count * OCTEP_OQ_DESC_SIZE,237 oq->desc_ring, oq->desc_ring_dma);238 239 vfree(oq);240 oct->oq[q_no] = NULL;241 oct->num_oqs--;242 return 0;243}244 245/**246 * octep_setup_oqs() - setup resources for all Rx queues.247 *248 * @oct: Octeon device private data structure.249 */250int octep_setup_oqs(struct octep_device *oct)251{252 int i, retval = 0;253 254 oct->num_oqs = 0;255 for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {256 retval = octep_setup_oq(oct, i);257 if (retval) {258 dev_err(&oct->pdev->dev,259 "Failed to setup OQ(RxQ)-%d.\n", i);260 goto oq_setup_err;261 }262 dev_dbg(&oct->pdev->dev, "Successfully setup OQ(RxQ)-%d.\n", i);263 }264 265 return 0;266 267oq_setup_err:268 while (i) {269 i--;270 octep_free_oq(oct->oq[i]);271 }272 return -1;273}274 275/**276 * octep_oq_dbell_init() - Initialize Rx queue doorbell.277 *278 * @oct: Octeon device private data structure.279 *280 * Write number of descriptors to Rx queue doorbell register.281 */282void octep_oq_dbell_init(struct octep_device *oct)283{284 int i;285 286 for (i = 0; i < oct->num_oqs; i++)287 writel(oct->oq[i]->max_count, oct->oq[i]->pkts_credit_reg);288}289 290/**291 * octep_free_oqs() - Free resources of all Rx queues.292 *293 * @oct: Octeon device private data structure.294 */295void octep_free_oqs(struct octep_device *oct)296{297 int i;298 299 for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {300 if (!oct->oq[i])301 continue;302 octep_free_oq(oct->oq[i]);303 dev_dbg(&oct->pdev->dev,304 "Successfully freed OQ(RxQ)-%d.\n", i);305 }306}307 308/**309 * octep_oq_check_hw_for_pkts() - Check for new Rx packets.310 *311 * @oct: Octeon device private data structure.312 * @oq: Octeon Rx queue data structure.313 *314 * Return: packets received after previous check.315 */316static int octep_oq_check_hw_for_pkts(struct octep_device *oct,317 struct octep_oq *oq)318{319 u32 pkt_count, new_pkts;320 321 pkt_count = readl(oq->pkts_sent_reg);322 new_pkts = pkt_count - oq->last_pkt_count;323 324 /* Clear the hardware packets counter register if the rx queue is325 * being processed continuously with-in a single interrupt and326 * reached half its max value.327 * this counter is not cleared every time read, to save write cycles.328 */329 if (unlikely(pkt_count > 0xF0000000U)) {330 writel(pkt_count, oq->pkts_sent_reg);331 pkt_count = readl(oq->pkts_sent_reg);332 new_pkts += pkt_count;333 }334 oq->last_pkt_count = pkt_count;335 oq->pkts_pending += new_pkts;336 return new_pkts;337}338 339/**340 * octep_oq_next_pkt() - Move to the next packet in Rx queue.341 *342 * @oq: Octeon Rx queue data structure.343 * @buff_info: Current packet buffer info.344 * @read_idx: Current packet index in the ring.345 * @desc_used: Current packet descriptor number.346 *347 * Free the resources associated with a packet.348 * Increment packet index in the ring and packet descriptor number.349 */350static void octep_oq_next_pkt(struct octep_oq *oq,351 struct octep_rx_buffer *buff_info,352 u32 *read_idx, u32 *desc_used)353{354 dma_unmap_page(oq->dev, oq->desc_ring[*read_idx].buffer_ptr,355 PAGE_SIZE, DMA_FROM_DEVICE);356 buff_info->page = NULL;357 (*read_idx)++;358 (*desc_used)++;359 if (*read_idx == oq->max_count)360 *read_idx = 0;361}362 363/**364 * octep_oq_drop_rx() - Free the resources associated with a packet.365 *366 * @oq: Octeon Rx queue data structure.367 * @buff_info: Current packet buffer info.368 * @read_idx: Current packet index in the ring.369 * @desc_used: Current packet descriptor number.370 *371 */372static void octep_oq_drop_rx(struct octep_oq *oq,373 struct octep_rx_buffer *buff_info,374 u32 *read_idx, u32 *desc_used)375{376 int data_len = buff_info->len - oq->max_single_buffer_size;377 378 while (data_len > 0) {379 octep_oq_next_pkt(oq, buff_info, read_idx, desc_used);380 data_len -= oq->buffer_size;381 };382}383 384/**385 * __octep_oq_process_rx() - Process hardware Rx queue and push to stack.386 *387 * @oct: Octeon device private data structure.388 * @oq: Octeon Rx queue data structure.389 * @pkts_to_process: number of packets to be processed.390 *391 * Process the new packets in Rx queue.392 * Packets larger than single Rx buffer arrive in consecutive descriptors.393 * But, count returned by the API only accounts full packets, not fragments.394 *395 * Return: number of packets processed and pushed to stack.396 */397static int __octep_oq_process_rx(struct octep_device *oct,398 struct octep_oq *oq, u16 pkts_to_process)399{400 struct octep_oq_resp_hw_ext *resp_hw_ext = NULL;401 netdev_features_t feat = oq->netdev->features;402 struct octep_rx_buffer *buff_info;403 struct octep_oq_resp_hw *resp_hw;404 u32 pkt, rx_bytes, desc_used;405 struct sk_buff *skb;406 u16 data_offset;407 u16 rx_ol_flags;408 u32 read_idx;409 410 read_idx = oq->host_read_idx;411 rx_bytes = 0;412 desc_used = 0;413 for (pkt = 0; pkt < pkts_to_process; pkt++) {414 buff_info = (struct octep_rx_buffer *)&oq->buff_info[read_idx];415 resp_hw = page_address(buff_info->page);416 417 /* Swap the length field that is in Big-Endian to CPU */418 buff_info->len = be64_to_cpu(resp_hw->length);419 if (oct->conf->fw_info.rx_ol_flags) {420 /* Extended response header is immediately after421 * response header (resp_hw)422 */423 resp_hw_ext = (struct octep_oq_resp_hw_ext *)424 (resp_hw + 1);425 buff_info->len -= OCTEP_OQ_RESP_HW_EXT_SIZE;426 /* Packet Data is immediately after427 * extended response header.428 */429 data_offset = OCTEP_OQ_RESP_HW_SIZE +430 OCTEP_OQ_RESP_HW_EXT_SIZE;431 rx_ol_flags = resp_hw_ext->rx_ol_flags;432 } else {433 /* Data is immediately after434 * Hardware Rx response header.435 */436 data_offset = OCTEP_OQ_RESP_HW_SIZE;437 rx_ol_flags = 0;438 }439 440 octep_oq_next_pkt(oq, buff_info, &read_idx, &desc_used);441 442 skb = build_skb((void *)resp_hw, PAGE_SIZE);443 if (!skb) {444 octep_oq_drop_rx(oq, buff_info,445 &read_idx, &desc_used);446 oq->stats.alloc_failures++;447 continue;448 }449 skb_reserve(skb, data_offset);450 451 rx_bytes += buff_info->len;452 453 if (buff_info->len <= oq->max_single_buffer_size) {454 skb_put(skb, buff_info->len);455 } else {456 struct skb_shared_info *shinfo;457 u16 data_len;458 459 /* Head fragment includes response header(s);460 * subsequent fragments contains only data.461 */462 skb_put(skb, oq->max_single_buffer_size);463 shinfo = skb_shinfo(skb);464 data_len = buff_info->len - oq->max_single_buffer_size;465 while (data_len) {466 buff_info = (struct octep_rx_buffer *)467 &oq->buff_info[read_idx];468 if (data_len < oq->buffer_size) {469 buff_info->len = data_len;470 data_len = 0;471 } else {472 buff_info->len = oq->buffer_size;473 data_len -= oq->buffer_size;474 }475 476 skb_add_rx_frag(skb, shinfo->nr_frags,477 buff_info->page, 0,478 buff_info->len,479 buff_info->len);480 481 octep_oq_next_pkt(oq, buff_info, &read_idx, &desc_used);482 }483 }484 485 skb->dev = oq->netdev;486 skb->protocol = eth_type_trans(skb, skb->dev);487 if (feat & NETIF_F_RXCSUM &&488 OCTEP_RX_CSUM_VERIFIED(rx_ol_flags))489 skb->ip_summed = CHECKSUM_UNNECESSARY;490 else491 skb->ip_summed = CHECKSUM_NONE;492 napi_gro_receive(oq->napi, skb);493 }494 495 oq->host_read_idx = read_idx;496 oq->refill_count += desc_used;497 oq->stats.packets += pkt;498 oq->stats.bytes += rx_bytes;499 500 return pkt;501}502 503/**504 * octep_oq_process_rx() - Process Rx queue.505 *506 * @oq: Octeon Rx queue data structure.507 * @budget: max number of packets can be processed in one invocation.508 *509 * Check for newly received packets and process them.510 * Keeps checking for new packets until budget is used or no new packets seen.511 *512 * Return: number of packets processed.513 */514int octep_oq_process_rx(struct octep_oq *oq, int budget)515{516 u32 pkts_available, pkts_processed, total_pkts_processed;517 struct octep_device *oct = oq->octep_dev;518 519 pkts_available = 0;520 pkts_processed = 0;521 total_pkts_processed = 0;522 while (total_pkts_processed < budget) {523 /* update pending count only when current one exhausted */524 if (oq->pkts_pending == 0)525 octep_oq_check_hw_for_pkts(oct, oq);526 pkts_available = min(budget - total_pkts_processed,527 oq->pkts_pending);528 if (!pkts_available)529 break;530 531 pkts_processed = __octep_oq_process_rx(oct, oq,532 pkts_available);533 oq->pkts_pending -= pkts_processed;534 total_pkts_processed += pkts_processed;535 }536 537 if (oq->refill_count >= oq->refill_threshold) {538 u32 desc_refilled = octep_oq_refill(oct, oq);539 540 /* flush pending writes before updating credits */541 wmb();542 writel(desc_refilled, oq->pkts_credit_reg);543 }544 545 return total_pkts_processed;546}547