511 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Marvell Octeon EP (EndPoint) VF Ethernet Driver3 *4 * Copyright (C) 2020 Marvell.5 *6 */7 8#include <linux/pci.h>9#include <linux/etherdevice.h>10#include <linux/vmalloc.h>11 12#include "octep_vf_config.h"13#include "octep_vf_main.h"14 15static void octep_vf_oq_reset_indices(struct octep_vf_oq *oq)16{17 oq->host_read_idx = 0;18 oq->host_refill_idx = 0;19 oq->refill_count = 0;20 oq->last_pkt_count = 0;21 oq->pkts_pending = 0;22}23 24/**25 * octep_vf_oq_fill_ring_buffers() - fill initial receive buffers for Rx ring.26 *27 * @oq: Octeon Rx queue data structure.28 *29 * Return: 0, if successfully filled receive buffers for all descriptors.30 * -ENOMEM, if failed to allocate a buffer or failed to map for DMA.31 */32static int octep_vf_oq_fill_ring_buffers(struct octep_vf_oq *oq)33{34 struct octep_vf_oq_desc_hw *desc_ring = oq->desc_ring;35 struct page *page;36 u32 i;37 38 for (i = 0; i < oq->max_count; i++) {39 page = dev_alloc_page();40 if (unlikely(!page)) {41 dev_err(oq->dev, "Rx buffer alloc failed\n");42 goto rx_buf_alloc_err;43 }44 desc_ring[i].buffer_ptr = dma_map_page(oq->dev, page, 0,45 PAGE_SIZE,46 DMA_FROM_DEVICE);47 if (dma_mapping_error(oq->dev, desc_ring[i].buffer_ptr)) {48 dev_err(oq->dev,49 "OQ-%d buffer alloc: DMA mapping error!\n",50 oq->q_no);51 goto dma_map_err;52 }53 oq->buff_info[i].page = page;54 }55 56 return 0;57 58dma_map_err:59 put_page(page);60rx_buf_alloc_err:61 while (i) {62 i--;63 dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr, PAGE_SIZE, DMA_FROM_DEVICE);64 put_page(oq->buff_info[i].page);65 oq->buff_info[i].page = NULL;66 }67 68 return -ENOMEM;69}70 71/**72 * octep_vf_oq_refill() - refill buffers for used Rx ring descriptors.73 *74 * @oct: Octeon device private data structure.75 * @oq: Octeon Rx queue data structure.76 *77 * Return: number of descriptors successfully refilled with receive buffers.78 */79static int octep_vf_oq_refill(struct octep_vf_device *oct, struct octep_vf_oq *oq)80{81 struct octep_vf_oq_desc_hw *desc_ring = oq->desc_ring;82 struct page *page;83 u32 refill_idx, i;84 85 refill_idx = oq->host_refill_idx;86 for (i = 0; i < oq->refill_count; i++) {87 page = dev_alloc_page();88 if (unlikely(!page)) {89 dev_err(oq->dev, "refill: rx buffer alloc failed\n");90 oq->stats.alloc_failures++;91 break;92 }93 94 desc_ring[refill_idx].buffer_ptr = dma_map_page(oq->dev, page, 0,95 PAGE_SIZE, DMA_FROM_DEVICE);96 if (dma_mapping_error(oq->dev, desc_ring[refill_idx].buffer_ptr)) {97 dev_err(oq->dev,98 "OQ-%d buffer refill: DMA mapping error!\n",99 oq->q_no);100 put_page(page);101 oq->stats.alloc_failures++;102 break;103 }104 oq->buff_info[refill_idx].page = page;105 refill_idx++;106 if (refill_idx == oq->max_count)107 refill_idx = 0;108 }109 oq->host_refill_idx = refill_idx;110 oq->refill_count -= i;111 112 return i;113}114 115/**116 * octep_vf_setup_oq() - Setup a Rx queue.117 *118 * @oct: Octeon device private data structure.119 * @q_no: Rx queue number to be setup.120 *121 * Allocate resources for a Rx queue.122 */123static int octep_vf_setup_oq(struct octep_vf_device *oct, int q_no)124{125 struct octep_vf_oq *oq;126 u32 desc_ring_size;127 128 oq = vzalloc(sizeof(*oq));129 if (!oq)130 goto create_oq_fail;131 oct->oq[q_no] = oq;132 133 oq->octep_vf_dev = oct;134 oq->netdev = oct->netdev;135 oq->dev = &oct->pdev->dev;136 oq->q_no = q_no;137 oq->max_count = CFG_GET_OQ_NUM_DESC(oct->conf);138 oq->ring_size_mask = oq->max_count - 1;139 oq->buffer_size = CFG_GET_OQ_BUF_SIZE(oct->conf);140 oq->max_single_buffer_size = oq->buffer_size - OCTEP_VF_OQ_RESP_HW_SIZE;141 142 /* When the hardware/firmware supports additional capabilities,143 * additional header is filled-in by Octeon after length field in144 * Rx packets. this header contains additional packet information.145 */146 if (oct->fw_info.rx_ol_flags)147 oq->max_single_buffer_size -= OCTEP_VF_OQ_RESP_HW_EXT_SIZE;148 149 oq->refill_threshold = CFG_GET_OQ_REFILL_THRESHOLD(oct->conf);150 151 desc_ring_size = oq->max_count * OCTEP_VF_OQ_DESC_SIZE;152 oq->desc_ring = dma_alloc_coherent(oq->dev, desc_ring_size,153 &oq->desc_ring_dma, GFP_KERNEL);154 155 if (unlikely(!oq->desc_ring)) {156 dev_err(oq->dev,157 "Failed to allocate DMA memory for OQ-%d !!\n", q_no);158 goto desc_dma_alloc_err;159 }160 161 oq->buff_info = vzalloc(oq->max_count * OCTEP_VF_OQ_RECVBUF_SIZE);162 163 if (unlikely(!oq->buff_info)) {164 dev_err(&oct->pdev->dev,165 "Failed to allocate buffer info for OQ-%d\n", q_no);166 goto buf_list_err;167 }168 169 if (octep_vf_oq_fill_ring_buffers(oq))170 goto oq_fill_buff_err;171 172 octep_vf_oq_reset_indices(oq);173 oct->hw_ops.setup_oq_regs(oct, q_no);174 oct->num_oqs++;175 176 return 0;177 178oq_fill_buff_err:179 vfree(oq->buff_info);180 oq->buff_info = NULL;181buf_list_err:182 dma_free_coherent(oq->dev, desc_ring_size,183 oq->desc_ring, oq->desc_ring_dma);184 oq->desc_ring = NULL;185desc_dma_alloc_err:186 vfree(oq);187 oct->oq[q_no] = NULL;188create_oq_fail:189 return -ENOMEM;190}191 192/**193 * octep_vf_oq_free_ring_buffers() - Free ring buffers.194 *195 * @oq: Octeon Rx queue data structure.196 *197 * Free receive buffers in unused Rx queue descriptors.198 */199static void octep_vf_oq_free_ring_buffers(struct octep_vf_oq *oq)200{201 struct octep_vf_oq_desc_hw *desc_ring = oq->desc_ring;202 int i;203 204 if (!oq->desc_ring || !oq->buff_info)205 return;206 207 for (i = 0; i < oq->max_count; i++) {208 if (oq->buff_info[i].page) {209 dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr,210 PAGE_SIZE, DMA_FROM_DEVICE);211 put_page(oq->buff_info[i].page);212 oq->buff_info[i].page = NULL;213 desc_ring[i].buffer_ptr = 0;214 }215 }216 octep_vf_oq_reset_indices(oq);217}218 219/**220 * octep_vf_free_oq() - Free Rx queue resources.221 *222 * @oq: Octeon Rx queue data structure.223 *224 * Free all resources of a Rx queue.225 */226static int octep_vf_free_oq(struct octep_vf_oq *oq)227{228 struct octep_vf_device *oct = oq->octep_vf_dev;229 int q_no = oq->q_no;230 231 octep_vf_oq_free_ring_buffers(oq);232 233 vfree(oq->buff_info);234 235 if (oq->desc_ring)236 dma_free_coherent(oq->dev,237 oq->max_count * OCTEP_VF_OQ_DESC_SIZE,238 oq->desc_ring, oq->desc_ring_dma);239 240 vfree(oq);241 oct->oq[q_no] = NULL;242 oct->num_oqs--;243 return 0;244}245 246/**247 * octep_vf_setup_oqs() - setup resources for all Rx queues.248 *249 * @oct: Octeon device private data structure.250 */251int octep_vf_setup_oqs(struct octep_vf_device *oct)252{253 int i, retval = 0;254 255 oct->num_oqs = 0;256 for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {257 retval = octep_vf_setup_oq(oct, i);258 if (retval) {259 dev_err(&oct->pdev->dev,260 "Failed to setup OQ(RxQ)-%d.\n", i);261 goto oq_setup_err;262 }263 dev_dbg(&oct->pdev->dev, "Successfully setup OQ(RxQ)-%d.\n", i);264 }265 266 return 0;267 268oq_setup_err:269 while (i) {270 i--;271 octep_vf_free_oq(oct->oq[i]);272 }273 return retval;274}275 276/**277 * octep_vf_oq_dbell_init() - Initialize Rx queue doorbell.278 *279 * @oct: Octeon device private data structure.280 *281 * Write number of descriptors to Rx queue doorbell register.282 */283void octep_vf_oq_dbell_init(struct octep_vf_device *oct)284{285 int i;286 287 for (i = 0; i < oct->num_oqs; i++)288 writel(oct->oq[i]->max_count, oct->oq[i]->pkts_credit_reg);289}290 291/**292 * octep_vf_free_oqs() - Free resources of all Rx queues.293 *294 * @oct: Octeon device private data structure.295 */296void octep_vf_free_oqs(struct octep_vf_device *oct)297{298 int i;299 300 for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {301 if (!oct->oq[i])302 continue;303 octep_vf_free_oq(oct->oq[i]);304 dev_dbg(&oct->pdev->dev,305 "Successfully freed OQ(RxQ)-%d.\n", i);306 }307}308 309/**310 * octep_vf_oq_check_hw_for_pkts() - Check for new Rx packets.311 *312 * @oct: Octeon device private data structure.313 * @oq: Octeon Rx queue data structure.314 *315 * Return: packets received after previous check.316 */317static int octep_vf_oq_check_hw_for_pkts(struct octep_vf_device *oct,318 struct octep_vf_oq *oq)319{320 u32 pkt_count, new_pkts;321 322 pkt_count = readl(oq->pkts_sent_reg);323 new_pkts = pkt_count - oq->last_pkt_count;324 325 /* Clear the hardware packets counter register if the rx queue is326 * being processed continuously with-in a single interrupt and327 * reached half its max value.328 * this counter is not cleared every time read, to save write cycles.329 */330 if (unlikely(pkt_count > 0xF0000000U)) {331 writel(pkt_count, oq->pkts_sent_reg);332 pkt_count = readl(oq->pkts_sent_reg);333 new_pkts += pkt_count;334 }335 oq->last_pkt_count = pkt_count;336 oq->pkts_pending += new_pkts;337 return new_pkts;338}339 340/**341 * __octep_vf_oq_process_rx() - Process hardware Rx queue and push to stack.342 *343 * @oct: Octeon device private data structure.344 * @oq: Octeon Rx queue data structure.345 * @pkts_to_process: number of packets to be processed.346 *347 * Process the new packets in Rx queue.348 * Packets larger than single Rx buffer arrive in consecutive descriptors.349 * But, count returned by the API only accounts full packets, not fragments.350 *351 * Return: number of packets processed and pushed to stack.352 */353static int __octep_vf_oq_process_rx(struct octep_vf_device *oct,354 struct octep_vf_oq *oq, u16 pkts_to_process)355{356 struct octep_vf_oq_resp_hw_ext *resp_hw_ext = NULL;357 netdev_features_t feat = oq->netdev->features;358 struct octep_vf_rx_buffer *buff_info;359 struct octep_vf_oq_resp_hw *resp_hw;360 u32 pkt, rx_bytes, desc_used;361 u16 data_offset, rx_ol_flags;362 struct sk_buff *skb;363 u32 read_idx;364 365 read_idx = oq->host_read_idx;366 rx_bytes = 0;367 desc_used = 0;368 for (pkt = 0; pkt < pkts_to_process; pkt++) {369 buff_info = (struct octep_vf_rx_buffer *)&oq->buff_info[read_idx];370 dma_unmap_page(oq->dev, oq->desc_ring[read_idx].buffer_ptr,371 PAGE_SIZE, DMA_FROM_DEVICE);372 resp_hw = page_address(buff_info->page);373 buff_info->page = NULL;374 375 /* Swap the length field that is in Big-Endian to CPU */376 buff_info->len = be64_to_cpu(resp_hw->length);377 if (oct->fw_info.rx_ol_flags) {378 /* Extended response header is immediately after379 * response header (resp_hw)380 */381 resp_hw_ext = (struct octep_vf_oq_resp_hw_ext *)382 (resp_hw + 1);383 buff_info->len -= OCTEP_VF_OQ_RESP_HW_EXT_SIZE;384 /* Packet Data is immediately after385 * extended response header.386 */387 data_offset = OCTEP_VF_OQ_RESP_HW_SIZE +388 OCTEP_VF_OQ_RESP_HW_EXT_SIZE;389 rx_ol_flags = resp_hw_ext->rx_ol_flags;390 } else {391 /* Data is immediately after392 * Hardware Rx response header.393 */394 data_offset = OCTEP_VF_OQ_RESP_HW_SIZE;395 rx_ol_flags = 0;396 }397 rx_bytes += buff_info->len;398 399 if (buff_info->len <= oq->max_single_buffer_size) {400 skb = napi_build_skb((void *)resp_hw, PAGE_SIZE);401 skb_reserve(skb, data_offset);402 skb_put(skb, buff_info->len);403 read_idx++;404 desc_used++;405 if (read_idx == oq->max_count)406 read_idx = 0;407 } else {408 struct skb_shared_info *shinfo;409 u16 data_len;410 411 skb = napi_build_skb((void *)resp_hw, PAGE_SIZE);412 skb_reserve(skb, data_offset);413 /* Head fragment includes response header(s);414 * subsequent fragments contains only data.415 */416 skb_put(skb, oq->max_single_buffer_size);417 read_idx++;418 desc_used++;419 if (read_idx == oq->max_count)420 read_idx = 0;421 422 shinfo = skb_shinfo(skb);423 data_len = buff_info->len - oq->max_single_buffer_size;424 while (data_len) {425 dma_unmap_page(oq->dev, oq->desc_ring[read_idx].buffer_ptr,426 PAGE_SIZE, DMA_FROM_DEVICE);427 buff_info = (struct octep_vf_rx_buffer *)428 &oq->buff_info[read_idx];429 if (data_len < oq->buffer_size) {430 buff_info->len = data_len;431 data_len = 0;432 } else {433 buff_info->len = oq->buffer_size;434 data_len -= oq->buffer_size;435 }436 437 skb_add_rx_frag(skb, shinfo->nr_frags,438 buff_info->page, 0,439 buff_info->len,440 buff_info->len);441 buff_info->page = NULL;442 read_idx++;443 desc_used++;444 if (read_idx == oq->max_count)445 read_idx = 0;446 }447 }448 449 skb->dev = oq->netdev;450 skb->protocol = eth_type_trans(skb, skb->dev);451 if (feat & NETIF_F_RXCSUM &&452 OCTEP_VF_RX_CSUM_VERIFIED(rx_ol_flags))453 skb->ip_summed = CHECKSUM_UNNECESSARY;454 else455 skb->ip_summed = CHECKSUM_NONE;456 napi_gro_receive(oq->napi, skb);457 }458 459 oq->host_read_idx = read_idx;460 oq->refill_count += desc_used;461 oq->stats.packets += pkt;462 oq->stats.bytes += rx_bytes;463 464 return pkt;465}466 467/**468 * octep_vf_oq_process_rx() - Process Rx queue.469 *470 * @oq: Octeon Rx queue data structure.471 * @budget: max number of packets can be processed in one invocation.472 *473 * Check for newly received packets and process them.474 * Keeps checking for new packets until budget is used or no new packets seen.475 *476 * Return: number of packets processed.477 */478int octep_vf_oq_process_rx(struct octep_vf_oq *oq, int budget)479{480 u32 pkts_available, pkts_processed, total_pkts_processed;481 struct octep_vf_device *oct = oq->octep_vf_dev;482 483 pkts_available = 0;484 pkts_processed = 0;485 total_pkts_processed = 0;486 while (total_pkts_processed < budget) {487 /* update pending count only when current one exhausted */488 if (oq->pkts_pending == 0)489 octep_vf_oq_check_hw_for_pkts(oct, oq);490 pkts_available = min(budget - total_pkts_processed,491 oq->pkts_pending);492 if (!pkts_available)493 break;494 495 pkts_processed = __octep_vf_oq_process_rx(oct, oq,496 pkts_available);497 oq->pkts_pending -= pkts_processed;498 total_pkts_processed += pkts_processed;499 }500 501 if (oq->refill_count >= oq->refill_threshold) {502 u32 desc_refilled = octep_vf_oq_refill(oct, oq);503 504 /* flush pending writes before updating credits */505 smp_wmb();506 writel(desc_refilled, oq->pkts_credit_reg);507 }508 509 return total_pkts_processed;510}511