1232 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Marvell Octeon EP (EndPoint) VF Ethernet Driver3 *4 * Copyright (C) 2020 Marvell.5 *6 */7 8#include <linux/types.h>9#include <linux/module.h>10#include <linux/pci.h>11#include <linux/aer.h>12#include <linux/netdevice.h>13#include <linux/etherdevice.h>14#include <linux/rtnetlink.h>15#include <linux/vmalloc.h>16#include <net/netdev_queues.h>17 18#include "octep_vf_config.h"19#include "octep_vf_main.h"20 21struct workqueue_struct *octep_vf_wq;22 23/* Supported Devices */24static const struct pci_device_id octep_vf_pci_id_tbl[] = {25 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN93_VF)},26 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF95N_VF)},27 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN98_VF)},28 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN10KA_VF)},29 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF10KA_VF)},30 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CNF10KB_VF)},31 {PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OCTEP_PCI_DEVICE_ID_CN10KB_VF)},32 {0, },33};34MODULE_DEVICE_TABLE(pci, octep_vf_pci_id_tbl);35 36MODULE_AUTHOR("Veerasenareddy Burru <vburru@marvell.com>");37MODULE_DESCRIPTION(OCTEP_VF_DRV_STRING);38MODULE_LICENSE("GPL");39 40/**41 * octep_vf_alloc_ioq_vectors() - Allocate Tx/Rx Queue interrupt info.42 *43 * @oct: Octeon device private data structure.44 *45 * Allocate resources to hold per Tx/Rx queue interrupt info.46 * This is the information passed to interrupt handler, from which napi poll47 * is scheduled and includes quick access to private data of Tx/Rx queue48 * corresponding to the interrupt being handled.49 *50 * Return: 0, on successful allocation of resources for all queue interrupts.51 * -1, if failed to allocate any resource.52 */53static int octep_vf_alloc_ioq_vectors(struct octep_vf_device *oct)54{55 struct octep_vf_ioq_vector *ioq_vector;56 int i;57 58 for (i = 0; i < oct->num_oqs; i++) {59 oct->ioq_vector[i] = vzalloc(sizeof(*oct->ioq_vector[i]));60 if (!oct->ioq_vector[i])61 goto free_ioq_vector;62 63 ioq_vector = oct->ioq_vector[i];64 ioq_vector->iq = oct->iq[i];65 ioq_vector->oq = oct->oq[i];66 ioq_vector->octep_vf_dev = oct;67 }68 69 dev_info(&oct->pdev->dev, "Allocated %d IOQ vectors\n", oct->num_oqs);70 return 0;71 72free_ioq_vector:73 while (i) {74 i--;75 vfree(oct->ioq_vector[i]);76 oct->ioq_vector[i] = NULL;77 }78 return -1;79}80 81/**82 * octep_vf_free_ioq_vectors() - Free Tx/Rx Queue interrupt vector info.83 *84 * @oct: Octeon device private data structure.85 */86static void octep_vf_free_ioq_vectors(struct octep_vf_device *oct)87{88 int i;89 90 for (i = 0; i < oct->num_oqs; i++) {91 if (oct->ioq_vector[i]) {92 vfree(oct->ioq_vector[i]);93 oct->ioq_vector[i] = NULL;94 }95 }96 netdev_info(oct->netdev, "Freed IOQ Vectors\n");97}98 99/**100 * octep_vf_enable_msix_range() - enable MSI-x interrupts.101 *102 * @oct: Octeon device private data structure.103 *104 * Allocate and enable all MSI-x interrupts (queue and non-queue interrupts)105 * for the Octeon device.106 *107 * Return: 0, on successfully enabling all MSI-x interrupts.108 * -1, if failed to enable any MSI-x interrupt.109 */110static int octep_vf_enable_msix_range(struct octep_vf_device *oct)111{112 int num_msix, msix_allocated;113 int i;114 115 /* Generic interrupts apart from input/output queues */116 //num_msix = oct->num_oqs + CFG_GET_NON_IOQ_MSIX(oct->conf);117 num_msix = oct->num_oqs;118 oct->msix_entries = kcalloc(num_msix, sizeof(struct msix_entry), GFP_KERNEL);119 if (!oct->msix_entries)120 goto msix_alloc_err;121 122 for (i = 0; i < num_msix; i++)123 oct->msix_entries[i].entry = i;124 125 msix_allocated = pci_enable_msix_range(oct->pdev, oct->msix_entries,126 num_msix, num_msix);127 if (msix_allocated != num_msix) {128 dev_err(&oct->pdev->dev,129 "Failed to enable %d msix irqs; got only %d\n",130 num_msix, msix_allocated);131 goto enable_msix_err;132 }133 oct->num_irqs = msix_allocated;134 dev_info(&oct->pdev->dev, "MSI-X enabled successfully\n");135 136 return 0;137 138enable_msix_err:139 if (msix_allocated > 0)140 pci_disable_msix(oct->pdev);141 kfree(oct->msix_entries);142 oct->msix_entries = NULL;143msix_alloc_err:144 return -1;145}146 147/**148 * octep_vf_disable_msix() - disable MSI-x interrupts.149 *150 * @oct: Octeon device private data structure.151 *152 * Disable MSI-x on the Octeon device.153 */154static void octep_vf_disable_msix(struct octep_vf_device *oct)155{156 pci_disable_msix(oct->pdev);157 kfree(oct->msix_entries);158 oct->msix_entries = NULL;159 dev_info(&oct->pdev->dev, "Disabled MSI-X\n");160}161 162/**163 * octep_vf_ioq_intr_handler() - handler for all Tx/Rx queue interrupts.164 *165 * @irq: Interrupt number.166 * @data: interrupt data contains pointers to Tx/Rx queue private data167 * and correspong NAPI context.168 *169 * this is common handler for all non-queue (generic) interrupts.170 */171static irqreturn_t octep_vf_ioq_intr_handler(int irq, void *data)172{173 struct octep_vf_ioq_vector *ioq_vector = data;174 struct octep_vf_device *oct = ioq_vector->octep_vf_dev;175 176 return oct->hw_ops.ioq_intr_handler(ioq_vector);177}178 179/**180 * octep_vf_request_irqs() - Register interrupt handlers.181 *182 * @oct: Octeon device private data structure.183 *184 * Register handlers for all queue and non-queue interrupts.185 *186 * Return: 0, on successful registration of all interrupt handlers.187 * -1, on any error.188 */189static int octep_vf_request_irqs(struct octep_vf_device *oct)190{191 struct net_device *netdev = oct->netdev;192 struct octep_vf_ioq_vector *ioq_vector;193 struct msix_entry *msix_entry;194 int ret, i;195 196 /* Request IRQs for Tx/Rx queues */197 for (i = 0; i < oct->num_oqs; i++) {198 ioq_vector = oct->ioq_vector[i];199 msix_entry = &oct->msix_entries[i];200 201 snprintf(ioq_vector->name, sizeof(ioq_vector->name),202 "%s-q%d", netdev->name, i);203 ret = request_irq(msix_entry->vector,204 octep_vf_ioq_intr_handler, 0,205 ioq_vector->name, ioq_vector);206 if (ret) {207 netdev_err(netdev,208 "request_irq failed for Q-%d; err=%d",209 i, ret);210 goto ioq_irq_err;211 }212 213 cpumask_set_cpu(i % num_online_cpus(),214 &ioq_vector->affinity_mask);215 irq_set_affinity_hint(msix_entry->vector,216 &ioq_vector->affinity_mask);217 }218 219 return 0;220ioq_irq_err:221 while (i) {222 --i;223 free_irq(oct->msix_entries[i].vector, oct);224 }225 return -1;226}227 228/**229 * octep_vf_free_irqs() - free all registered interrupts.230 *231 * @oct: Octeon device private data structure.232 *233 * Free all queue and non-queue interrupts of the Octeon device.234 */235static void octep_vf_free_irqs(struct octep_vf_device *oct)236{237 int i;238 239 for (i = 0; i < oct->num_irqs; i++) {240 irq_set_affinity_hint(oct->msix_entries[i].vector, NULL);241 free_irq(oct->msix_entries[i].vector, oct->ioq_vector[i]);242 }243 netdev_info(oct->netdev, "IRQs freed\n");244}245 246/**247 * octep_vf_setup_irqs() - setup interrupts for the Octeon device.248 *249 * @oct: Octeon device private data structure.250 *251 * Allocate data structures to hold per interrupt information, allocate/enable252 * MSI-x interrupt and register interrupt handlers.253 *254 * Return: 0, on successful allocation and registration of all interrupts.255 * -1, on any error.256 */257static int octep_vf_setup_irqs(struct octep_vf_device *oct)258{259 if (octep_vf_alloc_ioq_vectors(oct))260 goto ioq_vector_err;261 262 if (octep_vf_enable_msix_range(oct))263 goto enable_msix_err;264 265 if (octep_vf_request_irqs(oct))266 goto request_irq_err;267 268 return 0;269 270request_irq_err:271 octep_vf_disable_msix(oct);272enable_msix_err:273 octep_vf_free_ioq_vectors(oct);274ioq_vector_err:275 return -1;276}277 278/**279 * octep_vf_clean_irqs() - free all interrupts and its resources.280 *281 * @oct: Octeon device private data structure.282 */283static void octep_vf_clean_irqs(struct octep_vf_device *oct)284{285 octep_vf_free_irqs(oct);286 octep_vf_disable_msix(oct);287 octep_vf_free_ioq_vectors(oct);288}289 290/**291 * octep_vf_enable_ioq_irq() - Enable MSI-x interrupt of a Tx/Rx queue.292 *293 * @iq: Octeon Tx queue data structure.294 * @oq: Octeon Rx queue data structure.295 */296static void octep_vf_enable_ioq_irq(struct octep_vf_iq *iq, struct octep_vf_oq *oq)297{298 u32 pkts_pend = oq->pkts_pending;299 300 netdev_dbg(iq->netdev, "enabling intr for Q-%u\n", iq->q_no);301 if (iq->pkts_processed) {302 writel(iq->pkts_processed, iq->inst_cnt_reg);303 iq->pkt_in_done -= iq->pkts_processed;304 iq->pkts_processed = 0;305 }306 if (oq->last_pkt_count - pkts_pend) {307 writel(oq->last_pkt_count - pkts_pend, oq->pkts_sent_reg);308 oq->last_pkt_count = pkts_pend;309 }310 311 /* Flush the previous wrties before writing to RESEND bit */312 smp_wmb();313 writeq(1UL << OCTEP_VF_OQ_INTR_RESEND_BIT, oq->pkts_sent_reg);314 writeq(1UL << OCTEP_VF_IQ_INTR_RESEND_BIT, iq->inst_cnt_reg);315}316 317/**318 * octep_vf_napi_poll() - NAPI poll function for Tx/Rx.319 *320 * @napi: pointer to napi context.321 * @budget: max number of packets to be processed in single invocation.322 */323static int octep_vf_napi_poll(struct napi_struct *napi, int budget)324{325 struct octep_vf_ioq_vector *ioq_vector =326 container_of(napi, struct octep_vf_ioq_vector, napi);327 u32 tx_pending, rx_done;328 329 tx_pending = octep_vf_iq_process_completions(ioq_vector->iq, 64);330 rx_done = octep_vf_oq_process_rx(ioq_vector->oq, budget);331 332 /* need more polling if tx completion processing is still pending or333 * processed at least 'budget' number of rx packets.334 */335 if (tx_pending || rx_done >= budget)336 return budget;337 338 if (likely(napi_complete_done(napi, rx_done)))339 octep_vf_enable_ioq_irq(ioq_vector->iq, ioq_vector->oq);340 341 return rx_done;342}343 344/**345 * octep_vf_napi_add() - Add NAPI poll for all Tx/Rx queues.346 *347 * @oct: Octeon device private data structure.348 */349static void octep_vf_napi_add(struct octep_vf_device *oct)350{351 int i;352 353 for (i = 0; i < oct->num_oqs; i++) {354 netdev_dbg(oct->netdev, "Adding NAPI on Q-%d\n", i);355 netif_napi_add(oct->netdev, &oct->ioq_vector[i]->napi, octep_vf_napi_poll);356 oct->oq[i]->napi = &oct->ioq_vector[i]->napi;357 }358}359 360/**361 * octep_vf_napi_delete() - delete NAPI poll callback for all Tx/Rx queues.362 *363 * @oct: Octeon device private data structure.364 */365static void octep_vf_napi_delete(struct octep_vf_device *oct)366{367 int i;368 369 for (i = 0; i < oct->num_oqs; i++) {370 netdev_dbg(oct->netdev, "Deleting NAPI on Q-%d\n", i);371 netif_napi_del(&oct->ioq_vector[i]->napi);372 oct->oq[i]->napi = NULL;373 }374}375 376/**377 * octep_vf_napi_enable() - enable NAPI for all Tx/Rx queues.378 *379 * @oct: Octeon device private data structure.380 */381static void octep_vf_napi_enable(struct octep_vf_device *oct)382{383 int i;384 385 for (i = 0; i < oct->num_oqs; i++) {386 netdev_dbg(oct->netdev, "Enabling NAPI on Q-%d\n", i);387 napi_enable(&oct->ioq_vector[i]->napi);388 }389}390 391/**392 * octep_vf_napi_disable() - disable NAPI for all Tx/Rx queues.393 *394 * @oct: Octeon device private data structure.395 */396static void octep_vf_napi_disable(struct octep_vf_device *oct)397{398 int i;399 400 for (i = 0; i < oct->num_oqs; i++) {401 netdev_dbg(oct->netdev, "Disabling NAPI on Q-%d\n", i);402 napi_disable(&oct->ioq_vector[i]->napi);403 }404}405 406static void octep_vf_link_up(struct net_device *netdev)407{408 netif_carrier_on(netdev);409 netif_tx_start_all_queues(netdev);410}411 412static void octep_vf_set_rx_state(struct octep_vf_device *oct, bool up)413{414 int err;415 416 err = octep_vf_mbox_set_rx_state(oct, up);417 if (err)418 netdev_err(oct->netdev, "Set Rx state to %d failed with err:%d\n", up, err);419}420 421static int octep_vf_get_link_status(struct octep_vf_device *oct)422{423 int err;424 425 err = octep_vf_mbox_get_link_status(oct, &oct->link_info.oper_up);426 if (err)427 netdev_err(oct->netdev, "Get link status failed with err:%d\n", err);428 return oct->link_info.oper_up;429}430 431static void octep_vf_set_link_status(struct octep_vf_device *oct, bool up)432{433 int err;434 435 err = octep_vf_mbox_set_link_status(oct, up);436 if (err) {437 netdev_err(oct->netdev, "Set link status to %d failed with err:%d\n", up, err);438 return;439 }440 oct->link_info.oper_up = up;441}442 443/**444 * octep_vf_open() - start the octeon network device.445 *446 * @netdev: pointer to kernel network device.447 *448 * setup Tx/Rx queues, interrupts and enable hardware operation of Tx/Rx queues449 * and interrupts..450 *451 * Return: 0, on successfully setting up device and bring it up.452 * -1, on any error.453 */454static int octep_vf_open(struct net_device *netdev)455{456 struct octep_vf_device *oct = netdev_priv(netdev);457 int err, ret;458 459 netdev_info(netdev, "Starting netdev ...\n");460 netif_carrier_off(netdev);461 462 oct->hw_ops.reset_io_queues(oct);463 464 if (octep_vf_setup_iqs(oct))465 goto setup_iq_err;466 if (octep_vf_setup_oqs(oct))467 goto setup_oq_err;468 if (octep_vf_setup_irqs(oct))469 goto setup_irq_err;470 471 err = netif_set_real_num_tx_queues(netdev, oct->num_oqs);472 if (err)473 goto set_queues_err;474 err = netif_set_real_num_rx_queues(netdev, oct->num_iqs);475 if (err)476 goto set_queues_err;477 478 octep_vf_napi_add(oct);479 octep_vf_napi_enable(oct);480 481 oct->link_info.admin_up = 1;482 octep_vf_set_rx_state(oct, true);483 484 ret = octep_vf_get_link_status(oct);485 if (!ret)486 octep_vf_set_link_status(oct, true);487 488 /* Enable the input and output queues for this Octeon device */489 oct->hw_ops.enable_io_queues(oct);490 491 /* Enable Octeon device interrupts */492 oct->hw_ops.enable_interrupts(oct);493 494 octep_vf_oq_dbell_init(oct);495 496 ret = octep_vf_get_link_status(oct);497 if (ret)498 octep_vf_link_up(netdev);499 500 return 0;501 502set_queues_err:503 octep_vf_napi_disable(oct);504 octep_vf_napi_delete(oct);505 octep_vf_clean_irqs(oct);506setup_irq_err:507 octep_vf_free_oqs(oct);508setup_oq_err:509 octep_vf_free_iqs(oct);510setup_iq_err:511 return -1;512}513 514/**515 * octep_vf_stop() - stop the octeon network device.516 *517 * @netdev: pointer to kernel network device.518 *519 * stop the device Tx/Rx operations, bring down the link and520 * free up all resources allocated for Tx/Rx queues and interrupts.521 */522static int octep_vf_stop(struct net_device *netdev)523{524 struct octep_vf_device *oct = netdev_priv(netdev);525 526 netdev_info(netdev, "Stopping the device ...\n");527 528 /* Stop Tx from stack */529 netif_carrier_off(netdev);530 netif_tx_disable(netdev);531 532 octep_vf_set_link_status(oct, false);533 octep_vf_set_rx_state(oct, false);534 535 oct->link_info.admin_up = 0;536 oct->link_info.oper_up = 0;537 538 oct->hw_ops.disable_interrupts(oct);539 octep_vf_napi_disable(oct);540 octep_vf_napi_delete(oct);541 542 octep_vf_clean_irqs(oct);543 octep_vf_clean_iqs(oct);544 545 oct->hw_ops.disable_io_queues(oct);546 oct->hw_ops.reset_io_queues(oct);547 octep_vf_free_oqs(oct);548 octep_vf_free_iqs(oct);549 netdev_info(netdev, "Device stopped !!\n");550 return 0;551}552 553/**554 * octep_vf_iq_full_check() - check if a Tx queue is full.555 *556 * @iq: Octeon Tx queue data structure.557 *558 * Return: 0, if the Tx queue is not full.559 * 1, if the Tx queue is full.560 */561static int octep_vf_iq_full_check(struct octep_vf_iq *iq)562{563 int ret;564 565 ret = netif_subqueue_maybe_stop(iq->netdev, iq->q_no, IQ_INSTR_SPACE(iq),566 OCTEP_VF_WAKE_QUEUE_THRESHOLD,567 OCTEP_VF_WAKE_QUEUE_THRESHOLD);568 switch (ret) {569 case 0: /* Stopped the queue, since IQ is full */570 return 1;571 case -1: /*572 * Pending updates in write index from573 * iq_process_completion in other cpus574 * caused queues to get re-enabled after575 * being stopped576 */577 iq->stats.restart_cnt++;578 fallthrough;579 case 1: /* Queue left enabled, since IQ is not yet full*/580 return 0;581 }582 583 return 1;584}585 586/**587 * octep_vf_start_xmit() - Enqueue packet to Octoen hardware Tx Queue.588 *589 * @skb: packet skbuff pointer.590 * @netdev: kernel network device.591 *592 * Return: NETDEV_TX_BUSY, if Tx Queue is full.593 * NETDEV_TX_OK, if successfully enqueued to hardware Tx queue.594 */595static netdev_tx_t octep_vf_start_xmit(struct sk_buff *skb,596 struct net_device *netdev)597{598 struct octep_vf_device *oct = netdev_priv(netdev);599 netdev_features_t feat = netdev->features;600 struct octep_vf_tx_sglist_desc *sglist;601 struct octep_vf_tx_buffer *tx_buffer;602 struct octep_vf_tx_desc_hw *hw_desc;603 struct skb_shared_info *shinfo;604 struct octep_vf_instr_hdr *ih;605 struct octep_vf_iq *iq;606 skb_frag_t *frag;607 u16 nr_frags, si;608 int xmit_more;609 u16 q_no, wi;610 611 if (skb_put_padto(skb, ETH_ZLEN))612 return NETDEV_TX_OK;613 614 q_no = skb_get_queue_mapping(skb);615 if (q_no >= oct->num_iqs) {616 netdev_err(netdev, "Invalid Tx skb->queue_mapping=%d\n", q_no);617 q_no = q_no % oct->num_iqs;618 }619 620 iq = oct->iq[q_no];621 622 shinfo = skb_shinfo(skb);623 nr_frags = shinfo->nr_frags;624 625 wi = iq->host_write_index;626 hw_desc = &iq->desc_ring[wi];627 hw_desc->ih64 = 0;628 629 tx_buffer = iq->buff_info + wi;630 tx_buffer->skb = skb;631 632 ih = &hw_desc->ih;633 ih->tlen = skb->len;634 ih->pkind = oct->fw_info.pkind;635 ih->fsz = oct->fw_info.fsz;636 ih->tlen = skb->len + ih->fsz;637 638 if (!nr_frags) {639 tx_buffer->gather = 0;640 tx_buffer->dma = dma_map_single(iq->dev, skb->data,641 skb->len, DMA_TO_DEVICE);642 if (dma_mapping_error(iq->dev, tx_buffer->dma))643 goto dma_map_err;644 hw_desc->dptr = tx_buffer->dma;645 } else {646 /* Scatter/Gather */647 dma_addr_t dma;648 u16 len;649 650 sglist = tx_buffer->sglist;651 652 ih->gsz = nr_frags + 1;653 ih->gather = 1;654 tx_buffer->gather = 1;655 656 len = skb_headlen(skb);657 dma = dma_map_single(iq->dev, skb->data, len, DMA_TO_DEVICE);658 if (dma_mapping_error(iq->dev, dma))659 goto dma_map_err;660 661 memset(sglist, 0, OCTEP_VF_SGLIST_SIZE_PER_PKT);662 sglist[0].len[3] = len;663 sglist[0].dma_ptr[0] = dma;664 665 si = 1; /* entry 0 is main skb, mapped above */666 frag = &shinfo->frags[0];667 while (nr_frags--) {668 len = skb_frag_size(frag);669 dma = skb_frag_dma_map(iq->dev, frag, 0,670 len, DMA_TO_DEVICE);671 if (dma_mapping_error(iq->dev, dma))672 goto dma_map_sg_err;673 674 sglist[si >> 2].len[3 - (si & 3)] = len;675 sglist[si >> 2].dma_ptr[si & 3] = dma;676 677 frag++;678 si++;679 }680 hw_desc->dptr = tx_buffer->sglist_dma;681 }682 if (oct->fw_info.tx_ol_flags) {683 if ((feat & (NETIF_F_TSO)) && (skb_is_gso(skb))) {684 hw_desc->txm.ol_flags = OCTEP_VF_TX_OFFLOAD_CKSUM;685 hw_desc->txm.ol_flags |= OCTEP_VF_TX_OFFLOAD_TSO;686 hw_desc->txm.gso_size = skb_shinfo(skb)->gso_size;687 hw_desc->txm.gso_segs = skb_shinfo(skb)->gso_segs;688 } else if (feat & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {689 hw_desc->txm.ol_flags = OCTEP_VF_TX_OFFLOAD_CKSUM;690 }691 /* due to ESR txm will be swapped by hw */692 hw_desc->txm64[0] = (__force u64)cpu_to_be64(hw_desc->txm64[0]);693 }694 695 xmit_more = netdev_xmit_more();696 697 netdev_tx_sent_queue(iq->netdev_q, skb->len);698 699 skb_tx_timestamp(skb);700 iq->fill_cnt++;701 wi++;702 iq->host_write_index = wi & iq->ring_size_mask;703 704 /* octep_iq_full_check stops the queue and returns705 * true if so, in case the queue has become full706 * by inserting current packet. If so, we can707 * go ahead and ring doorbell.708 */709 if (!octep_vf_iq_full_check(iq) && xmit_more &&710 iq->fill_cnt < iq->fill_threshold)711 return NETDEV_TX_OK;712 713 goto ring_dbell;714 715dma_map_sg_err:716 if (si > 0) {717 dma_unmap_single(iq->dev, sglist[0].dma_ptr[0],718 sglist[0].len[0], DMA_TO_DEVICE);719 sglist[0].len[0] = 0;720 }721 while (si > 1) {722 dma_unmap_page(iq->dev, sglist[si >> 2].dma_ptr[si & 3],723 sglist[si >> 2].len[si & 3], DMA_TO_DEVICE);724 sglist[si >> 2].len[si & 3] = 0;725 si--;726 }727 tx_buffer->gather = 0;728dma_map_err:729 dev_kfree_skb_any(skb);730ring_dbell:731 /* Flush the hw descriptors before writing to doorbell */732 smp_wmb();733 writel(iq->fill_cnt, iq->doorbell_reg);734 iq->stats.instr_posted += iq->fill_cnt;735 iq->fill_cnt = 0;736 return NETDEV_TX_OK;737}738 739int octep_vf_get_if_stats(struct octep_vf_device *oct)740{741 struct octep_vf_iface_rxtx_stats vf_stats;742 int ret, size;743 744 memset(&vf_stats, 0, sizeof(struct octep_vf_iface_rxtx_stats));745 ret = octep_vf_mbox_bulk_read(oct, OCTEP_PFVF_MBOX_CMD_GET_STATS,746 (u8 *)&vf_stats, &size);747 748 if (ret)749 return ret;750 751 memcpy(&oct->iface_rx_stats, &vf_stats.iface_rx_stats,752 sizeof(struct octep_vf_iface_rx_stats));753 memcpy(&oct->iface_tx_stats, &vf_stats.iface_tx_stats,754 sizeof(struct octep_vf_iface_tx_stats));755 756 return 0;757}758 759int octep_vf_get_link_info(struct octep_vf_device *oct)760{761 int ret, size;762 763 ret = octep_vf_mbox_bulk_read(oct, OCTEP_PFVF_MBOX_CMD_GET_LINK_INFO,764 (u8 *)&oct->link_info, &size);765 if (ret) {766 dev_err(&oct->pdev->dev, "Get VF link info failed via VF Mbox\n");767 return ret;768 }769 return 0;770}771 772/**773 * octep_vf_get_stats64() - Get Octeon network device statistics.774 *775 * @netdev: kernel network device.776 * @stats: pointer to stats structure to be filled in.777 */778static void octep_vf_get_stats64(struct net_device *netdev,779 struct rtnl_link_stats64 *stats)780{781 struct octep_vf_device *oct = netdev_priv(netdev);782 u64 tx_packets, tx_bytes, rx_packets, rx_bytes;783 int q;784 785 tx_packets = 0;786 tx_bytes = 0;787 rx_packets = 0;788 rx_bytes = 0;789 for (q = 0; q < oct->num_oqs; q++) {790 struct octep_vf_iq *iq = oct->iq[q];791 struct octep_vf_oq *oq = oct->oq[q];792 793 tx_packets += iq->stats.instr_completed;794 tx_bytes += iq->stats.bytes_sent;795 rx_packets += oq->stats.packets;796 rx_bytes += oq->stats.bytes;797 }798 stats->tx_packets = tx_packets;799 stats->tx_bytes = tx_bytes;800 stats->rx_packets = rx_packets;801 stats->rx_bytes = rx_bytes;802 if (!octep_vf_get_if_stats(oct)) {803 stats->multicast = oct->iface_rx_stats.mcast_pkts;804 stats->rx_errors = oct->iface_rx_stats.err_pkts;805 stats->rx_dropped = oct->iface_rx_stats.dropped_pkts_fifo_full +806 oct->iface_rx_stats.err_pkts;807 stats->rx_missed_errors = oct->iface_rx_stats.dropped_pkts_fifo_full;808 stats->tx_dropped = oct->iface_tx_stats.dropped;809 }810}811 812/**813 * octep_vf_tx_timeout_task - work queue task to Handle Tx queue timeout.814 *815 * @work: pointer to Tx queue timeout work_struct816 *817 * Stop and start the device so that it frees up all queue resources818 * and restarts the queues, that potentially clears a Tx queue timeout819 * condition.820 **/821static void octep_vf_tx_timeout_task(struct work_struct *work)822{823 struct octep_vf_device *oct = container_of(work, struct octep_vf_device,824 tx_timeout_task);825 struct net_device *netdev = oct->netdev;826 827 rtnl_lock();828 if (netif_running(netdev)) {829 octep_vf_stop(netdev);830 octep_vf_open(netdev);831 }832 rtnl_unlock();833 netdev_put(netdev, NULL);834}835 836/**837 * octep_vf_tx_timeout() - Handle Tx Queue timeout.838 *839 * @netdev: pointer to kernel network device.840 * @txqueue: Timed out Tx queue number.841 *842 * Schedule a work to handle Tx queue timeout.843 */844static void octep_vf_tx_timeout(struct net_device *netdev, unsigned int txqueue)845{846 struct octep_vf_device *oct = netdev_priv(netdev);847 848 netdev_hold(netdev, NULL, GFP_ATOMIC);849 schedule_work(&oct->tx_timeout_task);850}851 852static int octep_vf_set_mac(struct net_device *netdev, void *p)853{854 struct octep_vf_device *oct = netdev_priv(netdev);855 struct sockaddr *addr = (struct sockaddr *)p;856 int err;857 858 if (!is_valid_ether_addr(addr->sa_data))859 return -EADDRNOTAVAIL;860 861 err = octep_vf_mbox_set_mac_addr(oct, addr->sa_data);862 if (err)863 return err;864 865 memcpy(oct->mac_addr, addr->sa_data, ETH_ALEN);866 eth_hw_addr_set(netdev, addr->sa_data);867 868 return 0;869}870 871static int octep_vf_change_mtu(struct net_device *netdev, int new_mtu)872{873 struct octep_vf_device *oct = netdev_priv(netdev);874 struct octep_vf_iface_link_info *link_info;875 int err;876 877 link_info = &oct->link_info;878 if (link_info->mtu == new_mtu)879 return 0;880 881 err = octep_vf_mbox_set_mtu(oct, new_mtu);882 if (!err) {883 oct->link_info.mtu = new_mtu;884 WRITE_ONCE(netdev->mtu, new_mtu);885 }886 return err;887}888 889static int octep_vf_set_features(struct net_device *netdev,890 netdev_features_t features)891{892 struct octep_vf_device *oct = netdev_priv(netdev);893 u16 rx_offloads = 0, tx_offloads = 0;894 int err;895 896 /* We only support features received from firmware */897 if ((features & netdev->hw_features) != features)898 return -EINVAL;899 900 if (features & NETIF_F_TSO)901 tx_offloads |= OCTEP_VF_TX_OFFLOAD_TSO;902 903 if (features & NETIF_F_TSO6)904 tx_offloads |= OCTEP_VF_TX_OFFLOAD_TSO;905 906 if (features & NETIF_F_IP_CSUM)907 tx_offloads |= OCTEP_VF_TX_OFFLOAD_CKSUM;908 909 if (features & NETIF_F_IPV6_CSUM)910 tx_offloads |= OCTEP_VF_TX_OFFLOAD_CKSUM;911 912 if (features & NETIF_F_RXCSUM)913 rx_offloads |= OCTEP_VF_RX_OFFLOAD_CKSUM;914 915 err = octep_vf_mbox_set_offloads(oct, tx_offloads, rx_offloads);916 if (!err)917 netdev->features = features;918 919 return err;920}921 922static const struct net_device_ops octep_vf_netdev_ops = {923 .ndo_open = octep_vf_open,924 .ndo_stop = octep_vf_stop,925 .ndo_start_xmit = octep_vf_start_xmit,926 .ndo_get_stats64 = octep_vf_get_stats64,927 .ndo_tx_timeout = octep_vf_tx_timeout,928 .ndo_set_mac_address = octep_vf_set_mac,929 .ndo_change_mtu = octep_vf_change_mtu,930 .ndo_set_features = octep_vf_set_features,931};932 933static const char *octep_vf_devid_to_str(struct octep_vf_device *oct)934{935 switch (oct->chip_id) {936 case OCTEP_PCI_DEVICE_ID_CN93_VF:937 return "CN93XX";938 case OCTEP_PCI_DEVICE_ID_CNF95N_VF:939 return "CNF95N";940 case OCTEP_PCI_DEVICE_ID_CN10KA_VF:941 return "CN10KA";942 case OCTEP_PCI_DEVICE_ID_CNF10KA_VF:943 return "CNF10KA";944 case OCTEP_PCI_DEVICE_ID_CNF10KB_VF:945 return "CNF10KB";946 case OCTEP_PCI_DEVICE_ID_CN10KB_VF:947 return "CN10KB";948 default:949 return "Unsupported";950 }951}952 953/**954 * octep_vf_device_setup() - Setup Octeon Device.955 *956 * @oct: Octeon device private data structure.957 *958 * Setup Octeon device hardware operations, configuration, etc ...959 */960int octep_vf_device_setup(struct octep_vf_device *oct)961{962 struct pci_dev *pdev = oct->pdev;963 964 /* allocate memory for oct->conf */965 oct->conf = kzalloc(sizeof(*oct->conf), GFP_KERNEL);966 if (!oct->conf)967 return -ENOMEM;968 969 /* Map BAR region 0 */970 oct->mmio.hw_addr = ioremap(pci_resource_start(oct->pdev, 0),971 pci_resource_len(oct->pdev, 0));972 if (!oct->mmio.hw_addr) {973 dev_err(&pdev->dev,974 "Failed to remap BAR0; start=0x%llx len=0x%llx\n",975 pci_resource_start(oct->pdev, 0),976 pci_resource_len(oct->pdev, 0));977 goto ioremap_err;978 }979 oct->mmio.mapped = 1;980 981 oct->chip_id = pdev->device;982 oct->rev_id = pdev->revision;983 dev_info(&pdev->dev, "chip_id = 0x%x\n", pdev->device);984 985 switch (oct->chip_id) {986 case OCTEP_PCI_DEVICE_ID_CN93_VF:987 case OCTEP_PCI_DEVICE_ID_CNF95N_VF:988 case OCTEP_PCI_DEVICE_ID_CN98_VF:989 dev_info(&pdev->dev, "Setting up OCTEON %s VF PASS%d.%d\n",990 octep_vf_devid_to_str(oct), OCTEP_VF_MAJOR_REV(oct),991 OCTEP_VF_MINOR_REV(oct));992 octep_vf_device_setup_cn93(oct);993 break;994 case OCTEP_PCI_DEVICE_ID_CNF10KA_VF:995 case OCTEP_PCI_DEVICE_ID_CN10KA_VF:996 case OCTEP_PCI_DEVICE_ID_CNF10KB_VF:997 case OCTEP_PCI_DEVICE_ID_CN10KB_VF:998 dev_info(&pdev->dev, "Setting up OCTEON %s VF PASS%d.%d\n",999 octep_vf_devid_to_str(oct), OCTEP_VF_MAJOR_REV(oct),1000 OCTEP_VF_MINOR_REV(oct));1001 octep_vf_device_setup_cnxk(oct);1002 break;1003 default:1004 dev_err(&pdev->dev, "Unsupported device\n");1005 goto unsupported_dev;1006 }1007 1008 return 0;1009 1010unsupported_dev:1011 iounmap(oct->mmio.hw_addr);1012ioremap_err:1013 kfree(oct->conf);1014 return -EOPNOTSUPP;1015}1016 1017/**1018 * octep_vf_device_cleanup() - Cleanup Octeon Device.1019 *1020 * @oct: Octeon device private data structure.1021 *1022 * Cleanup Octeon device allocated resources.1023 */1024static void octep_vf_device_cleanup(struct octep_vf_device *oct)1025{1026 dev_info(&oct->pdev->dev, "Cleaning up Octeon Device ...\n");1027 1028 if (oct->mmio.mapped)1029 iounmap(oct->mmio.hw_addr);1030 1031 kfree(oct->conf);1032 oct->conf = NULL;1033}1034 1035static int octep_vf_get_mac_addr(struct octep_vf_device *oct, u8 *addr)1036{1037 return octep_vf_mbox_get_mac_addr(oct, addr);1038}1039 1040/**1041 * octep_vf_probe() - Octeon PCI device probe handler.1042 *1043 * @pdev: PCI device structure.1044 * @ent: entry in Octeon PCI device ID table.1045 *1046 * Initializes and enables the Octeon PCI device for network operations.1047 * Initializes Octeon private data structure and registers a network device.1048 */1049static int octep_vf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)1050{1051 struct octep_vf_device *octep_vf_dev;1052 struct net_device *netdev;1053 int err;1054 1055 err = pci_enable_device(pdev);1056 if (err) {1057 dev_err(&pdev->dev, "Failed to enable PCI device\n");1058 return err;1059 }1060 1061 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));1062 if (err) {1063 dev_err(&pdev->dev, "Failed to set DMA mask !!\n");1064 goto disable_pci_device;1065 }1066 1067 err = pci_request_mem_regions(pdev, OCTEP_VF_DRV_NAME);1068 if (err) {1069 dev_err(&pdev->dev, "Failed to map PCI memory regions\n");1070 goto disable_pci_device;1071 }1072 1073 pci_set_master(pdev);1074 1075 netdev = alloc_etherdev_mq(sizeof(struct octep_vf_device),1076 OCTEP_VF_MAX_QUEUES);1077 if (!netdev) {1078 dev_err(&pdev->dev, "Failed to allocate netdev\n");1079 err = -ENOMEM;1080 goto mem_regions_release;1081 }1082 SET_NETDEV_DEV(netdev, &pdev->dev);1083 1084 octep_vf_dev = netdev_priv(netdev);1085 octep_vf_dev->netdev = netdev;1086 octep_vf_dev->pdev = pdev;1087 octep_vf_dev->dev = &pdev->dev;1088 pci_set_drvdata(pdev, octep_vf_dev);1089 1090 err = octep_vf_device_setup(octep_vf_dev);1091 if (err) {1092 dev_err(&pdev->dev, "Device setup failed\n");1093 goto netdevice_free;1094 }1095 INIT_WORK(&octep_vf_dev->tx_timeout_task, octep_vf_tx_timeout_task);1096 1097 netdev->netdev_ops = &octep_vf_netdev_ops;1098 octep_vf_set_ethtool_ops(netdev);1099 netif_carrier_off(netdev);1100 1101 if (octep_vf_setup_mbox(octep_vf_dev)) {1102 dev_err(&pdev->dev, "VF Mailbox setup failed\n");1103 err = -ENOMEM;1104 goto device_cleanup;1105 }1106 1107 if (octep_vf_mbox_version_check(octep_vf_dev)) {1108 dev_err(&pdev->dev, "PF VF Mailbox version mismatch\n");1109 err = -EINVAL;1110 goto delete_mbox;1111 }1112 1113 if (octep_vf_mbox_get_fw_info(octep_vf_dev)) {1114 dev_err(&pdev->dev, "unable to get fw info\n");1115 err = -EINVAL;1116 goto delete_mbox;1117 }1118 1119 netdev->hw_features = NETIF_F_SG;1120 if (OCTEP_VF_TX_IP_CSUM(octep_vf_dev->fw_info.tx_ol_flags))1121 netdev->hw_features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);1122 1123 if (OCTEP_VF_RX_IP_CSUM(octep_vf_dev->fw_info.rx_ol_flags))1124 netdev->hw_features |= NETIF_F_RXCSUM;1125 1126 netdev->min_mtu = OCTEP_VF_MIN_MTU;1127 netdev->max_mtu = OCTEP_VF_MAX_MTU;1128 netdev->mtu = OCTEP_VF_DEFAULT_MTU;1129 1130 if (OCTEP_VF_TX_TSO(octep_vf_dev->fw_info.tx_ol_flags)) {1131 netdev->hw_features |= NETIF_F_TSO;1132 netif_set_tso_max_size(netdev, netdev->max_mtu);1133 }1134 1135 netdev->features |= netdev->hw_features;1136 octep_vf_get_mac_addr(octep_vf_dev, octep_vf_dev->mac_addr);1137 eth_hw_addr_set(netdev, octep_vf_dev->mac_addr);1138 err = register_netdev(netdev);1139 if (err) {1140 dev_err(&pdev->dev, "Failed to register netdev\n");1141 goto delete_mbox;1142 }1143 dev_info(&pdev->dev, "Device probe successful\n");1144 return 0;1145 1146delete_mbox:1147 octep_vf_delete_mbox(octep_vf_dev);1148device_cleanup:1149 octep_vf_device_cleanup(octep_vf_dev);1150netdevice_free:1151 free_netdev(netdev);1152mem_regions_release:1153 pci_release_mem_regions(pdev);1154disable_pci_device:1155 pci_disable_device(pdev);1156 dev_err(&pdev->dev, "Device probe failed\n");1157 return err;1158}1159 1160/**1161 * octep_vf_remove() - Remove Octeon PCI device from driver control.1162 *1163 * @pdev: PCI device structure of the Octeon device.1164 *1165 * Cleanup all resources allocated for the Octeon device.1166 * Unregister from network device and disable the PCI device.1167 */1168static void octep_vf_remove(struct pci_dev *pdev)1169{1170 struct octep_vf_device *oct = pci_get_drvdata(pdev);1171 struct net_device *netdev;1172 1173 if (!oct)1174 return;1175 1176 octep_vf_mbox_dev_remove(oct);1177 cancel_work_sync(&oct->tx_timeout_task);1178 netdev = oct->netdev;1179 if (netdev->reg_state == NETREG_REGISTERED)1180 unregister_netdev(netdev);1181 octep_vf_delete_mbox(oct);1182 octep_vf_device_cleanup(oct);1183 pci_release_mem_regions(pdev);1184 free_netdev(netdev);1185 pci_disable_device(pdev);1186}1187 1188static struct pci_driver octep_vf_driver = {1189 .name = OCTEP_VF_DRV_NAME,1190 .id_table = octep_vf_pci_id_tbl,1191 .probe = octep_vf_probe,1192 .remove = octep_vf_remove,1193};1194 1195/**1196 * octep_vf_init_module() - Module initialization.1197 *1198 * create common resource for the driver and register PCI driver.1199 */1200static int __init octep_vf_init_module(void)1201{1202 int ret;1203 1204 pr_info("%s: Loading %s ...\n", OCTEP_VF_DRV_NAME, OCTEP_VF_DRV_STRING);1205 1206 ret = pci_register_driver(&octep_vf_driver);1207 if (ret < 0) {1208 pr_err("%s: Failed to register PCI driver; err=%d\n",1209 OCTEP_VF_DRV_NAME, ret);1210 return ret;1211 }1212 1213 return ret;1214}1215 1216/**1217 * octep_vf_exit_module() - Module exit routine.1218 *1219 * unregister the driver with PCI subsystem and cleanup common resources.1220 */1221static void __exit octep_vf_exit_module(void)1222{1223 pr_info("%s: Unloading ...\n", OCTEP_VF_DRV_NAME);1224 1225 pci_unregister_driver(&octep_vf_driver);1226 1227 pr_info("%s: Unloading complete\n", OCTEP_VF_DRV_NAME);1228}1229 1230module_init(octep_vf_init_module);1231module_exit(octep_vf_exit_module);1232