5438 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright(c) 2013 - 2018 Intel Corporation. */3 4#include <linux/net/intel/libie/rx.h>5 6#include "iavf.h"7#include "iavf_prototype.h"8/* All iavf tracepoints are defined by the include below, which must9 * be included exactly once across the whole kernel with10 * CREATE_TRACE_POINTS defined11 */12#define CREATE_TRACE_POINTS13#include "iavf_trace.h"14 15static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter);16static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter);17static int iavf_close(struct net_device *netdev);18static void iavf_init_get_resources(struct iavf_adapter *adapter);19static int iavf_check_reset_complete(struct iavf_hw *hw);20 21char iavf_driver_name[] = "iavf";22static const char iavf_driver_string[] =23 "Intel(R) Ethernet Adaptive Virtual Function Network Driver";24 25static const char iavf_copyright[] =26 "Copyright (c) 2013 - 2018 Intel Corporation.";27 28/* iavf_pci_tbl - PCI Device ID Table29 *30 * Wildcard entries (PCI_ANY_ID) should come last31 * Last entry must be all 0s32 *33 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,34 * Class, Class Mask, private data (not used) }35 */36static const struct pci_device_id iavf_pci_tbl[] = {37 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF), 0},38 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF_HV), 0},39 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_X722_VF), 0},40 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_ADAPTIVE_VF), 0},41 /* required last entry */42 {0, }43};44 45MODULE_DEVICE_TABLE(pci, iavf_pci_tbl);46 47MODULE_ALIAS("i40evf");48MODULE_DESCRIPTION("Intel(R) Ethernet Adaptive Virtual Function Network Driver");49MODULE_IMPORT_NS(LIBETH);50MODULE_IMPORT_NS(LIBIE);51MODULE_LICENSE("GPL v2");52 53static const struct net_device_ops iavf_netdev_ops;54 55int iavf_status_to_errno(enum iavf_status status)56{57 switch (status) {58 case IAVF_SUCCESS:59 return 0;60 case IAVF_ERR_PARAM:61 case IAVF_ERR_MAC_TYPE:62 case IAVF_ERR_INVALID_MAC_ADDR:63 case IAVF_ERR_INVALID_LINK_SETTINGS:64 case IAVF_ERR_INVALID_PD_ID:65 case IAVF_ERR_INVALID_QP_ID:66 case IAVF_ERR_INVALID_CQ_ID:67 case IAVF_ERR_INVALID_CEQ_ID:68 case IAVF_ERR_INVALID_AEQ_ID:69 case IAVF_ERR_INVALID_SIZE:70 case IAVF_ERR_INVALID_ARP_INDEX:71 case IAVF_ERR_INVALID_FPM_FUNC_ID:72 case IAVF_ERR_QP_INVALID_MSG_SIZE:73 case IAVF_ERR_INVALID_FRAG_COUNT:74 case IAVF_ERR_INVALID_ALIGNMENT:75 case IAVF_ERR_INVALID_PUSH_PAGE_INDEX:76 case IAVF_ERR_INVALID_IMM_DATA_SIZE:77 case IAVF_ERR_INVALID_VF_ID:78 case IAVF_ERR_INVALID_HMCFN_ID:79 case IAVF_ERR_INVALID_PBLE_INDEX:80 case IAVF_ERR_INVALID_SD_INDEX:81 case IAVF_ERR_INVALID_PAGE_DESC_INDEX:82 case IAVF_ERR_INVALID_SD_TYPE:83 case IAVF_ERR_INVALID_HMC_OBJ_INDEX:84 case IAVF_ERR_INVALID_HMC_OBJ_COUNT:85 case IAVF_ERR_INVALID_SRQ_ARM_LIMIT:86 return -EINVAL;87 case IAVF_ERR_NVM:88 case IAVF_ERR_NVM_CHECKSUM:89 case IAVF_ERR_PHY:90 case IAVF_ERR_CONFIG:91 case IAVF_ERR_UNKNOWN_PHY:92 case IAVF_ERR_LINK_SETUP:93 case IAVF_ERR_ADAPTER_STOPPED:94 case IAVF_ERR_PRIMARY_REQUESTS_PENDING:95 case IAVF_ERR_AUTONEG_NOT_COMPLETE:96 case IAVF_ERR_RESET_FAILED:97 case IAVF_ERR_BAD_PTR:98 case IAVF_ERR_SWFW_SYNC:99 case IAVF_ERR_QP_TOOMANY_WRS_POSTED:100 case IAVF_ERR_QUEUE_EMPTY:101 case IAVF_ERR_FLUSHED_QUEUE:102 case IAVF_ERR_OPCODE_MISMATCH:103 case IAVF_ERR_CQP_COMPL_ERROR:104 case IAVF_ERR_BACKING_PAGE_ERROR:105 case IAVF_ERR_NO_PBLCHUNKS_AVAILABLE:106 case IAVF_ERR_MEMCPY_FAILED:107 case IAVF_ERR_SRQ_ENABLED:108 case IAVF_ERR_ADMIN_QUEUE_ERROR:109 case IAVF_ERR_ADMIN_QUEUE_FULL:110 case IAVF_ERR_BAD_RDMA_CQE:111 case IAVF_ERR_NVM_BLANK_MODE:112 case IAVF_ERR_PE_DOORBELL_NOT_ENABLED:113 case IAVF_ERR_DIAG_TEST_FAILED:114 case IAVF_ERR_FIRMWARE_API_VERSION:115 case IAVF_ERR_ADMIN_QUEUE_CRITICAL_ERROR:116 return -EIO;117 case IAVF_ERR_DEVICE_NOT_SUPPORTED:118 return -ENODEV;119 case IAVF_ERR_NO_AVAILABLE_VSI:120 case IAVF_ERR_RING_FULL:121 return -ENOSPC;122 case IAVF_ERR_NO_MEMORY:123 return -ENOMEM;124 case IAVF_ERR_TIMEOUT:125 case IAVF_ERR_ADMIN_QUEUE_TIMEOUT:126 return -ETIMEDOUT;127 case IAVF_ERR_NOT_IMPLEMENTED:128 case IAVF_NOT_SUPPORTED:129 return -EOPNOTSUPP;130 case IAVF_ERR_ADMIN_QUEUE_NO_WORK:131 return -EALREADY;132 case IAVF_ERR_NOT_READY:133 return -EBUSY;134 case IAVF_ERR_BUF_TOO_SHORT:135 return -EMSGSIZE;136 }137 138 return -EIO;139}140 141int virtchnl_status_to_errno(enum virtchnl_status_code v_status)142{143 switch (v_status) {144 case VIRTCHNL_STATUS_SUCCESS:145 return 0;146 case VIRTCHNL_STATUS_ERR_PARAM:147 case VIRTCHNL_STATUS_ERR_INVALID_VF_ID:148 return -EINVAL;149 case VIRTCHNL_STATUS_ERR_NO_MEMORY:150 return -ENOMEM;151 case VIRTCHNL_STATUS_ERR_OPCODE_MISMATCH:152 case VIRTCHNL_STATUS_ERR_CQP_COMPL_ERROR:153 case VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR:154 return -EIO;155 case VIRTCHNL_STATUS_ERR_NOT_SUPPORTED:156 return -EOPNOTSUPP;157 }158 159 return -EIO;160}161 162/**163 * iavf_pdev_to_adapter - go from pci_dev to adapter164 * @pdev: pci_dev pointer165 */166static struct iavf_adapter *iavf_pdev_to_adapter(struct pci_dev *pdev)167{168 return netdev_priv(pci_get_drvdata(pdev));169}170 171/**172 * iavf_is_reset_in_progress - Check if a reset is in progress173 * @adapter: board private structure174 */175static bool iavf_is_reset_in_progress(struct iavf_adapter *adapter)176{177 if (adapter->state == __IAVF_RESETTING ||178 adapter->flags & (IAVF_FLAG_RESET_PENDING |179 IAVF_FLAG_RESET_NEEDED))180 return true;181 182 return false;183}184 185/**186 * iavf_wait_for_reset - Wait for reset to finish.187 * @adapter: board private structure188 *189 * Returns 0 if reset finished successfully, negative on timeout or interrupt.190 */191int iavf_wait_for_reset(struct iavf_adapter *adapter)192{193 int ret = wait_event_interruptible_timeout(adapter->reset_waitqueue,194 !iavf_is_reset_in_progress(adapter),195 msecs_to_jiffies(5000));196 197 /* If ret < 0 then it means wait was interrupted.198 * If ret == 0 then it means we got a timeout while waiting199 * for reset to finish.200 * If ret > 0 it means reset has finished.201 */202 if (ret > 0)203 return 0;204 else if (ret < 0)205 return -EINTR;206 else207 return -EBUSY;208}209 210/**211 * iavf_allocate_dma_mem_d - OS specific memory alloc for shared code212 * @hw: pointer to the HW structure213 * @mem: ptr to mem struct to fill out214 * @size: size of memory requested215 * @alignment: what to align the allocation to216 **/217enum iavf_status iavf_allocate_dma_mem_d(struct iavf_hw *hw,218 struct iavf_dma_mem *mem,219 u64 size, u32 alignment)220{221 struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back;222 223 if (!mem)224 return IAVF_ERR_PARAM;225 226 mem->size = ALIGN(size, alignment);227 mem->va = dma_alloc_coherent(&adapter->pdev->dev, mem->size,228 (dma_addr_t *)&mem->pa, GFP_KERNEL);229 if (mem->va)230 return 0;231 else232 return IAVF_ERR_NO_MEMORY;233}234 235/**236 * iavf_free_dma_mem - wrapper for DMA memory freeing237 * @hw: pointer to the HW structure238 * @mem: ptr to mem struct to free239 **/240enum iavf_status iavf_free_dma_mem(struct iavf_hw *hw, struct iavf_dma_mem *mem)241{242 struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back;243 244 if (!mem || !mem->va)245 return IAVF_ERR_PARAM;246 dma_free_coherent(&adapter->pdev->dev, mem->size,247 mem->va, (dma_addr_t)mem->pa);248 return 0;249}250 251/**252 * iavf_allocate_virt_mem - virt memory alloc wrapper253 * @hw: pointer to the HW structure254 * @mem: ptr to mem struct to fill out255 * @size: size of memory requested256 **/257enum iavf_status iavf_allocate_virt_mem(struct iavf_hw *hw,258 struct iavf_virt_mem *mem, u32 size)259{260 if (!mem)261 return IAVF_ERR_PARAM;262 263 mem->size = size;264 mem->va = kzalloc(size, GFP_KERNEL);265 266 if (mem->va)267 return 0;268 else269 return IAVF_ERR_NO_MEMORY;270}271 272/**273 * iavf_free_virt_mem - virt memory free wrapper274 * @hw: pointer to the HW structure275 * @mem: ptr to mem struct to free276 **/277void iavf_free_virt_mem(struct iavf_hw *hw, struct iavf_virt_mem *mem)278{279 kfree(mem->va);280}281 282/**283 * iavf_schedule_reset - Set the flags and schedule a reset event284 * @adapter: board private structure285 * @flags: IAVF_FLAG_RESET_PENDING or IAVF_FLAG_RESET_NEEDED286 **/287void iavf_schedule_reset(struct iavf_adapter *adapter, u64 flags)288{289 if (!test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section) &&290 !(adapter->flags &291 (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED))) {292 adapter->flags |= flags;293 queue_work(adapter->wq, &adapter->reset_task);294 }295}296 297/**298 * iavf_schedule_aq_request - Set the flags and schedule aq request299 * @adapter: board private structure300 * @flags: requested aq flags301 **/302void iavf_schedule_aq_request(struct iavf_adapter *adapter, u64 flags)303{304 adapter->aq_required |= flags;305 mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);306}307 308/**309 * iavf_tx_timeout - Respond to a Tx Hang310 * @netdev: network interface device structure311 * @txqueue: queue number that is timing out312 **/313static void iavf_tx_timeout(struct net_device *netdev, unsigned int txqueue)314{315 struct iavf_adapter *adapter = netdev_priv(netdev);316 317 adapter->tx_timeout_count++;318 iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED);319}320 321/**322 * iavf_misc_irq_disable - Mask off interrupt generation on the NIC323 * @adapter: board private structure324 **/325static void iavf_misc_irq_disable(struct iavf_adapter *adapter)326{327 struct iavf_hw *hw = &adapter->hw;328 329 if (!adapter->msix_entries)330 return;331 332 wr32(hw, IAVF_VFINT_DYN_CTL01, 0);333 334 iavf_flush(hw);335 336 synchronize_irq(adapter->msix_entries[0].vector);337}338 339/**340 * iavf_misc_irq_enable - Enable default interrupt generation settings341 * @adapter: board private structure342 **/343static void iavf_misc_irq_enable(struct iavf_adapter *adapter)344{345 struct iavf_hw *hw = &adapter->hw;346 347 wr32(hw, IAVF_VFINT_DYN_CTL01, IAVF_VFINT_DYN_CTL01_INTENA_MASK |348 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK);349 wr32(hw, IAVF_VFINT_ICR0_ENA1, IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK);350 351 iavf_flush(hw);352}353 354/**355 * iavf_irq_disable - Mask off interrupt generation on the NIC356 * @adapter: board private structure357 **/358static void iavf_irq_disable(struct iavf_adapter *adapter)359{360 int i;361 struct iavf_hw *hw = &adapter->hw;362 363 if (!adapter->msix_entries)364 return;365 366 for (i = 1; i < adapter->num_msix_vectors; i++) {367 wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1), 0);368 synchronize_irq(adapter->msix_entries[i].vector);369 }370 iavf_flush(hw);371}372 373/**374 * iavf_irq_enable_queues - Enable interrupt for all queues375 * @adapter: board private structure376 **/377static void iavf_irq_enable_queues(struct iavf_adapter *adapter)378{379 struct iavf_hw *hw = &adapter->hw;380 int i;381 382 for (i = 1; i < adapter->num_msix_vectors; i++) {383 wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1),384 IAVF_VFINT_DYN_CTLN1_INTENA_MASK |385 IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK);386 }387}388 389/**390 * iavf_irq_enable - Enable default interrupt generation settings391 * @adapter: board private structure392 * @flush: boolean value whether to run rd32()393 **/394void iavf_irq_enable(struct iavf_adapter *adapter, bool flush)395{396 struct iavf_hw *hw = &adapter->hw;397 398 iavf_misc_irq_enable(adapter);399 iavf_irq_enable_queues(adapter);400 401 if (flush)402 iavf_flush(hw);403}404 405/**406 * iavf_msix_aq - Interrupt handler for vector 0407 * @irq: interrupt number408 * @data: pointer to netdev409 **/410static irqreturn_t iavf_msix_aq(int irq, void *data)411{412 struct net_device *netdev = data;413 struct iavf_adapter *adapter = netdev_priv(netdev);414 struct iavf_hw *hw = &adapter->hw;415 416 /* handle non-queue interrupts, these reads clear the registers */417 rd32(hw, IAVF_VFINT_ICR01);418 rd32(hw, IAVF_VFINT_ICR0_ENA1);419 420 if (adapter->state != __IAVF_REMOVE)421 /* schedule work on the private workqueue */422 queue_work(adapter->wq, &adapter->adminq_task);423 424 return IRQ_HANDLED;425}426 427/**428 * iavf_msix_clean_rings - MSIX mode Interrupt Handler429 * @irq: interrupt number430 * @data: pointer to a q_vector431 **/432static irqreturn_t iavf_msix_clean_rings(int irq, void *data)433{434 struct iavf_q_vector *q_vector = data;435 436 if (!q_vector->tx.ring && !q_vector->rx.ring)437 return IRQ_HANDLED;438 439 napi_schedule_irqoff(&q_vector->napi);440 441 return IRQ_HANDLED;442}443 444/**445 * iavf_map_vector_to_rxq - associate irqs with rx queues446 * @adapter: board private structure447 * @v_idx: interrupt number448 * @r_idx: queue number449 **/450static void451iavf_map_vector_to_rxq(struct iavf_adapter *adapter, int v_idx, int r_idx)452{453 struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx];454 struct iavf_ring *rx_ring = &adapter->rx_rings[r_idx];455 struct iavf_hw *hw = &adapter->hw;456 457 rx_ring->q_vector = q_vector;458 rx_ring->next = q_vector->rx.ring;459 rx_ring->vsi = &adapter->vsi;460 q_vector->rx.ring = rx_ring;461 q_vector->rx.count++;462 q_vector->rx.next_update = jiffies + 1;463 q_vector->rx.target_itr = ITR_TO_REG(rx_ring->itr_setting);464 q_vector->ring_mask |= BIT(r_idx);465 wr32(hw, IAVF_VFINT_ITRN1(IAVF_RX_ITR, q_vector->reg_idx),466 q_vector->rx.current_itr >> 1);467 q_vector->rx.current_itr = q_vector->rx.target_itr;468}469 470/**471 * iavf_map_vector_to_txq - associate irqs with tx queues472 * @adapter: board private structure473 * @v_idx: interrupt number474 * @t_idx: queue number475 **/476static void477iavf_map_vector_to_txq(struct iavf_adapter *adapter, int v_idx, int t_idx)478{479 struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx];480 struct iavf_ring *tx_ring = &adapter->tx_rings[t_idx];481 struct iavf_hw *hw = &adapter->hw;482 483 tx_ring->q_vector = q_vector;484 tx_ring->next = q_vector->tx.ring;485 tx_ring->vsi = &adapter->vsi;486 q_vector->tx.ring = tx_ring;487 q_vector->tx.count++;488 q_vector->tx.next_update = jiffies + 1;489 q_vector->tx.target_itr = ITR_TO_REG(tx_ring->itr_setting);490 q_vector->num_ringpairs++;491 wr32(hw, IAVF_VFINT_ITRN1(IAVF_TX_ITR, q_vector->reg_idx),492 q_vector->tx.target_itr >> 1);493 q_vector->tx.current_itr = q_vector->tx.target_itr;494}495 496/**497 * iavf_map_rings_to_vectors - Maps descriptor rings to vectors498 * @adapter: board private structure to initialize499 *500 * This function maps descriptor rings to the queue-specific vectors501 * we were allotted through the MSI-X enabling code. Ideally, we'd have502 * one vector per ring/queue, but on a constrained vector budget, we503 * group the rings as "efficiently" as possible. You would add new504 * mapping configurations in here.505 **/506static void iavf_map_rings_to_vectors(struct iavf_adapter *adapter)507{508 int rings_remaining = adapter->num_active_queues;509 int ridx = 0, vidx = 0;510 int q_vectors;511 512 q_vectors = adapter->num_msix_vectors - NONQ_VECS;513 514 for (; ridx < rings_remaining; ridx++) {515 iavf_map_vector_to_rxq(adapter, vidx, ridx);516 iavf_map_vector_to_txq(adapter, vidx, ridx);517 518 /* In the case where we have more queues than vectors, continue519 * round-robin on vectors until all queues are mapped.520 */521 if (++vidx >= q_vectors)522 vidx = 0;523 }524 525 adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS;526}527 528/**529 * iavf_irq_affinity_notify - Callback for affinity changes530 * @notify: context as to what irq was changed531 * @mask: the new affinity mask532 *533 * This is a callback function used by the irq_set_affinity_notifier function534 * so that we may register to receive changes to the irq affinity masks.535 **/536static void iavf_irq_affinity_notify(struct irq_affinity_notify *notify,537 const cpumask_t *mask)538{539 struct iavf_q_vector *q_vector =540 container_of(notify, struct iavf_q_vector, affinity_notify);541 542 cpumask_copy(&q_vector->affinity_mask, mask);543}544 545/**546 * iavf_irq_affinity_release - Callback for affinity notifier release547 * @ref: internal core kernel usage548 *549 * This is a callback function used by the irq_set_affinity_notifier function550 * to inform the current notification subscriber that they will no longer551 * receive notifications.552 **/553static void iavf_irq_affinity_release(struct kref *ref) {}554 555/**556 * iavf_request_traffic_irqs - Initialize MSI-X interrupts557 * @adapter: board private structure558 * @basename: device basename559 *560 * Allocates MSI-X vectors for tx and rx handling, and requests561 * interrupts from the kernel.562 **/563static int564iavf_request_traffic_irqs(struct iavf_adapter *adapter, char *basename)565{566 unsigned int vector, q_vectors;567 unsigned int rx_int_idx = 0, tx_int_idx = 0;568 int irq_num, err;569 int cpu;570 571 iavf_irq_disable(adapter);572 /* Decrement for Other and TCP Timer vectors */573 q_vectors = adapter->num_msix_vectors - NONQ_VECS;574 575 for (vector = 0; vector < q_vectors; vector++) {576 struct iavf_q_vector *q_vector = &adapter->q_vectors[vector];577 578 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;579 580 if (q_vector->tx.ring && q_vector->rx.ring) {581 snprintf(q_vector->name, sizeof(q_vector->name),582 "iavf-%s-TxRx-%u", basename, rx_int_idx++);583 tx_int_idx++;584 } else if (q_vector->rx.ring) {585 snprintf(q_vector->name, sizeof(q_vector->name),586 "iavf-%s-rx-%u", basename, rx_int_idx++);587 } else if (q_vector->tx.ring) {588 snprintf(q_vector->name, sizeof(q_vector->name),589 "iavf-%s-tx-%u", basename, tx_int_idx++);590 } else {591 /* skip this unused q_vector */592 continue;593 }594 err = request_irq(irq_num,595 iavf_msix_clean_rings,596 0,597 q_vector->name,598 q_vector);599 if (err) {600 dev_info(&adapter->pdev->dev,601 "Request_irq failed, error: %d\n", err);602 goto free_queue_irqs;603 }604 /* register for affinity change notifications */605 q_vector->affinity_notify.notify = iavf_irq_affinity_notify;606 q_vector->affinity_notify.release =607 iavf_irq_affinity_release;608 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);609 /* Spread the IRQ affinity hints across online CPUs. Note that610 * get_cpu_mask returns a mask with a permanent lifetime so611 * it's safe to use as a hint for irq_update_affinity_hint.612 */613 cpu = cpumask_local_spread(q_vector->v_idx, -1);614 irq_update_affinity_hint(irq_num, get_cpu_mask(cpu));615 }616 617 return 0;618 619free_queue_irqs:620 while (vector) {621 vector--;622 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;623 irq_set_affinity_notifier(irq_num, NULL);624 irq_update_affinity_hint(irq_num, NULL);625 free_irq(irq_num, &adapter->q_vectors[vector]);626 }627 return err;628}629 630/**631 * iavf_request_misc_irq - Initialize MSI-X interrupts632 * @adapter: board private structure633 *634 * Allocates MSI-X vector 0 and requests interrupts from the kernel. This635 * vector is only for the admin queue, and stays active even when the netdev636 * is closed.637 **/638static int iavf_request_misc_irq(struct iavf_adapter *adapter)639{640 struct net_device *netdev = adapter->netdev;641 int err;642 643 snprintf(adapter->misc_vector_name,644 sizeof(adapter->misc_vector_name) - 1, "iavf-%s:mbx",645 dev_name(&adapter->pdev->dev));646 err = request_irq(adapter->msix_entries[0].vector,647 &iavf_msix_aq, 0,648 adapter->misc_vector_name, netdev);649 if (err) {650 dev_err(&adapter->pdev->dev,651 "request_irq for %s failed: %d\n",652 adapter->misc_vector_name, err);653 free_irq(adapter->msix_entries[0].vector, netdev);654 }655 return err;656}657 658/**659 * iavf_free_traffic_irqs - Free MSI-X interrupts660 * @adapter: board private structure661 *662 * Frees all MSI-X vectors other than 0.663 **/664static void iavf_free_traffic_irqs(struct iavf_adapter *adapter)665{666 int vector, irq_num, q_vectors;667 668 if (!adapter->msix_entries)669 return;670 671 q_vectors = adapter->num_msix_vectors - NONQ_VECS;672 673 for (vector = 0; vector < q_vectors; vector++) {674 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;675 irq_set_affinity_notifier(irq_num, NULL);676 irq_update_affinity_hint(irq_num, NULL);677 free_irq(irq_num, &adapter->q_vectors[vector]);678 }679}680 681/**682 * iavf_free_misc_irq - Free MSI-X miscellaneous vector683 * @adapter: board private structure684 *685 * Frees MSI-X vector 0.686 **/687static void iavf_free_misc_irq(struct iavf_adapter *adapter)688{689 struct net_device *netdev = adapter->netdev;690 691 if (!adapter->msix_entries)692 return;693 694 free_irq(adapter->msix_entries[0].vector, netdev);695}696 697/**698 * iavf_configure_tx - Configure Transmit Unit after Reset699 * @adapter: board private structure700 *701 * Configure the Tx unit of the MAC after a reset.702 **/703static void iavf_configure_tx(struct iavf_adapter *adapter)704{705 struct iavf_hw *hw = &adapter->hw;706 int i;707 708 for (i = 0; i < adapter->num_active_queues; i++)709 adapter->tx_rings[i].tail = hw->hw_addr + IAVF_QTX_TAIL1(i);710}711 712/**713 * iavf_configure_rx - Configure Receive Unit after Reset714 * @adapter: board private structure715 *716 * Configure the Rx unit of the MAC after a reset.717 **/718static void iavf_configure_rx(struct iavf_adapter *adapter)719{720 struct iavf_hw *hw = &adapter->hw;721 722 for (u32 i = 0; i < adapter->num_active_queues; i++)723 adapter->rx_rings[i].tail = hw->hw_addr + IAVF_QRX_TAIL1(i);724}725 726/**727 * iavf_find_vlan - Search filter list for specific vlan filter728 * @adapter: board private structure729 * @vlan: vlan tag730 *731 * Returns ptr to the filter object or NULL. Must be called while holding the732 * mac_vlan_list_lock.733 **/734static struct735iavf_vlan_filter *iavf_find_vlan(struct iavf_adapter *adapter,736 struct iavf_vlan vlan)737{738 struct iavf_vlan_filter *f;739 740 list_for_each_entry(f, &adapter->vlan_filter_list, list) {741 if (f->vlan.vid == vlan.vid &&742 f->vlan.tpid == vlan.tpid)743 return f;744 }745 746 return NULL;747}748 749/**750 * iavf_add_vlan - Add a vlan filter to the list751 * @adapter: board private structure752 * @vlan: VLAN tag753 *754 * Returns ptr to the filter object or NULL when no memory available.755 **/756static struct757iavf_vlan_filter *iavf_add_vlan(struct iavf_adapter *adapter,758 struct iavf_vlan vlan)759{760 struct iavf_vlan_filter *f = NULL;761 762 spin_lock_bh(&adapter->mac_vlan_list_lock);763 764 f = iavf_find_vlan(adapter, vlan);765 if (!f) {766 f = kzalloc(sizeof(*f), GFP_ATOMIC);767 if (!f)768 goto clearout;769 770 f->vlan = vlan;771 772 list_add_tail(&f->list, &adapter->vlan_filter_list);773 f->state = IAVF_VLAN_ADD;774 adapter->num_vlan_filters++;775 iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_ADD_VLAN_FILTER);776 }777 778clearout:779 spin_unlock_bh(&adapter->mac_vlan_list_lock);780 return f;781}782 783/**784 * iavf_del_vlan - Remove a vlan filter from the list785 * @adapter: board private structure786 * @vlan: VLAN tag787 **/788static void iavf_del_vlan(struct iavf_adapter *adapter, struct iavf_vlan vlan)789{790 struct iavf_vlan_filter *f;791 792 spin_lock_bh(&adapter->mac_vlan_list_lock);793 794 f = iavf_find_vlan(adapter, vlan);795 if (f) {796 f->state = IAVF_VLAN_REMOVE;797 iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_DEL_VLAN_FILTER);798 }799 800 spin_unlock_bh(&adapter->mac_vlan_list_lock);801}802 803/**804 * iavf_restore_filters805 * @adapter: board private structure806 *807 * Restore existing non MAC filters when VF netdev comes back up808 **/809static void iavf_restore_filters(struct iavf_adapter *adapter)810{811 struct iavf_vlan_filter *f;812 813 /* re-add all VLAN filters */814 spin_lock_bh(&adapter->mac_vlan_list_lock);815 816 list_for_each_entry(f, &adapter->vlan_filter_list, list) {817 if (f->state == IAVF_VLAN_INACTIVE)818 f->state = IAVF_VLAN_ADD;819 }820 821 spin_unlock_bh(&adapter->mac_vlan_list_lock);822 adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER;823}824 825/**826 * iavf_get_num_vlans_added - get number of VLANs added827 * @adapter: board private structure828 */829u16 iavf_get_num_vlans_added(struct iavf_adapter *adapter)830{831 return adapter->num_vlan_filters;832}833 834/**835 * iavf_get_max_vlans_allowed - get maximum VLANs allowed for this VF836 * @adapter: board private structure837 *838 * This depends on the negotiated VLAN capability. For VIRTCHNL_VF_OFFLOAD_VLAN,839 * do not impose a limit as that maintains current behavior and for840 * VIRTCHNL_VF_OFFLOAD_VLAN_V2, use the maximum allowed sent from the PF.841 **/842static u16 iavf_get_max_vlans_allowed(struct iavf_adapter *adapter)843{844 /* don't impose any limit for VIRTCHNL_VF_OFFLOAD_VLAN since there has845 * never been a limit on the VF driver side846 */847 if (VLAN_ALLOWED(adapter))848 return VLAN_N_VID;849 else if (VLAN_V2_ALLOWED(adapter))850 return adapter->vlan_v2_caps.filtering.max_filters;851 852 return 0;853}854 855/**856 * iavf_max_vlans_added - check if maximum VLANs allowed already exist857 * @adapter: board private structure858 **/859static bool iavf_max_vlans_added(struct iavf_adapter *adapter)860{861 if (iavf_get_num_vlans_added(adapter) <862 iavf_get_max_vlans_allowed(adapter))863 return false;864 865 return true;866}867 868/**869 * iavf_vlan_rx_add_vid - Add a VLAN filter to a device870 * @netdev: network device struct871 * @proto: unused protocol data872 * @vid: VLAN tag873 **/874static int iavf_vlan_rx_add_vid(struct net_device *netdev,875 __always_unused __be16 proto, u16 vid)876{877 struct iavf_adapter *adapter = netdev_priv(netdev);878 879 /* Do not track VLAN 0 filter, always added by the PF on VF init */880 if (!vid)881 return 0;882 883 if (!VLAN_FILTERING_ALLOWED(adapter))884 return -EIO;885 886 if (iavf_max_vlans_added(adapter)) {887 netdev_err(netdev, "Max allowed VLAN filters %u. Remove existing VLANs or disable filtering via Ethtool if supported.\n",888 iavf_get_max_vlans_allowed(adapter));889 return -EIO;890 }891 892 if (!iavf_add_vlan(adapter, IAVF_VLAN(vid, be16_to_cpu(proto))))893 return -ENOMEM;894 895 return 0;896}897 898/**899 * iavf_vlan_rx_kill_vid - Remove a VLAN filter from a device900 * @netdev: network device struct901 * @proto: unused protocol data902 * @vid: VLAN tag903 **/904static int iavf_vlan_rx_kill_vid(struct net_device *netdev,905 __always_unused __be16 proto, u16 vid)906{907 struct iavf_adapter *adapter = netdev_priv(netdev);908 909 /* We do not track VLAN 0 filter */910 if (!vid)911 return 0;912 913 iavf_del_vlan(adapter, IAVF_VLAN(vid, be16_to_cpu(proto)));914 return 0;915}916 917/**918 * iavf_find_filter - Search filter list for specific mac filter919 * @adapter: board private structure920 * @macaddr: the MAC address921 *922 * Returns ptr to the filter object or NULL. Must be called while holding the923 * mac_vlan_list_lock.924 **/925static struct926iavf_mac_filter *iavf_find_filter(struct iavf_adapter *adapter,927 const u8 *macaddr)928{929 struct iavf_mac_filter *f;930 931 if (!macaddr)932 return NULL;933 934 list_for_each_entry(f, &adapter->mac_filter_list, list) {935 if (ether_addr_equal(macaddr, f->macaddr))936 return f;937 }938 return NULL;939}940 941/**942 * iavf_add_filter - Add a mac filter to the filter list943 * @adapter: board private structure944 * @macaddr: the MAC address945 *946 * Returns ptr to the filter object or NULL when no memory available.947 **/948struct iavf_mac_filter *iavf_add_filter(struct iavf_adapter *adapter,949 const u8 *macaddr)950{951 struct iavf_mac_filter *f;952 953 if (!macaddr)954 return NULL;955 956 f = iavf_find_filter(adapter, macaddr);957 if (!f) {958 f = kzalloc(sizeof(*f), GFP_ATOMIC);959 if (!f)960 return f;961 962 ether_addr_copy(f->macaddr, macaddr);963 964 list_add_tail(&f->list, &adapter->mac_filter_list);965 f->add = true;966 f->add_handled = false;967 f->is_new_mac = true;968 f->is_primary = ether_addr_equal(macaddr, adapter->hw.mac.addr);969 adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;970 } else {971 f->remove = false;972 }973 974 return f;975}976 977/**978 * iavf_replace_primary_mac - Replace current primary address979 * @adapter: board private structure980 * @new_mac: new MAC address to be applied981 *982 * Replace current dev_addr and send request to PF for removal of previous983 * primary MAC address filter and addition of new primary MAC filter.984 * Return 0 for success, -ENOMEM for failure.985 *986 * Do not call this with mac_vlan_list_lock!987 **/988static int iavf_replace_primary_mac(struct iavf_adapter *adapter,989 const u8 *new_mac)990{991 struct iavf_hw *hw = &adapter->hw;992 struct iavf_mac_filter *new_f;993 struct iavf_mac_filter *old_f;994 995 spin_lock_bh(&adapter->mac_vlan_list_lock);996 997 new_f = iavf_add_filter(adapter, new_mac);998 if (!new_f) {999 spin_unlock_bh(&adapter->mac_vlan_list_lock);1000 return -ENOMEM;1001 }1002 1003 old_f = iavf_find_filter(adapter, hw->mac.addr);1004 if (old_f) {1005 old_f->is_primary = false;1006 old_f->remove = true;1007 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;1008 }1009 /* Always send the request to add if changing primary MAC,1010 * even if filter is already present on the list1011 */1012 new_f->is_primary = true;1013 new_f->add = true;1014 ether_addr_copy(hw->mac.addr, new_mac);1015 1016 spin_unlock_bh(&adapter->mac_vlan_list_lock);1017 1018 /* schedule the watchdog task to immediately process the request */1019 iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_ADD_MAC_FILTER);1020 return 0;1021}1022 1023/**1024 * iavf_is_mac_set_handled - wait for a response to set MAC from PF1025 * @netdev: network interface device structure1026 * @macaddr: MAC address to set1027 *1028 * Returns true on success, false on failure1029 */1030static bool iavf_is_mac_set_handled(struct net_device *netdev,1031 const u8 *macaddr)1032{1033 struct iavf_adapter *adapter = netdev_priv(netdev);1034 struct iavf_mac_filter *f;1035 bool ret = false;1036 1037 spin_lock_bh(&adapter->mac_vlan_list_lock);1038 1039 f = iavf_find_filter(adapter, macaddr);1040 1041 if (!f || (!f->add && f->add_handled))1042 ret = true;1043 1044 spin_unlock_bh(&adapter->mac_vlan_list_lock);1045 1046 return ret;1047}1048 1049/**1050 * iavf_set_mac - NDO callback to set port MAC address1051 * @netdev: network interface device structure1052 * @p: pointer to an address structure1053 *1054 * Returns 0 on success, negative on failure1055 */1056static int iavf_set_mac(struct net_device *netdev, void *p)1057{1058 struct iavf_adapter *adapter = netdev_priv(netdev);1059 struct sockaddr *addr = p;1060 int ret;1061 1062 if (!is_valid_ether_addr(addr->sa_data))1063 return -EADDRNOTAVAIL;1064 1065 ret = iavf_replace_primary_mac(adapter, addr->sa_data);1066 1067 if (ret)1068 return ret;1069 1070 ret = wait_event_interruptible_timeout(adapter->vc_waitqueue,1071 iavf_is_mac_set_handled(netdev, addr->sa_data),1072 msecs_to_jiffies(2500));1073 1074 /* If ret < 0 then it means wait was interrupted.1075 * If ret == 0 then it means we got a timeout.1076 * else it means we got response for set MAC from PF,1077 * check if netdev MAC was updated to requested MAC,1078 * if yes then set MAC succeeded otherwise it failed return -EACCES1079 */1080 if (ret < 0)1081 return ret;1082 1083 if (!ret)1084 return -EAGAIN;1085 1086 if (!ether_addr_equal(netdev->dev_addr, addr->sa_data))1087 return -EACCES;1088 1089 return 0;1090}1091 1092/**1093 * iavf_addr_sync - Callback for dev_(mc|uc)_sync to add address1094 * @netdev: the netdevice1095 * @addr: address to add1096 *1097 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call1098 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.1099 */1100static int iavf_addr_sync(struct net_device *netdev, const u8 *addr)1101{1102 struct iavf_adapter *adapter = netdev_priv(netdev);1103 1104 if (iavf_add_filter(adapter, addr))1105 return 0;1106 else1107 return -ENOMEM;1108}1109 1110/**1111 * iavf_addr_unsync - Callback for dev_(mc|uc)_sync to remove address1112 * @netdev: the netdevice1113 * @addr: address to add1114 *1115 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call1116 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.1117 */1118static int iavf_addr_unsync(struct net_device *netdev, const u8 *addr)1119{1120 struct iavf_adapter *adapter = netdev_priv(netdev);1121 struct iavf_mac_filter *f;1122 1123 /* Under some circumstances, we might receive a request to delete1124 * our own device address from our uc list. Because we store the1125 * device address in the VSI's MAC/VLAN filter list, we need to ignore1126 * such requests and not delete our device address from this list.1127 */1128 if (ether_addr_equal(addr, netdev->dev_addr))1129 return 0;1130 1131 f = iavf_find_filter(adapter, addr);1132 if (f) {1133 f->remove = true;1134 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;1135 }1136 return 0;1137}1138 1139/**1140 * iavf_promiscuous_mode_changed - check if promiscuous mode bits changed1141 * @adapter: device specific adapter1142 */1143bool iavf_promiscuous_mode_changed(struct iavf_adapter *adapter)1144{1145 return (adapter->current_netdev_promisc_flags ^ adapter->netdev->flags) &1146 (IFF_PROMISC | IFF_ALLMULTI);1147}1148 1149/**1150 * iavf_set_rx_mode - NDO callback to set the netdev filters1151 * @netdev: network interface device structure1152 **/1153static void iavf_set_rx_mode(struct net_device *netdev)1154{1155 struct iavf_adapter *adapter = netdev_priv(netdev);1156 1157 spin_lock_bh(&adapter->mac_vlan_list_lock);1158 __dev_uc_sync(netdev, iavf_addr_sync, iavf_addr_unsync);1159 __dev_mc_sync(netdev, iavf_addr_sync, iavf_addr_unsync);1160 spin_unlock_bh(&adapter->mac_vlan_list_lock);1161 1162 spin_lock_bh(&adapter->current_netdev_promisc_flags_lock);1163 if (iavf_promiscuous_mode_changed(adapter))1164 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_PROMISC_MODE;1165 spin_unlock_bh(&adapter->current_netdev_promisc_flags_lock);1166}1167 1168/**1169 * iavf_napi_enable_all - enable NAPI on all queue vectors1170 * @adapter: board private structure1171 **/1172static void iavf_napi_enable_all(struct iavf_adapter *adapter)1173{1174 int q_idx;1175 struct iavf_q_vector *q_vector;1176 int q_vectors = adapter->num_msix_vectors - NONQ_VECS;1177 1178 for (q_idx = 0; q_idx < q_vectors; q_idx++) {1179 struct napi_struct *napi;1180 1181 q_vector = &adapter->q_vectors[q_idx];1182 napi = &q_vector->napi;1183 napi_enable(napi);1184 }1185}1186 1187/**1188 * iavf_napi_disable_all - disable NAPI on all queue vectors1189 * @adapter: board private structure1190 **/1191static void iavf_napi_disable_all(struct iavf_adapter *adapter)1192{1193 int q_idx;1194 struct iavf_q_vector *q_vector;1195 int q_vectors = adapter->num_msix_vectors - NONQ_VECS;1196 1197 for (q_idx = 0; q_idx < q_vectors; q_idx++) {1198 q_vector = &adapter->q_vectors[q_idx];1199 napi_disable(&q_vector->napi);1200 }1201}1202 1203/**1204 * iavf_configure - set up transmit and receive data structures1205 * @adapter: board private structure1206 **/1207static void iavf_configure(struct iavf_adapter *adapter)1208{1209 struct net_device *netdev = adapter->netdev;1210 int i;1211 1212 iavf_set_rx_mode(netdev);1213 1214 iavf_configure_tx(adapter);1215 iavf_configure_rx(adapter);1216 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_QUEUES;1217 1218 for (i = 0; i < adapter->num_active_queues; i++) {1219 struct iavf_ring *ring = &adapter->rx_rings[i];1220 1221 iavf_alloc_rx_buffers(ring, IAVF_DESC_UNUSED(ring));1222 }1223}1224 1225/**1226 * iavf_up_complete - Finish the last steps of bringing up a connection1227 * @adapter: board private structure1228 *1229 * Expects to be called while holding crit_lock.1230 **/1231static void iavf_up_complete(struct iavf_adapter *adapter)1232{1233 iavf_change_state(adapter, __IAVF_RUNNING);1234 clear_bit(__IAVF_VSI_DOWN, adapter->vsi.state);1235 1236 iavf_napi_enable_all(adapter);1237 1238 iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_ENABLE_QUEUES);1239}1240 1241/**1242 * iavf_clear_mac_vlan_filters - Remove mac and vlan filters not sent to PF1243 * yet and mark other to be removed.1244 * @adapter: board private structure1245 **/1246static void iavf_clear_mac_vlan_filters(struct iavf_adapter *adapter)1247{1248 struct iavf_vlan_filter *vlf, *vlftmp;1249 struct iavf_mac_filter *f, *ftmp;1250 1251 spin_lock_bh(&adapter->mac_vlan_list_lock);1252 /* clear the sync flag on all filters */1253 __dev_uc_unsync(adapter->netdev, NULL);1254 __dev_mc_unsync(adapter->netdev, NULL);1255 1256 /* remove all MAC filters */1257 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list,1258 list) {1259 if (f->add) {1260 list_del(&f->list);1261 kfree(f);1262 } else {1263 f->remove = true;1264 }1265 }1266 1267 /* disable all VLAN filters */1268 list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list,1269 list)1270 vlf->state = IAVF_VLAN_DISABLE;1271 1272 spin_unlock_bh(&adapter->mac_vlan_list_lock);1273}1274 1275/**1276 * iavf_clear_cloud_filters - Remove cloud filters not sent to PF yet and1277 * mark other to be removed.1278 * @adapter: board private structure1279 **/1280static void iavf_clear_cloud_filters(struct iavf_adapter *adapter)1281{1282 struct iavf_cloud_filter *cf, *cftmp;1283 1284 /* remove all cloud filters */1285 spin_lock_bh(&adapter->cloud_filter_list_lock);1286 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list,1287 list) {1288 if (cf->add) {1289 list_del(&cf->list);1290 kfree(cf);1291 adapter->num_cloud_filters--;1292 } else {1293 cf->del = true;1294 }1295 }1296 spin_unlock_bh(&adapter->cloud_filter_list_lock);1297}1298 1299/**1300 * iavf_clear_fdir_filters - Remove fdir filters not sent to PF yet and mark1301 * other to be removed.1302 * @adapter: board private structure1303 **/1304static void iavf_clear_fdir_filters(struct iavf_adapter *adapter)1305{1306 struct iavf_fdir_fltr *fdir;1307 1308 /* remove all Flow Director filters */1309 spin_lock_bh(&adapter->fdir_fltr_lock);1310 list_for_each_entry(fdir, &adapter->fdir_list_head, list) {1311 if (fdir->state == IAVF_FDIR_FLTR_ADD_REQUEST) {1312 /* Cancel a request, keep filter as inactive */1313 fdir->state = IAVF_FDIR_FLTR_INACTIVE;1314 } else if (fdir->state == IAVF_FDIR_FLTR_ADD_PENDING ||1315 fdir->state == IAVF_FDIR_FLTR_ACTIVE) {1316 /* Disable filters which are active or have a pending1317 * request to PF to be added1318 */1319 fdir->state = IAVF_FDIR_FLTR_DIS_REQUEST;1320 }1321 }1322 spin_unlock_bh(&adapter->fdir_fltr_lock);1323}1324 1325/**1326 * iavf_clear_adv_rss_conf - Remove adv rss conf not sent to PF yet and mark1327 * other to be removed.1328 * @adapter: board private structure1329 **/1330static void iavf_clear_adv_rss_conf(struct iavf_adapter *adapter)1331{1332 struct iavf_adv_rss *rss, *rsstmp;1333 1334 /* remove all advance RSS configuration */1335 spin_lock_bh(&adapter->adv_rss_lock);1336 list_for_each_entry_safe(rss, rsstmp, &adapter->adv_rss_list_head,1337 list) {1338 if (rss->state == IAVF_ADV_RSS_ADD_REQUEST) {1339 list_del(&rss->list);1340 kfree(rss);1341 } else {1342 rss->state = IAVF_ADV_RSS_DEL_REQUEST;1343 }1344 }1345 spin_unlock_bh(&adapter->adv_rss_lock);1346}1347 1348/**1349 * iavf_down - Shutdown the connection processing1350 * @adapter: board private structure1351 *1352 * Expects to be called while holding crit_lock.1353 **/1354void iavf_down(struct iavf_adapter *adapter)1355{1356 struct net_device *netdev = adapter->netdev;1357 1358 if (adapter->state <= __IAVF_DOWN_PENDING)1359 return;1360 1361 netif_carrier_off(netdev);1362 netif_tx_disable(netdev);1363 adapter->link_up = false;1364 iavf_napi_disable_all(adapter);1365 iavf_irq_disable(adapter);1366 1367 iavf_clear_mac_vlan_filters(adapter);1368 iavf_clear_cloud_filters(adapter);1369 iavf_clear_fdir_filters(adapter);1370 iavf_clear_adv_rss_conf(adapter);1371 1372 if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)1373 return;1374 1375 if (!test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section)) {1376 /* cancel any current operation */1377 adapter->current_op = VIRTCHNL_OP_UNKNOWN;1378 /* Schedule operations to close down the HW. Don't wait1379 * here for this to complete. The watchdog is still running1380 * and it will take care of this.1381 */1382 if (!list_empty(&adapter->mac_filter_list))1383 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;1384 if (!list_empty(&adapter->vlan_filter_list))1385 adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER;1386 if (!list_empty(&adapter->cloud_filter_list))1387 adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;1388 if (!list_empty(&adapter->fdir_list_head))1389 adapter->aq_required |= IAVF_FLAG_AQ_DEL_FDIR_FILTER;1390 if (!list_empty(&adapter->adv_rss_list_head))1391 adapter->aq_required |= IAVF_FLAG_AQ_DEL_ADV_RSS_CFG;1392 }1393 1394 iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_DISABLE_QUEUES);1395}1396 1397/**1398 * iavf_acquire_msix_vectors - Setup the MSIX capability1399 * @adapter: board private structure1400 * @vectors: number of vectors to request1401 *1402 * Work with the OS to set up the MSIX vectors needed.1403 *1404 * Returns 0 on success, negative on failure1405 **/1406static int1407iavf_acquire_msix_vectors(struct iavf_adapter *adapter, int vectors)1408{1409 int err, vector_threshold;1410 1411 /* We'll want at least 3 (vector_threshold):1412 * 0) Other (Admin Queue and link, mostly)1413 * 1) TxQ[0] Cleanup1414 * 2) RxQ[0] Cleanup1415 */1416 vector_threshold = MIN_MSIX_COUNT;1417 1418 /* The more we get, the more we will assign to Tx/Rx Cleanup1419 * for the separate queues...where Rx Cleanup >= Tx Cleanup.1420 * Right now, we simply care about how many we'll get; we'll1421 * set them up later while requesting irq's.1422 */1423 err = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,1424 vector_threshold, vectors);1425 if (err < 0) {1426 dev_err(&adapter->pdev->dev, "Unable to allocate MSI-X interrupts\n");1427 kfree(adapter->msix_entries);1428 adapter->msix_entries = NULL;1429 return err;1430 }1431 1432 /* Adjust for only the vectors we'll use, which is minimum1433 * of max_msix_q_vectors + NONQ_VECS, or the number of1434 * vectors we were allocated.1435 */1436 adapter->num_msix_vectors = err;1437 return 0;1438}1439 1440/**1441 * iavf_free_queues - Free memory for all rings1442 * @adapter: board private structure to initialize1443 *1444 * Free all of the memory associated with queue pairs.1445 **/1446static void iavf_free_queues(struct iavf_adapter *adapter)1447{1448 if (!adapter->vsi_res)1449 return;1450 adapter->num_active_queues = 0;1451 kfree(adapter->tx_rings);1452 adapter->tx_rings = NULL;1453 kfree(adapter->rx_rings);1454 adapter->rx_rings = NULL;1455}1456 1457/**1458 * iavf_set_queue_vlan_tag_loc - set location for VLAN tag offload1459 * @adapter: board private structure1460 *1461 * Based on negotiated capabilities, the VLAN tag needs to be inserted and/or1462 * stripped in certain descriptor fields. Instead of checking the offload1463 * capability bits in the hot path, cache the location the ring specific1464 * flags.1465 */1466void iavf_set_queue_vlan_tag_loc(struct iavf_adapter *adapter)1467{1468 int i;1469 1470 for (i = 0; i < adapter->num_active_queues; i++) {1471 struct iavf_ring *tx_ring = &adapter->tx_rings[i];1472 struct iavf_ring *rx_ring = &adapter->rx_rings[i];1473 1474 /* prevent multiple L2TAG bits being set after VFR */1475 tx_ring->flags &=1476 ~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 |1477 IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2);1478 rx_ring->flags &=1479 ~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 |1480 IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2);1481 1482 if (VLAN_ALLOWED(adapter)) {1483 tx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;1484 rx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;1485 } else if (VLAN_V2_ALLOWED(adapter)) {1486 struct virtchnl_vlan_supported_caps *stripping_support;1487 struct virtchnl_vlan_supported_caps *insertion_support;1488 1489 stripping_support =1490 &adapter->vlan_v2_caps.offloads.stripping_support;1491 insertion_support =1492 &adapter->vlan_v2_caps.offloads.insertion_support;1493 1494 if (stripping_support->outer) {1495 if (stripping_support->outer &1496 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)1497 rx_ring->flags |=1498 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;1499 else if (stripping_support->outer &1500 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2)1501 rx_ring->flags |=1502 IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2;1503 } else if (stripping_support->inner) {1504 if (stripping_support->inner &1505 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)1506 rx_ring->flags |=1507 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;1508 else if (stripping_support->inner &1509 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2)1510 rx_ring->flags |=1511 IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2;1512 }1513 1514 if (insertion_support->outer) {1515 if (insertion_support->outer &1516 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)1517 tx_ring->flags |=1518 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;1519 else if (insertion_support->outer &1520 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2)1521 tx_ring->flags |=1522 IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2;1523 } else if (insertion_support->inner) {1524 if (insertion_support->inner &1525 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)1526 tx_ring->flags |=1527 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;1528 else if (insertion_support->inner &1529 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2)1530 tx_ring->flags |=1531 IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2;1532 }1533 }1534 }1535}1536 1537/**1538 * iavf_alloc_queues - Allocate memory for all rings1539 * @adapter: board private structure to initialize1540 *1541 * We allocate one ring per queue at run-time since we don't know the1542 * number of queues at compile-time. The polling_netdev array is1543 * intended for Multiqueue, but should work fine with a single queue.1544 **/1545static int iavf_alloc_queues(struct iavf_adapter *adapter)1546{1547 int i, num_active_queues;1548 1549 /* If we're in reset reallocating queues we don't actually know yet for1550 * certain the PF gave us the number of queues we asked for but we'll1551 * assume it did. Once basic reset is finished we'll confirm once we1552 * start negotiating config with PF.1553 */1554 if (adapter->num_req_queues)1555 num_active_queues = adapter->num_req_queues;1556 else if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&1557 adapter->num_tc)1558 num_active_queues = adapter->ch_config.total_qps;1559 else1560 num_active_queues = min_t(int,1561 adapter->vsi_res->num_queue_pairs,1562 (int)(num_online_cpus()));1563 1564 1565 adapter->tx_rings = kcalloc(num_active_queues,1566 sizeof(struct iavf_ring), GFP_KERNEL);1567 if (!adapter->tx_rings)1568 goto err_out;1569 adapter->rx_rings = kcalloc(num_active_queues,1570 sizeof(struct iavf_ring), GFP_KERNEL);1571 if (!adapter->rx_rings)1572 goto err_out;1573 1574 for (i = 0; i < num_active_queues; i++) {1575 struct iavf_ring *tx_ring;1576 struct iavf_ring *rx_ring;1577 1578 tx_ring = &adapter->tx_rings[i];1579 1580 tx_ring->queue_index = i;1581 tx_ring->netdev = adapter->netdev;1582 tx_ring->dev = &adapter->pdev->dev;1583 tx_ring->count = adapter->tx_desc_count;1584 tx_ring->itr_setting = IAVF_ITR_TX_DEF;1585 if (adapter->flags & IAVF_FLAG_WB_ON_ITR_CAPABLE)1586 tx_ring->flags |= IAVF_TXR_FLAGS_WB_ON_ITR;1587 1588 rx_ring = &adapter->rx_rings[i];1589 rx_ring->queue_index = i;1590 rx_ring->netdev = adapter->netdev;1591 rx_ring->count = adapter->rx_desc_count;1592 rx_ring->itr_setting = IAVF_ITR_RX_DEF;1593 }1594 1595 adapter->num_active_queues = num_active_queues;1596 1597 iavf_set_queue_vlan_tag_loc(adapter);1598 1599 return 0;1600 1601err_out:1602 iavf_free_queues(adapter);1603 return -ENOMEM;1604}1605 1606/**1607 * iavf_set_interrupt_capability - set MSI-X or FAIL if not supported1608 * @adapter: board private structure to initialize1609 *1610 * Attempt to configure the interrupts using the best available1611 * capabilities of the hardware and the kernel.1612 **/1613static int iavf_set_interrupt_capability(struct iavf_adapter *adapter)1614{1615 int vector, v_budget;1616 int pairs = 0;1617 int err = 0;1618 1619 if (!adapter->vsi_res) {1620 err = -EIO;1621 goto out;1622 }1623 pairs = adapter->num_active_queues;1624 1625 /* It's easy to be greedy for MSI-X vectors, but it really doesn't do1626 * us much good if we have more vectors than CPUs. However, we already1627 * limit the total number of queues by the number of CPUs so we do not1628 * need any further limiting here.1629 */1630 v_budget = min_t(int, pairs + NONQ_VECS,1631 (int)adapter->vf_res->max_vectors);1632 1633 adapter->msix_entries = kcalloc(v_budget,1634 sizeof(struct msix_entry), GFP_KERNEL);1635 if (!adapter->msix_entries) {1636 err = -ENOMEM;1637 goto out;1638 }1639 1640 for (vector = 0; vector < v_budget; vector++)1641 adapter->msix_entries[vector].entry = vector;1642 1643 err = iavf_acquire_msix_vectors(adapter, v_budget);1644 if (!err)1645 iavf_schedule_finish_config(adapter);1646 1647out:1648 return err;1649}1650 1651/**1652 * iavf_config_rss_aq - Configure RSS keys and lut by using AQ commands1653 * @adapter: board private structure1654 *1655 * Return 0 on success, negative on failure1656 **/1657static int iavf_config_rss_aq(struct iavf_adapter *adapter)1658{1659 struct iavf_aqc_get_set_rss_key_data *rss_key =1660 (struct iavf_aqc_get_set_rss_key_data *)adapter->rss_key;1661 struct iavf_hw *hw = &adapter->hw;1662 enum iavf_status status;1663 1664 if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {1665 /* bail because we already have a command pending */1666 dev_err(&adapter->pdev->dev, "Cannot configure RSS, command %d pending\n",1667 adapter->current_op);1668 return -EBUSY;1669 }1670 1671 status = iavf_aq_set_rss_key(hw, adapter->vsi.id, rss_key);1672 if (status) {1673 dev_err(&adapter->pdev->dev, "Cannot set RSS key, err %s aq_err %s\n",1674 iavf_stat_str(hw, status),1675 iavf_aq_str(hw, hw->aq.asq_last_status));1676 return iavf_status_to_errno(status);1677 1678 }1679 1680 status = iavf_aq_set_rss_lut(hw, adapter->vsi.id, false,1681 adapter->rss_lut, adapter->rss_lut_size);1682 if (status) {1683 dev_err(&adapter->pdev->dev, "Cannot set RSS lut, err %s aq_err %s\n",1684 iavf_stat_str(hw, status),1685 iavf_aq_str(hw, hw->aq.asq_last_status));1686 return iavf_status_to_errno(status);1687 }1688 1689 return 0;1690 1691}1692 1693/**1694 * iavf_config_rss_reg - Configure RSS keys and lut by writing registers1695 * @adapter: board private structure1696 *1697 * Returns 0 on success, negative on failure1698 **/1699static int iavf_config_rss_reg(struct iavf_adapter *adapter)1700{1701 struct iavf_hw *hw = &adapter->hw;1702 u32 *dw;1703 u16 i;1704 1705 dw = (u32 *)adapter->rss_key;1706 for (i = 0; i <= adapter->rss_key_size / 4; i++)1707 wr32(hw, IAVF_VFQF_HKEY(i), dw[i]);1708 1709 dw = (u32 *)adapter->rss_lut;1710 for (i = 0; i <= adapter->rss_lut_size / 4; i++)1711 wr32(hw, IAVF_VFQF_HLUT(i), dw[i]);1712 1713 iavf_flush(hw);1714 1715 return 0;1716}1717 1718/**1719 * iavf_config_rss - Configure RSS keys and lut1720 * @adapter: board private structure1721 *1722 * Returns 0 on success, negative on failure1723 **/1724int iavf_config_rss(struct iavf_adapter *adapter)1725{1726 1727 if (RSS_PF(adapter)) {1728 adapter->aq_required |= IAVF_FLAG_AQ_SET_RSS_LUT |1729 IAVF_FLAG_AQ_SET_RSS_KEY;1730 return 0;1731 } else if (RSS_AQ(adapter)) {1732 return iavf_config_rss_aq(adapter);1733 } else {1734 return iavf_config_rss_reg(adapter);1735 }1736}1737 1738/**1739 * iavf_fill_rss_lut - Fill the lut with default values1740 * @adapter: board private structure1741 **/1742static void iavf_fill_rss_lut(struct iavf_adapter *adapter)1743{1744 u16 i;1745 1746 for (i = 0; i < adapter->rss_lut_size; i++)1747 adapter->rss_lut[i] = i % adapter->num_active_queues;1748}1749 1750/**1751 * iavf_init_rss - Prepare for RSS1752 * @adapter: board private structure1753 *1754 * Return 0 on success, negative on failure1755 **/1756static int iavf_init_rss(struct iavf_adapter *adapter)1757{1758 struct iavf_hw *hw = &adapter->hw;1759 1760 if (!RSS_PF(adapter)) {1761 /* Enable PCTYPES for RSS, TCP/UDP with IPv4/IPv6 */1762 if (adapter->vf_res->vf_cap_flags &1763 VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2)1764 adapter->hena = IAVF_DEFAULT_RSS_HENA_EXPANDED;1765 else1766 adapter->hena = IAVF_DEFAULT_RSS_HENA;1767 1768 wr32(hw, IAVF_VFQF_HENA(0), (u32)adapter->hena);1769 wr32(hw, IAVF_VFQF_HENA(1), (u32)(adapter->hena >> 32));1770 }1771 1772 iavf_fill_rss_lut(adapter);1773 netdev_rss_key_fill((void *)adapter->rss_key, adapter->rss_key_size);1774 1775 return iavf_config_rss(adapter);1776}1777 1778/**1779 * iavf_alloc_q_vectors - Allocate memory for interrupt vectors1780 * @adapter: board private structure to initialize1781 *1782 * We allocate one q_vector per queue interrupt. If allocation fails we1783 * return -ENOMEM.1784 **/1785static int iavf_alloc_q_vectors(struct iavf_adapter *adapter)1786{1787 int q_idx = 0, num_q_vectors;1788 struct iavf_q_vector *q_vector;1789 1790 num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;1791 adapter->q_vectors = kcalloc(num_q_vectors, sizeof(*q_vector),1792 GFP_KERNEL);1793 if (!adapter->q_vectors)1794 return -ENOMEM;1795 1796 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {1797 q_vector = &adapter->q_vectors[q_idx];1798 q_vector->adapter = adapter;1799 q_vector->vsi = &adapter->vsi;1800 q_vector->v_idx = q_idx;1801 q_vector->reg_idx = q_idx;1802 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);1803 netif_napi_add(adapter->netdev, &q_vector->napi,1804 iavf_napi_poll);1805 }1806 1807 return 0;1808}1809 1810/**1811 * iavf_free_q_vectors - Free memory allocated for interrupt vectors1812 * @adapter: board private structure to initialize1813 *1814 * This function frees the memory allocated to the q_vectors. In addition if1815 * NAPI is enabled it will delete any references to the NAPI struct prior1816 * to freeing the q_vector.1817 **/1818static void iavf_free_q_vectors(struct iavf_adapter *adapter)1819{1820 int q_idx, num_q_vectors;1821 1822 if (!adapter->q_vectors)1823 return;1824 1825 num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;1826 1827 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {1828 struct iavf_q_vector *q_vector = &adapter->q_vectors[q_idx];1829 1830 netif_napi_del(&q_vector->napi);1831 }1832 kfree(adapter->q_vectors);1833 adapter->q_vectors = NULL;1834}1835 1836/**1837 * iavf_reset_interrupt_capability - Reset MSIX setup1838 * @adapter: board private structure1839 *1840 **/1841static void iavf_reset_interrupt_capability(struct iavf_adapter *adapter)1842{1843 if (!adapter->msix_entries)1844 return;1845 1846 pci_disable_msix(adapter->pdev);1847 kfree(adapter->msix_entries);1848 adapter->msix_entries = NULL;1849}1850 1851/**1852 * iavf_init_interrupt_scheme - Determine if MSIX is supported and init1853 * @adapter: board private structure to initialize1854 *1855 **/1856static int iavf_init_interrupt_scheme(struct iavf_adapter *adapter)1857{1858 int err;1859 1860 err = iavf_alloc_queues(adapter);1861 if (err) {1862 dev_err(&adapter->pdev->dev,1863 "Unable to allocate memory for queues\n");1864 goto err_alloc_queues;1865 }1866 1867 err = iavf_set_interrupt_capability(adapter);1868 if (err) {1869 dev_err(&adapter->pdev->dev,1870 "Unable to setup interrupt capabilities\n");1871 goto err_set_interrupt;1872 }1873 1874 err = iavf_alloc_q_vectors(adapter);1875 if (err) {1876 dev_err(&adapter->pdev->dev,1877 "Unable to allocate memory for queue vectors\n");1878 goto err_alloc_q_vectors;1879 }1880 1881 /* If we've made it so far while ADq flag being ON, then we haven't1882 * bailed out anywhere in middle. And ADq isn't just enabled but actual1883 * resources have been allocated in the reset path.1884 * Now we can truly claim that ADq is enabled.1885 */1886 if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&1887 adapter->num_tc)1888 dev_info(&adapter->pdev->dev, "ADq Enabled, %u TCs created",1889 adapter->num_tc);1890 1891 dev_info(&adapter->pdev->dev, "Multiqueue %s: Queue pair count = %u",1892 (adapter->num_active_queues > 1) ? "Enabled" : "Disabled",1893 adapter->num_active_queues);1894 1895 return 0;1896err_alloc_q_vectors:1897 iavf_reset_interrupt_capability(adapter);1898err_set_interrupt:1899 iavf_free_queues(adapter);1900err_alloc_queues:1901 return err;1902}1903 1904/**1905 * iavf_free_interrupt_scheme - Undo what iavf_init_interrupt_scheme does1906 * @adapter: board private structure1907 **/1908static void iavf_free_interrupt_scheme(struct iavf_adapter *adapter)1909{1910 iavf_free_q_vectors(adapter);1911 iavf_reset_interrupt_capability(adapter);1912 iavf_free_queues(adapter);1913}1914 1915/**1916 * iavf_free_rss - Free memory used by RSS structs1917 * @adapter: board private structure1918 **/1919static void iavf_free_rss(struct iavf_adapter *adapter)1920{1921 kfree(adapter->rss_key);1922 adapter->rss_key = NULL;1923 1924 kfree(adapter->rss_lut);1925 adapter->rss_lut = NULL;1926}1927 1928/**1929 * iavf_reinit_interrupt_scheme - Reallocate queues and vectors1930 * @adapter: board private structure1931 * @running: true if adapter->state == __IAVF_RUNNING1932 *1933 * Returns 0 on success, negative on failure1934 **/1935static int iavf_reinit_interrupt_scheme(struct iavf_adapter *adapter, bool running)1936{1937 struct net_device *netdev = adapter->netdev;1938 int err;1939 1940 if (running)1941 iavf_free_traffic_irqs(adapter);1942 iavf_free_misc_irq(adapter);1943 iavf_free_interrupt_scheme(adapter);1944 1945 err = iavf_init_interrupt_scheme(adapter);1946 if (err)1947 goto err;1948 1949 netif_tx_stop_all_queues(netdev);1950 1951 err = iavf_request_misc_irq(adapter);1952 if (err)1953 goto err;1954 1955 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);1956 1957 iavf_map_rings_to_vectors(adapter);1958err:1959 return err;1960}1961 1962/**1963 * iavf_finish_config - do all netdev work that needs RTNL1964 * @work: our work_struct1965 *1966 * Do work that needs both RTNL and crit_lock.1967 **/1968static void iavf_finish_config(struct work_struct *work)1969{1970 struct iavf_adapter *adapter;1971 int pairs, err;1972 1973 adapter = container_of(work, struct iavf_adapter, finish_config);1974 1975 /* Always take RTNL first to prevent circular lock dependency */1976 rtnl_lock();1977 mutex_lock(&adapter->crit_lock);1978 1979 if ((adapter->flags & IAVF_FLAG_SETUP_NETDEV_FEATURES) &&1980 adapter->netdev->reg_state == NETREG_REGISTERED &&1981 !test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section)) {1982 netdev_update_features(adapter->netdev);1983 adapter->flags &= ~IAVF_FLAG_SETUP_NETDEV_FEATURES;1984 }1985 1986 switch (adapter->state) {1987 case __IAVF_DOWN:1988 if (adapter->netdev->reg_state != NETREG_REGISTERED) {1989 err = register_netdevice(adapter->netdev);1990 if (err) {1991 dev_err(&adapter->pdev->dev, "Unable to register netdev (%d)\n",1992 err);1993 1994 /* go back and try again.*/1995 iavf_free_rss(adapter);1996 iavf_free_misc_irq(adapter);1997 iavf_reset_interrupt_capability(adapter);1998 iavf_change_state(adapter,1999 __IAVF_INIT_CONFIG_ADAPTER);2000 goto out;2001 }2002 }2003 2004 /* Set the real number of queues when reset occurs while2005 * state == __IAVF_DOWN2006 */2007 fallthrough;2008 case __IAVF_RUNNING:2009 pairs = adapter->num_active_queues;2010 netif_set_real_num_rx_queues(adapter->netdev, pairs);2011 netif_set_real_num_tx_queues(adapter->netdev, pairs);2012 break;2013 2014 default:2015 break;2016 }2017 2018out:2019 mutex_unlock(&adapter->crit_lock);2020 rtnl_unlock();2021}2022 2023/**2024 * iavf_schedule_finish_config - Set the flags and schedule a reset event2025 * @adapter: board private structure2026 **/2027void iavf_schedule_finish_config(struct iavf_adapter *adapter)2028{2029 if (!test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section))2030 queue_work(adapter->wq, &adapter->finish_config);2031}2032 2033/**2034 * iavf_process_aq_command - process aq_required flags2035 * and sends aq command2036 * @adapter: pointer to iavf adapter structure2037 *2038 * Returns 0 on success2039 * Returns error code if no command was sent2040 * or error code if the command failed.2041 **/2042static int iavf_process_aq_command(struct iavf_adapter *adapter)2043{2044 if (adapter->aq_required & IAVF_FLAG_AQ_GET_CONFIG)2045 return iavf_send_vf_config_msg(adapter);2046 if (adapter->aq_required & IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS)2047 return iavf_send_vf_offload_vlan_v2_msg(adapter);2048 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_QUEUES) {2049 iavf_disable_queues(adapter);2050 return 0;2051 }2052 2053 if (adapter->aq_required & IAVF_FLAG_AQ_MAP_VECTORS) {2054 iavf_map_queues(adapter);2055 return 0;2056 }2057 2058 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_MAC_FILTER) {2059 iavf_add_ether_addrs(adapter);2060 return 0;2061 }2062 2063 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_VLAN_FILTER) {2064 iavf_add_vlans(adapter);2065 return 0;2066 }2067 2068 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_MAC_FILTER) {2069 iavf_del_ether_addrs(adapter);2070 return 0;2071 }2072 2073 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_VLAN_FILTER) {2074 iavf_del_vlans(adapter);2075 return 0;2076 }2077 2078 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING) {2079 iavf_enable_vlan_stripping(adapter);2080 return 0;2081 }2082 2083 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING) {2084 iavf_disable_vlan_stripping(adapter);2085 return 0;2086 }2087 2088 if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_QUEUES) {2089 iavf_configure_queues(adapter);2090 return 0;2091 }2092 2093 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_QUEUES) {2094 iavf_enable_queues(adapter);2095 return 0;2096 }2097 2098 if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_RSS) {2099 /* This message goes straight to the firmware, not the2100 * PF, so we don't have to set current_op as we will2101 * not get a response through the ARQ.2102 */2103 adapter->aq_required &= ~IAVF_FLAG_AQ_CONFIGURE_RSS;2104 return 0;2105 }2106 if (adapter->aq_required & IAVF_FLAG_AQ_GET_HENA) {2107 iavf_get_hena(adapter);2108 return 0;2109 }2110 if (adapter->aq_required & IAVF_FLAG_AQ_SET_HENA) {2111 iavf_set_hena(adapter);2112 return 0;2113 }2114 if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_KEY) {2115 iavf_set_rss_key(adapter);2116 return 0;2117 }2118 if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_LUT) {2119 iavf_set_rss_lut(adapter);2120 return 0;2121 }2122 if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_HFUNC) {2123 iavf_set_rss_hfunc(adapter);2124 return 0;2125 }2126 2127 if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_PROMISC_MODE) {2128 iavf_set_promiscuous(adapter);2129 return 0;2130 }2131 2132 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CHANNELS) {2133 iavf_enable_channels(adapter);2134 return 0;2135 }2136 2137 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CHANNELS) {2138 iavf_disable_channels(adapter);2139 return 0;2140 }2141 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) {2142 iavf_add_cloud_filter(adapter);2143 return 0;2144 }2145 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) {2146 iavf_del_cloud_filter(adapter);2147 return 0;2148 }2149 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_FDIR_FILTER) {2150 iavf_add_fdir_filter(adapter);2151 return IAVF_SUCCESS;2152 }2153 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_FDIR_FILTER) {2154 iavf_del_fdir_filter(adapter);2155 return IAVF_SUCCESS;2156 }2157 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_ADV_RSS_CFG) {2158 iavf_add_adv_rss_cfg(adapter);2159 return 0;2160 }2161 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_ADV_RSS_CFG) {2162 iavf_del_adv_rss_cfg(adapter);2163 return 0;2164 }2165 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING) {2166 iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021Q);2167 return 0;2168 }2169 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING) {2170 iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021AD);2171 return 0;2172 }2173 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING) {2174 iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021Q);2175 return 0;2176 }2177 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING) {2178 iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021AD);2179 return 0;2180 }2181 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION) {2182 iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021Q);2183 return 0;2184 }2185 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION) {2186 iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021AD);2187 return 0;2188 }2189 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION) {2190 iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021Q);2191 return 0;2192 }2193 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION) {2194 iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021AD);2195 return 0;2196 }2197 2198 if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_STATS) {2199 iavf_request_stats(adapter);2200 return 0;2201 }2202 2203 return -EAGAIN;2204}2205 2206/**2207 * iavf_set_vlan_offload_features - set VLAN offload configuration2208 * @adapter: board private structure2209 * @prev_features: previous features used for comparison2210 * @features: updated features used for configuration2211 *2212 * Set the aq_required bit(s) based on the requested features passed in to2213 * configure VLAN stripping and/or VLAN insertion if supported. Also, schedule2214 * the watchdog if any changes are requested to expedite the request via2215 * virtchnl.2216 **/2217static void2218iavf_set_vlan_offload_features(struct iavf_adapter *adapter,2219 netdev_features_t prev_features,2220 netdev_features_t features)2221{2222 bool enable_stripping = true, enable_insertion = true;2223 u16 vlan_ethertype = 0;2224 u64 aq_required = 0;2225 2226 /* keep cases separate because one ethertype for offloads can be2227 * disabled at the same time as another is disabled, so check for an2228 * enabled ethertype first, then check for disabled. Default to2229 * ETH_P_8021Q so an ethertype is specified if disabling insertion and2230 * stripping.2231 */2232 if (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))2233 vlan_ethertype = ETH_P_8021AD;2234 else if (features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX))2235 vlan_ethertype = ETH_P_8021Q;2236 else if (prev_features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))2237 vlan_ethertype = ETH_P_8021AD;2238 else if (prev_features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX))2239 vlan_ethertype = ETH_P_8021Q;2240 else2241 vlan_ethertype = ETH_P_8021Q;2242 2243 if (!(features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_CTAG_RX)))2244 enable_stripping = false;2245 if (!(features & (NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_CTAG_TX)))2246 enable_insertion = false;2247 2248 if (VLAN_ALLOWED(adapter)) {2249 /* VIRTCHNL_VF_OFFLOAD_VLAN only has support for toggling VLAN2250 * stripping via virtchnl. VLAN insertion can be toggled on the2251 * netdev, but it doesn't require a virtchnl message2252 */2253 if (enable_stripping)2254 aq_required |= IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING;2255 else2256 aq_required |= IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING;2257 2258 } else if (VLAN_V2_ALLOWED(adapter)) {2259 switch (vlan_ethertype) {2260 case ETH_P_8021Q:2261 if (enable_stripping)2262 aq_required |= IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING;2263 else2264 aq_required |= IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING;2265 2266 if (enable_insertion)2267 aq_required |= IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION;2268 else2269 aq_required |= IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION;2270 break;2271 case ETH_P_8021AD:2272 if (enable_stripping)2273 aq_required |= IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING;2274 else2275 aq_required |= IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING;2276 2277 if (enable_insertion)2278 aq_required |= IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION;2279 else2280 aq_required |= IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION;2281 break;2282 }2283 }2284 2285 if (aq_required)2286 iavf_schedule_aq_request(adapter, aq_required);2287}2288 2289/**2290 * iavf_startup - first step of driver startup2291 * @adapter: board private structure2292 *2293 * Function process __IAVF_STARTUP driver state.2294 * When success the state is changed to __IAVF_INIT_VERSION_CHECK2295 * when fails the state is changed to __IAVF_INIT_FAILED2296 **/2297static void iavf_startup(struct iavf_adapter *adapter)2298{2299 struct pci_dev *pdev = adapter->pdev;2300 struct iavf_hw *hw = &adapter->hw;2301 enum iavf_status status;2302 int ret;2303 2304 WARN_ON(adapter->state != __IAVF_STARTUP);2305 2306 /* driver loaded, probe complete */2307 adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;2308 adapter->flags &= ~IAVF_FLAG_RESET_PENDING;2309 2310 ret = iavf_check_reset_complete(hw);2311 if (ret) {2312 dev_info(&pdev->dev, "Device is still in reset (%d), retrying\n",2313 ret);2314 goto err;2315 }2316 hw->aq.num_arq_entries = IAVF_AQ_LEN;2317 hw->aq.num_asq_entries = IAVF_AQ_LEN;2318 hw->aq.arq_buf_size = IAVF_MAX_AQ_BUF_SIZE;2319 hw->aq.asq_buf_size = IAVF_MAX_AQ_BUF_SIZE;2320 2321 status = iavf_init_adminq(hw);2322 if (status) {2323 dev_err(&pdev->dev, "Failed to init Admin Queue (%d)\n",2324 status);2325 goto err;2326 }2327 ret = iavf_send_api_ver(adapter);2328 if (ret) {2329 dev_err(&pdev->dev, "Unable to send to PF (%d)\n", ret);2330 iavf_shutdown_adminq(hw);2331 goto err;2332 }2333 iavf_change_state(adapter, __IAVF_INIT_VERSION_CHECK);2334 return;2335err:2336 iavf_change_state(adapter, __IAVF_INIT_FAILED);2337}2338 2339/**2340 * iavf_init_version_check - second step of driver startup2341 * @adapter: board private structure2342 *2343 * Function process __IAVF_INIT_VERSION_CHECK driver state.2344 * When success the state is changed to __IAVF_INIT_GET_RESOURCES2345 * when fails the state is changed to __IAVF_INIT_FAILED2346 **/2347static void iavf_init_version_check(struct iavf_adapter *adapter)2348{2349 struct pci_dev *pdev = adapter->pdev;2350 struct iavf_hw *hw = &adapter->hw;2351 int err = -EAGAIN;2352 2353 WARN_ON(adapter->state != __IAVF_INIT_VERSION_CHECK);2354 2355 if (!iavf_asq_done(hw)) {2356 dev_err(&pdev->dev, "Admin queue command never completed\n");2357 iavf_shutdown_adminq(hw);2358 iavf_change_state(adapter, __IAVF_STARTUP);2359 goto err;2360 }2361 2362 /* aq msg sent, awaiting reply */2363 err = iavf_verify_api_ver(adapter);2364 if (err) {2365 if (err == -EALREADY)2366 err = iavf_send_api_ver(adapter);2367 else2368 dev_err(&pdev->dev, "Unsupported PF API version %d.%d, expected %d.%d\n",2369 adapter->pf_version.major,2370 adapter->pf_version.minor,2371 VIRTCHNL_VERSION_MAJOR,2372 VIRTCHNL_VERSION_MINOR);2373 goto err;2374 }2375 err = iavf_send_vf_config_msg(adapter);2376 if (err) {2377 dev_err(&pdev->dev, "Unable to send config request (%d)\n",2378 err);2379 goto err;2380 }2381 iavf_change_state(adapter, __IAVF_INIT_GET_RESOURCES);2382 return;2383err:2384 iavf_change_state(adapter, __IAVF_INIT_FAILED);2385}2386 2387/**2388 * iavf_parse_vf_resource_msg - parse response from VIRTCHNL_OP_GET_VF_RESOURCES2389 * @adapter: board private structure2390 */2391int iavf_parse_vf_resource_msg(struct iavf_adapter *adapter)2392{2393 int i, num_req_queues = adapter->num_req_queues;2394 struct iavf_vsi *vsi = &adapter->vsi;2395 2396 for (i = 0; i < adapter->vf_res->num_vsis; i++) {2397 if (adapter->vf_res->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV)2398 adapter->vsi_res = &adapter->vf_res->vsi_res[i];2399 }2400 if (!adapter->vsi_res) {2401 dev_err(&adapter->pdev->dev, "No LAN VSI found\n");2402 return -ENODEV;2403 }2404 2405 if (num_req_queues &&2406 num_req_queues > adapter->vsi_res->num_queue_pairs) {2407 /* Problem. The PF gave us fewer queues than what we had2408 * negotiated in our request. Need a reset to see if we can't2409 * get back to a working state.2410 */2411 dev_err(&adapter->pdev->dev,2412 "Requested %d queues, but PF only gave us %d.\n",2413 num_req_queues,2414 adapter->vsi_res->num_queue_pairs);2415 adapter->flags |= IAVF_FLAG_REINIT_MSIX_NEEDED;2416 adapter->num_req_queues = adapter->vsi_res->num_queue_pairs;2417 iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED);2418 2419 return -EAGAIN;2420 }2421 adapter->num_req_queues = 0;2422 adapter->vsi.id = adapter->vsi_res->vsi_id;2423 2424 adapter->vsi.back = adapter;2425 adapter->vsi.base_vector = 1;2426 vsi->netdev = adapter->netdev;2427 vsi->qs_handle = adapter->vsi_res->qset_handle;2428 if (adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {2429 adapter->rss_key_size = adapter->vf_res->rss_key_size;2430 adapter->rss_lut_size = adapter->vf_res->rss_lut_size;2431 } else {2432 adapter->rss_key_size = IAVF_HKEY_ARRAY_SIZE;2433 adapter->rss_lut_size = IAVF_HLUT_ARRAY_SIZE;2434 }2435 2436 return 0;2437}2438 2439/**2440 * iavf_init_get_resources - third step of driver startup2441 * @adapter: board private structure2442 *2443 * Function process __IAVF_INIT_GET_RESOURCES driver state and2444 * finishes driver initialization procedure.2445 * When success the state is changed to __IAVF_DOWN2446 * when fails the state is changed to __IAVF_INIT_FAILED2447 **/2448static void iavf_init_get_resources(struct iavf_adapter *adapter)2449{2450 struct pci_dev *pdev = adapter->pdev;2451 struct iavf_hw *hw = &adapter->hw;2452 int err;2453 2454 WARN_ON(adapter->state != __IAVF_INIT_GET_RESOURCES);2455 /* aq msg sent, awaiting reply */2456 if (!adapter->vf_res) {2457 adapter->vf_res = kzalloc(IAVF_VIRTCHNL_VF_RESOURCE_SIZE,2458 GFP_KERNEL);2459 if (!adapter->vf_res) {2460 err = -ENOMEM;2461 goto err;2462 }2463 }2464 err = iavf_get_vf_config(adapter);2465 if (err == -EALREADY) {2466 err = iavf_send_vf_config_msg(adapter);2467 goto err;2468 } else if (err == -EINVAL) {2469 /* We only get -EINVAL if the device is in a very bad2470 * state or if we've been disabled for previous bad2471 * behavior. Either way, we're done now.2472 */2473 iavf_shutdown_adminq(hw);2474 dev_err(&pdev->dev, "Unable to get VF config due to PF error condition, not retrying\n");2475 return;2476 }2477 if (err) {2478 dev_err(&pdev->dev, "Unable to get VF config (%d)\n", err);2479 goto err_alloc;2480 }2481 2482 err = iavf_parse_vf_resource_msg(adapter);2483 if (err) {2484 dev_err(&pdev->dev, "Failed to parse VF resource message from PF (%d)\n",2485 err);2486 goto err_alloc;2487 }2488 /* Some features require additional messages to negotiate extended2489 * capabilities. These are processed in sequence by the2490 * __IAVF_INIT_EXTENDED_CAPS driver state.2491 */2492 adapter->extended_caps = IAVF_EXTENDED_CAPS;2493 2494 iavf_change_state(adapter, __IAVF_INIT_EXTENDED_CAPS);2495 return;2496 2497err_alloc:2498 kfree(adapter->vf_res);2499 adapter->vf_res = NULL;2500err:2501 iavf_change_state(adapter, __IAVF_INIT_FAILED);2502}2503 2504/**2505 * iavf_init_send_offload_vlan_v2_caps - part of initializing VLAN V2 caps2506 * @adapter: board private structure2507 *2508 * Function processes send of the extended VLAN V2 capability message to the2509 * PF. Must clear IAVF_EXTENDED_CAP_RECV_VLAN_V2 if the message is not sent,2510 * e.g. due to PF not negotiating VIRTCHNL_VF_OFFLOAD_VLAN_V2.2511 */2512static void iavf_init_send_offload_vlan_v2_caps(struct iavf_adapter *adapter)2513{2514 int ret;2515 2516 WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_VLAN_V2));2517 2518 ret = iavf_send_vf_offload_vlan_v2_msg(adapter);2519 if (ret && ret == -EOPNOTSUPP) {2520 /* PF does not support VIRTCHNL_VF_OFFLOAD_V2. In this case,2521 * we did not send the capability exchange message and do not2522 * expect a response.2523 */2524 adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_VLAN_V2;2525 }2526 2527 /* We sent the message, so move on to the next step */2528 adapter->extended_caps &= ~IAVF_EXTENDED_CAP_SEND_VLAN_V2;2529}2530 2531/**2532 * iavf_init_recv_offload_vlan_v2_caps - part of initializing VLAN V2 caps2533 * @adapter: board private structure2534 *2535 * Function processes receipt of the extended VLAN V2 capability message from2536 * the PF.2537 **/2538static void iavf_init_recv_offload_vlan_v2_caps(struct iavf_adapter *adapter)2539{2540 int ret;2541 2542 WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_VLAN_V2));2543 2544 memset(&adapter->vlan_v2_caps, 0, sizeof(adapter->vlan_v2_caps));2545 2546 ret = iavf_get_vf_vlan_v2_caps(adapter);2547 if (ret)2548 goto err;2549 2550 /* We've processed receipt of the VLAN V2 caps message */2551 adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_VLAN_V2;2552 return;2553err:2554 /* We didn't receive a reply. Make sure we try sending again when2555 * __IAVF_INIT_FAILED attempts to recover.2556 */2557 adapter->extended_caps |= IAVF_EXTENDED_CAP_SEND_VLAN_V2;2558 iavf_change_state(adapter, __IAVF_INIT_FAILED);2559}2560 2561/**2562 * iavf_init_process_extended_caps - Part of driver startup2563 * @adapter: board private structure2564 *2565 * Function processes __IAVF_INIT_EXTENDED_CAPS driver state. This state2566 * handles negotiating capabilities for features which require an additional2567 * message.2568 *2569 * Once all extended capabilities exchanges are finished, the driver will2570 * transition into __IAVF_INIT_CONFIG_ADAPTER.2571 */2572static void iavf_init_process_extended_caps(struct iavf_adapter *adapter)2573{2574 WARN_ON(adapter->state != __IAVF_INIT_EXTENDED_CAPS);2575 2576 /* Process capability exchange for VLAN V2 */2577 if (adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_VLAN_V2) {2578 iavf_init_send_offload_vlan_v2_caps(adapter);2579 return;2580 } else if (adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_VLAN_V2) {2581 iavf_init_recv_offload_vlan_v2_caps(adapter);2582 return;2583 }2584 2585 /* When we reach here, no further extended capabilities exchanges are2586 * necessary, so we finally transition into __IAVF_INIT_CONFIG_ADAPTER2587 */2588 iavf_change_state(adapter, __IAVF_INIT_CONFIG_ADAPTER);2589}2590 2591/**2592 * iavf_init_config_adapter - last part of driver startup2593 * @adapter: board private structure2594 *2595 * After all the supported capabilities are negotiated, then the2596 * __IAVF_INIT_CONFIG_ADAPTER state will finish driver initialization.2597 */2598static void iavf_init_config_adapter(struct iavf_adapter *adapter)2599{2600 struct net_device *netdev = adapter->netdev;2601 struct pci_dev *pdev = adapter->pdev;2602 int err;2603 2604 WARN_ON(adapter->state != __IAVF_INIT_CONFIG_ADAPTER);2605 2606 if (iavf_process_config(adapter))2607 goto err;2608 2609 adapter->current_op = VIRTCHNL_OP_UNKNOWN;2610 2611 adapter->flags |= IAVF_FLAG_RX_CSUM_ENABLED;2612 2613 netdev->netdev_ops = &iavf_netdev_ops;2614 iavf_set_ethtool_ops(netdev);2615 netdev->watchdog_timeo = 5 * HZ;2616 2617 netdev->min_mtu = ETH_MIN_MTU;2618 netdev->max_mtu = LIBIE_MAX_MTU;2619 2620 if (!is_valid_ether_addr(adapter->hw.mac.addr)) {2621 dev_info(&pdev->dev, "Invalid MAC address %pM, using random\n",2622 adapter->hw.mac.addr);2623 eth_hw_addr_random(netdev);2624 ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr);2625 } else {2626 eth_hw_addr_set(netdev, adapter->hw.mac.addr);2627 ether_addr_copy(netdev->perm_addr, adapter->hw.mac.addr);2628 }2629 2630 adapter->tx_desc_count = IAVF_DEFAULT_TXD;2631 adapter->rx_desc_count = IAVF_DEFAULT_RXD;2632 err = iavf_init_interrupt_scheme(adapter);2633 if (err)2634 goto err_sw_init;2635 iavf_map_rings_to_vectors(adapter);2636 if (adapter->vf_res->vf_cap_flags &2637 VIRTCHNL_VF_OFFLOAD_WB_ON_ITR)2638 adapter->flags |= IAVF_FLAG_WB_ON_ITR_CAPABLE;2639 2640 err = iavf_request_misc_irq(adapter);2641 if (err)2642 goto err_sw_init;2643 2644 netif_carrier_off(netdev);2645 adapter->link_up = false;2646 netif_tx_stop_all_queues(netdev);2647 2648 dev_info(&pdev->dev, "MAC address: %pM\n", adapter->hw.mac.addr);2649 if (netdev->features & NETIF_F_GRO)2650 dev_info(&pdev->dev, "GRO is enabled\n");2651 2652 iavf_change_state(adapter, __IAVF_DOWN);2653 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);2654 2655 iavf_misc_irq_enable(adapter);2656 wake_up(&adapter->down_waitqueue);2657 2658 adapter->rss_key = kzalloc(adapter->rss_key_size, GFP_KERNEL);2659 adapter->rss_lut = kzalloc(adapter->rss_lut_size, GFP_KERNEL);2660 if (!adapter->rss_key || !adapter->rss_lut) {2661 err = -ENOMEM;2662 goto err_mem;2663 }2664 if (RSS_AQ(adapter))2665 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS;2666 else2667 iavf_init_rss(adapter);2668 2669 if (VLAN_V2_ALLOWED(adapter))2670 /* request initial VLAN offload settings */2671 iavf_set_vlan_offload_features(adapter, 0, netdev->features);2672 2673 iavf_schedule_finish_config(adapter);2674 return;2675 2676err_mem:2677 iavf_free_rss(adapter);2678 iavf_free_misc_irq(adapter);2679err_sw_init:2680 iavf_reset_interrupt_capability(adapter);2681err:2682 iavf_change_state(adapter, __IAVF_INIT_FAILED);2683}2684 2685/**2686 * iavf_watchdog_task - Periodic call-back task2687 * @work: pointer to work_struct2688 **/2689static void iavf_watchdog_task(struct work_struct *work)2690{2691 struct iavf_adapter *adapter = container_of(work,2692 struct iavf_adapter,2693 watchdog_task.work);2694 struct iavf_hw *hw = &adapter->hw;2695 u32 reg_val;2696 2697 if (!mutex_trylock(&adapter->crit_lock)) {2698 if (adapter->state == __IAVF_REMOVE)2699 return;2700 2701 goto restart_watchdog;2702 }2703 2704 if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)2705 iavf_change_state(adapter, __IAVF_COMM_FAILED);2706 2707 switch (adapter->state) {2708 case __IAVF_STARTUP:2709 iavf_startup(adapter);2710 mutex_unlock(&adapter->crit_lock);2711 queue_delayed_work(adapter->wq, &adapter->watchdog_task,2712 msecs_to_jiffies(30));2713 return;2714 case __IAVF_INIT_VERSION_CHECK:2715 iavf_init_version_check(adapter);2716 mutex_unlock(&adapter->crit_lock);2717 queue_delayed_work(adapter->wq, &adapter->watchdog_task,2718 msecs_to_jiffies(30));2719 return;2720 case __IAVF_INIT_GET_RESOURCES:2721 iavf_init_get_resources(adapter);2722 mutex_unlock(&adapter->crit_lock);2723 queue_delayed_work(adapter->wq, &adapter->watchdog_task,2724 msecs_to_jiffies(1));2725 return;2726 case __IAVF_INIT_EXTENDED_CAPS:2727 iavf_init_process_extended_caps(adapter);2728 mutex_unlock(&adapter->crit_lock);2729 queue_delayed_work(adapter->wq, &adapter->watchdog_task,2730 msecs_to_jiffies(1));2731 return;2732 case __IAVF_INIT_CONFIG_ADAPTER:2733 iavf_init_config_adapter(adapter);2734 mutex_unlock(&adapter->crit_lock);2735 queue_delayed_work(adapter->wq, &adapter->watchdog_task,2736 msecs_to_jiffies(1));2737 return;2738 case __IAVF_INIT_FAILED:2739 if (test_bit(__IAVF_IN_REMOVE_TASK,2740 &adapter->crit_section)) {2741 /* Do not update the state and do not reschedule2742 * watchdog task, iavf_remove should handle this state2743 * as it can loop forever2744 */2745 mutex_unlock(&adapter->crit_lock);2746 return;2747 }2748 if (++adapter->aq_wait_count > IAVF_AQ_MAX_ERR) {2749 dev_err(&adapter->pdev->dev,2750 "Failed to communicate with PF; waiting before retry\n");2751 adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;2752 iavf_shutdown_adminq(hw);2753 mutex_unlock(&adapter->crit_lock);2754 queue_delayed_work(adapter->wq,2755 &adapter->watchdog_task, (5 * HZ));2756 return;2757 }2758 /* Try again from failed step*/2759 iavf_change_state(adapter, adapter->last_state);2760 mutex_unlock(&adapter->crit_lock);2761 queue_delayed_work(adapter->wq, &adapter->watchdog_task, HZ);2762 return;2763 case __IAVF_COMM_FAILED:2764 if (test_bit(__IAVF_IN_REMOVE_TASK,2765 &adapter->crit_section)) {2766 /* Set state to __IAVF_INIT_FAILED and perform remove2767 * steps. Remove IAVF_FLAG_PF_COMMS_FAILED so the task2768 * doesn't bring the state back to __IAVF_COMM_FAILED.2769 */2770 iavf_change_state(adapter, __IAVF_INIT_FAILED);2771 adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;2772 mutex_unlock(&adapter->crit_lock);2773 return;2774 }2775 reg_val = rd32(hw, IAVF_VFGEN_RSTAT) &2776 IAVF_VFGEN_RSTAT_VFR_STATE_MASK;2777 if (reg_val == VIRTCHNL_VFR_VFACTIVE ||2778 reg_val == VIRTCHNL_VFR_COMPLETED) {2779 /* A chance for redemption! */2780 dev_err(&adapter->pdev->dev,2781 "Hardware came out of reset. Attempting reinit.\n");2782 /* When init task contacts the PF and2783 * gets everything set up again, it'll restart the2784 * watchdog for us. Down, boy. Sit. Stay. Woof.2785 */2786 iavf_change_state(adapter, __IAVF_STARTUP);2787 adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;2788 }2789 adapter->aq_required = 0;2790 adapter->current_op = VIRTCHNL_OP_UNKNOWN;2791 mutex_unlock(&adapter->crit_lock);2792 queue_delayed_work(adapter->wq,2793 &adapter->watchdog_task,2794 msecs_to_jiffies(10));2795 return;2796 case __IAVF_RESETTING:2797 mutex_unlock(&adapter->crit_lock);2798 queue_delayed_work(adapter->wq, &adapter->watchdog_task,2799 HZ * 2);2800 return;2801 case __IAVF_DOWN:2802 case __IAVF_DOWN_PENDING:2803 case __IAVF_TESTING:2804 case __IAVF_RUNNING:2805 if (adapter->current_op) {2806 if (!iavf_asq_done(hw)) {2807 dev_dbg(&adapter->pdev->dev,2808 "Admin queue timeout\n");2809 iavf_send_api_ver(adapter);2810 }2811 } else {2812 int ret = iavf_process_aq_command(adapter);2813 2814 /* An error will be returned if no commands were2815 * processed; use this opportunity to update stats2816 * if the error isn't -ENOTSUPP2817 */2818 if (ret && ret != -EOPNOTSUPP &&2819 adapter->state == __IAVF_RUNNING)2820 iavf_request_stats(adapter);2821 }2822 if (adapter->state == __IAVF_RUNNING)2823 iavf_detect_recover_hung(&adapter->vsi);2824 break;2825 case __IAVF_REMOVE:2826 default:2827 mutex_unlock(&adapter->crit_lock);2828 return;2829 }2830 2831 /* check for hw reset */2832 reg_val = rd32(hw, IAVF_VF_ARQLEN1) & IAVF_VF_ARQLEN1_ARQENABLE_MASK;2833 if (!reg_val) {2834 adapter->aq_required = 0;2835 adapter->current_op = VIRTCHNL_OP_UNKNOWN;2836 dev_err(&adapter->pdev->dev, "Hardware reset detected\n");2837 iavf_schedule_reset(adapter, IAVF_FLAG_RESET_PENDING);2838 mutex_unlock(&adapter->crit_lock);2839 queue_delayed_work(adapter->wq,2840 &adapter->watchdog_task, HZ * 2);2841 return;2842 }2843 2844 mutex_unlock(&adapter->crit_lock);2845restart_watchdog:2846 if (adapter->state >= __IAVF_DOWN)2847 queue_work(adapter->wq, &adapter->adminq_task);2848 if (adapter->aq_required)2849 queue_delayed_work(adapter->wq, &adapter->watchdog_task,2850 msecs_to_jiffies(20));2851 else2852 queue_delayed_work(adapter->wq, &adapter->watchdog_task,2853 HZ * 2);2854}2855 2856/**2857 * iavf_disable_vf - disable VF2858 * @adapter: board private structure2859 *2860 * Set communication failed flag and free all resources.2861 * NOTE: This function is expected to be called with crit_lock being held.2862 **/2863static void iavf_disable_vf(struct iavf_adapter *adapter)2864{2865 struct iavf_mac_filter *f, *ftmp;2866 struct iavf_vlan_filter *fv, *fvtmp;2867 struct iavf_cloud_filter *cf, *cftmp;2868 2869 adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;2870 2871 /* We don't use netif_running() because it may be true prior to2872 * ndo_open() returning, so we can't assume it means all our open2873 * tasks have finished, since we're not holding the rtnl_lock here.2874 */2875 if (adapter->state == __IAVF_RUNNING) {2876 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);2877 netif_carrier_off(adapter->netdev);2878 netif_tx_disable(adapter->netdev);2879 adapter->link_up = false;2880 iavf_napi_disable_all(adapter);2881 iavf_irq_disable(adapter);2882 iavf_free_traffic_irqs(adapter);2883 iavf_free_all_tx_resources(adapter);2884 iavf_free_all_rx_resources(adapter);2885 }2886 2887 spin_lock_bh(&adapter->mac_vlan_list_lock);2888 2889 /* Delete all of the filters */2890 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {2891 list_del(&f->list);2892 kfree(f);2893 }2894 2895 list_for_each_entry_safe(fv, fvtmp, &adapter->vlan_filter_list, list) {2896 list_del(&fv->list);2897 kfree(fv);2898 }2899 adapter->num_vlan_filters = 0;2900 2901 spin_unlock_bh(&adapter->mac_vlan_list_lock);2902 2903 spin_lock_bh(&adapter->cloud_filter_list_lock);2904 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {2905 list_del(&cf->list);2906 kfree(cf);2907 adapter->num_cloud_filters--;2908 }2909 spin_unlock_bh(&adapter->cloud_filter_list_lock);2910 2911 iavf_free_misc_irq(adapter);2912 iavf_free_interrupt_scheme(adapter);2913 memset(adapter->vf_res, 0, IAVF_VIRTCHNL_VF_RESOURCE_SIZE);2914 iavf_shutdown_adminq(&adapter->hw);2915 adapter->flags &= ~IAVF_FLAG_RESET_PENDING;2916 iavf_change_state(adapter, __IAVF_DOWN);2917 wake_up(&adapter->down_waitqueue);2918 dev_info(&adapter->pdev->dev, "Reset task did not complete, VF disabled\n");2919}2920 2921/**2922 * iavf_reset_task - Call-back task to handle hardware reset2923 * @work: pointer to work_struct2924 *2925 * During reset we need to shut down and reinitialize the admin queue2926 * before we can use it to communicate with the PF again. We also clear2927 * and reinit the rings because that context is lost as well.2928 **/2929static void iavf_reset_task(struct work_struct *work)2930{2931 struct iavf_adapter *adapter = container_of(work,2932 struct iavf_adapter,2933 reset_task);2934 struct virtchnl_vf_resource *vfres = adapter->vf_res;2935 struct net_device *netdev = adapter->netdev;2936 struct iavf_hw *hw = &adapter->hw;2937 struct iavf_mac_filter *f, *ftmp;2938 struct iavf_cloud_filter *cf;2939 enum iavf_status status;2940 u32 reg_val;2941 int i = 0, err;2942 bool running;2943 2944 /* When device is being removed it doesn't make sense to run the reset2945 * task, just return in such a case.2946 */2947 if (!mutex_trylock(&adapter->crit_lock)) {2948 if (adapter->state != __IAVF_REMOVE)2949 queue_work(adapter->wq, &adapter->reset_task);2950 2951 return;2952 }2953 2954 iavf_misc_irq_disable(adapter);2955 if (adapter->flags & IAVF_FLAG_RESET_NEEDED) {2956 adapter->flags &= ~IAVF_FLAG_RESET_NEEDED;2957 /* Restart the AQ here. If we have been reset but didn't2958 * detect it, or if the PF had to reinit, our AQ will be hosed.2959 */2960 iavf_shutdown_adminq(hw);2961 iavf_init_adminq(hw);2962 iavf_request_reset(adapter);2963 }2964 adapter->flags |= IAVF_FLAG_RESET_PENDING;2965 2966 /* poll until we see the reset actually happen */2967 for (i = 0; i < IAVF_RESET_WAIT_DETECTED_COUNT; i++) {2968 reg_val = rd32(hw, IAVF_VF_ARQLEN1) &2969 IAVF_VF_ARQLEN1_ARQENABLE_MASK;2970 if (!reg_val)2971 break;2972 usleep_range(5000, 10000);2973 }2974 if (i == IAVF_RESET_WAIT_DETECTED_COUNT) {2975 dev_info(&adapter->pdev->dev, "Never saw reset\n");2976 goto continue_reset; /* act like the reset happened */2977 }2978 2979 /* wait until the reset is complete and the PF is responding to us */2980 for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) {2981 /* sleep first to make sure a minimum wait time is met */2982 msleep(IAVF_RESET_WAIT_MS);2983 2984 reg_val = rd32(hw, IAVF_VFGEN_RSTAT) &2985 IAVF_VFGEN_RSTAT_VFR_STATE_MASK;2986 if (reg_val == VIRTCHNL_VFR_VFACTIVE)2987 break;2988 }2989 2990 pci_set_master(adapter->pdev);2991 pci_restore_msi_state(adapter->pdev);2992 2993 if (i == IAVF_RESET_WAIT_COMPLETE_COUNT) {2994 dev_err(&adapter->pdev->dev, "Reset never finished (%x)\n",2995 reg_val);2996 iavf_disable_vf(adapter);2997 mutex_unlock(&adapter->crit_lock);2998 return; /* Do not attempt to reinit. It's dead, Jim. */2999 }3000 3001continue_reset:3002 /* We don't use netif_running() because it may be true prior to3003 * ndo_open() returning, so we can't assume it means all our open3004 * tasks have finished, since we're not holding the rtnl_lock here.3005 */3006 running = adapter->state == __IAVF_RUNNING;3007 3008 if (running) {3009 netif_carrier_off(netdev);3010 netif_tx_stop_all_queues(netdev);3011 adapter->link_up = false;3012 iavf_napi_disable_all(adapter);3013 }3014 iavf_irq_disable(adapter);3015 3016 iavf_change_state(adapter, __IAVF_RESETTING);3017 adapter->flags &= ~IAVF_FLAG_RESET_PENDING;3018 3019 /* free the Tx/Rx rings and descriptors, might be better to just3020 * re-use them sometime in the future3021 */3022 iavf_free_all_rx_resources(adapter);3023 iavf_free_all_tx_resources(adapter);3024 3025 adapter->flags |= IAVF_FLAG_QUEUES_DISABLED;3026 /* kill and reinit the admin queue */3027 iavf_shutdown_adminq(hw);3028 adapter->current_op = VIRTCHNL_OP_UNKNOWN;3029 status = iavf_init_adminq(hw);3030 if (status) {3031 dev_info(&adapter->pdev->dev, "Failed to init adminq: %d\n",3032 status);3033 goto reset_err;3034 }3035 adapter->aq_required = 0;3036 3037 if ((adapter->flags & IAVF_FLAG_REINIT_MSIX_NEEDED) ||3038 (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED)) {3039 err = iavf_reinit_interrupt_scheme(adapter, running);3040 if (err)3041 goto reset_err;3042 }3043 3044 if (RSS_AQ(adapter)) {3045 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS;3046 } else {3047 err = iavf_init_rss(adapter);3048 if (err)3049 goto reset_err;3050 }3051 3052 adapter->aq_required |= IAVF_FLAG_AQ_GET_CONFIG;3053 /* always set since VIRTCHNL_OP_GET_VF_RESOURCES has not been3054 * sent/received yet, so VLAN_V2_ALLOWED() cannot is not reliable here,3055 * however the VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS won't be sent until3056 * VIRTCHNL_OP_GET_VF_RESOURCES and VIRTCHNL_VF_OFFLOAD_VLAN_V2 have3057 * been successfully sent and negotiated3058 */3059 adapter->aq_required |= IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS;3060 adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS;3061 3062 spin_lock_bh(&adapter->mac_vlan_list_lock);3063 3064 /* Delete filter for the current MAC address, it could have3065 * been changed by the PF via administratively set MAC.3066 * Will be re-added via VIRTCHNL_OP_GET_VF_RESOURCES.3067 */3068 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {3069 if (ether_addr_equal(f->macaddr, adapter->hw.mac.addr)) {3070 list_del(&f->list);3071 kfree(f);3072 }3073 }3074 /* re-add all MAC filters */3075 list_for_each_entry(f, &adapter->mac_filter_list, list) {3076 f->add = true;3077 }3078 spin_unlock_bh(&adapter->mac_vlan_list_lock);3079 3080 /* check if TCs are running and re-add all cloud filters */3081 spin_lock_bh(&adapter->cloud_filter_list_lock);3082 if ((vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&3083 adapter->num_tc) {3084 list_for_each_entry(cf, &adapter->cloud_filter_list, list) {3085 cf->add = true;3086 }3087 }3088 spin_unlock_bh(&adapter->cloud_filter_list_lock);3089 3090 adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;3091 adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER;3092 iavf_misc_irq_enable(adapter);3093 3094 mod_delayed_work(adapter->wq, &adapter->watchdog_task, 2);3095 3096 /* We were running when the reset started, so we need to restore some3097 * state here.3098 */3099 if (running) {3100 /* allocate transmit descriptors */3101 err = iavf_setup_all_tx_resources(adapter);3102 if (err)3103 goto reset_err;3104 3105 /* allocate receive descriptors */3106 err = iavf_setup_all_rx_resources(adapter);3107 if (err)3108 goto reset_err;3109 3110 if ((adapter->flags & IAVF_FLAG_REINIT_MSIX_NEEDED) ||3111 (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED)) {3112 err = iavf_request_traffic_irqs(adapter, netdev->name);3113 if (err)3114 goto reset_err;3115 3116 adapter->flags &= ~IAVF_FLAG_REINIT_MSIX_NEEDED;3117 }3118 3119 iavf_configure(adapter);3120 3121 /* iavf_up_complete() will switch device back3122 * to __IAVF_RUNNING3123 */3124 iavf_up_complete(adapter);3125 3126 iavf_irq_enable(adapter, true);3127 } else {3128 iavf_change_state(adapter, __IAVF_DOWN);3129 wake_up(&adapter->down_waitqueue);3130 }3131 3132 adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;3133 3134 wake_up(&adapter->reset_waitqueue);3135 mutex_unlock(&adapter->crit_lock);3136 3137 return;3138reset_err:3139 if (running) {3140 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);3141 iavf_free_traffic_irqs(adapter);3142 }3143 iavf_disable_vf(adapter);3144 3145 mutex_unlock(&adapter->crit_lock);3146 dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n");3147}3148 3149/**3150 * iavf_adminq_task - worker thread to clean the admin queue3151 * @work: pointer to work_struct containing our data3152 **/3153static void iavf_adminq_task(struct work_struct *work)3154{3155 struct iavf_adapter *adapter =3156 container_of(work, struct iavf_adapter, adminq_task);3157 struct iavf_hw *hw = &adapter->hw;3158 struct iavf_arq_event_info event;3159 enum virtchnl_ops v_op;3160 enum iavf_status ret, v_ret;3161 u32 val, oldval;3162 u16 pending;3163 3164 if (!mutex_trylock(&adapter->crit_lock)) {3165 if (adapter->state == __IAVF_REMOVE)3166 return;3167 3168 queue_work(adapter->wq, &adapter->adminq_task);3169 goto out;3170 }3171 3172 if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)3173 goto unlock;3174 3175 event.buf_len = IAVF_MAX_AQ_BUF_SIZE;3176 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);3177 if (!event.msg_buf)3178 goto unlock;3179 3180 do {3181 ret = iavf_clean_arq_element(hw, &event, &pending);3182 v_op = (enum virtchnl_ops)le32_to_cpu(event.desc.cookie_high);3183 v_ret = (enum iavf_status)le32_to_cpu(event.desc.cookie_low);3184 3185 if (ret || !v_op)3186 break; /* No event to process or error cleaning ARQ */3187 3188 iavf_virtchnl_completion(adapter, v_op, v_ret, event.msg_buf,3189 event.msg_len);3190 if (pending != 0)3191 memset(event.msg_buf, 0, IAVF_MAX_AQ_BUF_SIZE);3192 } while (pending);3193 3194 if (iavf_is_reset_in_progress(adapter))3195 goto freedom;3196 3197 /* check for error indications */3198 val = rd32(hw, IAVF_VF_ARQLEN1);3199 if (val == 0xdeadbeef || val == 0xffffffff) /* device in reset */3200 goto freedom;3201 oldval = val;3202 if (val & IAVF_VF_ARQLEN1_ARQVFE_MASK) {3203 dev_info(&adapter->pdev->dev, "ARQ VF Error detected\n");3204 val &= ~IAVF_VF_ARQLEN1_ARQVFE_MASK;3205 }3206 if (val & IAVF_VF_ARQLEN1_ARQOVFL_MASK) {3207 dev_info(&adapter->pdev->dev, "ARQ Overflow Error detected\n");3208 val &= ~IAVF_VF_ARQLEN1_ARQOVFL_MASK;3209 }3210 if (val & IAVF_VF_ARQLEN1_ARQCRIT_MASK) {3211 dev_info(&adapter->pdev->dev, "ARQ Critical Error detected\n");3212 val &= ~IAVF_VF_ARQLEN1_ARQCRIT_MASK;3213 }3214 if (oldval != val)3215 wr32(hw, IAVF_VF_ARQLEN1, val);3216 3217 val = rd32(hw, IAVF_VF_ATQLEN1);3218 oldval = val;3219 if (val & IAVF_VF_ATQLEN1_ATQVFE_MASK) {3220 dev_info(&adapter->pdev->dev, "ASQ VF Error detected\n");3221 val &= ~IAVF_VF_ATQLEN1_ATQVFE_MASK;3222 }3223 if (val & IAVF_VF_ATQLEN1_ATQOVFL_MASK) {3224 dev_info(&adapter->pdev->dev, "ASQ Overflow Error detected\n");3225 val &= ~IAVF_VF_ATQLEN1_ATQOVFL_MASK;3226 }3227 if (val & IAVF_VF_ATQLEN1_ATQCRIT_MASK) {3228 dev_info(&adapter->pdev->dev, "ASQ Critical Error detected\n");3229 val &= ~IAVF_VF_ATQLEN1_ATQCRIT_MASK;3230 }3231 if (oldval != val)3232 wr32(hw, IAVF_VF_ATQLEN1, val);3233 3234freedom:3235 kfree(event.msg_buf);3236unlock:3237 mutex_unlock(&adapter->crit_lock);3238out:3239 /* re-enable Admin queue interrupt cause */3240 iavf_misc_irq_enable(adapter);3241}3242 3243/**3244 * iavf_free_all_tx_resources - Free Tx Resources for All Queues3245 * @adapter: board private structure3246 *3247 * Free all transmit software resources3248 **/3249void iavf_free_all_tx_resources(struct iavf_adapter *adapter)3250{3251 int i;3252 3253 if (!adapter->tx_rings)3254 return;3255 3256 for (i = 0; i < adapter->num_active_queues; i++)3257 if (adapter->tx_rings[i].desc)3258 iavf_free_tx_resources(&adapter->tx_rings[i]);3259}3260 3261/**3262 * iavf_setup_all_tx_resources - allocate all queues Tx resources3263 * @adapter: board private structure3264 *3265 * If this function returns with an error, then it's possible one or3266 * more of the rings is populated (while the rest are not). It is the3267 * callers duty to clean those orphaned rings.3268 *3269 * Return 0 on success, negative on failure3270 **/3271static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter)3272{3273 int i, err = 0;3274 3275 for (i = 0; i < adapter->num_active_queues; i++) {3276 adapter->tx_rings[i].count = adapter->tx_desc_count;3277 err = iavf_setup_tx_descriptors(&adapter->tx_rings[i]);3278 if (!err)3279 continue;3280 dev_err(&adapter->pdev->dev,3281 "Allocation for Tx Queue %u failed\n", i);3282 break;3283 }3284 3285 return err;3286}3287 3288/**3289 * iavf_setup_all_rx_resources - allocate all queues Rx resources3290 * @adapter: board private structure3291 *3292 * If this function returns with an error, then it's possible one or3293 * more of the rings is populated (while the rest are not). It is the3294 * callers duty to clean those orphaned rings.3295 *3296 * Return 0 on success, negative on failure3297 **/3298static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter)3299{3300 int i, err = 0;3301 3302 for (i = 0; i < adapter->num_active_queues; i++) {3303 adapter->rx_rings[i].count = adapter->rx_desc_count;3304 err = iavf_setup_rx_descriptors(&adapter->rx_rings[i]);3305 if (!err)3306 continue;3307 dev_err(&adapter->pdev->dev,3308 "Allocation for Rx Queue %u failed\n", i);3309 break;3310 }3311 return err;3312}3313 3314/**3315 * iavf_free_all_rx_resources - Free Rx Resources for All Queues3316 * @adapter: board private structure3317 *3318 * Free all receive software resources3319 **/3320void iavf_free_all_rx_resources(struct iavf_adapter *adapter)3321{3322 int i;3323 3324 if (!adapter->rx_rings)3325 return;3326 3327 for (i = 0; i < adapter->num_active_queues; i++)3328 if (adapter->rx_rings[i].desc)3329 iavf_free_rx_resources(&adapter->rx_rings[i]);3330}3331 3332/**3333 * iavf_validate_tx_bandwidth - validate the max Tx bandwidth3334 * @adapter: board private structure3335 * @max_tx_rate: max Tx bw for a tc3336 **/3337static int iavf_validate_tx_bandwidth(struct iavf_adapter *adapter,3338 u64 max_tx_rate)3339{3340 int speed = 0, ret = 0;3341 3342 if (ADV_LINK_SUPPORT(adapter)) {3343 if (adapter->link_speed_mbps < U32_MAX) {3344 speed = adapter->link_speed_mbps;3345 goto validate_bw;3346 } else {3347 dev_err(&adapter->pdev->dev, "Unknown link speed\n");3348 return -EINVAL;3349 }3350 }3351 3352 switch (adapter->link_speed) {3353 case VIRTCHNL_LINK_SPEED_40GB:3354 speed = SPEED_40000;3355 break;3356 case VIRTCHNL_LINK_SPEED_25GB:3357 speed = SPEED_25000;3358 break;3359 case VIRTCHNL_LINK_SPEED_20GB:3360 speed = SPEED_20000;3361 break;3362 case VIRTCHNL_LINK_SPEED_10GB:3363 speed = SPEED_10000;3364 break;3365 case VIRTCHNL_LINK_SPEED_5GB:3366 speed = SPEED_5000;3367 break;3368 case VIRTCHNL_LINK_SPEED_2_5GB:3369 speed = SPEED_2500;3370 break;3371 case VIRTCHNL_LINK_SPEED_1GB:3372 speed = SPEED_1000;3373 break;3374 case VIRTCHNL_LINK_SPEED_100MB:3375 speed = SPEED_100;3376 break;3377 default:3378 break;3379 }3380 3381validate_bw:3382 if (max_tx_rate > speed) {3383 dev_err(&adapter->pdev->dev,3384 "Invalid tx rate specified\n");3385 ret = -EINVAL;3386 }3387 3388 return ret;3389}3390 3391/**3392 * iavf_validate_ch_config - validate queue mapping info3393 * @adapter: board private structure3394 * @mqprio_qopt: queue parameters3395 *3396 * This function validates if the config provided by the user to3397 * configure queue channels is valid or not. Returns 0 on a valid3398 * config.3399 **/3400static int iavf_validate_ch_config(struct iavf_adapter *adapter,3401 struct tc_mqprio_qopt_offload *mqprio_qopt)3402{3403 u64 total_max_rate = 0;3404 u32 tx_rate_rem = 0;3405 int i, num_qps = 0;3406 u64 tx_rate = 0;3407 int ret = 0;3408 3409 if (mqprio_qopt->qopt.num_tc > IAVF_MAX_TRAFFIC_CLASS ||3410 mqprio_qopt->qopt.num_tc < 1)3411 return -EINVAL;3412 3413 for (i = 0; i <= mqprio_qopt->qopt.num_tc - 1; i++) {3414 if (!mqprio_qopt->qopt.count[i] ||3415 mqprio_qopt->qopt.offset[i] != num_qps)3416 return -EINVAL;3417 if (mqprio_qopt->min_rate[i]) {3418 dev_err(&adapter->pdev->dev,3419 "Invalid min tx rate (greater than 0) specified for TC%d\n",3420 i);3421 return -EINVAL;3422 }3423 3424 /* convert to Mbps */3425 tx_rate = div_u64(mqprio_qopt->max_rate[i],3426 IAVF_MBPS_DIVISOR);3427 3428 if (mqprio_qopt->max_rate[i] &&3429 tx_rate < IAVF_MBPS_QUANTA) {3430 dev_err(&adapter->pdev->dev,3431 "Invalid max tx rate for TC%d, minimum %dMbps\n",3432 i, IAVF_MBPS_QUANTA);3433 return -EINVAL;3434 }3435 3436 (void)div_u64_rem(tx_rate, IAVF_MBPS_QUANTA, &tx_rate_rem);3437 3438 if (tx_rate_rem != 0) {3439 dev_err(&adapter->pdev->dev,3440 "Invalid max tx rate for TC%d, not divisible by %d\n",3441 i, IAVF_MBPS_QUANTA);3442 return -EINVAL;3443 }3444 3445 total_max_rate += tx_rate;3446 num_qps += mqprio_qopt->qopt.count[i];3447 }3448 if (num_qps > adapter->num_active_queues) {3449 dev_err(&adapter->pdev->dev,3450 "Cannot support requested number of queues\n");3451 return -EINVAL;3452 }3453 3454 ret = iavf_validate_tx_bandwidth(adapter, total_max_rate);3455 return ret;3456}3457 3458/**3459 * iavf_del_all_cloud_filters - delete all cloud filters on the traffic classes3460 * @adapter: board private structure3461 **/3462static void iavf_del_all_cloud_filters(struct iavf_adapter *adapter)3463{3464 struct iavf_cloud_filter *cf, *cftmp;3465 3466 spin_lock_bh(&adapter->cloud_filter_list_lock);3467 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list,3468 list) {3469 list_del(&cf->list);3470 kfree(cf);3471 adapter->num_cloud_filters--;3472 }3473 spin_unlock_bh(&adapter->cloud_filter_list_lock);3474}3475 3476/**3477 * iavf_is_tc_config_same - Compare the mqprio TC config with the3478 * TC config already configured on this adapter.3479 * @adapter: board private structure3480 * @mqprio_qopt: TC config received from kernel.3481 *3482 * This function compares the TC config received from the kernel3483 * with the config already configured on the adapter.3484 *3485 * Return: True if configuration is same, false otherwise.3486 **/3487static bool iavf_is_tc_config_same(struct iavf_adapter *adapter,3488 struct tc_mqprio_qopt *mqprio_qopt)3489{3490 struct virtchnl_channel_info *ch = &adapter->ch_config.ch_info[0];3491 int i;3492 3493 if (adapter->num_tc != mqprio_qopt->num_tc)3494 return false;3495 3496 for (i = 0; i < adapter->num_tc; i++) {3497 if (ch[i].count != mqprio_qopt->count[i] ||3498 ch[i].offset != mqprio_qopt->offset[i])3499 return false;3500 }3501 return true;3502}3503 3504/**3505 * __iavf_setup_tc - configure multiple traffic classes3506 * @netdev: network interface device structure3507 * @type_data: tc offload data3508 *3509 * This function processes the config information provided by the3510 * user to configure traffic classes/queue channels and packages the3511 * information to request the PF to setup traffic classes.3512 *3513 * Returns 0 on success.3514 **/3515static int __iavf_setup_tc(struct net_device *netdev, void *type_data)3516{3517 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;3518 struct iavf_adapter *adapter = netdev_priv(netdev);3519 struct virtchnl_vf_resource *vfres = adapter->vf_res;3520 u8 num_tc = 0, total_qps = 0;3521 int ret = 0, netdev_tc = 0;3522 u64 max_tx_rate;3523 u16 mode;3524 int i;3525 3526 num_tc = mqprio_qopt->qopt.num_tc;3527 mode = mqprio_qopt->mode;3528 3529 /* delete queue_channel */3530 if (!mqprio_qopt->qopt.hw) {3531 if (adapter->ch_config.state == __IAVF_TC_RUNNING) {3532 /* reset the tc configuration */3533 netdev_reset_tc(netdev);3534 adapter->num_tc = 0;3535 netif_tx_stop_all_queues(netdev);3536 netif_tx_disable(netdev);3537 iavf_del_all_cloud_filters(adapter);3538 adapter->aq_required = IAVF_FLAG_AQ_DISABLE_CHANNELS;3539 total_qps = adapter->orig_num_active_queues;3540 goto exit;3541 } else {3542 return -EINVAL;3543 }3544 }3545 3546 /* add queue channel */3547 if (mode == TC_MQPRIO_MODE_CHANNEL) {3548 if (!(vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)) {3549 dev_err(&adapter->pdev->dev, "ADq not supported\n");3550 return -EOPNOTSUPP;3551 }3552 if (adapter->ch_config.state != __IAVF_TC_INVALID) {3553 dev_err(&adapter->pdev->dev, "TC configuration already exists\n");3554 return -EINVAL;3555 }3556 3557 ret = iavf_validate_ch_config(adapter, mqprio_qopt);3558 if (ret)3559 return ret;3560 /* Return if same TC config is requested */3561 if (iavf_is_tc_config_same(adapter, &mqprio_qopt->qopt))3562 return 0;3563 adapter->num_tc = num_tc;3564 3565 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) {3566 if (i < num_tc) {3567 adapter->ch_config.ch_info[i].count =3568 mqprio_qopt->qopt.count[i];3569 adapter->ch_config.ch_info[i].offset =3570 mqprio_qopt->qopt.offset[i];3571 total_qps += mqprio_qopt->qopt.count[i];3572 max_tx_rate = mqprio_qopt->max_rate[i];3573 /* convert to Mbps */3574 max_tx_rate = div_u64(max_tx_rate,3575 IAVF_MBPS_DIVISOR);3576 adapter->ch_config.ch_info[i].max_tx_rate =3577 max_tx_rate;3578 } else {3579 adapter->ch_config.ch_info[i].count = 1;3580 adapter->ch_config.ch_info[i].offset = 0;3581 }3582 }3583 3584 /* Take snapshot of original config such as "num_active_queues"3585 * It is used later when delete ADQ flow is exercised, so that3586 * once delete ADQ flow completes, VF shall go back to its3587 * original queue configuration3588 */3589 3590 adapter->orig_num_active_queues = adapter->num_active_queues;3591 3592 /* Store queue info based on TC so that VF gets configured3593 * with correct number of queues when VF completes ADQ config3594 * flow3595 */3596 adapter->ch_config.total_qps = total_qps;3597 3598 netif_tx_stop_all_queues(netdev);3599 netif_tx_disable(netdev);3600 adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_CHANNELS;3601 netdev_reset_tc(netdev);3602 /* Report the tc mapping up the stack */3603 netdev_set_num_tc(adapter->netdev, num_tc);3604 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) {3605 u16 qcount = mqprio_qopt->qopt.count[i];3606 u16 qoffset = mqprio_qopt->qopt.offset[i];3607 3608 if (i < num_tc)3609 netdev_set_tc_queue(netdev, netdev_tc++, qcount,3610 qoffset);3611 }3612 }3613exit:3614 if (test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section))3615 return 0;3616 3617 netif_set_real_num_rx_queues(netdev, total_qps);3618 netif_set_real_num_tx_queues(netdev, total_qps);3619 3620 return ret;3621}3622 3623/**3624 * iavf_parse_cls_flower - Parse tc flower filters provided by kernel3625 * @adapter: board private structure3626 * @f: pointer to struct flow_cls_offload3627 * @filter: pointer to cloud filter structure3628 */3629static int iavf_parse_cls_flower(struct iavf_adapter *adapter,3630 struct flow_cls_offload *f,3631 struct iavf_cloud_filter *filter)3632{3633 struct flow_rule *rule = flow_cls_offload_flow_rule(f);3634 struct flow_dissector *dissector = rule->match.dissector;3635 u16 n_proto_mask = 0;3636 u16 n_proto_key = 0;3637 u8 field_flags = 0;3638 u16 addr_type = 0;3639 u16 n_proto = 0;3640 int i = 0;3641 struct virtchnl_filter *vf = &filter->f;3642 3643 if (dissector->used_keys &3644 ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |3645 BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |3646 BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |3647 BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) |3648 BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |3649 BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |3650 BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |3651 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID))) {3652 dev_err(&adapter->pdev->dev, "Unsupported key used: 0x%llx\n",3653 dissector->used_keys);3654 return -EOPNOTSUPP;3655 }3656 3657 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {3658 struct flow_match_enc_keyid match;3659 3660 flow_rule_match_enc_keyid(rule, &match);3661 if (match.mask->keyid != 0)3662 field_flags |= IAVF_CLOUD_FIELD_TEN_ID;3663 }3664 3665 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {3666 struct flow_match_basic match;3667 3668 flow_rule_match_basic(rule, &match);3669 n_proto_key = ntohs(match.key->n_proto);3670 n_proto_mask = ntohs(match.mask->n_proto);3671 3672 if (n_proto_key == ETH_P_ALL) {3673 n_proto_key = 0;3674 n_proto_mask = 0;3675 }3676 n_proto = n_proto_key & n_proto_mask;3677 if (n_proto != ETH_P_IP && n_proto != ETH_P_IPV6)3678 return -EINVAL;3679 if (n_proto == ETH_P_IPV6) {3680 /* specify flow type as TCP IPv6 */3681 vf->flow_type = VIRTCHNL_TCP_V6_FLOW;3682 }3683 3684 if (match.key->ip_proto != IPPROTO_TCP) {3685 dev_info(&adapter->pdev->dev, "Only TCP transport is supported\n");3686 return -EINVAL;3687 }3688 }3689 3690 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {3691 struct flow_match_eth_addrs match;3692 3693 flow_rule_match_eth_addrs(rule, &match);3694 3695 /* use is_broadcast and is_zero to check for all 0xf or 0 */3696 if (!is_zero_ether_addr(match.mask->dst)) {3697 if (is_broadcast_ether_addr(match.mask->dst)) {3698 field_flags |= IAVF_CLOUD_FIELD_OMAC;3699 } else {3700 dev_err(&adapter->pdev->dev, "Bad ether dest mask %pM\n",3701 match.mask->dst);3702 return -EINVAL;3703 }3704 }3705 3706 if (!is_zero_ether_addr(match.mask->src)) {3707 if (is_broadcast_ether_addr(match.mask->src)) {3708 field_flags |= IAVF_CLOUD_FIELD_IMAC;3709 } else {3710 dev_err(&adapter->pdev->dev, "Bad ether src mask %pM\n",3711 match.mask->src);3712 return -EINVAL;3713 }3714 }3715 3716 if (!is_zero_ether_addr(match.key->dst))3717 if (is_valid_ether_addr(match.key->dst) ||3718 is_multicast_ether_addr(match.key->dst)) {3719 /* set the mask if a valid dst_mac address */3720 for (i = 0; i < ETH_ALEN; i++)3721 vf->mask.tcp_spec.dst_mac[i] |= 0xff;3722 ether_addr_copy(vf->data.tcp_spec.dst_mac,3723 match.key->dst);3724 }3725 3726 if (!is_zero_ether_addr(match.key->src))3727 if (is_valid_ether_addr(match.key->src) ||3728 is_multicast_ether_addr(match.key->src)) {3729 /* set the mask if a valid dst_mac address */3730 for (i = 0; i < ETH_ALEN; i++)3731 vf->mask.tcp_spec.src_mac[i] |= 0xff;3732 ether_addr_copy(vf->data.tcp_spec.src_mac,3733 match.key->src);3734 }3735 }3736 3737 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {3738 struct flow_match_vlan match;3739 3740 flow_rule_match_vlan(rule, &match);3741 if (match.mask->vlan_id) {3742 if (match.mask->vlan_id == VLAN_VID_MASK) {3743 field_flags |= IAVF_CLOUD_FIELD_IVLAN;3744 } else {3745 dev_err(&adapter->pdev->dev, "Bad vlan mask %u\n",3746 match.mask->vlan_id);3747 return -EINVAL;3748 }3749 }3750 vf->mask.tcp_spec.vlan_id |= cpu_to_be16(0xffff);3751 vf->data.tcp_spec.vlan_id = cpu_to_be16(match.key->vlan_id);3752 }3753 3754 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {3755 struct flow_match_control match;3756 3757 flow_rule_match_control(rule, &match);3758 addr_type = match.key->addr_type;3759 3760 if (flow_rule_has_control_flags(match.mask->flags,3761 f->common.extack))3762 return -EOPNOTSUPP;3763 }3764 3765 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {3766 struct flow_match_ipv4_addrs match;3767 3768 flow_rule_match_ipv4_addrs(rule, &match);3769 if (match.mask->dst) {3770 if (match.mask->dst == cpu_to_be32(0xffffffff)) {3771 field_flags |= IAVF_CLOUD_FIELD_IIP;3772 } else {3773 dev_err(&adapter->pdev->dev, "Bad ip dst mask 0x%08x\n",3774 be32_to_cpu(match.mask->dst));3775 return -EINVAL;3776 }3777 }3778 3779 if (match.mask->src) {3780 if (match.mask->src == cpu_to_be32(0xffffffff)) {3781 field_flags |= IAVF_CLOUD_FIELD_IIP;3782 } else {3783 dev_err(&adapter->pdev->dev, "Bad ip src mask 0x%08x\n",3784 be32_to_cpu(match.mask->src));3785 return -EINVAL;3786 }3787 }3788 3789 if (field_flags & IAVF_CLOUD_FIELD_TEN_ID) {3790 dev_info(&adapter->pdev->dev, "Tenant id not allowed for ip filter\n");3791 return -EINVAL;3792 }3793 if (match.key->dst) {3794 vf->mask.tcp_spec.dst_ip[0] |= cpu_to_be32(0xffffffff);3795 vf->data.tcp_spec.dst_ip[0] = match.key->dst;3796 }3797 if (match.key->src) {3798 vf->mask.tcp_spec.src_ip[0] |= cpu_to_be32(0xffffffff);3799 vf->data.tcp_spec.src_ip[0] = match.key->src;3800 }3801 }3802 3803 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {3804 struct flow_match_ipv6_addrs match;3805 3806 flow_rule_match_ipv6_addrs(rule, &match);3807 3808 /* validate mask, make sure it is not IPV6_ADDR_ANY */3809 if (ipv6_addr_any(&match.mask->dst)) {3810 dev_err(&adapter->pdev->dev, "Bad ipv6 dst mask 0x%02x\n",3811 IPV6_ADDR_ANY);3812 return -EINVAL;3813 }3814 3815 /* src and dest IPv6 address should not be LOOPBACK3816 * (0:0:0:0:0:0:0:1) which can be represented as ::13817 */3818 if (ipv6_addr_loopback(&match.key->dst) ||3819 ipv6_addr_loopback(&match.key->src)) {3820 dev_err(&adapter->pdev->dev,3821 "ipv6 addr should not be loopback\n");3822 return -EINVAL;3823 }3824 if (!ipv6_addr_any(&match.mask->dst) ||3825 !ipv6_addr_any(&match.mask->src))3826 field_flags |= IAVF_CLOUD_FIELD_IIP;3827 3828 for (i = 0; i < 4; i++)3829 vf->mask.tcp_spec.dst_ip[i] |= cpu_to_be32(0xffffffff);3830 memcpy(&vf->data.tcp_spec.dst_ip, &match.key->dst.s6_addr32,3831 sizeof(vf->data.tcp_spec.dst_ip));3832 for (i = 0; i < 4; i++)3833 vf->mask.tcp_spec.src_ip[i] |= cpu_to_be32(0xffffffff);3834 memcpy(&vf->data.tcp_spec.src_ip, &match.key->src.s6_addr32,3835 sizeof(vf->data.tcp_spec.src_ip));3836 }3837 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {3838 struct flow_match_ports match;3839 3840 flow_rule_match_ports(rule, &match);3841 if (match.mask->src) {3842 if (match.mask->src == cpu_to_be16(0xffff)) {3843 field_flags |= IAVF_CLOUD_FIELD_IIP;3844 } else {3845 dev_err(&adapter->pdev->dev, "Bad src port mask %u\n",3846 be16_to_cpu(match.mask->src));3847 return -EINVAL;3848 }3849 }3850 3851 if (match.mask->dst) {3852 if (match.mask->dst == cpu_to_be16(0xffff)) {3853 field_flags |= IAVF_CLOUD_FIELD_IIP;3854 } else {3855 dev_err(&adapter->pdev->dev, "Bad dst port mask %u\n",3856 be16_to_cpu(match.mask->dst));3857 return -EINVAL;3858 }3859 }3860 if (match.key->dst) {3861 vf->mask.tcp_spec.dst_port |= cpu_to_be16(0xffff);3862 vf->data.tcp_spec.dst_port = match.key->dst;3863 }3864 3865 if (match.key->src) {3866 vf->mask.tcp_spec.src_port |= cpu_to_be16(0xffff);3867 vf->data.tcp_spec.src_port = match.key->src;3868 }3869 }3870 vf->field_flags = field_flags;3871 3872 return 0;3873}3874 3875/**3876 * iavf_handle_tclass - Forward to a traffic class on the device3877 * @adapter: board private structure3878 * @tc: traffic class index on the device3879 * @filter: pointer to cloud filter structure3880 */3881static int iavf_handle_tclass(struct iavf_adapter *adapter, u32 tc,3882 struct iavf_cloud_filter *filter)3883{3884 if (tc == 0)3885 return 0;3886 if (tc < adapter->num_tc) {3887 if (!filter->f.data.tcp_spec.dst_port) {3888 dev_err(&adapter->pdev->dev,3889 "Specify destination port to redirect to traffic class other than TC0\n");3890 return -EINVAL;3891 }3892 }3893 /* redirect to a traffic class on the same device */3894 filter->f.action = VIRTCHNL_ACTION_TC_REDIRECT;3895 filter->f.action_meta = tc;3896 return 0;3897}3898 3899/**3900 * iavf_find_cf - Find the cloud filter in the list3901 * @adapter: Board private structure3902 * @cookie: filter specific cookie3903 *3904 * Returns ptr to the filter object or NULL. Must be called while holding the3905 * cloud_filter_list_lock.3906 */3907static struct iavf_cloud_filter *iavf_find_cf(struct iavf_adapter *adapter,3908 unsigned long *cookie)3909{3910 struct iavf_cloud_filter *filter = NULL;3911 3912 if (!cookie)3913 return NULL;3914 3915 list_for_each_entry(filter, &adapter->cloud_filter_list, list) {3916 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))3917 return filter;3918 }3919 return NULL;3920}3921 3922/**3923 * iavf_configure_clsflower - Add tc flower filters3924 * @adapter: board private structure3925 * @cls_flower: Pointer to struct flow_cls_offload3926 */3927static int iavf_configure_clsflower(struct iavf_adapter *adapter,3928 struct flow_cls_offload *cls_flower)3929{3930 int tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid);3931 struct iavf_cloud_filter *filter = NULL;3932 int err = -EINVAL, count = 50;3933 3934 if (tc < 0) {3935 dev_err(&adapter->pdev->dev, "Invalid traffic class\n");3936 return -EINVAL;3937 }3938 3939 filter = kzalloc(sizeof(*filter), GFP_KERNEL);3940 if (!filter)3941 return -ENOMEM;3942 3943 while (!mutex_trylock(&adapter->crit_lock)) {3944 if (--count == 0) {3945 kfree(filter);3946 return err;3947 }3948 udelay(1);3949 }3950 3951 filter->cookie = cls_flower->cookie;3952 3953 /* bail out here if filter already exists */3954 spin_lock_bh(&adapter->cloud_filter_list_lock);3955 if (iavf_find_cf(adapter, &cls_flower->cookie)) {3956 dev_err(&adapter->pdev->dev, "Failed to add TC Flower filter, it already exists\n");3957 err = -EEXIST;3958 goto spin_unlock;3959 }3960 spin_unlock_bh(&adapter->cloud_filter_list_lock);3961 3962 /* set the mask to all zeroes to begin with */3963 memset(&filter->f.mask.tcp_spec, 0, sizeof(struct virtchnl_l4_spec));3964 /* start out with flow type and eth type IPv4 to begin with */3965 filter->f.flow_type = VIRTCHNL_TCP_V4_FLOW;3966 err = iavf_parse_cls_flower(adapter, cls_flower, filter);3967 if (err)3968 goto err;3969 3970 err = iavf_handle_tclass(adapter, tc, filter);3971 if (err)3972 goto err;3973 3974 /* add filter to the list */3975 spin_lock_bh(&adapter->cloud_filter_list_lock);3976 list_add_tail(&filter->list, &adapter->cloud_filter_list);3977 adapter->num_cloud_filters++;3978 filter->add = true;3979 adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER;3980spin_unlock:3981 spin_unlock_bh(&adapter->cloud_filter_list_lock);3982err:3983 if (err)3984 kfree(filter);3985 3986 mutex_unlock(&adapter->crit_lock);3987 return err;3988}3989 3990/**3991 * iavf_delete_clsflower - Remove tc flower filters3992 * @adapter: board private structure3993 * @cls_flower: Pointer to struct flow_cls_offload3994 */3995static int iavf_delete_clsflower(struct iavf_adapter *adapter,3996 struct flow_cls_offload *cls_flower)3997{3998 struct iavf_cloud_filter *filter = NULL;3999 int err = 0;4000 4001 spin_lock_bh(&adapter->cloud_filter_list_lock);4002 filter = iavf_find_cf(adapter, &cls_flower->cookie);4003 if (filter) {4004 filter->del = true;4005 adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;4006 } else {4007 err = -EINVAL;4008 }4009 spin_unlock_bh(&adapter->cloud_filter_list_lock);4010 4011 return err;4012}4013 4014/**4015 * iavf_setup_tc_cls_flower - flower classifier offloads4016 * @adapter: pointer to iavf adapter structure4017 * @cls_flower: pointer to flow_cls_offload struct with flow info4018 */4019static int iavf_setup_tc_cls_flower(struct iavf_adapter *adapter,4020 struct flow_cls_offload *cls_flower)4021{4022 switch (cls_flower->command) {4023 case FLOW_CLS_REPLACE:4024 return iavf_configure_clsflower(adapter, cls_flower);4025 case FLOW_CLS_DESTROY:4026 return iavf_delete_clsflower(adapter, cls_flower);4027 case FLOW_CLS_STATS:4028 return -EOPNOTSUPP;4029 default:4030 return -EOPNOTSUPP;4031 }4032}4033 4034/**4035 * iavf_add_cls_u32 - Add U32 classifier offloads4036 * @adapter: pointer to iavf adapter structure4037 * @cls_u32: pointer to tc_cls_u32_offload struct with flow info4038 *4039 * Return: 0 on success or negative errno on failure.4040 */4041static int iavf_add_cls_u32(struct iavf_adapter *adapter,4042 struct tc_cls_u32_offload *cls_u32)4043{4044 struct netlink_ext_ack *extack = cls_u32->common.extack;4045 struct virtchnl_fdir_rule *rule_cfg;4046 struct virtchnl_filter_action *vact;4047 struct virtchnl_proto_hdrs *hdrs;4048 struct ethhdr *spec_h, *mask_h;4049 const struct tc_action *act;4050 struct iavf_fdir_fltr *fltr;4051 struct tcf_exts *exts;4052 unsigned int q_index;4053 int i, status = 0;4054 int off_base = 0;4055 4056 if (cls_u32->knode.link_handle) {4057 NL_SET_ERR_MSG_MOD(extack, "Linking not supported");4058 return -EOPNOTSUPP;4059 }4060 4061 fltr = kzalloc(sizeof(*fltr), GFP_KERNEL);4062 if (!fltr)4063 return -ENOMEM;4064 4065 rule_cfg = &fltr->vc_add_msg.rule_cfg;4066 hdrs = &rule_cfg->proto_hdrs;4067 hdrs->count = 0;4068 4069 /* The parser lib at the PF expects the packet starting with MAC hdr */4070 switch (ntohs(cls_u32->common.protocol)) {4071 case ETH_P_802_3:4072 break;4073 case ETH_P_IP:4074 spec_h = (struct ethhdr *)hdrs->raw.spec;4075 mask_h = (struct ethhdr *)hdrs->raw.mask;4076 spec_h->h_proto = htons(ETH_P_IP);4077 mask_h->h_proto = htons(0xFFFF);4078 off_base += ETH_HLEN;4079 break;4080 default:4081 NL_SET_ERR_MSG_MOD(extack, "Only 802_3 and ip filter protocols are supported");4082 status = -EOPNOTSUPP;4083 goto free_alloc;4084 }4085 4086 for (i = 0; i < cls_u32->knode.sel->nkeys; i++) {4087 __be32 val, mask;4088 int off;4089 4090 off = off_base + cls_u32->knode.sel->keys[i].off;4091 val = cls_u32->knode.sel->keys[i].val;4092 mask = cls_u32->knode.sel->keys[i].mask;4093 4094 if (off >= sizeof(hdrs->raw.spec)) {4095 NL_SET_ERR_MSG_MOD(extack, "Input exceeds maximum allowed.");4096 status = -EINVAL;4097 goto free_alloc;4098 }4099 4100 memcpy(&hdrs->raw.spec[off], &val, sizeof(val));4101 memcpy(&hdrs->raw.mask[off], &mask, sizeof(mask));4102 hdrs->raw.pkt_len = off + sizeof(val);4103 }4104 4105 /* Only one action is allowed */4106 rule_cfg->action_set.count = 1;4107 vact = &rule_cfg->action_set.actions[0];4108 exts = cls_u32->knode.exts;4109 4110 tcf_exts_for_each_action(i, act, exts) {4111 /* FDIR queue */4112 if (is_tcf_skbedit_rx_queue_mapping(act)) {4113 q_index = tcf_skbedit_rx_queue_mapping(act);4114 if (q_index >= adapter->num_active_queues) {4115 status = -EINVAL;4116 goto free_alloc;4117 }4118 4119 vact->type = VIRTCHNL_ACTION_QUEUE;4120 vact->act_conf.queue.index = q_index;4121 break;4122 }4123 4124 /* Drop */4125 if (is_tcf_gact_shot(act)) {4126 vact->type = VIRTCHNL_ACTION_DROP;4127 break;4128 }4129 4130 /* Unsupported action */4131 NL_SET_ERR_MSG_MOD(extack, "Unsupported action.");4132 status = -EOPNOTSUPP;4133 goto free_alloc;4134 }4135 4136 fltr->vc_add_msg.vsi_id = adapter->vsi.id;4137 fltr->cls_u32_handle = cls_u32->knode.handle;4138 return iavf_fdir_add_fltr(adapter, fltr);4139 4140free_alloc:4141 kfree(fltr);4142 return status;4143}4144 4145/**4146 * iavf_del_cls_u32 - Delete U32 classifier offloads4147 * @adapter: pointer to iavf adapter structure4148 * @cls_u32: pointer to tc_cls_u32_offload struct with flow info4149 *4150 * Return: 0 on success or negative errno on failure.4151 */4152static int iavf_del_cls_u32(struct iavf_adapter *adapter,4153 struct tc_cls_u32_offload *cls_u32)4154{4155 return iavf_fdir_del_fltr(adapter, true, cls_u32->knode.handle);4156}4157 4158/**4159 * iavf_setup_tc_cls_u32 - U32 filter offloads4160 * @adapter: pointer to iavf adapter structure4161 * @cls_u32: pointer to tc_cls_u32_offload struct with flow info4162 *4163 * Return: 0 on success or negative errno on failure.4164 */4165static int iavf_setup_tc_cls_u32(struct iavf_adapter *adapter,4166 struct tc_cls_u32_offload *cls_u32)4167{4168 if (!TC_U32_SUPPORT(adapter) || !FDIR_FLTR_SUPPORT(adapter))4169 return -EOPNOTSUPP;4170 4171 switch (cls_u32->command) {4172 case TC_CLSU32_NEW_KNODE:4173 case TC_CLSU32_REPLACE_KNODE:4174 return iavf_add_cls_u32(adapter, cls_u32);4175 case TC_CLSU32_DELETE_KNODE:4176 return iavf_del_cls_u32(adapter, cls_u32);4177 default:4178 return -EOPNOTSUPP;4179 }4180}4181 4182/**4183 * iavf_setup_tc_block_cb - block callback for tc4184 * @type: type of offload4185 * @type_data: offload data4186 * @cb_priv:4187 *4188 * This function is the block callback for traffic classes4189 **/4190static int iavf_setup_tc_block_cb(enum tc_setup_type type, void *type_data,4191 void *cb_priv)4192{4193 struct iavf_adapter *adapter = cb_priv;4194 4195 if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data))4196 return -EOPNOTSUPP;4197 4198 switch (type) {4199 case TC_SETUP_CLSFLOWER:4200 return iavf_setup_tc_cls_flower(cb_priv, type_data);4201 case TC_SETUP_CLSU32:4202 return iavf_setup_tc_cls_u32(cb_priv, type_data);4203 default:4204 return -EOPNOTSUPP;4205 }4206}4207 4208static LIST_HEAD(iavf_block_cb_list);4209 4210/**4211 * iavf_setup_tc - configure multiple traffic classes4212 * @netdev: network interface device structure4213 * @type: type of offload4214 * @type_data: tc offload data4215 *4216 * This function is the callback to ndo_setup_tc in the4217 * netdev_ops.4218 *4219 * Returns 0 on success4220 **/4221static int iavf_setup_tc(struct net_device *netdev, enum tc_setup_type type,4222 void *type_data)4223{4224 struct iavf_adapter *adapter = netdev_priv(netdev);4225 4226 switch (type) {4227 case TC_SETUP_QDISC_MQPRIO:4228 return __iavf_setup_tc(netdev, type_data);4229 case TC_SETUP_BLOCK:4230 return flow_block_cb_setup_simple(type_data,4231 &iavf_block_cb_list,4232 iavf_setup_tc_block_cb,4233 adapter, adapter, true);4234 default:4235 return -EOPNOTSUPP;4236 }4237}4238 4239/**4240 * iavf_restore_fdir_filters4241 * @adapter: board private structure4242 *4243 * Restore existing FDIR filters when VF netdev comes back up.4244 **/4245static void iavf_restore_fdir_filters(struct iavf_adapter *adapter)4246{4247 struct iavf_fdir_fltr *f;4248 4249 spin_lock_bh(&adapter->fdir_fltr_lock);4250 list_for_each_entry(f, &adapter->fdir_list_head, list) {4251 if (f->state == IAVF_FDIR_FLTR_DIS_REQUEST) {4252 /* Cancel a request, keep filter as active */4253 f->state = IAVF_FDIR_FLTR_ACTIVE;4254 } else if (f->state == IAVF_FDIR_FLTR_DIS_PENDING ||4255 f->state == IAVF_FDIR_FLTR_INACTIVE) {4256 /* Add filters which are inactive or have a pending4257 * request to PF to be deleted4258 */4259 f->state = IAVF_FDIR_FLTR_ADD_REQUEST;4260 adapter->aq_required |= IAVF_FLAG_AQ_ADD_FDIR_FILTER;4261 }4262 }4263 spin_unlock_bh(&adapter->fdir_fltr_lock);4264}4265 4266/**4267 * iavf_open - Called when a network interface is made active4268 * @netdev: network interface device structure4269 *4270 * Returns 0 on success, negative value on failure4271 *4272 * The open entry point is called when a network interface is made4273 * active by the system (IFF_UP). At this point all resources needed4274 * for transmit and receive operations are allocated, the interrupt4275 * handler is registered with the OS, the watchdog is started,4276 * and the stack is notified that the interface is ready.4277 **/4278static int iavf_open(struct net_device *netdev)4279{4280 struct iavf_adapter *adapter = netdev_priv(netdev);4281 int err;4282 4283 if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) {4284 dev_err(&adapter->pdev->dev, "Unable to open device due to PF driver failure.\n");4285 return -EIO;4286 }4287 4288 while (!mutex_trylock(&adapter->crit_lock)) {4289 /* If we are in __IAVF_INIT_CONFIG_ADAPTER state the crit_lock4290 * is already taken and iavf_open is called from an upper4291 * device's notifier reacting on NETDEV_REGISTER event.4292 * We have to leave here to avoid dead lock.4293 */4294 if (adapter->state == __IAVF_INIT_CONFIG_ADAPTER)4295 return -EBUSY;4296 4297 usleep_range(500, 1000);4298 }4299 4300 if (adapter->state != __IAVF_DOWN) {4301 err = -EBUSY;4302 goto err_unlock;4303 }4304 4305 if (adapter->state == __IAVF_RUNNING &&4306 !test_bit(__IAVF_VSI_DOWN, adapter->vsi.state)) {4307 dev_dbg(&adapter->pdev->dev, "VF is already open.\n");4308 err = 0;4309 goto err_unlock;4310 }4311 4312 /* allocate transmit descriptors */4313 err = iavf_setup_all_tx_resources(adapter);4314 if (err)4315 goto err_setup_tx;4316 4317 /* allocate receive descriptors */4318 err = iavf_setup_all_rx_resources(adapter);4319 if (err)4320 goto err_setup_rx;4321 4322 /* clear any pending interrupts, may auto mask */4323 err = iavf_request_traffic_irqs(adapter, netdev->name);4324 if (err)4325 goto err_req_irq;4326 4327 spin_lock_bh(&adapter->mac_vlan_list_lock);4328 4329 iavf_add_filter(adapter, adapter->hw.mac.addr);4330 4331 spin_unlock_bh(&adapter->mac_vlan_list_lock);4332 4333 /* Restore filters that were removed with IFF_DOWN */4334 iavf_restore_filters(adapter);4335 iavf_restore_fdir_filters(adapter);4336 4337 iavf_configure(adapter);4338 4339 iavf_up_complete(adapter);4340 4341 iavf_irq_enable(adapter, true);4342 4343 mutex_unlock(&adapter->crit_lock);4344 4345 return 0;4346 4347err_req_irq:4348 iavf_down(adapter);4349 iavf_free_traffic_irqs(adapter);4350err_setup_rx:4351 iavf_free_all_rx_resources(adapter);4352err_setup_tx:4353 iavf_free_all_tx_resources(adapter);4354err_unlock:4355 mutex_unlock(&adapter->crit_lock);4356 4357 return err;4358}4359 4360/**4361 * iavf_close - Disables a network interface4362 * @netdev: network interface device structure4363 *4364 * Returns 0, this is not allowed to fail4365 *4366 * The close entry point is called when an interface is de-activated4367 * by the OS. The hardware is still under the drivers control, but4368 * needs to be disabled. All IRQs except vector 0 (reserved for admin queue)4369 * are freed, along with all transmit and receive resources.4370 **/4371static int iavf_close(struct net_device *netdev)4372{4373 struct iavf_adapter *adapter = netdev_priv(netdev);4374 u64 aq_to_restore;4375 int status;4376 4377 mutex_lock(&adapter->crit_lock);4378 4379 if (adapter->state <= __IAVF_DOWN_PENDING) {4380 mutex_unlock(&adapter->crit_lock);4381 return 0;4382 }4383 4384 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);4385 /* We cannot send IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS before4386 * IAVF_FLAG_AQ_DISABLE_QUEUES because in such case there is rtnl4387 * deadlock with adminq_task() until iavf_close timeouts. We must send4388 * IAVF_FLAG_AQ_GET_CONFIG before IAVF_FLAG_AQ_DISABLE_QUEUES to make4389 * disable queues possible for vf. Give only necessary flags to4390 * iavf_down and save other to set them right before iavf_close()4391 * returns, when IAVF_FLAG_AQ_DISABLE_QUEUES will be already sent and4392 * iavf will be in DOWN state.4393 */4394 aq_to_restore = adapter->aq_required;4395 adapter->aq_required &= IAVF_FLAG_AQ_GET_CONFIG;4396 4397 /* Remove flags which we do not want to send after close or we want to4398 * send before disable queues.4399 */4400 aq_to_restore &= ~(IAVF_FLAG_AQ_GET_CONFIG |4401 IAVF_FLAG_AQ_ENABLE_QUEUES |4402 IAVF_FLAG_AQ_CONFIGURE_QUEUES |4403 IAVF_FLAG_AQ_ADD_VLAN_FILTER |4404 IAVF_FLAG_AQ_ADD_MAC_FILTER |4405 IAVF_FLAG_AQ_ADD_CLOUD_FILTER |4406 IAVF_FLAG_AQ_ADD_FDIR_FILTER |4407 IAVF_FLAG_AQ_ADD_ADV_RSS_CFG);4408 4409 iavf_down(adapter);4410 iavf_change_state(adapter, __IAVF_DOWN_PENDING);4411 iavf_free_traffic_irqs(adapter);4412 4413 mutex_unlock(&adapter->crit_lock);4414 4415 /* We explicitly don't free resources here because the hardware is4416 * still active and can DMA into memory. Resources are cleared in4417 * iavf_virtchnl_completion() after we get confirmation from the PF4418 * driver that the rings have been stopped.4419 *4420 * Also, we wait for state to transition to __IAVF_DOWN before4421 * returning. State change occurs in iavf_virtchnl_completion() after4422 * VF resources are released (which occurs after PF driver processes and4423 * responds to admin queue commands).4424 */4425 4426 status = wait_event_timeout(adapter->down_waitqueue,4427 adapter->state == __IAVF_DOWN,4428 msecs_to_jiffies(500));4429 if (!status)4430 netdev_warn(netdev, "Device resources not yet released\n");4431 4432 mutex_lock(&adapter->crit_lock);4433 adapter->aq_required |= aq_to_restore;4434 mutex_unlock(&adapter->crit_lock);4435 return 0;4436}4437 4438/**4439 * iavf_change_mtu - Change the Maximum Transfer Unit4440 * @netdev: network interface device structure4441 * @new_mtu: new value for maximum frame size4442 *4443 * Returns 0 on success, negative on failure4444 **/4445static int iavf_change_mtu(struct net_device *netdev, int new_mtu)4446{4447 struct iavf_adapter *adapter = netdev_priv(netdev);4448 int ret = 0;4449 4450 netdev_dbg(netdev, "changing MTU from %d to %d\n",4451 netdev->mtu, new_mtu);4452 WRITE_ONCE(netdev->mtu, new_mtu);4453 4454 if (netif_running(netdev)) {4455 iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED);4456 ret = iavf_wait_for_reset(adapter);4457 if (ret < 0)4458 netdev_warn(netdev, "MTU change interrupted waiting for reset");4459 else if (ret)4460 netdev_warn(netdev, "MTU change timed out waiting for reset");4461 }4462 4463 return ret;4464}4465 4466/**4467 * iavf_disable_fdir - disable Flow Director and clear existing filters4468 * @adapter: board private structure4469 **/4470static void iavf_disable_fdir(struct iavf_adapter *adapter)4471{4472 struct iavf_fdir_fltr *fdir, *fdirtmp;4473 bool del_filters = false;4474 4475 adapter->flags &= ~IAVF_FLAG_FDIR_ENABLED;4476 4477 /* remove all Flow Director filters */4478 spin_lock_bh(&adapter->fdir_fltr_lock);4479 list_for_each_entry_safe(fdir, fdirtmp, &adapter->fdir_list_head,4480 list) {4481 if (fdir->state == IAVF_FDIR_FLTR_ADD_REQUEST ||4482 fdir->state == IAVF_FDIR_FLTR_INACTIVE) {4483 /* Delete filters not registered in PF */4484 list_del(&fdir->list);4485 iavf_dec_fdir_active_fltr(adapter, fdir);4486 kfree(fdir);4487 } else if (fdir->state == IAVF_FDIR_FLTR_ADD_PENDING ||4488 fdir->state == IAVF_FDIR_FLTR_DIS_REQUEST ||4489 fdir->state == IAVF_FDIR_FLTR_ACTIVE) {4490 /* Filters registered in PF, schedule their deletion */4491 fdir->state = IAVF_FDIR_FLTR_DEL_REQUEST;4492 del_filters = true;4493 } else if (fdir->state == IAVF_FDIR_FLTR_DIS_PENDING) {4494 /* Request to delete filter already sent to PF, change4495 * state to DEL_PENDING to delete filter after PF's4496 * response, not set as INACTIVE4497 */4498 fdir->state = IAVF_FDIR_FLTR_DEL_PENDING;4499 }4500 }4501 spin_unlock_bh(&adapter->fdir_fltr_lock);4502 4503 if (del_filters) {4504 adapter->aq_required |= IAVF_FLAG_AQ_DEL_FDIR_FILTER;4505 mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);4506 }4507}4508 4509#define NETIF_VLAN_OFFLOAD_FEATURES (NETIF_F_HW_VLAN_CTAG_RX | \4510 NETIF_F_HW_VLAN_CTAG_TX | \4511 NETIF_F_HW_VLAN_STAG_RX | \4512 NETIF_F_HW_VLAN_STAG_TX)4513 4514/**4515 * iavf_set_features - set the netdev feature flags4516 * @netdev: ptr to the netdev being adjusted4517 * @features: the feature set that the stack is suggesting4518 * Note: expects to be called while under rtnl_lock()4519 **/4520static int iavf_set_features(struct net_device *netdev,4521 netdev_features_t features)4522{4523 struct iavf_adapter *adapter = netdev_priv(netdev);4524 4525 /* trigger update on any VLAN feature change */4526 if ((netdev->features & NETIF_VLAN_OFFLOAD_FEATURES) ^4527 (features & NETIF_VLAN_OFFLOAD_FEATURES))4528 iavf_set_vlan_offload_features(adapter, netdev->features,4529 features);4530 if (CRC_OFFLOAD_ALLOWED(adapter) &&4531 ((netdev->features & NETIF_F_RXFCS) ^ (features & NETIF_F_RXFCS)))4532 iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED);4533 4534 if ((netdev->features & NETIF_F_NTUPLE) ^ (features & NETIF_F_NTUPLE)) {4535 if (features & NETIF_F_NTUPLE)4536 adapter->flags |= IAVF_FLAG_FDIR_ENABLED;4537 else4538 iavf_disable_fdir(adapter);4539 }4540 4541 return 0;4542}4543 4544/**4545 * iavf_features_check - Validate encapsulated packet conforms to limits4546 * @skb: skb buff4547 * @dev: This physical port's netdev4548 * @features: Offload features that the stack believes apply4549 **/4550static netdev_features_t iavf_features_check(struct sk_buff *skb,4551 struct net_device *dev,4552 netdev_features_t features)4553{4554 size_t len;4555 4556 /* No point in doing any of this if neither checksum nor GSO are4557 * being requested for this frame. We can rule out both by just4558 * checking for CHECKSUM_PARTIAL4559 */4560 if (skb->ip_summed != CHECKSUM_PARTIAL)4561 return features;4562 4563 /* We cannot support GSO if the MSS is going to be less than4564 * 64 bytes. If it is then we need to drop support for GSO.4565 */4566 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))4567 features &= ~NETIF_F_GSO_MASK;4568 4569 /* MACLEN can support at most 63 words */4570 len = skb_network_offset(skb);4571 if (len & ~(63 * 2))4572 goto out_err;4573 4574 /* IPLEN and EIPLEN can support at most 127 dwords */4575 len = skb_network_header_len(skb);4576 if (len & ~(127 * 4))4577 goto out_err;4578 4579 if (skb->encapsulation) {4580 /* L4TUNLEN can support 127 words */4581 len = skb_inner_network_header(skb) - skb_transport_header(skb);4582 if (len & ~(127 * 2))4583 goto out_err;4584 4585 /* IPLEN can support at most 127 dwords */4586 len = skb_inner_transport_header(skb) -4587 skb_inner_network_header(skb);4588 if (len & ~(127 * 4))4589 goto out_err;4590 }4591 4592 /* No need to validate L4LEN as TCP is the only protocol with a4593 * flexible value and we support all possible values supported4594 * by TCP, which is at most 15 dwords4595 */4596 4597 return features;4598out_err:4599 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);4600}4601 4602/**4603 * iavf_get_netdev_vlan_hw_features - get NETDEV VLAN features that can toggle on/off4604 * @adapter: board private structure4605 *4606 * Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V24607 * were negotiated determine the VLAN features that can be toggled on and off.4608 **/4609static netdev_features_t4610iavf_get_netdev_vlan_hw_features(struct iavf_adapter *adapter)4611{4612 netdev_features_t hw_features = 0;4613 4614 if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags)4615 return hw_features;4616 4617 /* Enable VLAN features if supported */4618 if (VLAN_ALLOWED(adapter)) {4619 hw_features |= (NETIF_F_HW_VLAN_CTAG_TX |4620 NETIF_F_HW_VLAN_CTAG_RX);4621 } else if (VLAN_V2_ALLOWED(adapter)) {4622 struct virtchnl_vlan_caps *vlan_v2_caps =4623 &adapter->vlan_v2_caps;4624 struct virtchnl_vlan_supported_caps *stripping_support =4625 &vlan_v2_caps->offloads.stripping_support;4626 struct virtchnl_vlan_supported_caps *insertion_support =4627 &vlan_v2_caps->offloads.insertion_support;4628 4629 if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED &&4630 stripping_support->outer & VIRTCHNL_VLAN_TOGGLE) {4631 if (stripping_support->outer &4632 VIRTCHNL_VLAN_ETHERTYPE_8100)4633 hw_features |= NETIF_F_HW_VLAN_CTAG_RX;4634 if (stripping_support->outer &4635 VIRTCHNL_VLAN_ETHERTYPE_88A8)4636 hw_features |= NETIF_F_HW_VLAN_STAG_RX;4637 } else if (stripping_support->inner !=4638 VIRTCHNL_VLAN_UNSUPPORTED &&4639 stripping_support->inner & VIRTCHNL_VLAN_TOGGLE) {4640 if (stripping_support->inner &4641 VIRTCHNL_VLAN_ETHERTYPE_8100)4642 hw_features |= NETIF_F_HW_VLAN_CTAG_RX;4643 }4644 4645 if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED &&4646 insertion_support->outer & VIRTCHNL_VLAN_TOGGLE) {4647 if (insertion_support->outer &4648 VIRTCHNL_VLAN_ETHERTYPE_8100)4649 hw_features |= NETIF_F_HW_VLAN_CTAG_TX;4650 if (insertion_support->outer &4651 VIRTCHNL_VLAN_ETHERTYPE_88A8)4652 hw_features |= NETIF_F_HW_VLAN_STAG_TX;4653 } else if (insertion_support->inner &&4654 insertion_support->inner & VIRTCHNL_VLAN_TOGGLE) {4655 if (insertion_support->inner &4656 VIRTCHNL_VLAN_ETHERTYPE_8100)4657 hw_features |= NETIF_F_HW_VLAN_CTAG_TX;4658 }4659 }4660 4661 if (CRC_OFFLOAD_ALLOWED(adapter))4662 hw_features |= NETIF_F_RXFCS;4663 4664 return hw_features;4665}4666 4667/**4668 * iavf_get_netdev_vlan_features - get the enabled NETDEV VLAN fetures4669 * @adapter: board private structure4670 *4671 * Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V24672 * were negotiated determine the VLAN features that are enabled by default.4673 **/4674static netdev_features_t4675iavf_get_netdev_vlan_features(struct iavf_adapter *adapter)4676{4677 netdev_features_t features = 0;4678 4679 if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags)4680 return features;4681 4682 if (VLAN_ALLOWED(adapter)) {4683 features |= NETIF_F_HW_VLAN_CTAG_FILTER |4684 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX;4685 } else if (VLAN_V2_ALLOWED(adapter)) {4686 struct virtchnl_vlan_caps *vlan_v2_caps =4687 &adapter->vlan_v2_caps;4688 struct virtchnl_vlan_supported_caps *filtering_support =4689 &vlan_v2_caps->filtering.filtering_support;4690 struct virtchnl_vlan_supported_caps *stripping_support =4691 &vlan_v2_caps->offloads.stripping_support;4692 struct virtchnl_vlan_supported_caps *insertion_support =4693 &vlan_v2_caps->offloads.insertion_support;4694 u32 ethertype_init;4695 4696 /* give priority to outer stripping and don't support both outer4697 * and inner stripping4698 */4699 ethertype_init = vlan_v2_caps->offloads.ethertype_init;4700 if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {4701 if (stripping_support->outer &4702 VIRTCHNL_VLAN_ETHERTYPE_8100 &&4703 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)4704 features |= NETIF_F_HW_VLAN_CTAG_RX;4705 else if (stripping_support->outer &4706 VIRTCHNL_VLAN_ETHERTYPE_88A8 &&4707 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)4708 features |= NETIF_F_HW_VLAN_STAG_RX;4709 } else if (stripping_support->inner !=4710 VIRTCHNL_VLAN_UNSUPPORTED) {4711 if (stripping_support->inner &4712 VIRTCHNL_VLAN_ETHERTYPE_8100 &&4713 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)4714 features |= NETIF_F_HW_VLAN_CTAG_RX;4715 }4716 4717 /* give priority to outer insertion and don't support both outer4718 * and inner insertion4719 */4720 if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {4721 if (insertion_support->outer &4722 VIRTCHNL_VLAN_ETHERTYPE_8100 &&4723 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)4724 features |= NETIF_F_HW_VLAN_CTAG_TX;4725 else if (insertion_support->outer &4726 VIRTCHNL_VLAN_ETHERTYPE_88A8 &&4727 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)4728 features |= NETIF_F_HW_VLAN_STAG_TX;4729 } else if (insertion_support->inner !=4730 VIRTCHNL_VLAN_UNSUPPORTED) {4731 if (insertion_support->inner &4732 VIRTCHNL_VLAN_ETHERTYPE_8100 &&4733 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)4734 features |= NETIF_F_HW_VLAN_CTAG_TX;4735 }4736 4737 /* give priority to outer filtering and don't bother if both4738 * outer and inner filtering are enabled4739 */4740 ethertype_init = vlan_v2_caps->filtering.ethertype_init;4741 if (filtering_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {4742 if (filtering_support->outer &4743 VIRTCHNL_VLAN_ETHERTYPE_8100 &&4744 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)4745 features |= NETIF_F_HW_VLAN_CTAG_FILTER;4746 if (filtering_support->outer &4747 VIRTCHNL_VLAN_ETHERTYPE_88A8 &&4748 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)4749 features |= NETIF_F_HW_VLAN_STAG_FILTER;4750 } else if (filtering_support->inner !=4751 VIRTCHNL_VLAN_UNSUPPORTED) {4752 if (filtering_support->inner &4753 VIRTCHNL_VLAN_ETHERTYPE_8100 &&4754 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)4755 features |= NETIF_F_HW_VLAN_CTAG_FILTER;4756 if (filtering_support->inner &4757 VIRTCHNL_VLAN_ETHERTYPE_88A8 &&4758 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)4759 features |= NETIF_F_HW_VLAN_STAG_FILTER;4760 }4761 }4762 4763 return features;4764}4765 4766#define IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested, allowed, feature_bit) \4767 (!(((requested) & (feature_bit)) && \4768 !((allowed) & (feature_bit))))4769 4770/**4771 * iavf_fix_netdev_vlan_features - fix NETDEV VLAN features based on support4772 * @adapter: board private structure4773 * @requested_features: stack requested NETDEV features4774 **/4775static netdev_features_t4776iavf_fix_netdev_vlan_features(struct iavf_adapter *adapter,4777 netdev_features_t requested_features)4778{4779 netdev_features_t allowed_features;4780 4781 allowed_features = iavf_get_netdev_vlan_hw_features(adapter) |4782 iavf_get_netdev_vlan_features(adapter);4783 4784 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,4785 allowed_features,4786 NETIF_F_HW_VLAN_CTAG_TX))4787 requested_features &= ~NETIF_F_HW_VLAN_CTAG_TX;4788 4789 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,4790 allowed_features,4791 NETIF_F_HW_VLAN_CTAG_RX))4792 requested_features &= ~NETIF_F_HW_VLAN_CTAG_RX;4793 4794 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,4795 allowed_features,4796 NETIF_F_HW_VLAN_STAG_TX))4797 requested_features &= ~NETIF_F_HW_VLAN_STAG_TX;4798 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,4799 allowed_features,4800 NETIF_F_HW_VLAN_STAG_RX))4801 requested_features &= ~NETIF_F_HW_VLAN_STAG_RX;4802 4803 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,4804 allowed_features,4805 NETIF_F_HW_VLAN_CTAG_FILTER))4806 requested_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;4807 4808 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,4809 allowed_features,4810 NETIF_F_HW_VLAN_STAG_FILTER))4811 requested_features &= ~NETIF_F_HW_VLAN_STAG_FILTER;4812 4813 if ((requested_features &4814 (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) &&4815 (requested_features &4816 (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX)) &&4817 adapter->vlan_v2_caps.offloads.ethertype_match ==4818 VIRTCHNL_ETHERTYPE_STRIPPING_MATCHES_INSERTION) {4819 netdev_warn(adapter->netdev, "cannot support CTAG and STAG VLAN stripping and/or insertion simultaneously since CTAG and STAG offloads are mutually exclusive, clearing STAG offload settings\n");4820 requested_features &= ~(NETIF_F_HW_VLAN_STAG_RX |4821 NETIF_F_HW_VLAN_STAG_TX);4822 }4823 4824 return requested_features;4825}4826 4827/**4828 * iavf_fix_strip_features - fix NETDEV CRC and VLAN strip features4829 * @adapter: board private structure4830 * @requested_features: stack requested NETDEV features4831 *4832 * Returns fixed-up features bits4833 **/4834static netdev_features_t4835iavf_fix_strip_features(struct iavf_adapter *adapter,4836 netdev_features_t requested_features)4837{4838 struct net_device *netdev = adapter->netdev;4839 bool crc_offload_req, is_vlan_strip;4840 netdev_features_t vlan_strip;4841 int num_non_zero_vlan;4842 4843 crc_offload_req = CRC_OFFLOAD_ALLOWED(adapter) &&4844 (requested_features & NETIF_F_RXFCS);4845 num_non_zero_vlan = iavf_get_num_vlans_added(adapter);4846 vlan_strip = (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX);4847 is_vlan_strip = requested_features & vlan_strip;4848 4849 if (!crc_offload_req)4850 return requested_features;4851 4852 if (!num_non_zero_vlan && (netdev->features & vlan_strip) &&4853 !(netdev->features & NETIF_F_RXFCS) && is_vlan_strip) {4854 requested_features &= ~vlan_strip;4855 netdev_info(netdev, "Disabling VLAN stripping as FCS/CRC stripping is also disabled and there is no VLAN configured\n");4856 return requested_features;4857 }4858 4859 if ((netdev->features & NETIF_F_RXFCS) && is_vlan_strip) {4860 requested_features &= ~vlan_strip;4861 if (!(netdev->features & vlan_strip))4862 netdev_info(netdev, "To enable VLAN stripping, first need to enable FCS/CRC stripping");4863 4864 return requested_features;4865 }4866 4867 if (num_non_zero_vlan && is_vlan_strip &&4868 !(netdev->features & NETIF_F_RXFCS)) {4869 requested_features &= ~NETIF_F_RXFCS;4870 netdev_info(netdev, "To disable FCS/CRC stripping, first need to disable VLAN stripping");4871 }4872 4873 return requested_features;4874}4875 4876/**4877 * iavf_fix_features - fix up the netdev feature bits4878 * @netdev: our net device4879 * @features: desired feature bits4880 *4881 * Returns fixed-up features bits4882 **/4883static netdev_features_t iavf_fix_features(struct net_device *netdev,4884 netdev_features_t features)4885{4886 struct iavf_adapter *adapter = netdev_priv(netdev);4887 4888 features = iavf_fix_netdev_vlan_features(adapter, features);4889 4890 if (!FDIR_FLTR_SUPPORT(adapter))4891 features &= ~NETIF_F_NTUPLE;4892 4893 return iavf_fix_strip_features(adapter, features);4894}4895 4896static const struct net_device_ops iavf_netdev_ops = {4897 .ndo_open = iavf_open,4898 .ndo_stop = iavf_close,4899 .ndo_start_xmit = iavf_xmit_frame,4900 .ndo_set_rx_mode = iavf_set_rx_mode,4901 .ndo_validate_addr = eth_validate_addr,4902 .ndo_set_mac_address = iavf_set_mac,4903 .ndo_change_mtu = iavf_change_mtu,4904 .ndo_tx_timeout = iavf_tx_timeout,4905 .ndo_vlan_rx_add_vid = iavf_vlan_rx_add_vid,4906 .ndo_vlan_rx_kill_vid = iavf_vlan_rx_kill_vid,4907 .ndo_features_check = iavf_features_check,4908 .ndo_fix_features = iavf_fix_features,4909 .ndo_set_features = iavf_set_features,4910 .ndo_setup_tc = iavf_setup_tc,4911};4912 4913/**4914 * iavf_check_reset_complete - check that VF reset is complete4915 * @hw: pointer to hw struct4916 *4917 * Returns 0 if device is ready to use, or -EBUSY if it's in reset.4918 **/4919static int iavf_check_reset_complete(struct iavf_hw *hw)4920{4921 u32 rstat;4922 int i;4923 4924 for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) {4925 rstat = rd32(hw, IAVF_VFGEN_RSTAT) &4926 IAVF_VFGEN_RSTAT_VFR_STATE_MASK;4927 if ((rstat == VIRTCHNL_VFR_VFACTIVE) ||4928 (rstat == VIRTCHNL_VFR_COMPLETED))4929 return 0;4930 msleep(IAVF_RESET_WAIT_MS);4931 }4932 return -EBUSY;4933}4934 4935/**4936 * iavf_process_config - Process the config information we got from the PF4937 * @adapter: board private structure4938 *4939 * Verify that we have a valid config struct, and set up our netdev features4940 * and our VSI struct.4941 **/4942int iavf_process_config(struct iavf_adapter *adapter)4943{4944 struct virtchnl_vf_resource *vfres = adapter->vf_res;4945 netdev_features_t hw_vlan_features, vlan_features;4946 struct net_device *netdev = adapter->netdev;4947 netdev_features_t hw_enc_features;4948 netdev_features_t hw_features;4949 4950 hw_enc_features = NETIF_F_SG |4951 NETIF_F_IP_CSUM |4952 NETIF_F_IPV6_CSUM |4953 NETIF_F_HIGHDMA |4954 NETIF_F_SOFT_FEATURES |4955 NETIF_F_TSO |4956 NETIF_F_TSO_ECN |4957 NETIF_F_TSO6 |4958 NETIF_F_SCTP_CRC |4959 NETIF_F_RXHASH |4960 NETIF_F_RXCSUM |4961 0;4962 4963 /* advertise to stack only if offloads for encapsulated packets is4964 * supported4965 */4966 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ENCAP) {4967 hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL |4968 NETIF_F_GSO_GRE |4969 NETIF_F_GSO_GRE_CSUM |4970 NETIF_F_GSO_IPXIP4 |4971 NETIF_F_GSO_IPXIP6 |4972 NETIF_F_GSO_UDP_TUNNEL_CSUM |4973 NETIF_F_GSO_PARTIAL |4974 0;4975 4976 if (!(vfres->vf_cap_flags &4977 VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM))4978 netdev->gso_partial_features |=4979 NETIF_F_GSO_UDP_TUNNEL_CSUM;4980 4981 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;4982 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;4983 netdev->hw_enc_features |= hw_enc_features;4984 }4985 /* record features VLANs can make use of */4986 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;4987 4988 /* Write features and hw_features separately to avoid polluting4989 * with, or dropping, features that are set when we registered.4990 */4991 hw_features = hw_enc_features;4992 4993 /* get HW VLAN features that can be toggled */4994 hw_vlan_features = iavf_get_netdev_vlan_hw_features(adapter);4995 4996 /* Enable HW TC offload if ADQ or tc U32 is supported */4997 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ ||4998 TC_U32_SUPPORT(adapter))4999 hw_features |= NETIF_F_HW_TC;5000 5001 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_USO)5002 hw_features |= NETIF_F_GSO_UDP_L4;5003 5004 netdev->hw_features |= hw_features | hw_vlan_features;5005 vlan_features = iavf_get_netdev_vlan_features(adapter);5006 5007 netdev->features |= hw_features | vlan_features;5008 5009 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)5010 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;5011 5012 if (FDIR_FLTR_SUPPORT(adapter)) {5013 netdev->hw_features |= NETIF_F_NTUPLE;5014 netdev->features |= NETIF_F_NTUPLE;5015 adapter->flags |= IAVF_FLAG_FDIR_ENABLED;5016 }5017 5018 netdev->priv_flags |= IFF_UNICAST_FLT;5019 5020 /* Do not turn on offloads when they are requested to be turned off.5021 * TSO needs minimum 576 bytes to work correctly.5022 */5023 if (netdev->wanted_features) {5024 if (!(netdev->wanted_features & NETIF_F_TSO) ||5025 netdev->mtu < 576)5026 netdev->features &= ~NETIF_F_TSO;5027 if (!(netdev->wanted_features & NETIF_F_TSO6) ||5028 netdev->mtu < 576)5029 netdev->features &= ~NETIF_F_TSO6;5030 if (!(netdev->wanted_features & NETIF_F_TSO_ECN))5031 netdev->features &= ~NETIF_F_TSO_ECN;5032 if (!(netdev->wanted_features & NETIF_F_GRO))5033 netdev->features &= ~NETIF_F_GRO;5034 if (!(netdev->wanted_features & NETIF_F_GSO))5035 netdev->features &= ~NETIF_F_GSO;5036 }5037 5038 return 0;5039}5040 5041/**5042 * iavf_probe - Device Initialization Routine5043 * @pdev: PCI device information struct5044 * @ent: entry in iavf_pci_tbl5045 *5046 * Returns 0 on success, negative on failure5047 *5048 * iavf_probe initializes an adapter identified by a pci_dev structure.5049 * The OS initialization, configuring of the adapter private structure,5050 * and a hardware reset occur.5051 **/5052static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)5053{5054 struct net_device *netdev;5055 struct iavf_adapter *adapter = NULL;5056 struct iavf_hw *hw = NULL;5057 int err;5058 5059 err = pci_enable_device(pdev);5060 if (err)5061 return err;5062 5063 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));5064 if (err) {5065 dev_err(&pdev->dev,5066 "DMA configuration failed: 0x%x\n", err);5067 goto err_dma;5068 }5069 5070 err = pci_request_regions(pdev, iavf_driver_name);5071 if (err) {5072 dev_err(&pdev->dev,5073 "pci_request_regions failed 0x%x\n", err);5074 goto err_pci_reg;5075 }5076 5077 pci_set_master(pdev);5078 5079 netdev = alloc_etherdev_mq(sizeof(struct iavf_adapter),5080 IAVF_MAX_REQ_QUEUES);5081 if (!netdev) {5082 err = -ENOMEM;5083 goto err_alloc_etherdev;5084 }5085 5086 SET_NETDEV_DEV(netdev, &pdev->dev);5087 5088 pci_set_drvdata(pdev, netdev);5089 adapter = netdev_priv(netdev);5090 5091 adapter->netdev = netdev;5092 adapter->pdev = pdev;5093 5094 hw = &adapter->hw;5095 hw->back = adapter;5096 5097 adapter->wq = alloc_ordered_workqueue("%s", WQ_MEM_RECLAIM,5098 iavf_driver_name);5099 if (!adapter->wq) {5100 err = -ENOMEM;5101 goto err_alloc_wq;5102 }5103 5104 adapter->msg_enable = BIT(DEFAULT_DEBUG_LEVEL_SHIFT) - 1;5105 iavf_change_state(adapter, __IAVF_STARTUP);5106 5107 /* Call save state here because it relies on the adapter struct. */5108 pci_save_state(pdev);5109 5110 hw->hw_addr = ioremap(pci_resource_start(pdev, 0),5111 pci_resource_len(pdev, 0));5112 if (!hw->hw_addr) {5113 err = -EIO;5114 goto err_ioremap;5115 }5116 hw->vendor_id = pdev->vendor;5117 hw->device_id = pdev->device;5118 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);5119 hw->subsystem_vendor_id = pdev->subsystem_vendor;5120 hw->subsystem_device_id = pdev->subsystem_device;5121 hw->bus.device = PCI_SLOT(pdev->devfn);5122 hw->bus.func = PCI_FUNC(pdev->devfn);5123 hw->bus.bus_id = pdev->bus->number;5124 5125 /* set up the locks for the AQ, do this only once in probe5126 * and destroy them only once in remove5127 */5128 mutex_init(&adapter->crit_lock);5129 mutex_init(&hw->aq.asq_mutex);5130 mutex_init(&hw->aq.arq_mutex);5131 5132 spin_lock_init(&adapter->mac_vlan_list_lock);5133 spin_lock_init(&adapter->cloud_filter_list_lock);5134 spin_lock_init(&adapter->fdir_fltr_lock);5135 spin_lock_init(&adapter->adv_rss_lock);5136 spin_lock_init(&adapter->current_netdev_promisc_flags_lock);5137 5138 INIT_LIST_HEAD(&adapter->mac_filter_list);5139 INIT_LIST_HEAD(&adapter->vlan_filter_list);5140 INIT_LIST_HEAD(&adapter->cloud_filter_list);5141 INIT_LIST_HEAD(&adapter->fdir_list_head);5142 INIT_LIST_HEAD(&adapter->adv_rss_list_head);5143 5144 INIT_WORK(&adapter->reset_task, iavf_reset_task);5145 INIT_WORK(&adapter->adminq_task, iavf_adminq_task);5146 INIT_WORK(&adapter->finish_config, iavf_finish_config);5147 INIT_DELAYED_WORK(&adapter->watchdog_task, iavf_watchdog_task);5148 5149 /* Setup the wait queue for indicating transition to down status */5150 init_waitqueue_head(&adapter->down_waitqueue);5151 5152 /* Setup the wait queue for indicating transition to running state */5153 init_waitqueue_head(&adapter->reset_waitqueue);5154 5155 /* Setup the wait queue for indicating virtchannel events */5156 init_waitqueue_head(&adapter->vc_waitqueue);5157 5158 queue_delayed_work(adapter->wq, &adapter->watchdog_task,5159 msecs_to_jiffies(5 * (pdev->devfn & 0x07)));5160 /* Initialization goes on in the work. Do not add more of it below. */5161 return 0;5162 5163err_ioremap:5164 destroy_workqueue(adapter->wq);5165err_alloc_wq:5166 free_netdev(netdev);5167err_alloc_etherdev:5168 pci_release_regions(pdev);5169err_pci_reg:5170err_dma:5171 pci_disable_device(pdev);5172 return err;5173}5174 5175/**5176 * iavf_suspend - Power management suspend routine5177 * @dev_d: device info pointer5178 *5179 * Called when the system (VM) is entering sleep/suspend.5180 **/5181static int iavf_suspend(struct device *dev_d)5182{5183 struct net_device *netdev = dev_get_drvdata(dev_d);5184 struct iavf_adapter *adapter = netdev_priv(netdev);5185 5186 netif_device_detach(netdev);5187 5188 mutex_lock(&adapter->crit_lock);5189 5190 if (netif_running(netdev)) {5191 rtnl_lock();5192 iavf_down(adapter);5193 rtnl_unlock();5194 }5195 iavf_free_misc_irq(adapter);5196 iavf_reset_interrupt_capability(adapter);5197 5198 mutex_unlock(&adapter->crit_lock);5199 5200 return 0;5201}5202 5203/**5204 * iavf_resume - Power management resume routine5205 * @dev_d: device info pointer5206 *5207 * Called when the system (VM) is resumed from sleep/suspend.5208 **/5209static int iavf_resume(struct device *dev_d)5210{5211 struct pci_dev *pdev = to_pci_dev(dev_d);5212 struct iavf_adapter *adapter;5213 u32 err;5214 5215 adapter = iavf_pdev_to_adapter(pdev);5216 5217 pci_set_master(pdev);5218 5219 rtnl_lock();5220 err = iavf_set_interrupt_capability(adapter);5221 if (err) {5222 rtnl_unlock();5223 dev_err(&pdev->dev, "Cannot enable MSI-X interrupts.\n");5224 return err;5225 }5226 err = iavf_request_misc_irq(adapter);5227 rtnl_unlock();5228 if (err) {5229 dev_err(&pdev->dev, "Cannot get interrupt vector.\n");5230 return err;5231 }5232 5233 queue_work(adapter->wq, &adapter->reset_task);5234 5235 netif_device_attach(adapter->netdev);5236 5237 return err;5238}5239 5240/**5241 * iavf_remove - Device Removal Routine5242 * @pdev: PCI device information struct5243 *5244 * iavf_remove is called by the PCI subsystem to alert the driver5245 * that it should release a PCI device. The could be caused by a5246 * Hot-Plug event, or because the driver is going to be removed from5247 * memory.5248 **/5249static void iavf_remove(struct pci_dev *pdev)5250{5251 struct iavf_fdir_fltr *fdir, *fdirtmp;5252 struct iavf_vlan_filter *vlf, *vlftmp;5253 struct iavf_cloud_filter *cf, *cftmp;5254 struct iavf_adv_rss *rss, *rsstmp;5255 struct iavf_mac_filter *f, *ftmp;5256 struct iavf_adapter *adapter;5257 struct net_device *netdev;5258 struct iavf_hw *hw;5259 5260 /* Don't proceed with remove if netdev is already freed */5261 netdev = pci_get_drvdata(pdev);5262 if (!netdev)5263 return;5264 5265 adapter = iavf_pdev_to_adapter(pdev);5266 hw = &adapter->hw;5267 5268 if (test_and_set_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section))5269 return;5270 5271 /* Wait until port initialization is complete.5272 * There are flows where register/unregister netdev may race.5273 */5274 while (1) {5275 mutex_lock(&adapter->crit_lock);5276 if (adapter->state == __IAVF_RUNNING ||5277 adapter->state == __IAVF_DOWN ||5278 adapter->state == __IAVF_INIT_FAILED) {5279 mutex_unlock(&adapter->crit_lock);5280 break;5281 }5282 /* Simply return if we already went through iavf_shutdown */5283 if (adapter->state == __IAVF_REMOVE) {5284 mutex_unlock(&adapter->crit_lock);5285 return;5286 }5287 5288 mutex_unlock(&adapter->crit_lock);5289 usleep_range(500, 1000);5290 }5291 cancel_delayed_work_sync(&adapter->watchdog_task);5292 cancel_work_sync(&adapter->finish_config);5293 5294 if (netdev->reg_state == NETREG_REGISTERED)5295 unregister_netdev(netdev);5296 5297 mutex_lock(&adapter->crit_lock);5298 dev_info(&adapter->pdev->dev, "Removing device\n");5299 iavf_change_state(adapter, __IAVF_REMOVE);5300 5301 iavf_request_reset(adapter);5302 msleep(50);5303 /* If the FW isn't responding, kick it once, but only once. */5304 if (!iavf_asq_done(hw)) {5305 iavf_request_reset(adapter);5306 msleep(50);5307 }5308 5309 iavf_misc_irq_disable(adapter);5310 /* Shut down all the garbage mashers on the detention level */5311 cancel_work_sync(&adapter->reset_task);5312 cancel_delayed_work_sync(&adapter->watchdog_task);5313 cancel_work_sync(&adapter->adminq_task);5314 5315 adapter->aq_required = 0;5316 adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;5317 5318 iavf_free_all_tx_resources(adapter);5319 iavf_free_all_rx_resources(adapter);5320 iavf_free_misc_irq(adapter);5321 iavf_free_interrupt_scheme(adapter);5322 5323 iavf_free_rss(adapter);5324 5325 if (hw->aq.asq.count)5326 iavf_shutdown_adminq(hw);5327 5328 /* destroy the locks only once, here */5329 mutex_destroy(&hw->aq.arq_mutex);5330 mutex_destroy(&hw->aq.asq_mutex);5331 mutex_unlock(&adapter->crit_lock);5332 mutex_destroy(&adapter->crit_lock);5333 5334 iounmap(hw->hw_addr);5335 pci_release_regions(pdev);5336 kfree(adapter->vf_res);5337 spin_lock_bh(&adapter->mac_vlan_list_lock);5338 /* If we got removed before an up/down sequence, we've got a filter5339 * hanging out there that we need to get rid of.5340 */5341 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {5342 list_del(&f->list);5343 kfree(f);5344 }5345 list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list,5346 list) {5347 list_del(&vlf->list);5348 kfree(vlf);5349 }5350 5351 spin_unlock_bh(&adapter->mac_vlan_list_lock);5352 5353 spin_lock_bh(&adapter->cloud_filter_list_lock);5354 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {5355 list_del(&cf->list);5356 kfree(cf);5357 }5358 spin_unlock_bh(&adapter->cloud_filter_list_lock);5359 5360 spin_lock_bh(&adapter->fdir_fltr_lock);5361 list_for_each_entry_safe(fdir, fdirtmp, &adapter->fdir_list_head, list) {5362 list_del(&fdir->list);5363 kfree(fdir);5364 }5365 spin_unlock_bh(&adapter->fdir_fltr_lock);5366 5367 spin_lock_bh(&adapter->adv_rss_lock);5368 list_for_each_entry_safe(rss, rsstmp, &adapter->adv_rss_list_head,5369 list) {5370 list_del(&rss->list);5371 kfree(rss);5372 }5373 spin_unlock_bh(&adapter->adv_rss_lock);5374 5375 destroy_workqueue(adapter->wq);5376 5377 pci_set_drvdata(pdev, NULL);5378 5379 free_netdev(netdev);5380 5381 pci_disable_device(pdev);5382}5383 5384/**5385 * iavf_shutdown - Shutdown the device in preparation for a reboot5386 * @pdev: pci device structure5387 **/5388static void iavf_shutdown(struct pci_dev *pdev)5389{5390 iavf_remove(pdev);5391 5392 if (system_state == SYSTEM_POWER_OFF)5393 pci_set_power_state(pdev, PCI_D3hot);5394}5395 5396static DEFINE_SIMPLE_DEV_PM_OPS(iavf_pm_ops, iavf_suspend, iavf_resume);5397 5398static struct pci_driver iavf_driver = {5399 .name = iavf_driver_name,5400 .id_table = iavf_pci_tbl,5401 .probe = iavf_probe,5402 .remove = iavf_remove,5403 .driver.pm = pm_sleep_ptr(&iavf_pm_ops),5404 .shutdown = iavf_shutdown,5405};5406 5407/**5408 * iavf_init_module - Driver Registration Routine5409 *5410 * iavf_init_module is the first routine called when the driver is5411 * loaded. All it does is register with the PCI subsystem.5412 **/5413static int __init iavf_init_module(void)5414{5415 pr_info("iavf: %s\n", iavf_driver_string);5416 5417 pr_info("%s\n", iavf_copyright);5418 5419 return pci_register_driver(&iavf_driver);5420}5421 5422module_init(iavf_init_module);5423 5424/**5425 * iavf_exit_module - Driver Exit Cleanup Routine5426 *5427 * iavf_exit_module is called just before the driver is removed5428 * from memory.5429 **/5430static void __exit iavf_exit_module(void)5431{5432 pci_unregister_driver(&iavf_driver);5433}5434 5435module_exit(iavf_exit_module);5436 5437/* iavf_main.c */5438