brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB2/*3 * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved.4 */5 6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt7 8#ifdef CONFIG_RFS_ACCEL9#include <linux/cpu_rmap.h>10#endif /* CONFIG_RFS_ACCEL */11#include <linux/ethtool.h>12#include <linux/kernel.h>13#include <linux/module.h>14#include <linux/numa.h>15#include <linux/pci.h>16#include <linux/utsname.h>17#include <linux/version.h>18#include <linux/vmalloc.h>19#include <net/ip.h>20 21#include "ena_netdev.h"22#include "ena_pci_id_tbl.h"23#include "ena_xdp.h"24 25MODULE_AUTHOR("Amazon.com, Inc. or its affiliates");26MODULE_DESCRIPTION(DEVICE_NAME);27MODULE_LICENSE("GPL");28 29/* Time in jiffies before concluding the transmitter is hung. */30#define TX_TIMEOUT  (5 * HZ)31 32#define ENA_MAX_RINGS min_t(unsigned int, ENA_MAX_NUM_IO_QUEUES, num_possible_cpus())33 34#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_IFUP | \35		NETIF_MSG_IFDOWN | NETIF_MSG_TX_ERR | NETIF_MSG_RX_ERR)36 37static struct ena_aenq_handlers aenq_handlers;38 39static struct workqueue_struct *ena_wq;40 41MODULE_DEVICE_TABLE(pci, ena_pci_tbl);42 43static int ena_rss_init_default(struct ena_adapter *adapter);44static void check_for_admin_com_state(struct ena_adapter *adapter);45static int ena_destroy_device(struct ena_adapter *adapter, bool graceful);46static int ena_restore_device(struct ena_adapter *adapter);47 48static void ena_tx_timeout(struct net_device *dev, unsigned int txqueue)49{50	enum ena_regs_reset_reason_types reset_reason = ENA_REGS_RESET_OS_NETDEV_WD;51	struct ena_adapter *adapter = netdev_priv(dev);52	unsigned int time_since_last_napi, threshold;53	struct ena_ring *tx_ring;54	int napi_scheduled;55 56	if (txqueue >= adapter->num_io_queues) {57		netdev_err(dev, "TX timeout on invalid queue %u\n", txqueue);58		goto schedule_reset;59	}60 61	threshold = jiffies_to_usecs(dev->watchdog_timeo);62	tx_ring = &adapter->tx_ring[txqueue];63 64	time_since_last_napi = jiffies_to_usecs(jiffies - tx_ring->tx_stats.last_napi_jiffies);65	napi_scheduled = !!(tx_ring->napi->state & NAPIF_STATE_SCHED);66 67	netdev_err(dev,68		   "TX q %d is paused for too long (threshold %u). Time since last napi %u usec. napi scheduled: %d\n",69		   txqueue,70		   threshold,71		   time_since_last_napi,72		   napi_scheduled);73 74	if (threshold < time_since_last_napi && napi_scheduled) {75		netdev_err(dev,76			   "napi handler hasn't been called for a long time but is scheduled\n");77			   reset_reason = ENA_REGS_RESET_SUSPECTED_POLL_STARVATION;78	}79schedule_reset:80	/* Change the state of the device to trigger reset81	 * Check that we are not in the middle or a trigger already82	 */83	if (test_and_set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))84		return;85 86	ena_reset_device(adapter, reset_reason);87	ena_increase_stat(&adapter->dev_stats.tx_timeout, 1, &adapter->syncp);88}89 90static void update_rx_ring_mtu(struct ena_adapter *adapter, int mtu)91{92	int i;93 94	for (i = 0; i < adapter->num_io_queues; i++)95		adapter->rx_ring[i].mtu = mtu;96}97 98static int ena_change_mtu(struct net_device *dev, int new_mtu)99{100	struct ena_adapter *adapter = netdev_priv(dev);101	int ret;102 103	ret = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu);104	if (!ret) {105		netif_dbg(adapter, drv, dev, "Set MTU to %d\n", new_mtu);106		update_rx_ring_mtu(adapter, new_mtu);107		WRITE_ONCE(dev->mtu, new_mtu);108	} else {109		netif_err(adapter, drv, dev, "Failed to set MTU to %d\n",110			  new_mtu);111	}112 113	return ret;114}115 116int ena_xmit_common(struct ena_adapter *adapter,117		    struct ena_ring *ring,118		    struct ena_tx_buffer *tx_info,119		    struct ena_com_tx_ctx *ena_tx_ctx,120		    u16 next_to_use,121		    u32 bytes)122{123	int rc, nb_hw_desc;124 125	if (unlikely(ena_com_is_doorbell_needed(ring->ena_com_io_sq,126						ena_tx_ctx))) {127		netif_dbg(adapter, tx_queued, adapter->netdev,128			  "llq tx max burst size of queue %d achieved, writing doorbell to send burst\n",129			  ring->qid);130		ena_ring_tx_doorbell(ring);131	}132 133	/* prepare the packet's descriptors to dma engine */134	rc = ena_com_prepare_tx(ring->ena_com_io_sq, ena_tx_ctx,135				&nb_hw_desc);136 137	/* In case there isn't enough space in the queue for the packet,138	 * we simply drop it. All other failure reasons of139	 * ena_com_prepare_tx() are fatal and therefore require a device reset.140	 */141	if (unlikely(rc)) {142		netif_err(adapter, tx_queued, adapter->netdev,143			  "Failed to prepare tx bufs\n");144		ena_increase_stat(&ring->tx_stats.prepare_ctx_err, 1, &ring->syncp);145		if (rc != -ENOMEM)146			ena_reset_device(adapter, ENA_REGS_RESET_DRIVER_INVALID_STATE);147		return rc;148	}149 150	u64_stats_update_begin(&ring->syncp);151	ring->tx_stats.cnt++;152	ring->tx_stats.bytes += bytes;153	u64_stats_update_end(&ring->syncp);154 155	tx_info->tx_descs = nb_hw_desc;156	tx_info->total_tx_size = bytes;157	tx_info->last_jiffies = jiffies;158	tx_info->print_once = 0;159 160	ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use,161						 ring->ring_size);162	return 0;163}164 165static int ena_init_rx_cpu_rmap(struct ena_adapter *adapter)166{167#ifdef CONFIG_RFS_ACCEL168	u32 i;169	int rc;170 171	adapter->netdev->rx_cpu_rmap = alloc_irq_cpu_rmap(adapter->num_io_queues);172	if (!adapter->netdev->rx_cpu_rmap)173		return -ENOMEM;174	for (i = 0; i < adapter->num_io_queues; i++) {175		int irq_idx = ENA_IO_IRQ_IDX(i);176 177		rc = irq_cpu_rmap_add(adapter->netdev->rx_cpu_rmap,178				      pci_irq_vector(adapter->pdev, irq_idx));179		if (rc) {180			free_irq_cpu_rmap(adapter->netdev->rx_cpu_rmap);181			adapter->netdev->rx_cpu_rmap = NULL;182			return rc;183		}184	}185#endif /* CONFIG_RFS_ACCEL */186	return 0;187}188 189static void ena_init_io_rings_common(struct ena_adapter *adapter,190				     struct ena_ring *ring, u16 qid)191{192	ring->qid = qid;193	ring->pdev = adapter->pdev;194	ring->dev = &adapter->pdev->dev;195	ring->netdev = adapter->netdev;196	ring->napi = &adapter->ena_napi[qid].napi;197	ring->adapter = adapter;198	ring->ena_dev = adapter->ena_dev;199	ring->per_napi_packets = 0;200	ring->cpu = 0;201	ring->numa_node = 0;202	ring->no_interrupt_event_cnt = 0;203	u64_stats_init(&ring->syncp);204}205 206void ena_init_io_rings(struct ena_adapter *adapter,207		       int first_index, int count)208{209	struct ena_com_dev *ena_dev;210	struct ena_ring *txr, *rxr;211	int i;212 213	ena_dev = adapter->ena_dev;214 215	for (i = first_index; i < first_index + count; i++) {216		txr = &adapter->tx_ring[i];217		rxr = &adapter->rx_ring[i];218 219		/* TX common ring state */220		ena_init_io_rings_common(adapter, txr, i);221 222		/* TX specific ring state */223		txr->ring_size = adapter->requested_tx_ring_size;224		txr->tx_max_header_size = ena_dev->tx_max_header_size;225		txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type;226		txr->sgl_size = adapter->max_tx_sgl_size;227		txr->smoothed_interval =228			ena_com_get_nonadaptive_moderation_interval_tx(ena_dev);229		txr->disable_meta_caching = adapter->disable_meta_caching;230		spin_lock_init(&txr->xdp_tx_lock);231 232		/* Don't init RX queues for xdp queues */233		if (!ENA_IS_XDP_INDEX(adapter, i)) {234			/* RX common ring state */235			ena_init_io_rings_common(adapter, rxr, i);236 237			/* RX specific ring state */238			rxr->ring_size = adapter->requested_rx_ring_size;239			rxr->rx_copybreak = adapter->rx_copybreak;240			rxr->sgl_size = adapter->max_rx_sgl_size;241			rxr->smoothed_interval =242				ena_com_get_nonadaptive_moderation_interval_rx(ena_dev);243			rxr->empty_rx_queue = 0;244			rxr->rx_headroom = NET_SKB_PAD;245			adapter->ena_napi[i].dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;246			rxr->xdp_ring = &adapter->tx_ring[i + adapter->num_io_queues];247		}248	}249}250 251/* ena_setup_tx_resources - allocate I/O Tx resources (Descriptors)252 * @adapter: network interface device structure253 * @qid: queue index254 *255 * Return 0 on success, negative on failure256 */257static int ena_setup_tx_resources(struct ena_adapter *adapter, int qid)258{259	struct ena_ring *tx_ring = &adapter->tx_ring[qid];260	struct ena_irq *ena_irq = &adapter->irq_tbl[ENA_IO_IRQ_IDX(qid)];261	int size, i, node;262 263	if (tx_ring->tx_buffer_info) {264		netif_err(adapter, ifup,265			  adapter->netdev, "tx_buffer_info info is not NULL");266		return -EEXIST;267	}268 269	size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size;270	node = cpu_to_node(ena_irq->cpu);271 272	tx_ring->tx_buffer_info = vzalloc_node(size, node);273	if (!tx_ring->tx_buffer_info) {274		tx_ring->tx_buffer_info = vzalloc(size);275		if (!tx_ring->tx_buffer_info)276			goto err_tx_buffer_info;277	}278 279	size = sizeof(u16) * tx_ring->ring_size;280	tx_ring->free_ids = vzalloc_node(size, node);281	if (!tx_ring->free_ids) {282		tx_ring->free_ids = vzalloc(size);283		if (!tx_ring->free_ids)284			goto err_tx_free_ids;285	}286 287	size = tx_ring->tx_max_header_size;288	tx_ring->push_buf_intermediate_buf = vzalloc_node(size, node);289	if (!tx_ring->push_buf_intermediate_buf) {290		tx_ring->push_buf_intermediate_buf = vzalloc(size);291		if (!tx_ring->push_buf_intermediate_buf)292			goto err_push_buf_intermediate_buf;293	}294 295	/* Req id ring for TX out of order completions */296	for (i = 0; i < tx_ring->ring_size; i++)297		tx_ring->free_ids[i] = i;298 299	/* Reset tx statistics */300	memset(&tx_ring->tx_stats, 0x0, sizeof(tx_ring->tx_stats));301 302	tx_ring->next_to_use = 0;303	tx_ring->next_to_clean = 0;304	tx_ring->cpu = ena_irq->cpu;305	tx_ring->numa_node = node;306	return 0;307 308err_push_buf_intermediate_buf:309	vfree(tx_ring->free_ids);310	tx_ring->free_ids = NULL;311err_tx_free_ids:312	vfree(tx_ring->tx_buffer_info);313	tx_ring->tx_buffer_info = NULL;314err_tx_buffer_info:315	return -ENOMEM;316}317 318/* ena_free_tx_resources - Free I/O Tx Resources per Queue319 * @adapter: network interface device structure320 * @qid: queue index321 *322 * Free all transmit software resources323 */324static void ena_free_tx_resources(struct ena_adapter *adapter, int qid)325{326	struct ena_ring *tx_ring = &adapter->tx_ring[qid];327 328	vfree(tx_ring->tx_buffer_info);329	tx_ring->tx_buffer_info = NULL;330 331	vfree(tx_ring->free_ids);332	tx_ring->free_ids = NULL;333 334	vfree(tx_ring->push_buf_intermediate_buf);335	tx_ring->push_buf_intermediate_buf = NULL;336}337 338int ena_setup_tx_resources_in_range(struct ena_adapter *adapter,339				    int first_index, int count)340{341	int i, rc = 0;342 343	for (i = first_index; i < first_index + count; i++) {344		rc = ena_setup_tx_resources(adapter, i);345		if (rc)346			goto err_setup_tx;347	}348 349	return 0;350 351err_setup_tx:352 353	netif_err(adapter, ifup, adapter->netdev,354		  "Tx queue %d: allocation failed\n", i);355 356	/* rewind the index freeing the rings as we go */357	while (first_index < i--)358		ena_free_tx_resources(adapter, i);359	return rc;360}361 362void ena_free_all_io_tx_resources_in_range(struct ena_adapter *adapter,363					   int first_index, int count)364{365	int i;366 367	for (i = first_index; i < first_index + count; i++)368		ena_free_tx_resources(adapter, i);369}370 371/* ena_free_all_io_tx_resources - Free I/O Tx Resources for All Queues372 * @adapter: board private structure373 *374 * Free all transmit software resources375 */376void ena_free_all_io_tx_resources(struct ena_adapter *adapter)377{378	ena_free_all_io_tx_resources_in_range(adapter,379					      0,380					      adapter->xdp_num_queues +381					      adapter->num_io_queues);382}383 384/* ena_setup_rx_resources - allocate I/O Rx resources (Descriptors)385 * @adapter: network interface device structure386 * @qid: queue index387 *388 * Returns 0 on success, negative on failure389 */390static int ena_setup_rx_resources(struct ena_adapter *adapter,391				  u32 qid)392{393	struct ena_ring *rx_ring = &adapter->rx_ring[qid];394	struct ena_irq *ena_irq = &adapter->irq_tbl[ENA_IO_IRQ_IDX(qid)];395	int size, node, i;396 397	if (rx_ring->rx_buffer_info) {398		netif_err(adapter, ifup, adapter->netdev,399			  "rx_buffer_info is not NULL");400		return -EEXIST;401	}402 403	/* alloc extra element so in rx path404	 * we can always prefetch rx_info + 1405	 */406	size = sizeof(struct ena_rx_buffer) * (rx_ring->ring_size + 1);407	node = cpu_to_node(ena_irq->cpu);408 409	rx_ring->rx_buffer_info = vzalloc_node(size, node);410	if (!rx_ring->rx_buffer_info) {411		rx_ring->rx_buffer_info = vzalloc(size);412		if (!rx_ring->rx_buffer_info)413			return -ENOMEM;414	}415 416	size = sizeof(u16) * rx_ring->ring_size;417	rx_ring->free_ids = vzalloc_node(size, node);418	if (!rx_ring->free_ids) {419		rx_ring->free_ids = vzalloc(size);420		if (!rx_ring->free_ids) {421			vfree(rx_ring->rx_buffer_info);422			rx_ring->rx_buffer_info = NULL;423			return -ENOMEM;424		}425	}426 427	/* Req id ring for receiving RX pkts out of order */428	for (i = 0; i < rx_ring->ring_size; i++)429		rx_ring->free_ids[i] = i;430 431	/* Reset rx statistics */432	memset(&rx_ring->rx_stats, 0x0, sizeof(rx_ring->rx_stats));433 434	rx_ring->next_to_clean = 0;435	rx_ring->next_to_use = 0;436	rx_ring->cpu = ena_irq->cpu;437	rx_ring->numa_node = node;438 439	return 0;440}441 442/* ena_free_rx_resources - Free I/O Rx Resources443 * @adapter: network interface device structure444 * @qid: queue index445 *446 * Free all receive software resources447 */448static void ena_free_rx_resources(struct ena_adapter *adapter,449				  u32 qid)450{451	struct ena_ring *rx_ring = &adapter->rx_ring[qid];452 453	vfree(rx_ring->rx_buffer_info);454	rx_ring->rx_buffer_info = NULL;455 456	vfree(rx_ring->free_ids);457	rx_ring->free_ids = NULL;458}459 460/* ena_setup_all_rx_resources - allocate I/O Rx queues resources for all queues461 * @adapter: board private structure462 *463 * Return 0 on success, negative on failure464 */465static int ena_setup_all_rx_resources(struct ena_adapter *adapter)466{467	int i, rc = 0;468 469	for (i = 0; i < adapter->num_io_queues; i++) {470		rc = ena_setup_rx_resources(adapter, i);471		if (rc)472			goto err_setup_rx;473	}474 475	return 0;476 477err_setup_rx:478 479	netif_err(adapter, ifup, adapter->netdev,480		  "Rx queue %d: allocation failed\n", i);481 482	/* rewind the index freeing the rings as we go */483	while (i--)484		ena_free_rx_resources(adapter, i);485	return rc;486}487 488/* ena_free_all_io_rx_resources - Free I/O Rx Resources for All Queues489 * @adapter: board private structure490 *491 * Free all receive software resources492 */493static void ena_free_all_io_rx_resources(struct ena_adapter *adapter)494{495	int i;496 497	for (i = 0; i < adapter->num_io_queues; i++)498		ena_free_rx_resources(adapter, i);499}500 501static struct page *ena_alloc_map_page(struct ena_ring *rx_ring,502				       dma_addr_t *dma)503{504	struct page *page;505 506	/* This would allocate the page on the same NUMA node the executing code507	 * is running on.508	 */509	page = dev_alloc_page();510	if (!page) {511		ena_increase_stat(&rx_ring->rx_stats.page_alloc_fail, 1, &rx_ring->syncp);512		return ERR_PTR(-ENOSPC);513	}514 515	/* To enable NIC-side port-mirroring, AKA SPAN port,516	 * we make the buffer readable from the nic as well517	 */518	*dma = dma_map_page(rx_ring->dev, page, 0, ENA_PAGE_SIZE,519			    DMA_BIDIRECTIONAL);520	if (unlikely(dma_mapping_error(rx_ring->dev, *dma))) {521		ena_increase_stat(&rx_ring->rx_stats.dma_mapping_err, 1,522				  &rx_ring->syncp);523		__free_page(page);524		return ERR_PTR(-EIO);525	}526 527	return page;528}529 530static int ena_alloc_rx_buffer(struct ena_ring *rx_ring,531			       struct ena_rx_buffer *rx_info)532{533	int headroom = rx_ring->rx_headroom;534	struct ena_com_buf *ena_buf;535	struct page *page;536	dma_addr_t dma;537	int tailroom;538 539	/* restore page offset value in case it has been changed by device */540	rx_info->buf_offset = headroom;541 542	/* if previous allocated page is not used */543	if (unlikely(rx_info->page))544		return 0;545 546	/* We handle DMA here */547	page = ena_alloc_map_page(rx_ring, &dma);548	if (IS_ERR(page))549		return PTR_ERR(page);550 551	netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,552		  "Allocate page %p, rx_info %p\n", page, rx_info);553 554	tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));555 556	rx_info->page = page;557	rx_info->dma_addr = dma;558	rx_info->page_offset = 0;559	ena_buf = &rx_info->ena_buf;560	ena_buf->paddr = dma + headroom;561	ena_buf->len = ENA_PAGE_SIZE - headroom - tailroom;562 563	return 0;564}565 566static void ena_unmap_rx_buff_attrs(struct ena_ring *rx_ring,567				    struct ena_rx_buffer *rx_info,568				    unsigned long attrs)569{570	dma_unmap_page_attrs(rx_ring->dev, rx_info->dma_addr, ENA_PAGE_SIZE, DMA_BIDIRECTIONAL,571			     attrs);572}573 574static void ena_free_rx_page(struct ena_ring *rx_ring,575			     struct ena_rx_buffer *rx_info)576{577	struct page *page = rx_info->page;578 579	if (unlikely(!page)) {580		netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev,581			   "Trying to free unallocated buffer\n");582		return;583	}584 585	ena_unmap_rx_buff_attrs(rx_ring, rx_info, 0);586 587	__free_page(page);588	rx_info->page = NULL;589}590 591static int ena_refill_rx_bufs(struct ena_ring *rx_ring, u32 num)592{593	u16 next_to_use, req_id;594	u32 i;595	int rc;596 597	next_to_use = rx_ring->next_to_use;598 599	for (i = 0; i < num; i++) {600		struct ena_rx_buffer *rx_info;601 602		req_id = rx_ring->free_ids[next_to_use];603 604		rx_info = &rx_ring->rx_buffer_info[req_id];605 606		rc = ena_alloc_rx_buffer(rx_ring, rx_info);607		if (unlikely(rc < 0)) {608			netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev,609				   "Failed to allocate buffer for rx queue %d\n",610				   rx_ring->qid);611			break;612		}613		rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq,614						&rx_info->ena_buf,615						req_id);616		if (unlikely(rc)) {617			netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev,618				   "Failed to add buffer for rx queue %d\n",619				   rx_ring->qid);620			break;621		}622		next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use,623						   rx_ring->ring_size);624	}625 626	if (unlikely(i < num)) {627		ena_increase_stat(&rx_ring->rx_stats.refil_partial, 1,628				  &rx_ring->syncp);629		netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev,630			   "Refilled rx qid %d with only %d buffers (from %d)\n",631			   rx_ring->qid, i, num);632	}633 634	/* ena_com_write_sq_doorbell issues a wmb() */635	if (likely(i))636		ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq);637 638	rx_ring->next_to_use = next_to_use;639 640	return i;641}642 643static void ena_free_rx_bufs(struct ena_adapter *adapter,644			     u32 qid)645{646	struct ena_ring *rx_ring = &adapter->rx_ring[qid];647	u32 i;648 649	for (i = 0; i < rx_ring->ring_size; i++) {650		struct ena_rx_buffer *rx_info = &rx_ring->rx_buffer_info[i];651 652		if (rx_info->page)653			ena_free_rx_page(rx_ring, rx_info);654	}655}656 657/* ena_refill_all_rx_bufs - allocate all queues Rx buffers658 * @adapter: board private structure659 */660static void ena_refill_all_rx_bufs(struct ena_adapter *adapter)661{662	struct ena_ring *rx_ring;663	int i, rc, bufs_num;664 665	for (i = 0; i < adapter->num_io_queues; i++) {666		rx_ring = &adapter->rx_ring[i];667		bufs_num = rx_ring->ring_size - 1;668		rc = ena_refill_rx_bufs(rx_ring, bufs_num);669 670		if (unlikely(rc != bufs_num))671			netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev,672				   "Refilling Queue %d failed. allocated %d buffers from: %d\n",673				   i, rc, bufs_num);674	}675}676 677static void ena_free_all_rx_bufs(struct ena_adapter *adapter)678{679	int i;680 681	for (i = 0; i < adapter->num_io_queues; i++)682		ena_free_rx_bufs(adapter, i);683}684 685void ena_unmap_tx_buff(struct ena_ring *tx_ring,686		       struct ena_tx_buffer *tx_info)687{688	struct ena_com_buf *ena_buf;689	u32 cnt;690	int i;691 692	ena_buf = tx_info->bufs;693	cnt = tx_info->num_of_bufs;694 695	if (unlikely(!cnt))696		return;697 698	if (tx_info->map_linear_data) {699		dma_unmap_single(tx_ring->dev,700				 dma_unmap_addr(ena_buf, paddr),701				 dma_unmap_len(ena_buf, len),702				 DMA_TO_DEVICE);703		ena_buf++;704		cnt--;705	}706 707	/* unmap remaining mapped pages */708	for (i = 0; i < cnt; i++) {709		dma_unmap_page(tx_ring->dev, dma_unmap_addr(ena_buf, paddr),710			       dma_unmap_len(ena_buf, len), DMA_TO_DEVICE);711		ena_buf++;712	}713}714 715/* ena_free_tx_bufs - Free Tx Buffers per Queue716 * @tx_ring: TX ring for which buffers be freed717 */718static void ena_free_tx_bufs(struct ena_ring *tx_ring)719{720	bool print_once = true;721	bool is_xdp_ring;722	u32 i;723 724	is_xdp_ring = ENA_IS_XDP_INDEX(tx_ring->adapter, tx_ring->qid);725 726	for (i = 0; i < tx_ring->ring_size; i++) {727		struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i];728 729		if (!tx_info->skb)730			continue;731 732		if (print_once) {733			netif_notice(tx_ring->adapter, ifdown, tx_ring->netdev,734				     "Free uncompleted tx skb qid %d idx 0x%x\n",735				     tx_ring->qid, i);736			print_once = false;737		} else {738			netif_dbg(tx_ring->adapter, ifdown, tx_ring->netdev,739				  "Free uncompleted tx skb qid %d idx 0x%x\n",740				  tx_ring->qid, i);741		}742 743		ena_unmap_tx_buff(tx_ring, tx_info);744 745		if (is_xdp_ring)746			xdp_return_frame(tx_info->xdpf);747		else748			dev_kfree_skb_any(tx_info->skb);749	}750 751	if (!is_xdp_ring)752		netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev,753							  tx_ring->qid));754}755 756static void ena_free_all_tx_bufs(struct ena_adapter *adapter)757{758	struct ena_ring *tx_ring;759	int i;760 761	for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) {762		tx_ring = &adapter->tx_ring[i];763		ena_free_tx_bufs(tx_ring);764	}765}766 767static void ena_destroy_all_tx_queues(struct ena_adapter *adapter)768{769	u16 ena_qid;770	int i;771 772	for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) {773		ena_qid = ENA_IO_TXQ_IDX(i);774		ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);775	}776}777 778static void ena_destroy_all_rx_queues(struct ena_adapter *adapter)779{780	u16 ena_qid;781	int i;782 783	for (i = 0; i < adapter->num_io_queues; i++) {784		ena_qid = ENA_IO_RXQ_IDX(i);785		cancel_work_sync(&adapter->ena_napi[i].dim.work);786		ena_xdp_unregister_rxq_info(&adapter->rx_ring[i]);787		ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);788	}789}790 791static void ena_destroy_all_io_queues(struct ena_adapter *adapter)792{793	ena_destroy_all_tx_queues(adapter);794	ena_destroy_all_rx_queues(adapter);795}796 797int handle_invalid_req_id(struct ena_ring *ring, u16 req_id,798			  struct ena_tx_buffer *tx_info, bool is_xdp)799{800	if (tx_info)801		netif_err(ring->adapter,802			  tx_done,803			  ring->netdev,804			  "tx_info doesn't have valid %s. qid %u req_id %u",805			   is_xdp ? "xdp frame" : "skb", ring->qid, req_id);806	else807		netif_err(ring->adapter,808			  tx_done,809			  ring->netdev,810			  "Invalid req_id %u in qid %u\n",811			  req_id, ring->qid);812 813	ena_increase_stat(&ring->tx_stats.bad_req_id, 1, &ring->syncp);814	ena_reset_device(ring->adapter, ENA_REGS_RESET_INV_TX_REQ_ID);815 816	return -EFAULT;817}818 819static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id)820{821	struct ena_tx_buffer *tx_info;822 823	tx_info = &tx_ring->tx_buffer_info[req_id];824	if (likely(tx_info->skb))825		return 0;826 827	return handle_invalid_req_id(tx_ring, req_id, tx_info, false);828}829 830static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget)831{832	struct netdev_queue *txq;833	bool above_thresh;834	u32 tx_bytes = 0;835	u32 total_done = 0;836	u16 next_to_clean;837	u16 req_id;838	int tx_pkts = 0;839	int rc;840 841	next_to_clean = tx_ring->next_to_clean;842	txq = netdev_get_tx_queue(tx_ring->netdev, tx_ring->qid);843 844	while (tx_pkts < budget) {845		struct ena_tx_buffer *tx_info;846		struct sk_buff *skb;847 848		rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq,849						&req_id);850		if (rc) {851			if (unlikely(rc == -EINVAL))852				handle_invalid_req_id(tx_ring, req_id, NULL, false);853			break;854		}855 856		/* validate that the request id points to a valid skb */857		rc = validate_tx_req_id(tx_ring, req_id);858		if (rc)859			break;860 861		tx_info = &tx_ring->tx_buffer_info[req_id];862		skb = tx_info->skb;863 864		/* prefetch skb_end_pointer() to speedup skb_shinfo(skb) */865		prefetch(&skb->end);866 867		tx_info->skb = NULL;868		tx_info->last_jiffies = 0;869 870		ena_unmap_tx_buff(tx_ring, tx_info);871 872		netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,873			  "tx_poll: q %d skb %p completed\n", tx_ring->qid,874			  skb);875 876		tx_bytes += tx_info->total_tx_size;877		dev_kfree_skb(skb);878		tx_pkts++;879		total_done += tx_info->tx_descs;880 881		tx_ring->free_ids[next_to_clean] = req_id;882		next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,883						     tx_ring->ring_size);884	}885 886	tx_ring->next_to_clean = next_to_clean;887	ena_com_comp_ack(tx_ring->ena_com_io_sq, total_done);888 889	netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);890 891	netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,892		  "tx_poll: q %d done. total pkts: %d\n",893		  tx_ring->qid, tx_pkts);894 895	/* need to make the rings circular update visible to896	 * ena_start_xmit() before checking for netif_queue_stopped().897	 */898	smp_mb();899 900	above_thresh = ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,901						    ENA_TX_WAKEUP_THRESH);902	if (unlikely(netif_tx_queue_stopped(txq) && above_thresh)) {903		__netif_tx_lock(txq, smp_processor_id());904		above_thresh =905			ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,906						     ENA_TX_WAKEUP_THRESH);907		if (netif_tx_queue_stopped(txq) && above_thresh &&908		    test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags)) {909			netif_tx_wake_queue(txq);910			ena_increase_stat(&tx_ring->tx_stats.queue_wakeup, 1,911					  &tx_ring->syncp);912		}913		__netif_tx_unlock(txq);914	}915 916	return tx_pkts;917}918 919static struct sk_buff *ena_alloc_skb(struct ena_ring *rx_ring, void *first_frag, u16 len)920{921	struct sk_buff *skb;922 923	if (!first_frag)924		skb = napi_alloc_skb(rx_ring->napi, len);925	else926		skb = napi_build_skb(first_frag, len);927 928	if (unlikely(!skb)) {929		ena_increase_stat(&rx_ring->rx_stats.skb_alloc_fail, 1,930				  &rx_ring->syncp);931 932		netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev,933			  "Failed to allocate skb. first_frag %s\n",934			  first_frag ? "provided" : "not provided");935	}936 937	return skb;938}939 940static bool ena_try_rx_buf_page_reuse(struct ena_rx_buffer *rx_info, u16 buf_len,941				      u16 len, int pkt_offset)942{943	struct ena_com_buf *ena_buf = &rx_info->ena_buf;944 945	/* More than ENA_MIN_RX_BUF_SIZE left in the reused buffer946	 * for data + headroom + tailroom.947	 */948	if (SKB_DATA_ALIGN(len + pkt_offset) + ENA_MIN_RX_BUF_SIZE <= ena_buf->len) {949		page_ref_inc(rx_info->page);950		rx_info->page_offset += buf_len;951		ena_buf->paddr += buf_len;952		ena_buf->len -= buf_len;953		return true;954	}955 956	return false;957}958 959static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring,960				  struct ena_com_rx_buf_info *ena_bufs,961				  u32 descs,962				  u16 *next_to_clean)963{964	int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));965	bool is_xdp_loaded = ena_xdp_present_ring(rx_ring);966	struct ena_rx_buffer *rx_info;967	struct ena_adapter *adapter;968	int page_offset, pkt_offset;969	dma_addr_t pre_reuse_paddr;970	u16 len, req_id, buf = 0;971	bool reuse_rx_buf_page;972	struct sk_buff *skb;973	void *buf_addr;974	int buf_offset;975	u16 buf_len;976 977	len = ena_bufs[buf].len;978	req_id = ena_bufs[buf].req_id;979 980	rx_info = &rx_ring->rx_buffer_info[req_id];981 982	if (unlikely(!rx_info->page)) {983		adapter = rx_ring->adapter;984		netif_err(adapter, rx_err, rx_ring->netdev,985			  "Page is NULL. qid %u req_id %u\n", rx_ring->qid, req_id);986		ena_increase_stat(&rx_ring->rx_stats.bad_req_id, 1, &rx_ring->syncp);987		ena_reset_device(adapter, ENA_REGS_RESET_INV_RX_REQ_ID);988		return NULL;989	}990 991	netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,992		  "rx_info %p page %p\n",993		  rx_info, rx_info->page);994 995	buf_offset = rx_info->buf_offset;996	pkt_offset = buf_offset - rx_ring->rx_headroom;997	page_offset = rx_info->page_offset;998	buf_addr = page_address(rx_info->page) + page_offset;999 1000	if (len <= rx_ring->rx_copybreak) {1001		skb = ena_alloc_skb(rx_ring, NULL, len);1002		if (unlikely(!skb))1003			return NULL;1004 1005		skb_copy_to_linear_data(skb, buf_addr + buf_offset, len);1006		dma_sync_single_for_device(rx_ring->dev,1007					   dma_unmap_addr(&rx_info->ena_buf, paddr) + pkt_offset,1008					   len,1009					   DMA_FROM_DEVICE);1010 1011		skb_put(skb, len);1012		netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,1013			  "RX allocated small packet. len %d.\n", skb->len);1014		skb->protocol = eth_type_trans(skb, rx_ring->netdev);1015		rx_ring->free_ids[*next_to_clean] = req_id;1016		*next_to_clean = ENA_RX_RING_IDX_ADD(*next_to_clean, descs,1017						     rx_ring->ring_size);1018		return skb;1019	}1020 1021	buf_len = SKB_DATA_ALIGN(len + buf_offset + tailroom);1022 1023	/* If XDP isn't loaded try to reuse part of the RX buffer */1024	reuse_rx_buf_page = !is_xdp_loaded &&1025			    ena_try_rx_buf_page_reuse(rx_info, buf_len, len, pkt_offset);1026 1027	if (!reuse_rx_buf_page)1028		ena_unmap_rx_buff_attrs(rx_ring, rx_info, DMA_ATTR_SKIP_CPU_SYNC);1029 1030	skb = ena_alloc_skb(rx_ring, buf_addr, buf_len);1031	if (unlikely(!skb))1032		return NULL;1033 1034	/* Populate skb's linear part */1035	skb_reserve(skb, buf_offset);1036	skb_put(skb, len);1037	skb->protocol = eth_type_trans(skb, rx_ring->netdev);1038 1039	do {1040		netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,1041			  "RX skb updated. len %d. data_len %d\n",1042			  skb->len, skb->data_len);1043 1044		if (!reuse_rx_buf_page)1045			rx_info->page = NULL;1046 1047		rx_ring->free_ids[*next_to_clean] = req_id;1048		*next_to_clean =1049			ENA_RX_RING_IDX_NEXT(*next_to_clean,1050					     rx_ring->ring_size);1051		if (likely(--descs == 0))1052			break;1053 1054		buf++;1055		len = ena_bufs[buf].len;1056		req_id = ena_bufs[buf].req_id;1057 1058		rx_info = &rx_ring->rx_buffer_info[req_id];1059 1060		/* rx_info->buf_offset includes rx_ring->rx_headroom */1061		buf_offset = rx_info->buf_offset;1062		pkt_offset = buf_offset - rx_ring->rx_headroom;1063		buf_len = SKB_DATA_ALIGN(len + buf_offset + tailroom);1064		page_offset = rx_info->page_offset;1065 1066		pre_reuse_paddr = dma_unmap_addr(&rx_info->ena_buf, paddr);1067 1068		reuse_rx_buf_page = !is_xdp_loaded &&1069				    ena_try_rx_buf_page_reuse(rx_info, buf_len, len, pkt_offset);1070 1071		dma_sync_single_for_cpu(rx_ring->dev,1072					pre_reuse_paddr + pkt_offset,1073					len,1074					DMA_FROM_DEVICE);1075 1076		if (!reuse_rx_buf_page)1077			ena_unmap_rx_buff_attrs(rx_ring, rx_info, DMA_ATTR_SKIP_CPU_SYNC);1078 1079		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_info->page,1080				page_offset + buf_offset, len, buf_len);1081 1082	} while (1);1083 1084	return skb;1085}1086 1087/* ena_rx_checksum - indicate in skb if hw indicated a good cksum1088 * @adapter: structure containing adapter specific data1089 * @ena_rx_ctx: received packet context/metadata1090 * @skb: skb currently being received and modified1091 */1092static void ena_rx_checksum(struct ena_ring *rx_ring,1093				   struct ena_com_rx_ctx *ena_rx_ctx,1094				   struct sk_buff *skb)1095{1096	/* Rx csum disabled */1097	if (unlikely(!(rx_ring->netdev->features & NETIF_F_RXCSUM))) {1098		skb->ip_summed = CHECKSUM_NONE;1099		return;1100	}1101 1102	/* For fragmented packets the checksum isn't valid */1103	if (ena_rx_ctx->frag) {1104		skb->ip_summed = CHECKSUM_NONE;1105		return;1106	}1107 1108	/* if IP and error */1109	if (unlikely((ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) &&1110		     (ena_rx_ctx->l3_csum_err))) {1111		/* ipv4 checksum error */1112		skb->ip_summed = CHECKSUM_NONE;1113		ena_increase_stat(&rx_ring->rx_stats.csum_bad, 1,1114				  &rx_ring->syncp);1115		netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev,1116			  "RX IPv4 header checksum error\n");1117		return;1118	}1119 1120	/* if TCP/UDP */1121	if (likely((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||1122		   (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP))) {1123		if (unlikely(ena_rx_ctx->l4_csum_err)) {1124			/* TCP/UDP checksum error */1125			ena_increase_stat(&rx_ring->rx_stats.csum_bad, 1,1126					  &rx_ring->syncp);1127			netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev,1128				  "RX L4 checksum error\n");1129			skb->ip_summed = CHECKSUM_NONE;1130			return;1131		}1132 1133		if (likely(ena_rx_ctx->l4_csum_checked)) {1134			skb->ip_summed = CHECKSUM_UNNECESSARY;1135			ena_increase_stat(&rx_ring->rx_stats.csum_good, 1,1136					  &rx_ring->syncp);1137		} else {1138			ena_increase_stat(&rx_ring->rx_stats.csum_unchecked, 1,1139					  &rx_ring->syncp);1140			skb->ip_summed = CHECKSUM_NONE;1141		}1142	} else {1143		skb->ip_summed = CHECKSUM_NONE;1144		return;1145	}1146 1147}1148 1149static void ena_set_rx_hash(struct ena_ring *rx_ring,1150			    struct ena_com_rx_ctx *ena_rx_ctx,1151			    struct sk_buff *skb)1152{1153	enum pkt_hash_types hash_type;1154 1155	if (likely(rx_ring->netdev->features & NETIF_F_RXHASH)) {1156		if (likely((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||1157			   (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP)))1158 1159			hash_type = PKT_HASH_TYPE_L4;1160		else1161			hash_type = PKT_HASH_TYPE_NONE;1162 1163		/* Override hash type if the packet is fragmented */1164		if (ena_rx_ctx->frag)1165			hash_type = PKT_HASH_TYPE_NONE;1166 1167		skb_set_hash(skb, ena_rx_ctx->hash, hash_type);1168	}1169}1170 1171static int ena_xdp_handle_buff(struct ena_ring *rx_ring, struct xdp_buff *xdp, u16 num_descs)1172{1173	struct ena_rx_buffer *rx_info;1174	int ret;1175 1176	/* XDP multi-buffer packets not supported */1177	if (unlikely(num_descs > 1)) {1178		netdev_err_once(rx_ring->adapter->netdev,1179				"xdp: dropped unsupported multi-buffer packets\n");1180		ena_increase_stat(&rx_ring->rx_stats.xdp_drop, 1, &rx_ring->syncp);1181		return ENA_XDP_DROP;1182	}1183 1184	rx_info = &rx_ring->rx_buffer_info[rx_ring->ena_bufs[0].req_id];1185	xdp_prepare_buff(xdp, page_address(rx_info->page),1186			 rx_info->buf_offset,1187			 rx_ring->ena_bufs[0].len, false);1188 1189	ret = ena_xdp_execute(rx_ring, xdp);1190 1191	/* The xdp program might expand the headers */1192	if (ret == ENA_XDP_PASS) {1193		rx_info->buf_offset = xdp->data - xdp->data_hard_start;1194		rx_ring->ena_bufs[0].len = xdp->data_end - xdp->data;1195	}1196 1197	return ret;1198}1199 1200/* ena_clean_rx_irq - Cleanup RX irq1201 * @rx_ring: RX ring to clean1202 * @napi: napi handler1203 * @budget: how many packets driver is allowed to clean1204 *1205 * Returns the number of cleaned buffers.1206 */1207static int ena_clean_rx_irq(struct ena_ring *rx_ring, struct napi_struct *napi,1208			    u32 budget)1209{1210	u16 next_to_clean = rx_ring->next_to_clean;1211	struct ena_com_rx_ctx ena_rx_ctx;1212	struct ena_rx_buffer *rx_info;1213	struct ena_adapter *adapter;1214	u32 res_budget, work_done;1215	int rx_copybreak_pkt = 0;1216	int refill_threshold;1217	struct sk_buff *skb;1218	int refill_required;1219	struct xdp_buff xdp;1220	int xdp_flags = 0;1221	int total_len = 0;1222	int xdp_verdict;1223	u8 pkt_offset;1224	int rc = 0;1225	int i;1226 1227	netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,1228		  "%s qid %d\n", __func__, rx_ring->qid);1229	res_budget = budget;1230	xdp_init_buff(&xdp, ENA_PAGE_SIZE, &rx_ring->xdp_rxq);1231 1232	do {1233		xdp_verdict = ENA_XDP_PASS;1234		skb = NULL;1235		ena_rx_ctx.ena_bufs = rx_ring->ena_bufs;1236		ena_rx_ctx.max_bufs = rx_ring->sgl_size;1237		ena_rx_ctx.descs = 0;1238		ena_rx_ctx.pkt_offset = 0;1239		rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq,1240				    rx_ring->ena_com_io_sq,1241				    &ena_rx_ctx);1242		if (unlikely(rc))1243			goto error;1244 1245		if (unlikely(ena_rx_ctx.descs == 0))1246			break;1247 1248		/* First descriptor might have an offset set by the device */1249		rx_info = &rx_ring->rx_buffer_info[rx_ring->ena_bufs[0].req_id];1250		pkt_offset = ena_rx_ctx.pkt_offset;1251		rx_info->buf_offset += pkt_offset;1252 1253		netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,1254			  "rx_poll: q %d got packet from ena. descs #: %d l3 proto %d l4 proto %d hash: %x\n",1255			  rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto,1256			  ena_rx_ctx.l4_proto, ena_rx_ctx.hash);1257 1258		dma_sync_single_for_cpu(rx_ring->dev,1259					dma_unmap_addr(&rx_info->ena_buf, paddr) + pkt_offset,1260					rx_ring->ena_bufs[0].len,1261					DMA_FROM_DEVICE);1262 1263		if (ena_xdp_present_ring(rx_ring))1264			xdp_verdict = ena_xdp_handle_buff(rx_ring, &xdp, ena_rx_ctx.descs);1265 1266		/* allocate skb and fill it */1267		if (xdp_verdict == ENA_XDP_PASS)1268			skb = ena_rx_skb(rx_ring,1269					 rx_ring->ena_bufs,1270					 ena_rx_ctx.descs,1271					 &next_to_clean);1272 1273		if (unlikely(!skb)) {1274			for (i = 0; i < ena_rx_ctx.descs; i++) {1275				int req_id = rx_ring->ena_bufs[i].req_id;1276 1277				rx_ring->free_ids[next_to_clean] = req_id;1278				next_to_clean =1279					ENA_RX_RING_IDX_NEXT(next_to_clean,1280							     rx_ring->ring_size);1281 1282				/* Packets was passed for transmission, unmap it1283				 * from RX side.1284				 */1285				if (xdp_verdict & ENA_XDP_FORWARDED) {1286					ena_unmap_rx_buff_attrs(rx_ring,1287								&rx_ring->rx_buffer_info[req_id],1288								DMA_ATTR_SKIP_CPU_SYNC);1289					rx_ring->rx_buffer_info[req_id].page = NULL;1290				}1291			}1292			if (xdp_verdict != ENA_XDP_PASS) {1293				xdp_flags |= xdp_verdict;1294				total_len += ena_rx_ctx.ena_bufs[0].len;1295				res_budget--;1296				continue;1297			}1298			break;1299		}1300 1301		ena_rx_checksum(rx_ring, &ena_rx_ctx, skb);1302 1303		ena_set_rx_hash(rx_ring, &ena_rx_ctx, skb);1304 1305		skb_record_rx_queue(skb, rx_ring->qid);1306 1307		if (rx_ring->ena_bufs[0].len <= rx_ring->rx_copybreak)1308			rx_copybreak_pkt++;1309 1310		total_len += skb->len;1311 1312		napi_gro_receive(napi, skb);1313 1314		res_budget--;1315	} while (likely(res_budget));1316 1317	work_done = budget - res_budget;1318	rx_ring->per_napi_packets += work_done;1319	u64_stats_update_begin(&rx_ring->syncp);1320	rx_ring->rx_stats.bytes += total_len;1321	rx_ring->rx_stats.cnt += work_done;1322	rx_ring->rx_stats.rx_copybreak_pkt += rx_copybreak_pkt;1323	u64_stats_update_end(&rx_ring->syncp);1324 1325	rx_ring->next_to_clean = next_to_clean;1326 1327	refill_required = ena_com_free_q_entries(rx_ring->ena_com_io_sq);1328	refill_threshold =1329		min_t(int, rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER,1330		      ENA_RX_REFILL_THRESH_PACKET);1331 1332	/* Optimization, try to batch new rx buffers */1333	if (refill_required > refill_threshold)1334		ena_refill_rx_bufs(rx_ring, refill_required);1335 1336	if (xdp_flags & ENA_XDP_REDIRECT)1337		xdp_do_flush();1338 1339	return work_done;1340 1341error:1342	if (xdp_flags & ENA_XDP_REDIRECT)1343		xdp_do_flush();1344 1345	adapter = netdev_priv(rx_ring->netdev);1346 1347	if (rc == -ENOSPC) {1348		ena_increase_stat(&rx_ring->rx_stats.bad_desc_num, 1, &rx_ring->syncp);1349		ena_reset_device(adapter, ENA_REGS_RESET_TOO_MANY_RX_DESCS);1350	} else if (rc == -EFAULT) {1351		ena_reset_device(adapter, ENA_REGS_RESET_RX_DESCRIPTOR_MALFORMED);1352	} else {1353		ena_increase_stat(&rx_ring->rx_stats.bad_req_id, 1,1354				  &rx_ring->syncp);1355		ena_reset_device(adapter, ENA_REGS_RESET_INV_RX_REQ_ID);1356	}1357	return 0;1358}1359 1360static void ena_dim_work(struct work_struct *w)1361{1362	struct dim *dim = container_of(w, struct dim, work);1363	struct dim_cq_moder cur_moder =1364		net_dim_get_rx_moderation(dim->mode, dim->profile_ix);1365	struct ena_napi *ena_napi = container_of(dim, struct ena_napi, dim);1366 1367	ena_napi->rx_ring->smoothed_interval = cur_moder.usec;1368	dim->state = DIM_START_MEASURE;1369}1370 1371static void ena_adjust_adaptive_rx_intr_moderation(struct ena_napi *ena_napi)1372{1373	struct dim_sample dim_sample;1374	struct ena_ring *rx_ring = ena_napi->rx_ring;1375 1376	if (!rx_ring->per_napi_packets)1377		return;1378 1379	rx_ring->non_empty_napi_events++;1380 1381	dim_update_sample(rx_ring->non_empty_napi_events,1382			  rx_ring->rx_stats.cnt,1383			  rx_ring->rx_stats.bytes,1384			  &dim_sample);1385 1386	net_dim(&ena_napi->dim, dim_sample);1387 1388	rx_ring->per_napi_packets = 0;1389}1390 1391void ena_unmask_interrupt(struct ena_ring *tx_ring,1392			  struct ena_ring *rx_ring)1393{1394	u32 rx_interval = tx_ring->smoothed_interval;1395	struct ena_eth_io_intr_reg intr_reg;1396 1397	/* Rx ring can be NULL when for XDP tx queues which don't have an1398	 * accompanying rx_ring pair.1399	 */1400	if (rx_ring)1401		rx_interval = ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev) ?1402			rx_ring->smoothed_interval :1403			ena_com_get_nonadaptive_moderation_interval_rx(rx_ring->ena_dev);1404 1405	/* Update intr register: rx intr delay,1406	 * tx intr delay and interrupt unmask1407	 */1408	ena_com_update_intr_reg(&intr_reg,1409				rx_interval,1410				tx_ring->smoothed_interval,1411				true);1412 1413	ena_increase_stat(&tx_ring->tx_stats.unmask_interrupt, 1,1414			  &tx_ring->syncp);1415 1416	/* It is a shared MSI-X.1417	 * Tx and Rx CQ have pointer to it.1418	 * So we use one of them to reach the intr reg1419	 * The Tx ring is used because the rx_ring is NULL for XDP queues1420	 */1421	ena_com_unmask_intr(tx_ring->ena_com_io_cq, &intr_reg);1422}1423 1424void ena_update_ring_numa_node(struct ena_ring *tx_ring,1425			       struct ena_ring *rx_ring)1426{1427	int cpu = get_cpu();1428	int numa_node;1429 1430	/* Check only one ring since the 2 rings are running on the same cpu */1431	if (likely(tx_ring->cpu == cpu))1432		goto out;1433 1434	tx_ring->cpu = cpu;1435	if (rx_ring)1436		rx_ring->cpu = cpu;1437 1438	numa_node = cpu_to_node(cpu);1439 1440	if (likely(tx_ring->numa_node == numa_node))1441		goto out;1442 1443	put_cpu();1444 1445	if (numa_node != NUMA_NO_NODE) {1446		ena_com_update_numa_node(tx_ring->ena_com_io_cq, numa_node);1447		tx_ring->numa_node = numa_node;1448		if (rx_ring) {1449			rx_ring->numa_node = numa_node;1450			ena_com_update_numa_node(rx_ring->ena_com_io_cq,1451						 numa_node);1452		}1453	}1454 1455	return;1456out:1457	put_cpu();1458}1459 1460static int ena_io_poll(struct napi_struct *napi, int budget)1461{1462	struct ena_napi *ena_napi = container_of(napi, struct ena_napi, napi);1463	struct ena_ring *tx_ring, *rx_ring;1464	int tx_work_done;1465	int rx_work_done = 0;1466	int tx_budget;1467	int napi_comp_call = 0;1468	int ret;1469 1470	tx_ring = ena_napi->tx_ring;1471	rx_ring = ena_napi->rx_ring;1472 1473	tx_budget = tx_ring->ring_size / ENA_TX_POLL_BUDGET_DIVIDER;1474 1475	if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||1476	    test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags)) {1477		napi_complete_done(napi, 0);1478		return 0;1479	}1480 1481	tx_work_done = ena_clean_tx_irq(tx_ring, tx_budget);1482	/* On netpoll the budget is zero and the handler should only clean the1483	 * tx completions.1484	 */1485	if (likely(budget))1486		rx_work_done = ena_clean_rx_irq(rx_ring, napi, budget);1487 1488	/* If the device is about to reset or down, avoid unmask1489	 * the interrupt and return 0 so NAPI won't reschedule1490	 */1491	if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||1492		     test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags))) {1493		napi_complete_done(napi, 0);1494		ret = 0;1495 1496	} else if ((budget > rx_work_done) && (tx_budget > tx_work_done)) {1497		napi_comp_call = 1;1498 1499		/* Update numa and unmask the interrupt only when schedule1500		 * from the interrupt context (vs from sk_busy_loop)1501		 */1502		if (napi_complete_done(napi, rx_work_done) &&1503		    READ_ONCE(ena_napi->interrupts_masked)) {1504			smp_rmb(); /* make sure interrupts_masked is read */1505			WRITE_ONCE(ena_napi->interrupts_masked, false);1506			/* We apply adaptive moderation on Rx path only.1507			 * Tx uses static interrupt moderation.1508			 */1509			if (ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev))1510				ena_adjust_adaptive_rx_intr_moderation(ena_napi);1511 1512			ena_update_ring_numa_node(tx_ring, rx_ring);1513			ena_unmask_interrupt(tx_ring, rx_ring);1514		}1515 1516		ret = rx_work_done;1517	} else {1518		ret = budget;1519	}1520 1521	u64_stats_update_begin(&tx_ring->syncp);1522	tx_ring->tx_stats.napi_comp += napi_comp_call;1523	tx_ring->tx_stats.tx_poll++;1524	u64_stats_update_end(&tx_ring->syncp);1525 1526	tx_ring->tx_stats.last_napi_jiffies = jiffies;1527 1528	return ret;1529}1530 1531static irqreturn_t ena_intr_msix_mgmnt(int irq, void *data)1532{1533	struct ena_adapter *adapter = (struct ena_adapter *)data;1534 1535	ena_com_admin_q_comp_intr_handler(adapter->ena_dev);1536 1537	/* Don't call the aenq handler before probe is done */1538	if (likely(test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags)))1539		ena_com_aenq_intr_handler(adapter->ena_dev, data);1540 1541	return IRQ_HANDLED;1542}1543 1544/* ena_intr_msix_io - MSI-X Interrupt Handler for Tx/Rx1545 * @irq: interrupt number1546 * @data: pointer to a network interface private napi device structure1547 */1548static irqreturn_t ena_intr_msix_io(int irq, void *data)1549{1550	struct ena_napi *ena_napi = data;1551 1552	/* Used to check HW health */1553	WRITE_ONCE(ena_napi->first_interrupt, true);1554 1555	WRITE_ONCE(ena_napi->interrupts_masked, true);1556	smp_wmb(); /* write interrupts_masked before calling napi */1557 1558	napi_schedule_irqoff(&ena_napi->napi);1559 1560	return IRQ_HANDLED;1561}1562 1563/* Reserve a single MSI-X vector for management (admin + aenq).1564 * plus reserve one vector for each potential io queue.1565 * the number of potential io queues is the minimum of what the device1566 * supports and the number of vCPUs.1567 */1568static int ena_enable_msix(struct ena_adapter *adapter)1569{1570	int msix_vecs, irq_cnt;1571 1572	if (test_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags)) {1573		netif_err(adapter, probe, adapter->netdev,1574			  "Error, MSI-X is already enabled\n");1575		return -EPERM;1576	}1577 1578	/* Reserved the max msix vectors we might need */1579	msix_vecs = ENA_MAX_MSIX_VEC(adapter->max_num_io_queues);1580	netif_dbg(adapter, probe, adapter->netdev,1581		  "Trying to enable MSI-X, vectors %d\n", msix_vecs);1582 1583	irq_cnt = pci_alloc_irq_vectors(adapter->pdev, ENA_MIN_MSIX_VEC,1584					msix_vecs, PCI_IRQ_MSIX);1585 1586	if (irq_cnt < 0) {1587		netif_err(adapter, probe, adapter->netdev,1588			  "Failed to enable MSI-X. irq_cnt %d\n", irq_cnt);1589		return -ENOSPC;1590	}1591 1592	if (irq_cnt != msix_vecs) {1593		netif_notice(adapter, probe, adapter->netdev,1594			     "Enable only %d MSI-X (out of %d), reduce the number of queues\n",1595			     irq_cnt, msix_vecs);1596		adapter->num_io_queues = irq_cnt - ENA_ADMIN_MSIX_VEC;1597	}1598 1599	if (ena_init_rx_cpu_rmap(adapter))1600		netif_warn(adapter, probe, adapter->netdev,1601			   "Failed to map IRQs to CPUs\n");1602 1603	adapter->msix_vecs = irq_cnt;1604	set_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags);1605 1606	return 0;1607}1608 1609static void ena_setup_mgmnt_intr(struct ena_adapter *adapter)1610{1611	u32 cpu;1612 1613	snprintf(adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].name,1614		 ENA_IRQNAME_SIZE, "ena-mgmnt@pci:%s",1615		 pci_name(adapter->pdev));1616	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].handler =1617		ena_intr_msix_mgmnt;1618	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter;1619	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector =1620		pci_irq_vector(adapter->pdev, ENA_MGMNT_IRQ_IDX);1621	cpu = cpumask_first(cpu_online_mask);1622	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].cpu = cpu;1623	cpumask_set_cpu(cpu,1624			&adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].affinity_hint_mask);1625}1626 1627static void ena_setup_io_intr(struct ena_adapter *adapter)1628{1629	struct net_device *netdev;1630	int irq_idx, i, cpu;1631	int io_queue_count;1632 1633	netdev = adapter->netdev;1634	io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;1635 1636	for (i = 0; i < io_queue_count; i++) {1637		irq_idx = ENA_IO_IRQ_IDX(i);1638		cpu = i % num_online_cpus();1639 1640		snprintf(adapter->irq_tbl[irq_idx].name, ENA_IRQNAME_SIZE,1641			 "%s-Tx-Rx-%d", netdev->name, i);1642		adapter->irq_tbl[irq_idx].handler = ena_intr_msix_io;1643		adapter->irq_tbl[irq_idx].data = &adapter->ena_napi[i];1644		adapter->irq_tbl[irq_idx].vector =1645			pci_irq_vector(adapter->pdev, irq_idx);1646		adapter->irq_tbl[irq_idx].cpu = cpu;1647 1648		cpumask_set_cpu(cpu,1649				&adapter->irq_tbl[irq_idx].affinity_hint_mask);1650	}1651}1652 1653static int ena_request_mgmnt_irq(struct ena_adapter *adapter)1654{1655	unsigned long flags = 0;1656	struct ena_irq *irq;1657	int rc;1658 1659	irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];1660	rc = request_irq(irq->vector, irq->handler, flags, irq->name,1661			 irq->data);1662	if (rc) {1663		netif_err(adapter, probe, adapter->netdev,1664			  "Failed to request admin irq\n");1665		return rc;1666	}1667 1668	netif_dbg(adapter, probe, adapter->netdev,1669		  "Set affinity hint of mgmnt irq.to 0x%lx (irq vector: %d)\n",1670		  irq->affinity_hint_mask.bits[0], irq->vector);1671 1672	irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask);1673 1674	return rc;1675}1676 1677static int ena_request_io_irq(struct ena_adapter *adapter)1678{1679	u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;1680	unsigned long flags = 0;1681	struct ena_irq *irq;1682	int rc = 0, i, k;1683 1684	if (!test_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags)) {1685		netif_err(adapter, ifup, adapter->netdev,1686			  "Failed to request I/O IRQ: MSI-X is not enabled\n");1687		return -EINVAL;1688	}1689 1690	for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++) {1691		irq = &adapter->irq_tbl[i];1692		rc = request_irq(irq->vector, irq->handler, flags, irq->name,1693				 irq->data);1694		if (rc) {1695			netif_err(adapter, ifup, adapter->netdev,1696				  "Failed to request I/O IRQ. index %d rc %d\n",1697				   i, rc);1698			goto err;1699		}1700 1701		netif_dbg(adapter, ifup, adapter->netdev,1702			  "Set affinity hint of irq. index %d to 0x%lx (irq vector: %d)\n",1703			  i, irq->affinity_hint_mask.bits[0], irq->vector);1704 1705		irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask);1706	}1707 1708	return rc;1709 1710err:1711	for (k = ENA_IO_IRQ_FIRST_IDX; k < i; k++) {1712		irq = &adapter->irq_tbl[k];1713		free_irq(irq->vector, irq->data);1714	}1715 1716	return rc;1717}1718 1719static void ena_free_mgmnt_irq(struct ena_adapter *adapter)1720{1721	struct ena_irq *irq;1722 1723	irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];1724	synchronize_irq(irq->vector);1725	irq_set_affinity_hint(irq->vector, NULL);1726	free_irq(irq->vector, irq->data);1727}1728 1729static void ena_free_io_irq(struct ena_adapter *adapter)1730{1731	u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;1732	struct ena_irq *irq;1733	int i;1734 1735#ifdef CONFIG_RFS_ACCEL1736	if (adapter->msix_vecs >= 1) {1737		free_irq_cpu_rmap(adapter->netdev->rx_cpu_rmap);1738		adapter->netdev->rx_cpu_rmap = NULL;1739	}1740#endif /* CONFIG_RFS_ACCEL */1741 1742	for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++) {1743		irq = &adapter->irq_tbl[i];1744		irq_set_affinity_hint(irq->vector, NULL);1745		free_irq(irq->vector, irq->data);1746	}1747}1748 1749static void ena_disable_msix(struct ena_adapter *adapter)1750{1751	if (test_and_clear_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags))1752		pci_free_irq_vectors(adapter->pdev);1753}1754 1755static void ena_disable_io_intr_sync(struct ena_adapter *adapter)1756{1757	u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;1758	int i;1759 1760	if (!netif_running(adapter->netdev))1761		return;1762 1763	for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++)1764		synchronize_irq(adapter->irq_tbl[i].vector);1765}1766 1767static void ena_del_napi_in_range(struct ena_adapter *adapter,1768				  int first_index,1769				  int count)1770{1771	int i;1772 1773	for (i = first_index; i < first_index + count; i++) {1774		netif_napi_del(&adapter->ena_napi[i].napi);1775 1776		WARN_ON(ENA_IS_XDP_INDEX(adapter, i) &&1777			adapter->ena_napi[i].rx_ring);1778	}1779}1780 1781static void ena_init_napi_in_range(struct ena_adapter *adapter,1782				   int first_index, int count)1783{1784	int (*napi_handler)(struct napi_struct *napi, int budget);1785	int i;1786 1787	for (i = first_index; i < first_index + count; i++) {1788		struct ena_napi *napi = &adapter->ena_napi[i];1789		struct ena_ring *rx_ring, *tx_ring;1790 1791		memset(napi, 0, sizeof(*napi));1792 1793		rx_ring = &adapter->rx_ring[i];1794		tx_ring = &adapter->tx_ring[i];1795 1796		napi_handler = ena_io_poll;1797		if (ENA_IS_XDP_INDEX(adapter, i))1798			napi_handler = ena_xdp_io_poll;1799 1800		netif_napi_add(adapter->netdev, &napi->napi, napi_handler);1801 1802		if (!ENA_IS_XDP_INDEX(adapter, i))1803			napi->rx_ring = rx_ring;1804 1805		napi->tx_ring = tx_ring;1806		napi->qid = i;1807	}1808}1809 1810static void ena_napi_disable_in_range(struct ena_adapter *adapter,1811				      int first_index,1812				      int count)1813{1814	int i;1815 1816	for (i = first_index; i < first_index + count; i++)1817		napi_disable(&adapter->ena_napi[i].napi);1818}1819 1820static void ena_napi_enable_in_range(struct ena_adapter *adapter,1821				     int first_index,1822				     int count)1823{1824	int i;1825 1826	for (i = first_index; i < first_index + count; i++)1827		napi_enable(&adapter->ena_napi[i].napi);1828}1829 1830/* Configure the Rx forwarding */1831static int ena_rss_configure(struct ena_adapter *adapter)1832{1833	struct ena_com_dev *ena_dev = adapter->ena_dev;1834	int rc;1835 1836	/* In case the RSS table wasn't initialized by probe */1837	if (!ena_dev->rss.tbl_log_size) {1838		rc = ena_rss_init_default(adapter);1839		if (rc && (rc != -EOPNOTSUPP)) {1840			netif_err(adapter, ifup, adapter->netdev, "Failed to init RSS rc: %d\n", rc);1841			return rc;1842		}1843	}1844 1845	/* Set indirect table */1846	rc = ena_com_indirect_table_set(ena_dev);1847	if (unlikely(rc && rc != -EOPNOTSUPP))1848		return rc;1849 1850	/* Configure hash function (if supported) */1851	rc = ena_com_set_hash_function(ena_dev);1852	if (unlikely(rc && (rc != -EOPNOTSUPP)))1853		return rc;1854 1855	/* Configure hash inputs (if supported) */1856	rc = ena_com_set_hash_ctrl(ena_dev);1857	if (unlikely(rc && (rc != -EOPNOTSUPP)))1858		return rc;1859 1860	return 0;1861}1862 1863static int ena_up_complete(struct ena_adapter *adapter)1864{1865	int rc;1866 1867	rc = ena_rss_configure(adapter);1868	if (rc)1869		return rc;1870 1871	ena_change_mtu(adapter->netdev, adapter->netdev->mtu);1872 1873	ena_refill_all_rx_bufs(adapter);1874 1875	/* enable transmits */1876	netif_tx_start_all_queues(adapter->netdev);1877 1878	ena_napi_enable_in_range(adapter,1879				 0,1880				 adapter->xdp_num_queues + adapter->num_io_queues);1881 1882	return 0;1883}1884 1885static int ena_create_io_tx_queue(struct ena_adapter *adapter, int qid)1886{1887	struct ena_com_create_io_ctx ctx;1888	struct ena_com_dev *ena_dev;1889	struct ena_ring *tx_ring;1890	u32 msix_vector;1891	u16 ena_qid;1892	int rc;1893 1894	ena_dev = adapter->ena_dev;1895 1896	tx_ring = &adapter->tx_ring[qid];1897	msix_vector = ENA_IO_IRQ_IDX(qid);1898	ena_qid = ENA_IO_TXQ_IDX(qid);1899 1900	memset(&ctx, 0x0, sizeof(ctx));1901 1902	ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX;1903	ctx.qid = ena_qid;1904	ctx.mem_queue_type = ena_dev->tx_mem_queue_type;1905	ctx.msix_vector = msix_vector;1906	ctx.queue_size = tx_ring->ring_size;1907	ctx.numa_node = tx_ring->numa_node;1908 1909	rc = ena_com_create_io_queue(ena_dev, &ctx);1910	if (rc) {1911		netif_err(adapter, ifup, adapter->netdev,1912			  "Failed to create I/O TX queue num %d rc: %d\n",1913			  qid, rc);1914		return rc;1915	}1916 1917	rc = ena_com_get_io_handlers(ena_dev, ena_qid,1918				     &tx_ring->ena_com_io_sq,1919				     &tx_ring->ena_com_io_cq);1920	if (rc) {1921		netif_err(adapter, ifup, adapter->netdev,1922			  "Failed to get TX queue handlers. TX queue num %d rc: %d\n",1923			  qid, rc);1924		ena_com_destroy_io_queue(ena_dev, ena_qid);1925		return rc;1926	}1927 1928	ena_com_update_numa_node(tx_ring->ena_com_io_cq, ctx.numa_node);1929	return rc;1930}1931 1932int ena_create_io_tx_queues_in_range(struct ena_adapter *adapter,1933				     int first_index, int count)1934{1935	struct ena_com_dev *ena_dev = adapter->ena_dev;1936	int rc, i;1937 1938	for (i = first_index; i < first_index + count; i++) {1939		rc = ena_create_io_tx_queue(adapter, i);1940		if (rc)1941			goto create_err;1942	}1943 1944	return 0;1945 1946create_err:1947	while (i-- > first_index)1948		ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(i));1949 1950	return rc;1951}1952 1953static int ena_create_io_rx_queue(struct ena_adapter *adapter, int qid)1954{1955	struct ena_com_dev *ena_dev;1956	struct ena_com_create_io_ctx ctx;1957	struct ena_ring *rx_ring;1958	u32 msix_vector;1959	u16 ena_qid;1960	int rc;1961 1962	ena_dev = adapter->ena_dev;1963 1964	rx_ring = &adapter->rx_ring[qid];1965	msix_vector = ENA_IO_IRQ_IDX(qid);1966	ena_qid = ENA_IO_RXQ_IDX(qid);1967 1968	memset(&ctx, 0x0, sizeof(ctx));1969 1970	ctx.qid = ena_qid;1971	ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX;1972	ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;1973	ctx.msix_vector = msix_vector;1974	ctx.queue_size = rx_ring->ring_size;1975	ctx.numa_node = rx_ring->numa_node;1976 1977	rc = ena_com_create_io_queue(ena_dev, &ctx);1978	if (rc) {1979		netif_err(adapter, ifup, adapter->netdev,1980			  "Failed to create I/O RX queue num %d rc: %d\n",1981			  qid, rc);1982		return rc;1983	}1984 1985	rc = ena_com_get_io_handlers(ena_dev, ena_qid,1986				     &rx_ring->ena_com_io_sq,1987				     &rx_ring->ena_com_io_cq);1988	if (rc) {1989		netif_err(adapter, ifup, adapter->netdev,1990			  "Failed to get RX queue handlers. RX queue num %d rc: %d\n",1991			  qid, rc);1992		goto err;1993	}1994 1995	ena_com_update_numa_node(rx_ring->ena_com_io_cq, ctx.numa_node);1996 1997	return rc;1998err:1999	ena_com_destroy_io_queue(ena_dev, ena_qid);2000	return rc;2001}2002 2003static int ena_create_all_io_rx_queues(struct ena_adapter *adapter)2004{2005	struct ena_com_dev *ena_dev = adapter->ena_dev;2006	int rc, i;2007 2008	for (i = 0; i < adapter->num_io_queues; i++) {2009		rc = ena_create_io_rx_queue(adapter, i);2010		if (rc)2011			goto create_err;2012		INIT_WORK(&adapter->ena_napi[i].dim.work, ena_dim_work);2013 2014		ena_xdp_register_rxq_info(&adapter->rx_ring[i]);2015	}2016 2017	return 0;2018 2019create_err:2020	while (i--) {2021		ena_xdp_unregister_rxq_info(&adapter->rx_ring[i]);2022		cancel_work_sync(&adapter->ena_napi[i].dim.work);2023		ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(i));2024	}2025 2026	return rc;2027}2028 2029static void set_io_rings_size(struct ena_adapter *adapter,2030			      int new_tx_size,2031			      int new_rx_size)2032{2033	int i;2034 2035	for (i = 0; i < adapter->num_io_queues; i++) {2036		adapter->tx_ring[i].ring_size = new_tx_size;2037		adapter->rx_ring[i].ring_size = new_rx_size;2038	}2039}2040 2041/* This function allows queue allocation to backoff when the system is2042 * low on memory. If there is not enough memory to allocate io queues2043 * the driver will try to allocate smaller queues.2044 *2045 * The backoff algorithm is as follows:2046 *  1. Try to allocate TX and RX and if successful.2047 *  1.1. return success2048 *2049 *  2. Divide by 2 the size of the larger of RX and TX queues (or both if their size is the same).2050 *2051 *  3. If TX or RX is smaller than 2562052 *  3.1. return failure.2053 *  4. else2054 *  4.1. go back to 1.2055 */2056static int create_queues_with_size_backoff(struct ena_adapter *adapter)2057{2058	int rc, cur_rx_ring_size, cur_tx_ring_size;2059	int new_rx_ring_size, new_tx_ring_size;2060 2061	/* current queue sizes might be set to smaller than the requested2062	 * ones due to past queue allocation failures.2063	 */2064	set_io_rings_size(adapter, adapter->requested_tx_ring_size,2065			  adapter->requested_rx_ring_size);2066 2067	while (1) {2068		if (ena_xdp_present(adapter)) {2069			rc = ena_setup_and_create_all_xdp_queues(adapter);2070 2071			if (rc)2072				goto err_setup_tx;2073		}2074		rc = ena_setup_tx_resources_in_range(adapter,2075						     0,2076						     adapter->num_io_queues);2077		if (rc)2078			goto err_setup_tx;2079 2080		rc = ena_create_io_tx_queues_in_range(adapter,2081						      0,2082						      adapter->num_io_queues);2083		if (rc)2084			goto err_create_tx_queues;2085 2086		rc = ena_setup_all_rx_resources(adapter);2087		if (rc)2088			goto err_setup_rx;2089 2090		rc = ena_create_all_io_rx_queues(adapter);2091		if (rc)2092			goto err_create_rx_queues;2093 2094		return 0;2095 2096err_create_rx_queues:2097		ena_free_all_io_rx_resources(adapter);2098err_setup_rx:2099		ena_destroy_all_tx_queues(adapter);2100err_create_tx_queues:2101		ena_free_all_io_tx_resources(adapter);2102err_setup_tx:2103		if (rc != -ENOMEM) {2104			netif_err(adapter, ifup, adapter->netdev,2105				  "Queue creation failed with error code %d\n",2106				  rc);2107			return rc;2108		}2109 2110		cur_tx_ring_size = adapter->tx_ring[0].ring_size;2111		cur_rx_ring_size = adapter->rx_ring[0].ring_size;2112 2113		netif_err(adapter, ifup, adapter->netdev,2114			  "Not enough memory to create queues with sizes TX=%d, RX=%d\n",2115			  cur_tx_ring_size, cur_rx_ring_size);2116 2117		new_tx_ring_size = cur_tx_ring_size;2118		new_rx_ring_size = cur_rx_ring_size;2119 2120		/* Decrease the size of the larger queue, or2121		 * decrease both if they are the same size.2122		 */2123		if (cur_rx_ring_size <= cur_tx_ring_size)2124			new_tx_ring_size = cur_tx_ring_size / 2;2125		if (cur_rx_ring_size >= cur_tx_ring_size)2126			new_rx_ring_size = cur_rx_ring_size / 2;2127 2128		if (new_tx_ring_size < ENA_MIN_RING_SIZE ||2129		    new_rx_ring_size < ENA_MIN_RING_SIZE) {2130			netif_err(adapter, ifup, adapter->netdev,2131				  "Queue creation failed with the smallest possible queue size of %d for both queues. Not retrying with smaller queues\n",2132				  ENA_MIN_RING_SIZE);2133			return rc;2134		}2135 2136		netif_err(adapter, ifup, adapter->netdev,2137			  "Retrying queue creation with sizes TX=%d, RX=%d\n",2138			  new_tx_ring_size,2139			  new_rx_ring_size);2140 2141		set_io_rings_size(adapter, new_tx_ring_size,2142				  new_rx_ring_size);2143	}2144}2145 2146int ena_up(struct ena_adapter *adapter)2147{2148	int io_queue_count, rc, i;2149 2150	netif_dbg(adapter, ifup, adapter->netdev, "%s\n", __func__);2151 2152	io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;2153	ena_setup_io_intr(adapter);2154 2155	/* napi poll functions should be initialized before running2156	 * request_irq(), to handle a rare condition where there is a pending2157	 * interrupt, causing the ISR to fire immediately while the poll2158	 * function wasn't set yet, causing a null dereference2159	 */2160	ena_init_napi_in_range(adapter, 0, io_queue_count);2161 2162	/* Enabling DIM needs to happen before enabling IRQs since DIM2163	 * is run from napi routine2164	 */2165	if (ena_com_interrupt_moderation_supported(adapter->ena_dev))2166		ena_com_enable_adaptive_moderation(adapter->ena_dev);2167 2168	rc = ena_request_io_irq(adapter);2169	if (rc)2170		goto err_req_irq;2171 2172	rc = create_queues_with_size_backoff(adapter);2173	if (rc)2174		goto err_create_queues_with_backoff;2175 2176	rc = ena_up_complete(adapter);2177	if (rc)2178		goto err_up;2179 2180	if (test_bit(ENA_FLAG_LINK_UP, &adapter->flags))2181		netif_carrier_on(adapter->netdev);2182 2183	ena_increase_stat(&adapter->dev_stats.interface_up, 1,2184			  &adapter->syncp);2185 2186	set_bit(ENA_FLAG_DEV_UP, &adapter->flags);2187 2188	/* Enable completion queues interrupt */2189	for (i = 0; i < adapter->num_io_queues; i++)2190		ena_unmask_interrupt(&adapter->tx_ring[i],2191				     &adapter->rx_ring[i]);2192 2193	/* schedule napi in case we had pending packets2194	 * from the last time we disable napi2195	 */2196	for (i = 0; i < io_queue_count; i++)2197		napi_schedule(&adapter->ena_napi[i].napi);2198 2199	return rc;2200 2201err_up:2202	ena_destroy_all_tx_queues(adapter);2203	ena_free_all_io_tx_resources(adapter);2204	ena_destroy_all_rx_queues(adapter);2205	ena_free_all_io_rx_resources(adapter);2206err_create_queues_with_backoff:2207	ena_free_io_irq(adapter);2208err_req_irq:2209	ena_del_napi_in_range(adapter, 0, io_queue_count);2210 2211	return rc;2212}2213 2214void ena_down(struct ena_adapter *adapter)2215{2216	int io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;2217 2218	netif_dbg(adapter, ifdown, adapter->netdev, "%s\n", __func__);2219 2220	clear_bit(ENA_FLAG_DEV_UP, &adapter->flags);2221 2222	ena_increase_stat(&adapter->dev_stats.interface_down, 1,2223			  &adapter->syncp);2224 2225	netif_carrier_off(adapter->netdev);2226	netif_tx_disable(adapter->netdev);2227 2228	/* After this point the napi handler won't enable the tx queue */2229	ena_napi_disable_in_range(adapter, 0, io_queue_count);2230 2231	if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags)) {2232		int rc;2233 2234		rc = ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason);2235		if (rc)2236			netif_err(adapter, ifdown, adapter->netdev,2237				  "Device reset failed\n");2238		/* stop submitting admin commands on a device that was reset */2239		ena_com_set_admin_running_state(adapter->ena_dev, false);2240	}2241 2242	ena_destroy_all_io_queues(adapter);2243 2244	ena_disable_io_intr_sync(adapter);2245	ena_free_io_irq(adapter);2246	ena_del_napi_in_range(adapter, 0, io_queue_count);2247 2248	ena_free_all_tx_bufs(adapter);2249	ena_free_all_rx_bufs(adapter);2250	ena_free_all_io_tx_resources(adapter);2251	ena_free_all_io_rx_resources(adapter);2252}2253 2254/* ena_open - Called when a network interface is made active2255 * @netdev: network interface device structure2256 *2257 * Returns 0 on success, negative value on failure2258 *2259 * The open entry point is called when a network interface is made2260 * active by the system (IFF_UP).  At this point all resources needed2261 * for transmit and receive operations are allocated, the interrupt2262 * handler is registered with the OS, the watchdog timer is started,2263 * and the stack is notified that the interface is ready.2264 */2265static int ena_open(struct net_device *netdev)2266{2267	struct ena_adapter *adapter = netdev_priv(netdev);2268	int rc;2269 2270	/* Notify the stack of the actual queue counts. */2271	rc = netif_set_real_num_tx_queues(netdev, adapter->num_io_queues);2272	if (rc) {2273		netif_err(adapter, ifup, netdev, "Can't set num tx queues\n");2274		return rc;2275	}2276 2277	rc = netif_set_real_num_rx_queues(netdev, adapter->num_io_queues);2278	if (rc) {2279		netif_err(adapter, ifup, netdev, "Can't set num rx queues\n");2280		return rc;2281	}2282 2283	rc = ena_up(adapter);2284	if (rc)2285		return rc;2286 2287	return rc;2288}2289 2290/* ena_close - Disables a network interface2291 * @netdev: network interface device structure2292 *2293 * Returns 0, this is not allowed to fail2294 *2295 * The close entry point is called when an interface is de-activated2296 * by the OS.  The hardware is still under the drivers control, but2297 * needs to be disabled.  A global MAC reset is issued to stop the2298 * hardware, and all transmit and receive resources are freed.2299 */2300static int ena_close(struct net_device *netdev)2301{2302	struct ena_adapter *adapter = netdev_priv(netdev);2303 2304	netif_dbg(adapter, ifdown, netdev, "%s\n", __func__);2305 2306	if (!test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags))2307		return 0;2308 2309	if (test_bit(ENA_FLAG_DEV_UP, &adapter->flags))2310		ena_down(adapter);2311 2312	/* Check for device status and issue reset if needed*/2313	check_for_admin_com_state(adapter);2314	if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {2315		netif_err(adapter, ifdown, adapter->netdev,2316			  "Destroy failure, restarting device\n");2317		ena_dump_stats_to_dmesg(adapter);2318		/* rtnl lock already obtained in dev_ioctl() layer */2319		ena_destroy_device(adapter, false);2320		ena_restore_device(adapter);2321	}2322 2323	return 0;2324}2325 2326int ena_update_queue_params(struct ena_adapter *adapter,2327			    u32 new_tx_size,2328			    u32 new_rx_size,2329			    u32 new_llq_header_len)2330{2331	bool dev_was_up, large_llq_changed = false;2332	int rc = 0;2333 2334	dev_was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);2335	ena_close(adapter->netdev);2336	adapter->requested_tx_ring_size = new_tx_size;2337	adapter->requested_rx_ring_size = new_rx_size;2338	ena_init_io_rings(adapter,2339			  0,2340			  adapter->xdp_num_queues +2341			  adapter->num_io_queues);2342 2343	large_llq_changed = adapter->ena_dev->tx_mem_queue_type ==2344			    ENA_ADMIN_PLACEMENT_POLICY_DEV;2345	large_llq_changed &=2346		new_llq_header_len != adapter->ena_dev->tx_max_header_size;2347 2348	/* a check that the configuration is valid is done by caller */2349	if (large_llq_changed) {2350		adapter->large_llq_header_enabled = !adapter->large_llq_header_enabled;2351 2352		ena_destroy_device(adapter, false);2353		rc = ena_restore_device(adapter);2354	}2355 2356	return dev_was_up && !rc ? ena_up(adapter) : rc;2357}2358 2359int ena_set_rx_copybreak(struct ena_adapter *adapter, u32 rx_copybreak)2360{2361	struct ena_ring *rx_ring;2362	int i;2363 2364	if (rx_copybreak > min_t(u16, adapter->netdev->mtu, ENA_PAGE_SIZE))2365		return -EINVAL;2366 2367	adapter->rx_copybreak = rx_copybreak;2368 2369	for (i = 0; i < adapter->num_io_queues; i++) {2370		rx_ring = &adapter->rx_ring[i];2371		rx_ring->rx_copybreak = rx_copybreak;2372	}2373 2374	return 0;2375}2376 2377int ena_update_queue_count(struct ena_adapter *adapter, u32 new_channel_count)2378{2379	struct ena_com_dev *ena_dev = adapter->ena_dev;2380	int prev_channel_count;2381	bool dev_was_up;2382 2383	dev_was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);2384	ena_close(adapter->netdev);2385	prev_channel_count = adapter->num_io_queues;2386	adapter->num_io_queues = new_channel_count;2387	if (ena_xdp_present(adapter) &&2388	    ena_xdp_allowed(adapter) == ENA_XDP_ALLOWED) {2389		adapter->xdp_first_ring = new_channel_count;2390		adapter->xdp_num_queues = new_channel_count;2391		if (prev_channel_count > new_channel_count)2392			ena_xdp_exchange_program_rx_in_range(adapter,2393							     NULL,2394							     new_channel_count,2395							     prev_channel_count);2396		else2397			ena_xdp_exchange_program_rx_in_range(adapter,2398							     adapter->xdp_bpf_prog,2399							     prev_channel_count,2400							     new_channel_count);2401	}2402 2403	/* We need to destroy the rss table so that the indirection2404	 * table will be reinitialized by ena_up()2405	 */2406	ena_com_rss_destroy(ena_dev);2407	ena_init_io_rings(adapter,2408			  0,2409			  adapter->xdp_num_queues +2410			  adapter->num_io_queues);2411	return dev_was_up ? ena_open(adapter->netdev) : 0;2412}2413 2414static void ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx,2415			struct sk_buff *skb,2416			bool disable_meta_caching)2417{2418	u32 mss = skb_shinfo(skb)->gso_size;2419	struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta;2420	u8 l4_protocol = 0;2421 2422	if ((skb->ip_summed == CHECKSUM_PARTIAL) || mss) {2423		ena_tx_ctx->l4_csum_enable = 1;2424		if (mss) {2425			ena_tx_ctx->tso_enable = 1;2426			ena_meta->l4_hdr_len = tcp_hdr(skb)->doff;2427			ena_tx_ctx->l4_csum_partial = 0;2428		} else {2429			ena_tx_ctx->tso_enable = 0;2430			ena_meta->l4_hdr_len = 0;2431			ena_tx_ctx->l4_csum_partial = 1;2432		}2433 2434		switch (ip_hdr(skb)->version) {2435		case IPVERSION:2436			ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4;2437			if (ip_hdr(skb)->frag_off & htons(IP_DF))2438				ena_tx_ctx->df = 1;2439			if (mss)2440				ena_tx_ctx->l3_csum_enable = 1;2441			l4_protocol = ip_hdr(skb)->protocol;2442			break;2443		case 6:2444			ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6;2445			l4_protocol = ipv6_hdr(skb)->nexthdr;2446			break;2447		default:2448			break;2449		}2450 2451		if (l4_protocol == IPPROTO_TCP)2452			ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP;2453		else2454			ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP;2455 2456		ena_meta->mss = mss;2457		ena_meta->l3_hdr_len = skb_network_header_len(skb);2458		ena_meta->l3_hdr_offset = skb_network_offset(skb);2459		ena_tx_ctx->meta_valid = 1;2460	} else if (disable_meta_caching) {2461		memset(ena_meta, 0, sizeof(*ena_meta));2462		ena_tx_ctx->meta_valid = 1;2463	} else {2464		ena_tx_ctx->meta_valid = 0;2465	}2466}2467 2468static int ena_check_and_linearize_skb(struct ena_ring *tx_ring,2469				       struct sk_buff *skb)2470{2471	int num_frags, header_len, rc;2472 2473	num_frags = skb_shinfo(skb)->nr_frags;2474	header_len = skb_headlen(skb);2475 2476	if (num_frags < tx_ring->sgl_size)2477		return 0;2478 2479	if ((num_frags == tx_ring->sgl_size) &&2480	    (header_len < tx_ring->tx_max_header_size))2481		return 0;2482 2483	ena_increase_stat(&tx_ring->tx_stats.linearize, 1, &tx_ring->syncp);2484 2485	rc = skb_linearize(skb);2486	if (unlikely(rc)) {2487		ena_increase_stat(&tx_ring->tx_stats.linearize_failed, 1,2488				  &tx_ring->syncp);2489	}2490 2491	return rc;2492}2493 2494static int ena_tx_map_skb(struct ena_ring *tx_ring,2495			  struct ena_tx_buffer *tx_info,2496			  struct sk_buff *skb,2497			  void **push_hdr,2498			  u16 *header_len)2499{2500	struct ena_adapter *adapter = tx_ring->adapter;2501	struct ena_com_buf *ena_buf;2502	dma_addr_t dma;2503	u32 skb_head_len, frag_len, last_frag;2504	u16 push_len = 0;2505	u16 delta = 0;2506	int i = 0;2507 2508	skb_head_len = skb_headlen(skb);2509	tx_info->skb = skb;2510	ena_buf = tx_info->bufs;2511 2512	if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {2513		/* When the device is LLQ mode, the driver will copy2514		 * the header into the device memory space.2515		 * the ena_com layer assume the header is in a linear2516		 * memory space.2517		 * This assumption might be wrong since part of the header2518		 * can be in the fragmented buffers.2519		 * Use skb_header_pointer to make sure the header is in a2520		 * linear memory space.2521		 */2522 2523		push_len = min_t(u32, skb->len, tx_ring->tx_max_header_size);2524		*push_hdr = skb_header_pointer(skb, 0, push_len,2525					       tx_ring->push_buf_intermediate_buf);2526		*header_len = push_len;2527		if (unlikely(skb->data != *push_hdr)) {2528			ena_increase_stat(&tx_ring->tx_stats.llq_buffer_copy, 1,2529					  &tx_ring->syncp);2530 2531			delta = push_len - skb_head_len;2532		}2533	} else {2534		*push_hdr = NULL;2535		*header_len = min_t(u32, skb_head_len,2536				    tx_ring->tx_max_header_size);2537	}2538 2539	netif_dbg(adapter, tx_queued, adapter->netdev,2540		  "skb: %p header_buf->vaddr: %p push_len: %d\n", skb,2541		  *push_hdr, push_len);2542 2543	if (skb_head_len > push_len) {2544		dma = dma_map_single(tx_ring->dev, skb->data + push_len,2545				     skb_head_len - push_len, DMA_TO_DEVICE);2546		if (unlikely(dma_mapping_error(tx_ring->dev, dma)))2547			goto error_report_dma_error;2548 2549		ena_buf->paddr = dma;2550		ena_buf->len = skb_head_len - push_len;2551 2552		ena_buf++;2553		tx_info->num_of_bufs++;2554		tx_info->map_linear_data = 1;2555	} else {2556		tx_info->map_linear_data = 0;2557	}2558 2559	last_frag = skb_shinfo(skb)->nr_frags;2560 2561	for (i = 0; i < last_frag; i++) {2562		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];2563 2564		frag_len = skb_frag_size(frag);2565 2566		if (unlikely(delta >= frag_len)) {2567			delta -= frag_len;2568			continue;2569		}2570 2571		dma = skb_frag_dma_map(tx_ring->dev, frag, delta,2572				       frag_len - delta, DMA_TO_DEVICE);2573		if (unlikely(dma_mapping_error(tx_ring->dev, dma)))2574			goto error_report_dma_error;2575 2576		ena_buf->paddr = dma;2577		ena_buf->len = frag_len - delta;2578		ena_buf++;2579		tx_info->num_of_bufs++;2580		delta = 0;2581	}2582 2583	return 0;2584 2585error_report_dma_error:2586	ena_increase_stat(&tx_ring->tx_stats.dma_mapping_err, 1,2587			  &tx_ring->syncp);2588	netif_warn(adapter, tx_queued, adapter->netdev, "Failed to map skb\n");2589 2590	tx_info->skb = NULL;2591 2592	tx_info->num_of_bufs += i;2593	ena_unmap_tx_buff(tx_ring, tx_info);2594 2595	return -EINVAL;2596}2597 2598/* Called with netif_tx_lock. */2599static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev)2600{2601	struct ena_adapter *adapter = netdev_priv(dev);2602	struct ena_tx_buffer *tx_info;2603	struct ena_com_tx_ctx ena_tx_ctx;2604	struct ena_ring *tx_ring;2605	struct netdev_queue *txq;2606	void *push_hdr;2607	u16 next_to_use, req_id, header_len;2608	int qid, rc;2609 2610	netif_dbg(adapter, tx_queued, dev, "%s skb %p\n", __func__, skb);2611	/*  Determine which tx ring we will be placed on */2612	qid = skb_get_queue_mapping(skb);2613	tx_ring = &adapter->tx_ring[qid];2614	txq = netdev_get_tx_queue(dev, qid);2615 2616	rc = ena_check_and_linearize_skb(tx_ring, skb);2617	if (unlikely(rc))2618		goto error_drop_packet;2619 2620	next_to_use = tx_ring->next_to_use;2621	req_id = tx_ring->free_ids[next_to_use];2622	tx_info = &tx_ring->tx_buffer_info[req_id];2623	tx_info->num_of_bufs = 0;2624 2625	WARN(tx_info->skb, "SKB isn't NULL req_id %d\n", req_id);2626 2627	rc = ena_tx_map_skb(tx_ring, tx_info, skb, &push_hdr, &header_len);2628	if (unlikely(rc))2629		goto error_drop_packet;2630 2631	memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx));2632	ena_tx_ctx.ena_bufs = tx_info->bufs;2633	ena_tx_ctx.push_header = push_hdr;2634	ena_tx_ctx.num_bufs = tx_info->num_of_bufs;2635	ena_tx_ctx.req_id = req_id;2636	ena_tx_ctx.header_len = header_len;2637 2638	/* set flags and meta data */2639	ena_tx_csum(&ena_tx_ctx, skb, tx_ring->disable_meta_caching);2640 2641	rc = ena_xmit_common(adapter,2642			     tx_ring,2643			     tx_info,2644			     &ena_tx_ctx,2645			     next_to_use,2646			     skb->len);2647	if (rc)2648		goto error_unmap_dma;2649 2650	netdev_tx_sent_queue(txq, skb->len);2651 2652	/* stop the queue when no more space available, the packet can have up2653	 * to sgl_size + 2. one for the meta descriptor and one for header2654	 * (if the header is larger than tx_max_header_size).2655	 */2656	if (unlikely(!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,2657						   tx_ring->sgl_size + 2))) {2658		netif_dbg(adapter, tx_queued, dev, "%s stop queue %d\n",2659			  __func__, qid);2660 2661		netif_tx_stop_queue(txq);2662		ena_increase_stat(&tx_ring->tx_stats.queue_stop, 1,2663				  &tx_ring->syncp);2664 2665		/* There is a rare condition where this function decide to2666		 * stop the queue but meanwhile clean_tx_irq updates2667		 * next_to_completion and terminates.2668		 * The queue will remain stopped forever.2669		 * To solve this issue add a mb() to make sure that2670		 * netif_tx_stop_queue() write is vissible before checking if2671		 * there is additional space in the queue.2672		 */2673		smp_mb();2674 2675		if (ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,2676						 ENA_TX_WAKEUP_THRESH)) {2677			netif_tx_wake_queue(txq);2678			ena_increase_stat(&tx_ring->tx_stats.queue_wakeup, 1,2679					  &tx_ring->syncp);2680		}2681	}2682 2683	skb_tx_timestamp(skb);2684 2685	if (netif_xmit_stopped(txq) || !netdev_xmit_more())2686		/* trigger the dma engine. ena_ring_tx_doorbell()2687		 * calls a memory barrier inside it.2688		 */2689		ena_ring_tx_doorbell(tx_ring);2690 2691	return NETDEV_TX_OK;2692 2693error_unmap_dma:2694	ena_unmap_tx_buff(tx_ring, tx_info);2695	tx_info->skb = NULL;2696 2697error_drop_packet:2698	dev_kfree_skb(skb);2699	return NETDEV_TX_OK;2700}2701 2702static void ena_config_host_info(struct ena_com_dev *ena_dev, struct pci_dev *pdev)2703{2704	struct device *dev = &pdev->dev;2705	struct ena_admin_host_info *host_info;2706	ssize_t ret;2707	int rc;2708 2709	/* Allocate only the host info */2710	rc = ena_com_allocate_host_info(ena_dev);2711	if (rc) {2712		dev_err(dev, "Cannot allocate host info\n");2713		return;2714	}2715 2716	host_info = ena_dev->host_attr.host_info;2717 2718	host_info->bdf = pci_dev_id(pdev);2719	host_info->os_type = ENA_ADMIN_OS_LINUX;2720	host_info->kernel_ver = LINUX_VERSION_CODE;2721	ret = strscpy(host_info->kernel_ver_str, utsname()->version,2722		      sizeof(host_info->kernel_ver_str));2723	if (ret < 0)2724		dev_dbg(dev,2725			"kernel version string will be truncated, status = %zd\n", ret);2726 2727	host_info->os_dist = 0;2728	ret = strscpy(host_info->os_dist_str, utsname()->release,2729		      sizeof(host_info->os_dist_str));2730	if (ret < 0)2731		dev_dbg(dev,2732			"OS distribution string will be truncated, status = %zd\n", ret);2733 2734	host_info->driver_version =2735		(DRV_MODULE_GEN_MAJOR) |2736		(DRV_MODULE_GEN_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) |2737		(DRV_MODULE_GEN_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT) |2738		("K"[0] << ENA_ADMIN_HOST_INFO_MODULE_TYPE_SHIFT);2739	host_info->num_cpus = num_online_cpus();2740 2741	host_info->driver_supported_features =2742		ENA_ADMIN_HOST_INFO_RX_OFFSET_MASK |2743		ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_MASK |2744		ENA_ADMIN_HOST_INFO_RX_BUF_MIRRORING_MASK |2745		ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_MASK |2746		ENA_ADMIN_HOST_INFO_RX_PAGE_REUSE_MASK;2747 2748	rc = ena_com_set_host_attributes(ena_dev);2749	if (rc) {2750		if (rc == -EOPNOTSUPP)2751			dev_warn(dev, "Cannot set host attributes\n");2752		else2753			dev_err(dev, "Cannot set host attributes\n");2754 2755		goto err;2756	}2757 2758	return;2759 2760err:2761	ena_com_delete_host_info(ena_dev);2762}2763 2764static void ena_config_debug_area(struct ena_adapter *adapter)2765{2766	u32 debug_area_size;2767	int rc, ss_count;2768 2769	ss_count = ena_get_sset_count(adapter->netdev, ETH_SS_STATS);2770	if (ss_count <= 0) {2771		netif_err(adapter, drv, adapter->netdev,2772			  "SS count is negative\n");2773		return;2774	}2775 2776	/* allocate 32 bytes for each string and 64bit for the value */2777	debug_area_size = ss_count * ETH_GSTRING_LEN + sizeof(u64) * ss_count;2778 2779	rc = ena_com_allocate_debug_area(adapter->ena_dev, debug_area_size);2780	if (rc) {2781		netif_err(adapter, drv, adapter->netdev,2782			  "Cannot allocate debug area\n");2783		return;2784	}2785 2786	rc = ena_com_set_host_attributes(adapter->ena_dev);2787	if (rc) {2788		if (rc == -EOPNOTSUPP)2789			netif_warn(adapter, drv, adapter->netdev, "Cannot set host attributes\n");2790		else2791			netif_err(adapter, drv, adapter->netdev,2792				  "Cannot set host attributes\n");2793		goto err;2794	}2795 2796	return;2797err:2798	ena_com_delete_debug_area(adapter->ena_dev);2799}2800 2801static void ena_get_stats64(struct net_device *netdev,2802			    struct rtnl_link_stats64 *stats)2803{2804	struct ena_adapter *adapter = netdev_priv(netdev);2805	struct ena_ring *rx_ring, *tx_ring;2806	u64 total_xdp_rx_drops = 0;2807	unsigned int start;2808	u64 rx_drops;2809	u64 tx_drops;2810	int i;2811 2812	if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))2813		return;2814 2815	for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) {2816		u64 bytes, packets, xdp_rx_drops;2817 2818		tx_ring = &adapter->tx_ring[i];2819 2820		do {2821			start = u64_stats_fetch_begin(&tx_ring->syncp);2822			packets = tx_ring->tx_stats.cnt;2823			bytes = tx_ring->tx_stats.bytes;2824		} while (u64_stats_fetch_retry(&tx_ring->syncp, start));2825 2826		stats->tx_packets += packets;2827		stats->tx_bytes += bytes;2828 2829		/* In XDP there isn't an RX queue counterpart */2830		if (ENA_IS_XDP_INDEX(adapter, i))2831			continue;2832 2833		rx_ring = &adapter->rx_ring[i];2834 2835		do {2836			start = u64_stats_fetch_begin(&rx_ring->syncp);2837			packets = rx_ring->rx_stats.cnt;2838			bytes = rx_ring->rx_stats.bytes;2839			xdp_rx_drops = rx_ring->rx_stats.xdp_drop;2840		} while (u64_stats_fetch_retry(&rx_ring->syncp, start));2841 2842		stats->rx_packets += packets;2843		stats->rx_bytes += bytes;2844		total_xdp_rx_drops += xdp_rx_drops;2845	}2846 2847	do {2848		start = u64_stats_fetch_begin(&adapter->syncp);2849		rx_drops = adapter->dev_stats.rx_drops;2850		tx_drops = adapter->dev_stats.tx_drops;2851	} while (u64_stats_fetch_retry(&adapter->syncp, start));2852 2853	stats->rx_dropped = rx_drops + total_xdp_rx_drops;2854	stats->tx_dropped = tx_drops;2855 2856	stats->multicast = 0;2857	stats->collisions = 0;2858 2859	stats->rx_length_errors = 0;2860	stats->rx_crc_errors = 0;2861	stats->rx_frame_errors = 0;2862	stats->rx_fifo_errors = 0;2863	stats->rx_missed_errors = 0;2864	stats->tx_window_errors = 0;2865 2866	stats->rx_errors = 0;2867	stats->tx_errors = 0;2868}2869 2870static const struct net_device_ops ena_netdev_ops = {2871	.ndo_open		= ena_open,2872	.ndo_stop		= ena_close,2873	.ndo_start_xmit		= ena_start_xmit,2874	.ndo_get_stats64	= ena_get_stats64,2875	.ndo_tx_timeout		= ena_tx_timeout,2876	.ndo_change_mtu		= ena_change_mtu,2877	.ndo_validate_addr	= eth_validate_addr,2878	.ndo_bpf		= ena_xdp,2879	.ndo_xdp_xmit		= ena_xdp_xmit,2880};2881 2882static int ena_calc_io_queue_size(struct ena_adapter *adapter,2883				  struct ena_com_dev_get_features_ctx *get_feat_ctx)2884{2885	struct ena_admin_feature_llq_desc *llq = &get_feat_ctx->llq;2886	struct ena_com_dev *ena_dev = adapter->ena_dev;2887	u32 tx_queue_size = ENA_DEFAULT_RING_SIZE;2888	u32 rx_queue_size = ENA_DEFAULT_RING_SIZE;2889	u32 max_tx_queue_size;2890	u32 max_rx_queue_size;2891 2892	/* If this function is called after driver load, the ring sizes have already2893	 * been configured. Take it into account when recalculating ring size.2894	 */2895	if (adapter->tx_ring->ring_size)2896		tx_queue_size = adapter->tx_ring->ring_size;2897 2898	if (adapter->rx_ring->ring_size)2899		rx_queue_size = adapter->rx_ring->ring_size;2900 2901	if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {2902		struct ena_admin_queue_ext_feature_fields *max_queue_ext =2903			&get_feat_ctx->max_queue_ext.max_queue_ext;2904		max_rx_queue_size = min_t(u32, max_queue_ext->max_rx_cq_depth,2905					  max_queue_ext->max_rx_sq_depth);2906		max_tx_queue_size = max_queue_ext->max_tx_cq_depth;2907 2908		if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)2909			max_tx_queue_size = min_t(u32, max_tx_queue_size,2910						  llq->max_llq_depth);2911		else2912			max_tx_queue_size = min_t(u32, max_tx_queue_size,2913						  max_queue_ext->max_tx_sq_depth);2914 2915		adapter->max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,2916						 max_queue_ext->max_per_packet_tx_descs);2917		adapter->max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,2918						 max_queue_ext->max_per_packet_rx_descs);2919	} else {2920		struct ena_admin_queue_feature_desc *max_queues =2921			&get_feat_ctx->max_queues;2922		max_rx_queue_size = min_t(u32, max_queues->max_cq_depth,2923					  max_queues->max_sq_depth);2924		max_tx_queue_size = max_queues->max_cq_depth;2925 2926		if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)2927			max_tx_queue_size = min_t(u32, max_tx_queue_size,2928						  llq->max_llq_depth);2929		else2930			max_tx_queue_size = min_t(u32, max_tx_queue_size,2931						  max_queues->max_sq_depth);2932 2933		adapter->max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,2934						 max_queues->max_packet_tx_descs);2935		adapter->max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,2936						 max_queues->max_packet_rx_descs);2937	}2938 2939	max_tx_queue_size = rounddown_pow_of_two(max_tx_queue_size);2940	max_rx_queue_size = rounddown_pow_of_two(max_rx_queue_size);2941 2942	if (max_tx_queue_size < ENA_MIN_RING_SIZE) {2943		netdev_err(adapter->netdev, "Device max TX queue size: %d < minimum: %d\n",2944			   max_tx_queue_size, ENA_MIN_RING_SIZE);2945		return -EINVAL;2946	}2947 2948	if (max_rx_queue_size < ENA_MIN_RING_SIZE) {2949		netdev_err(adapter->netdev, "Device max RX queue size: %d < minimum: %d\n",2950			   max_rx_queue_size, ENA_MIN_RING_SIZE);2951		return -EINVAL;2952	}2953 2954	/* When forcing large headers, we multiply the entry size by 2, and therefore divide2955	 * the queue size by 2, leaving the amount of memory used by the queues unchanged.2956	 */2957	if (adapter->large_llq_header_enabled) {2958		if ((llq->entry_size_ctrl_supported & ENA_ADMIN_LIST_ENTRY_SIZE_256B) &&2959		    ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {2960			max_tx_queue_size /= 2;2961			dev_info(&adapter->pdev->dev,2962				 "Forcing large headers and decreasing maximum TX queue size to %d\n",2963				 max_tx_queue_size);2964		} else {2965			dev_err(&adapter->pdev->dev,2966				"Forcing large headers failed: LLQ is disabled or device does not support large headers\n");2967 2968			adapter->large_llq_header_enabled = false;2969		}2970	}2971 2972	tx_queue_size = clamp_val(tx_queue_size, ENA_MIN_RING_SIZE,2973				  max_tx_queue_size);2974	rx_queue_size = clamp_val(rx_queue_size, ENA_MIN_RING_SIZE,2975				  max_rx_queue_size);2976 2977	tx_queue_size = rounddown_pow_of_two(tx_queue_size);2978	rx_queue_size = rounddown_pow_of_two(rx_queue_size);2979 2980	adapter->max_tx_ring_size  = max_tx_queue_size;2981	adapter->max_rx_ring_size = max_rx_queue_size;2982	adapter->requested_tx_ring_size = tx_queue_size;2983	adapter->requested_rx_ring_size = rx_queue_size;2984 2985	return 0;2986}2987 2988static int ena_device_validate_params(struct ena_adapter *adapter,2989				      struct ena_com_dev_get_features_ctx *get_feat_ctx)2990{2991	struct net_device *netdev = adapter->netdev;2992	int rc;2993 2994	rc = ether_addr_equal(get_feat_ctx->dev_attr.mac_addr,2995			      adapter->mac_addr);2996	if (!rc) {2997		netif_err(adapter, drv, netdev,2998			  "Error, mac address are different\n");2999		return -EINVAL;3000	}3001 3002	if (get_feat_ctx->dev_attr.max_mtu < netdev->mtu) {3003		netif_err(adapter, drv, netdev,3004			  "Error, device max mtu is smaller than netdev MTU\n");3005		return -EINVAL;3006	}3007 3008	return 0;3009}3010 3011static void set_default_llq_configurations(struct ena_adapter *adapter,3012					   struct ena_llq_configurations *llq_config,3013					   struct ena_admin_feature_llq_desc *llq)3014{3015	struct ena_com_dev *ena_dev = adapter->ena_dev;3016 3017	llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER;3018	llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY;3019	llq_config->llq_num_decs_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2;3020 3021	adapter->large_llq_header_supported =3022		!!(ena_dev->supported_features & BIT(ENA_ADMIN_LLQ));3023	adapter->large_llq_header_supported &=3024		!!(llq->entry_size_ctrl_supported &3025			ENA_ADMIN_LIST_ENTRY_SIZE_256B);3026 3027	if ((llq->entry_size_ctrl_supported & ENA_ADMIN_LIST_ENTRY_SIZE_256B) &&3028	    adapter->large_llq_header_enabled) {3029		llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_256B;3030		llq_config->llq_ring_entry_size_value = 256;3031	} else {3032		llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_128B;3033		llq_config->llq_ring_entry_size_value = 128;3034	}3035}3036 3037static int ena_set_queues_placement_policy(struct pci_dev *pdev,3038					   struct ena_com_dev *ena_dev,3039					   struct ena_admin_feature_llq_desc *llq,3040					   struct ena_llq_configurations *llq_default_configurations)3041{3042	int rc;3043	u32 llq_feature_mask;3044 3045	llq_feature_mask = 1 << ENA_ADMIN_LLQ;3046	if (!(ena_dev->supported_features & llq_feature_mask)) {3047		dev_warn(&pdev->dev,3048			"LLQ is not supported Fallback to host mode policy.\n");3049		ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;3050		return 0;3051	}3052 3053	if (!ena_dev->mem_bar) {3054		netdev_err(ena_dev->net_device,3055			   "LLQ is advertised as supported but device doesn't expose mem bar\n");3056		ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;3057		return 0;3058	}3059 3060	rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations);3061	if (unlikely(rc)) {3062		dev_err(&pdev->dev,3063			"Failed to configure the device mode.  Fallback to host mode policy.\n");3064		ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;3065	}3066 3067	return 0;3068}3069 3070static int ena_map_llq_mem_bar(struct pci_dev *pdev, struct ena_com_dev *ena_dev,3071			       int bars)3072{3073	bool has_mem_bar = !!(bars & BIT(ENA_MEM_BAR));3074 3075	if (!has_mem_bar)3076		return 0;3077 3078	ena_dev->mem_bar = devm_ioremap_wc(&pdev->dev,3079					   pci_resource_start(pdev, ENA_MEM_BAR),3080					   pci_resource_len(pdev, ENA_MEM_BAR));3081 3082	if (!ena_dev->mem_bar)3083		return -EFAULT;3084 3085	return 0;3086}3087 3088static int ena_device_init(struct ena_adapter *adapter, struct pci_dev *pdev,3089			   struct ena_com_dev_get_features_ctx *get_feat_ctx,3090			   bool *wd_state)3091{3092	struct ena_com_dev *ena_dev = adapter->ena_dev;3093	struct net_device *netdev = adapter->netdev;3094	struct ena_llq_configurations llq_config;3095	struct device *dev = &pdev->dev;3096	bool readless_supported;3097	u32 aenq_groups;3098	int dma_width;3099	int rc;3100 3101	rc = ena_com_mmio_reg_read_request_init(ena_dev);3102	if (rc) {3103		dev_err(dev, "Failed to init mmio read less\n");3104		return rc;3105	}3106 3107	/* The PCIe configuration space revision id indicate if mmio reg3108	 * read is disabled3109	 */3110	readless_supported = !(pdev->revision & ENA_MMIO_DISABLE_REG_READ);3111	ena_com_set_mmio_read_mode(ena_dev, readless_supported);3112 3113	rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL);3114	if (rc) {3115		dev_err(dev, "Can not reset device\n");3116		goto err_mmio_read_less;3117	}3118 3119	rc = ena_com_validate_version(ena_dev);3120	if (rc) {3121		dev_err(dev, "Device version is too low\n");3122		goto err_mmio_read_less;3123	}3124 3125	dma_width = ena_com_get_dma_width(ena_dev);3126	if (dma_width < 0) {3127		dev_err(dev, "Invalid dma width value %d", dma_width);3128		rc = dma_width;3129		goto err_mmio_read_less;3130	}3131 3132	rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(dma_width));3133	if (rc) {3134		dev_err(dev, "dma_set_mask_and_coherent failed %d\n", rc);3135		goto err_mmio_read_less;3136	}3137 3138	/* ENA admin level init */3139	rc = ena_com_admin_init(ena_dev, &aenq_handlers);3140	if (rc) {3141		dev_err(dev,3142			"Can not initialize ena admin queue with device\n");3143		goto err_mmio_read_less;3144	}3145 3146	/* To enable the msix interrupts the driver needs to know the number3147	 * of queues. So the driver uses polling mode to retrieve this3148	 * information3149	 */3150	ena_com_set_admin_polling_mode(ena_dev, true);3151 3152	ena_config_host_info(ena_dev, pdev);3153 3154	/* Get Device Attributes*/3155	rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx);3156	if (rc) {3157		dev_err(dev, "Cannot get attribute for ena device rc=%d\n", rc);3158		goto err_admin_init;3159	}3160 3161	/* Try to turn all the available aenq groups */3162	aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) |3163		BIT(ENA_ADMIN_FATAL_ERROR) |3164		BIT(ENA_ADMIN_WARNING) |3165		BIT(ENA_ADMIN_NOTIFICATION) |3166		BIT(ENA_ADMIN_KEEP_ALIVE);3167 3168	aenq_groups &= get_feat_ctx->aenq.supported_groups;3169 3170	rc = ena_com_set_aenq_config(ena_dev, aenq_groups);3171	if (rc) {3172		dev_err(dev, "Cannot configure aenq groups rc= %d\n", rc);3173		goto err_admin_init;3174	}3175 3176	*wd_state = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE));3177 3178	set_default_llq_configurations(adapter, &llq_config, &get_feat_ctx->llq);3179 3180	rc = ena_set_queues_placement_policy(pdev, ena_dev, &get_feat_ctx->llq,3181					     &llq_config);3182	if (rc) {3183		netdev_err(netdev, "Cannot set queues placement policy rc= %d\n", rc);3184		goto err_admin_init;3185	}3186 3187	rc = ena_calc_io_queue_size(adapter, get_feat_ctx);3188	if (unlikely(rc))3189		goto err_admin_init;3190 3191	return 0;3192 3193err_admin_init:3194	ena_com_abort_admin_commands(ena_dev);3195	ena_com_wait_for_abort_completion(ena_dev);3196	ena_com_delete_host_info(ena_dev);3197	ena_com_admin_destroy(ena_dev);3198err_mmio_read_less:3199	ena_com_mmio_reg_read_request_destroy(ena_dev);3200 3201	return rc;3202}3203 3204static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter)3205{3206	struct ena_com_dev *ena_dev = adapter->ena_dev;3207	struct device *dev = &adapter->pdev->dev;3208	int rc;3209 3210	rc = ena_enable_msix(adapter);3211	if (rc) {3212		dev_err(dev, "Can not reserve msix vectors\n");3213		return rc;3214	}3215 3216	ena_setup_mgmnt_intr(adapter);3217 3218	rc = ena_request_mgmnt_irq(adapter);3219	if (rc) {3220		dev_err(dev, "Can not setup management interrupts\n");3221		goto err_disable_msix;3222	}3223 3224	ena_com_set_admin_polling_mode(ena_dev, false);3225 3226	ena_com_admin_aenq_enable(ena_dev);3227 3228	return 0;3229 3230err_disable_msix:3231	ena_disable_msix(adapter);3232 3233	return rc;3234}3235 3236static int ena_destroy_device(struct ena_adapter *adapter, bool graceful)3237{3238	struct net_device *netdev = adapter->netdev;3239	struct ena_com_dev *ena_dev = adapter->ena_dev;3240	bool dev_up;3241	int rc = 0;3242 3243	if (!test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags))3244		return 0;3245 3246	netif_carrier_off(netdev);3247 3248	del_timer_sync(&adapter->timer_service);3249 3250	dev_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);3251	adapter->dev_up_before_reset = dev_up;3252	if (!graceful)3253		ena_com_set_admin_running_state(ena_dev, false);3254 3255	if (dev_up)3256		ena_down(adapter);3257 3258	/* Stop the device from sending AENQ events (in case reset flag is set3259	 *  and device is up, ena_down() already reset the device.3260	 */3261	if (!(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags) && dev_up))3262		rc = ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason);3263 3264	ena_free_mgmnt_irq(adapter);3265 3266	ena_disable_msix(adapter);3267 3268	ena_com_abort_admin_commands(ena_dev);3269 3270	ena_com_wait_for_abort_completion(ena_dev);3271 3272	ena_com_admin_destroy(ena_dev);3273 3274	ena_com_mmio_reg_read_request_destroy(ena_dev);3275 3276	/* return reset reason to default value */3277	adapter->reset_reason = ENA_REGS_RESET_NORMAL;3278 3279	clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);3280	clear_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);3281 3282	return rc;3283}3284 3285static int ena_restore_device(struct ena_adapter *adapter)3286{3287	struct ena_com_dev_get_features_ctx get_feat_ctx;3288	struct ena_com_dev *ena_dev = adapter->ena_dev;3289	struct pci_dev *pdev = adapter->pdev;3290	struct ena_ring *txr;3291	int rc, count, i;3292	bool wd_state;3293 3294	set_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags);3295	rc = ena_device_init(adapter, adapter->pdev, &get_feat_ctx, &wd_state);3296	if (rc) {3297		dev_err(&pdev->dev, "Can not initialize device\n");3298		goto err;3299	}3300	adapter->wd_state = wd_state;3301 3302	count =  adapter->xdp_num_queues + adapter->num_io_queues;3303	for (i = 0 ; i < count; i++) {3304		txr = &adapter->tx_ring[i];3305		txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type;3306		txr->tx_max_header_size = ena_dev->tx_max_header_size;3307	}3308 3309	rc = ena_device_validate_params(adapter, &get_feat_ctx);3310	if (rc) {3311		dev_err(&pdev->dev, "Validation of device parameters failed\n");3312		goto err_device_destroy;3313	}3314 3315	rc = ena_enable_msix_and_set_admin_interrupts(adapter);3316	if (rc) {3317		dev_err(&pdev->dev, "Enable MSI-X failed\n");3318		goto err_device_destroy;3319	}3320	/* If the interface was up before the reset bring it up */3321	if (adapter->dev_up_before_reset) {3322		rc = ena_up(adapter);3323		if (rc) {3324			dev_err(&pdev->dev, "Failed to create I/O queues\n");3325			goto err_disable_msix;3326		}3327	}3328 3329	set_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);3330 3331	clear_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags);3332	if (test_bit(ENA_FLAG_LINK_UP, &adapter->flags))3333		netif_carrier_on(adapter->netdev);3334 3335	mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));3336	adapter->last_keep_alive_jiffies = jiffies;3337 3338	return rc;3339err_disable_msix:3340	ena_free_mgmnt_irq(adapter);3341	ena_disable_msix(adapter);3342err_device_destroy:3343	ena_com_abort_admin_commands(ena_dev);3344	ena_com_wait_for_abort_completion(ena_dev);3345	ena_com_admin_destroy(ena_dev);3346	ena_com_dev_reset(ena_dev, ENA_REGS_RESET_DRIVER_INVALID_STATE);3347	ena_com_mmio_reg_read_request_destroy(ena_dev);3348err:3349	clear_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);3350	clear_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags);3351	dev_err(&pdev->dev,3352		"Reset attempt failed. Can not reset the device\n");3353 3354	return rc;3355}3356 3357static void ena_fw_reset_device(struct work_struct *work)3358{3359	int rc = 0;3360 3361	struct ena_adapter *adapter =3362		container_of(work, struct ena_adapter, reset_task);3363 3364	rtnl_lock();3365 3366	if (likely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {3367		rc |= ena_destroy_device(adapter, false);3368		rc |= ena_restore_device(adapter);3369		adapter->dev_stats.reset_fail += !!rc;3370 3371		dev_err(&adapter->pdev->dev, "Device reset completed successfully\n");3372	}3373 3374	rtnl_unlock();3375}3376 3377static int check_for_rx_interrupt_queue(struct ena_adapter *adapter,3378					struct ena_ring *rx_ring)3379{3380	struct ena_napi *ena_napi = container_of(rx_ring->napi, struct ena_napi, napi);3381 3382	if (likely(READ_ONCE(ena_napi->first_interrupt)))3383		return 0;3384 3385	if (ena_com_cq_empty(rx_ring->ena_com_io_cq))3386		return 0;3387 3388	rx_ring->no_interrupt_event_cnt++;3389 3390	if (rx_ring->no_interrupt_event_cnt == ENA_MAX_NO_INTERRUPT_ITERATIONS) {3391		netif_err(adapter, rx_err, adapter->netdev,3392			  "Potential MSIX issue on Rx side Queue = %d. Reset the device\n",3393			  rx_ring->qid);3394 3395		ena_reset_device(adapter, ENA_REGS_RESET_MISS_INTERRUPT);3396		return -EIO;3397	}3398 3399	return 0;3400}3401 3402static int check_missing_comp_in_tx_queue(struct ena_adapter *adapter,3403					  struct ena_ring *tx_ring)3404{3405	struct ena_napi *ena_napi = container_of(tx_ring->napi, struct ena_napi, napi);3406	enum ena_regs_reset_reason_types reset_reason = ENA_REGS_RESET_MISS_TX_CMPL;3407	unsigned int time_since_last_napi;3408	unsigned int missing_tx_comp_to;3409	bool is_tx_comp_time_expired;3410	struct ena_tx_buffer *tx_buf;3411	unsigned long last_jiffies;3412	int napi_scheduled;3413	u32 missed_tx = 0;3414	int i, rc = 0;3415 3416	missing_tx_comp_to = jiffies_to_msecs(adapter->missing_tx_completion_to);3417 3418	for (i = 0; i < tx_ring->ring_size; i++) {3419		tx_buf = &tx_ring->tx_buffer_info[i];3420		last_jiffies = tx_buf->last_jiffies;3421 3422		if (last_jiffies == 0)3423			/* no pending Tx at this location */3424			continue;3425 3426		is_tx_comp_time_expired = time_is_before_jiffies(last_jiffies +3427			 2 * adapter->missing_tx_completion_to);3428 3429		if (unlikely(!READ_ONCE(ena_napi->first_interrupt) && is_tx_comp_time_expired)) {3430			/* If after graceful period interrupt is still not3431			 * received, we schedule a reset3432			 */3433			netif_err(adapter, tx_err, adapter->netdev,3434				  "Potential MSIX issue on Tx side Queue = %d. Reset the device\n",3435				  tx_ring->qid);3436			ena_reset_device(adapter, ENA_REGS_RESET_MISS_INTERRUPT);3437			return -EIO;3438		}3439 3440		is_tx_comp_time_expired = time_is_before_jiffies(last_jiffies +3441			adapter->missing_tx_completion_to);3442 3443		if (unlikely(is_tx_comp_time_expired)) {3444			time_since_last_napi =3445				jiffies_to_usecs(jiffies - tx_ring->tx_stats.last_napi_jiffies);3446			napi_scheduled = !!(ena_napi->napi.state & NAPIF_STATE_SCHED);3447 3448			if (missing_tx_comp_to < time_since_last_napi && napi_scheduled) {3449				/* We suspect napi isn't called because the3450				 * bottom half is not run. Require a bigger3451				 * timeout for these cases3452				 */3453				if (!time_is_before_jiffies(last_jiffies +3454					2 * adapter->missing_tx_completion_to))3455					continue;3456 3457				reset_reason = ENA_REGS_RESET_SUSPECTED_POLL_STARVATION;3458			}3459 3460			missed_tx++;3461 3462			if (tx_buf->print_once)3463				continue;3464 3465			netif_notice(adapter, tx_err, adapter->netdev,3466				     "TX hasn't completed, qid %d, index %d. %u usecs from last napi execution, napi scheduled: %d\n",3467				     tx_ring->qid, i, time_since_last_napi, napi_scheduled);3468 3469			tx_buf->print_once = 1;3470		}3471	}3472 3473	if (unlikely(missed_tx > adapter->missing_tx_completion_threshold)) {3474		netif_err(adapter, tx_err, adapter->netdev,3475			  "Lost TX completions are above the threshold (%d > %d). Completion transmission timeout: %u.\n",3476			  missed_tx,3477			  adapter->missing_tx_completion_threshold,3478			  missing_tx_comp_to);3479		netif_err(adapter, tx_err, adapter->netdev,3480			  "Resetting the device\n");3481 3482		ena_reset_device(adapter, reset_reason);3483		rc = -EIO;3484	}3485 3486	ena_increase_stat(&tx_ring->tx_stats.missed_tx, missed_tx,3487			  &tx_ring->syncp);3488 3489	return rc;3490}3491 3492static void check_for_missing_completions(struct ena_adapter *adapter)3493{3494	struct ena_ring *tx_ring;3495	struct ena_ring *rx_ring;3496	int qid, budget, rc;3497	int io_queue_count;3498 3499	io_queue_count = adapter->xdp_num_queues + adapter->num_io_queues;3500 3501	/* Make sure the driver doesn't turn the device in other process */3502	smp_rmb();3503 3504	if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))3505		return;3506 3507	if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))3508		return;3509 3510	if (adapter->missing_tx_completion_to == ENA_HW_HINTS_NO_TIMEOUT)3511		return;3512 3513	budget = min_t(u32, io_queue_count, ENA_MONITORED_TX_QUEUES);3514 3515	qid = adapter->last_monitored_tx_qid;3516 3517	while (budget) {3518		qid = (qid + 1) % io_queue_count;3519 3520		tx_ring = &adapter->tx_ring[qid];3521		rx_ring = &adapter->rx_ring[qid];3522 3523		rc = check_missing_comp_in_tx_queue(adapter, tx_ring);3524		if (unlikely(rc))3525			return;3526 3527		rc =  !ENA_IS_XDP_INDEX(adapter, qid) ?3528			check_for_rx_interrupt_queue(adapter, rx_ring) : 0;3529		if (unlikely(rc))3530			return;3531 3532		budget--;3533	}3534 3535	adapter->last_monitored_tx_qid = qid;3536}3537 3538/* trigger napi schedule after 2 consecutive detections */3539#define EMPTY_RX_REFILL 23540/* For the rare case where the device runs out of Rx descriptors and the3541 * napi handler failed to refill new Rx descriptors (due to a lack of memory3542 * for example).3543 * This case will lead to a deadlock:3544 * The device won't send interrupts since all the new Rx packets will be dropped3545 * The napi handler won't allocate new Rx descriptors so the device will be3546 * able to send new packets.3547 *3548 * This scenario can happen when the kernel's vm.min_free_kbytes is too small.3549 * It is recommended to have at least 512MB, with a minimum of 128MB for3550 * constrained environment).3551 *3552 * When such a situation is detected - Reschedule napi3553 */3554static void check_for_empty_rx_ring(struct ena_adapter *adapter)3555{3556	struct ena_ring *rx_ring;3557	int i, refill_required;3558 3559	if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))3560		return;3561 3562	if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))3563		return;3564 3565	for (i = 0; i < adapter->num_io_queues; i++) {3566		rx_ring = &adapter->rx_ring[i];3567 3568		refill_required = ena_com_free_q_entries(rx_ring->ena_com_io_sq);3569		if (unlikely(refill_required == (rx_ring->ring_size - 1))) {3570			rx_ring->empty_rx_queue++;3571 3572			if (rx_ring->empty_rx_queue >= EMPTY_RX_REFILL) {3573				ena_increase_stat(&rx_ring->rx_stats.empty_rx_ring, 1,3574						  &rx_ring->syncp);3575 3576				netif_err(adapter, drv, adapter->netdev,3577					  "Trigger refill for ring %d\n", i);3578 3579				napi_schedule(rx_ring->napi);3580				rx_ring->empty_rx_queue = 0;3581			}3582		} else {3583			rx_ring->empty_rx_queue = 0;3584		}3585	}3586}3587 3588/* Check for keep alive expiration */3589static void check_for_missing_keep_alive(struct ena_adapter *adapter)3590{3591	unsigned long keep_alive_expired;3592 3593	if (!adapter->wd_state)3594		return;3595 3596	if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT)3597		return;3598 3599	keep_alive_expired = adapter->last_keep_alive_jiffies +3600			     adapter->keep_alive_timeout;3601	if (unlikely(time_is_before_jiffies(keep_alive_expired))) {3602		netif_err(adapter, drv, adapter->netdev,3603			  "Keep alive watchdog timeout.\n");3604		ena_increase_stat(&adapter->dev_stats.wd_expired, 1,3605				  &adapter->syncp);3606		ena_reset_device(adapter, ENA_REGS_RESET_KEEP_ALIVE_TO);3607	}3608}3609 3610static void check_for_admin_com_state(struct ena_adapter *adapter)3611{3612	if (unlikely(!ena_com_get_admin_running_state(adapter->ena_dev))) {3613		netif_err(adapter, drv, adapter->netdev,3614			  "ENA admin queue is not in running state!\n");3615		ena_increase_stat(&adapter->dev_stats.admin_q_pause, 1,3616				  &adapter->syncp);3617		ena_reset_device(adapter, ENA_REGS_RESET_ADMIN_TO);3618	}3619}3620 3621static void ena_update_hints(struct ena_adapter *adapter,3622			     struct ena_admin_ena_hw_hints *hints)3623{3624	struct net_device *netdev = adapter->netdev;3625 3626	if (hints->admin_completion_tx_timeout)3627		adapter->ena_dev->admin_queue.completion_timeout =3628			hints->admin_completion_tx_timeout * 1000;3629 3630	if (hints->mmio_read_timeout)3631		/* convert to usec */3632		adapter->ena_dev->mmio_read.reg_read_to =3633			hints->mmio_read_timeout * 1000;3634 3635	if (hints->missed_tx_completion_count_threshold_to_reset)3636		adapter->missing_tx_completion_threshold =3637			hints->missed_tx_completion_count_threshold_to_reset;3638 3639	if (hints->missing_tx_completion_timeout) {3640		if (hints->missing_tx_completion_timeout == ENA_HW_HINTS_NO_TIMEOUT)3641			adapter->missing_tx_completion_to = ENA_HW_HINTS_NO_TIMEOUT;3642		else3643			adapter->missing_tx_completion_to =3644				msecs_to_jiffies(hints->missing_tx_completion_timeout);3645	}3646 3647	if (hints->netdev_wd_timeout)3648		netdev->watchdog_timeo = msecs_to_jiffies(hints->netdev_wd_timeout);3649 3650	if (hints->driver_watchdog_timeout) {3651		if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT)3652			adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT;3653		else3654			adapter->keep_alive_timeout =3655				msecs_to_jiffies(hints->driver_watchdog_timeout);3656	}3657}3658 3659static void ena_update_host_info(struct ena_admin_host_info *host_info,3660				 struct net_device *netdev)3661{3662	host_info->supported_network_features[0] =3663		netdev->features & GENMASK_ULL(31, 0);3664	host_info->supported_network_features[1] =3665		(netdev->features & GENMASK_ULL(63, 32)) >> 32;3666}3667 3668static void ena_timer_service(struct timer_list *t)3669{3670	struct ena_adapter *adapter = from_timer(adapter, t, timer_service);3671	u8 *debug_area = adapter->ena_dev->host_attr.debug_area_virt_addr;3672	struct ena_admin_host_info *host_info =3673		adapter->ena_dev->host_attr.host_info;3674 3675	check_for_missing_keep_alive(adapter);3676 3677	check_for_admin_com_state(adapter);3678 3679	check_for_missing_completions(adapter);3680 3681	check_for_empty_rx_ring(adapter);3682 3683	if (debug_area)3684		ena_dump_stats_to_buf(adapter, debug_area);3685 3686	if (host_info)3687		ena_update_host_info(host_info, adapter->netdev);3688 3689	if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {3690		netif_err(adapter, drv, adapter->netdev,3691			  "Trigger reset is on\n");3692		ena_dump_stats_to_dmesg(adapter);3693		queue_work(ena_wq, &adapter->reset_task);3694		return;3695	}3696 3697	/* Reset the timer */3698	mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));3699}3700 3701static u32 ena_calc_max_io_queue_num(struct pci_dev *pdev,3702				     struct ena_com_dev *ena_dev,3703				     struct ena_com_dev_get_features_ctx *get_feat_ctx)3704{3705	u32 io_tx_sq_num, io_tx_cq_num, io_rx_num, max_num_io_queues;3706 3707	if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {3708		struct ena_admin_queue_ext_feature_fields *max_queue_ext =3709			&get_feat_ctx->max_queue_ext.max_queue_ext;3710		io_rx_num = min_t(u32, max_queue_ext->max_rx_sq_num,3711				  max_queue_ext->max_rx_cq_num);3712 3713		io_tx_sq_num = max_queue_ext->max_tx_sq_num;3714		io_tx_cq_num = max_queue_ext->max_tx_cq_num;3715	} else {3716		struct ena_admin_queue_feature_desc *max_queues =3717			&get_feat_ctx->max_queues;3718		io_tx_sq_num = max_queues->max_sq_num;3719		io_tx_cq_num = max_queues->max_cq_num;3720		io_rx_num = min_t(u32, io_tx_sq_num, io_tx_cq_num);3721	}3722 3723	/* In case of LLQ use the llq fields for the tx SQ/CQ */3724	if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)3725		io_tx_sq_num = get_feat_ctx->llq.max_llq_num;3726 3727	max_num_io_queues = min_t(u32, num_online_cpus(), ENA_MAX_NUM_IO_QUEUES);3728	max_num_io_queues = min_t(u32, max_num_io_queues, io_rx_num);3729	max_num_io_queues = min_t(u32, max_num_io_queues, io_tx_sq_num);3730	max_num_io_queues = min_t(u32, max_num_io_queues, io_tx_cq_num);3731	/* 1 IRQ for mgmnt and 1 IRQs for each IO direction */3732	max_num_io_queues = min_t(u32, max_num_io_queues, pci_msix_vec_count(pdev) - 1);3733 3734	return max_num_io_queues;3735}3736 3737static void ena_set_dev_offloads(struct ena_com_dev_get_features_ctx *feat,3738				 struct net_device *netdev)3739{3740	netdev_features_t dev_features = 0;3741 3742	/* Set offload features */3743	if (feat->offload.tx &3744		ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK)3745		dev_features |= NETIF_F_IP_CSUM;3746 3747	if (feat->offload.tx &3748		ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK)3749		dev_features |= NETIF_F_IPV6_CSUM;3750 3751	if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK)3752		dev_features |= NETIF_F_TSO;3753 3754	if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK)3755		dev_features |= NETIF_F_TSO6;3756 3757	if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_MASK)3758		dev_features |= NETIF_F_TSO_ECN;3759 3760	if (feat->offload.rx_supported &3761		ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK)3762		dev_features |= NETIF_F_RXCSUM;3763 3764	if (feat->offload.rx_supported &3765		ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK)3766		dev_features |= NETIF_F_RXCSUM;3767 3768	netdev->features =3769		dev_features |3770		NETIF_F_SG |3771		NETIF_F_RXHASH |3772		NETIF_F_HIGHDMA;3773 3774	netdev->hw_features |= netdev->features;3775	netdev->vlan_features |= netdev->features;3776}3777 3778static void ena_set_conf_feat_params(struct ena_adapter *adapter,3779				     struct ena_com_dev_get_features_ctx *feat)3780{3781	struct net_device *netdev = adapter->netdev;3782 3783	/* Copy mac address */3784	if (!is_valid_ether_addr(feat->dev_attr.mac_addr)) {3785		eth_hw_addr_random(netdev);3786		ether_addr_copy(adapter->mac_addr, netdev->dev_addr);3787	} else {3788		ether_addr_copy(adapter->mac_addr, feat->dev_attr.mac_addr);3789		eth_hw_addr_set(netdev, adapter->mac_addr);3790	}3791 3792	/* Set offload features */3793	ena_set_dev_offloads(feat, netdev);3794 3795	adapter->max_mtu = feat->dev_attr.max_mtu;3796	netdev->max_mtu = adapter->max_mtu;3797	netdev->min_mtu = ENA_MIN_MTU;3798}3799 3800static int ena_rss_init_default(struct ena_adapter *adapter)3801{3802	struct ena_com_dev *ena_dev = adapter->ena_dev;3803	struct device *dev = &adapter->pdev->dev;3804	int rc, i;3805	u32 val;3806 3807	rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE);3808	if (unlikely(rc)) {3809		dev_err(dev, "Cannot init indirect table\n");3810		goto err_rss_init;3811	}3812 3813	for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) {3814		val = ethtool_rxfh_indir_default(i, adapter->num_io_queues);3815		rc = ena_com_indirect_table_fill_entry(ena_dev, i,3816						       ENA_IO_RXQ_IDX(val));3817		if (unlikely(rc)) {3818			dev_err(dev, "Cannot fill indirect table\n");3819			goto err_fill_indir;3820		}3821	}3822 3823	rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ, NULL, ENA_HASH_KEY_SIZE,3824					0xFFFFFFFF);3825	if (unlikely(rc && (rc != -EOPNOTSUPP))) {3826		dev_err(dev, "Cannot fill hash function\n");3827		goto err_fill_indir;3828	}3829 3830	rc = ena_com_set_default_hash_ctrl(ena_dev);3831	if (unlikely(rc && (rc != -EOPNOTSUPP))) {3832		dev_err(dev, "Cannot fill hash control\n");3833		goto err_fill_indir;3834	}3835 3836	return 0;3837 3838err_fill_indir:3839	ena_com_rss_destroy(ena_dev);3840err_rss_init:3841 3842	return rc;3843}3844 3845static void ena_release_bars(struct ena_com_dev *ena_dev, struct pci_dev *pdev)3846{3847	int release_bars = pci_select_bars(pdev, IORESOURCE_MEM) & ENA_BAR_MASK;3848 3849	pci_release_selected_regions(pdev, release_bars);3850}3851 3852/* ena_probe - Device Initialization Routine3853 * @pdev: PCI device information struct3854 * @ent: entry in ena_pci_tbl3855 *3856 * Returns 0 on success, negative on failure3857 *3858 * ena_probe initializes an adapter identified by a pci_dev structure.3859 * The OS initialization, configuring of the adapter private structure,3860 * and a hardware reset occur.3861 */3862static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)3863{3864	struct ena_com_dev_get_features_ctx get_feat_ctx;3865	struct ena_com_dev *ena_dev = NULL;3866	struct ena_adapter *adapter;3867	struct net_device *netdev;3868	static int adapters_found;3869	u32 max_num_io_queues;3870	bool wd_state;3871	int bars, rc;3872 3873	dev_dbg(&pdev->dev, "%s\n", __func__);3874 3875	rc = pci_enable_device_mem(pdev);3876	if (rc) {3877		dev_err(&pdev->dev, "pci_enable_device_mem() failed!\n");3878		return rc;3879	}3880 3881	rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(ENA_MAX_PHYS_ADDR_SIZE_BITS));3882	if (rc) {3883		dev_err(&pdev->dev, "dma_set_mask_and_coherent failed %d\n", rc);3884		goto err_disable_device;3885	}3886 3887	pci_set_master(pdev);3888 3889	ena_dev = vzalloc(sizeof(*ena_dev));3890	if (!ena_dev) {3891		rc = -ENOMEM;3892		goto err_disable_device;3893	}3894 3895	bars = pci_select_bars(pdev, IORESOURCE_MEM) & ENA_BAR_MASK;3896	rc = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);3897	if (rc) {3898		dev_err(&pdev->dev, "pci_request_selected_regions failed %d\n",3899			rc);3900		goto err_free_ena_dev;3901	}3902 3903	ena_dev->reg_bar = devm_ioremap(&pdev->dev,3904					pci_resource_start(pdev, ENA_REG_BAR),3905					pci_resource_len(pdev, ENA_REG_BAR));3906	if (!ena_dev->reg_bar) {3907		dev_err(&pdev->dev, "Failed to remap regs bar\n");3908		rc = -EFAULT;3909		goto err_free_region;3910	}3911 3912	ena_dev->ena_min_poll_delay_us = ENA_ADMIN_POLL_DELAY_US;3913 3914	ena_dev->dmadev = &pdev->dev;3915 3916	netdev = alloc_etherdev_mq(sizeof(struct ena_adapter), ENA_MAX_RINGS);3917	if (!netdev) {3918		dev_err(&pdev->dev, "alloc_etherdev_mq failed\n");3919		rc = -ENOMEM;3920		goto err_free_region;3921	}3922 3923	SET_NETDEV_DEV(netdev, &pdev->dev);3924	adapter = netdev_priv(netdev);3925	adapter->ena_dev = ena_dev;3926	adapter->netdev = netdev;3927	adapter->pdev = pdev;3928	adapter->msg_enable = DEFAULT_MSG_ENABLE;3929 3930	ena_dev->net_device = netdev;3931 3932	pci_set_drvdata(pdev, adapter);3933 3934	rc = ena_com_allocate_customer_metrics_buffer(ena_dev);3935	if (rc) {3936		netdev_err(netdev, "ena_com_allocate_customer_metrics_buffer failed\n");3937		goto err_netdev_destroy;3938	}3939 3940	rc = ena_map_llq_mem_bar(pdev, ena_dev, bars);3941	if (rc) {3942		dev_err(&pdev->dev, "ENA LLQ bar mapping failed\n");3943		goto err_metrics_destroy;3944	}3945 3946	rc = ena_device_init(adapter, pdev, &get_feat_ctx, &wd_state);3947	if (rc) {3948		dev_err(&pdev->dev, "ENA device init failed\n");3949		if (rc == -ETIME)3950			rc = -EPROBE_DEFER;3951		goto err_metrics_destroy;3952	}3953 3954	/* Initial TX and RX interrupt delay. Assumes 1 usec granularity.3955	 * Updated during device initialization with the real granularity3956	 */3957	ena_dev->intr_moder_tx_interval = ENA_INTR_INITIAL_TX_INTERVAL_USECS;3958	ena_dev->intr_moder_rx_interval = ENA_INTR_INITIAL_RX_INTERVAL_USECS;3959	ena_dev->intr_delay_resolution = ENA_DEFAULT_INTR_DELAY_RESOLUTION;3960	max_num_io_queues = ena_calc_max_io_queue_num(pdev, ena_dev, &get_feat_ctx);3961	if (unlikely(!max_num_io_queues)) {3962		rc = -EFAULT;3963		goto err_device_destroy;3964	}3965 3966	ena_set_conf_feat_params(adapter, &get_feat_ctx);3967 3968	adapter->reset_reason = ENA_REGS_RESET_NORMAL;3969 3970	adapter->num_io_queues = max_num_io_queues;3971	adapter->max_num_io_queues = max_num_io_queues;3972	adapter->last_monitored_tx_qid = 0;3973 3974	adapter->xdp_first_ring = 0;3975	adapter->xdp_num_queues = 0;3976 3977	adapter->rx_copybreak = ENA_DEFAULT_RX_COPYBREAK;3978	if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)3979		adapter->disable_meta_caching =3980			!!(get_feat_ctx.llq.accel_mode.u.get.supported_flags &3981			   BIT(ENA_ADMIN_DISABLE_META_CACHING));3982 3983	adapter->wd_state = wd_state;3984 3985	snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", adapters_found);3986 3987	rc = ena_com_init_interrupt_moderation(adapter->ena_dev);3988	if (rc) {3989		dev_err(&pdev->dev,3990			"Failed to query interrupt moderation feature\n");3991		goto err_device_destroy;3992	}3993 3994	ena_init_io_rings(adapter,3995			  0,3996			  adapter->xdp_num_queues +3997			  adapter->num_io_queues);3998 3999	netdev->netdev_ops = &ena_netdev_ops;4000	netdev->watchdog_timeo = TX_TIMEOUT;4001	ena_set_ethtool_ops(netdev);4002 4003	netdev->priv_flags |= IFF_UNICAST_FLT;4004 4005	u64_stats_init(&adapter->syncp);4006 4007	rc = ena_enable_msix_and_set_admin_interrupts(adapter);4008	if (rc) {4009		dev_err(&pdev->dev,4010			"Failed to enable and set the admin interrupts\n");4011		goto err_worker_destroy;4012	}4013	rc = ena_rss_init_default(adapter);4014	if (rc && (rc != -EOPNOTSUPP)) {4015		dev_err(&pdev->dev, "Cannot init RSS rc: %d\n", rc);4016		goto err_free_msix;4017	}4018 4019	ena_config_debug_area(adapter);4020 4021	if (ena_xdp_legal_queue_count(adapter, adapter->num_io_queues))4022		netdev->xdp_features = NETDEV_XDP_ACT_BASIC |4023				       NETDEV_XDP_ACT_REDIRECT;4024 4025	memcpy(adapter->netdev->perm_addr, adapter->mac_addr, netdev->addr_len);4026 4027	netif_carrier_off(netdev);4028 4029	rc = register_netdev(netdev);4030	if (rc) {4031		dev_err(&pdev->dev, "Cannot register net device\n");4032		goto err_rss;4033	}4034 4035	INIT_WORK(&adapter->reset_task, ena_fw_reset_device);4036 4037	adapter->last_keep_alive_jiffies = jiffies;4038	adapter->keep_alive_timeout = ENA_DEVICE_KALIVE_TIMEOUT;4039	adapter->missing_tx_completion_to = TX_TIMEOUT;4040	adapter->missing_tx_completion_threshold = MAX_NUM_OF_TIMEOUTED_PACKETS;4041 4042	ena_update_hints(adapter, &get_feat_ctx.hw_hints);4043 4044	timer_setup(&adapter->timer_service, ena_timer_service, 0);4045	mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));4046 4047	dev_info(&pdev->dev,4048		 "%s found at mem %lx, mac addr %pM\n",4049		 DEVICE_NAME, (long)pci_resource_start(pdev, 0),4050		 netdev->dev_addr);4051 4052	set_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);4053 4054	adapters_found++;4055 4056	return 0;4057 4058err_rss:4059	ena_com_delete_debug_area(ena_dev);4060	ena_com_rss_destroy(ena_dev);4061err_free_msix:4062	ena_com_dev_reset(ena_dev, ENA_REGS_RESET_INIT_ERR);4063	/* stop submitting admin commands on a device that was reset */4064	ena_com_set_admin_running_state(ena_dev, false);4065	ena_free_mgmnt_irq(adapter);4066	ena_disable_msix(adapter);4067err_worker_destroy:4068	del_timer(&adapter->timer_service);4069err_device_destroy:4070	ena_com_delete_host_info(ena_dev);4071	ena_com_admin_destroy(ena_dev);4072err_metrics_destroy:4073	ena_com_delete_customer_metrics_buffer(ena_dev);4074err_netdev_destroy:4075	free_netdev(netdev);4076err_free_region:4077	ena_release_bars(ena_dev, pdev);4078err_free_ena_dev:4079	vfree(ena_dev);4080err_disable_device:4081	pci_disable_device(pdev);4082	return rc;4083}4084 4085/*****************************************************************************/4086 4087/* __ena_shutoff - Helper used in both PCI remove/shutdown routines4088 * @pdev: PCI device information struct4089 * @shutdown: Is it a shutdown operation? If false, means it is a removal4090 *4091 * __ena_shutoff is a helper routine that does the real work on shutdown and4092 * removal paths; the difference between those paths is with regards to whether4093 * dettach or unregister the netdevice.4094 */4095static void __ena_shutoff(struct pci_dev *pdev, bool shutdown)4096{4097	struct ena_adapter *adapter = pci_get_drvdata(pdev);4098	struct ena_com_dev *ena_dev;4099	struct net_device *netdev;4100 4101	ena_dev = adapter->ena_dev;4102	netdev = adapter->netdev;4103 4104#ifdef CONFIG_RFS_ACCEL4105	if ((adapter->msix_vecs >= 1) && (netdev->rx_cpu_rmap)) {4106		free_irq_cpu_rmap(netdev->rx_cpu_rmap);4107		netdev->rx_cpu_rmap = NULL;4108	}4109 4110#endif /* CONFIG_RFS_ACCEL */4111	/* Make sure timer and reset routine won't be called after4112	 * freeing device resources.4113	 */4114	del_timer_sync(&adapter->timer_service);4115	cancel_work_sync(&adapter->reset_task);4116 4117	rtnl_lock(); /* lock released inside the below if-else block */4118	adapter->reset_reason = ENA_REGS_RESET_SHUTDOWN;4119	ena_destroy_device(adapter, true);4120 4121	if (shutdown) {4122		netif_device_detach(netdev);4123		dev_close(netdev);4124		rtnl_unlock();4125	} else {4126		rtnl_unlock();4127		unregister_netdev(netdev);4128		free_netdev(netdev);4129	}4130 4131	ena_com_rss_destroy(ena_dev);4132 4133	ena_com_delete_debug_area(ena_dev);4134 4135	ena_com_delete_host_info(ena_dev);4136 4137	ena_com_delete_customer_metrics_buffer(ena_dev);4138 4139	ena_release_bars(ena_dev, pdev);4140 4141	pci_disable_device(pdev);4142 4143	vfree(ena_dev);4144}4145 4146/* ena_remove - Device Removal Routine4147 * @pdev: PCI device information struct4148 *4149 * ena_remove is called by the PCI subsystem to alert the driver4150 * that it should release a PCI device.4151 */4152 4153static void ena_remove(struct pci_dev *pdev)4154{4155	__ena_shutoff(pdev, false);4156}4157 4158/* ena_shutdown - Device Shutdown Routine4159 * @pdev: PCI device information struct4160 *4161 * ena_shutdown is called by the PCI subsystem to alert the driver that4162 * a shutdown/reboot (or kexec) is happening and device must be disabled.4163 */4164 4165static void ena_shutdown(struct pci_dev *pdev)4166{4167	__ena_shutoff(pdev, true);4168}4169 4170/* ena_suspend - PM suspend callback4171 * @dev_d: Device information struct4172 */4173static int __maybe_unused ena_suspend(struct device *dev_d)4174{4175	struct pci_dev *pdev = to_pci_dev(dev_d);4176	struct ena_adapter *adapter = pci_get_drvdata(pdev);4177 4178	ena_increase_stat(&adapter->dev_stats.suspend, 1, &adapter->syncp);4179 4180	rtnl_lock();4181	if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {4182		dev_err(&pdev->dev,4183			"Ignoring device reset request as the device is being suspended\n");4184		clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);4185	}4186	ena_destroy_device(adapter, true);4187	rtnl_unlock();4188	return 0;4189}4190 4191/* ena_resume - PM resume callback4192 * @dev_d: Device information struct4193 */4194static int __maybe_unused ena_resume(struct device *dev_d)4195{4196	struct ena_adapter *adapter = dev_get_drvdata(dev_d);4197	int rc;4198 4199	ena_increase_stat(&adapter->dev_stats.resume, 1, &adapter->syncp);4200 4201	rtnl_lock();4202	rc = ena_restore_device(adapter);4203	rtnl_unlock();4204	return rc;4205}4206 4207static SIMPLE_DEV_PM_OPS(ena_pm_ops, ena_suspend, ena_resume);4208 4209static struct pci_driver ena_pci_driver = {4210	.name		= DRV_MODULE_NAME,4211	.id_table	= ena_pci_tbl,4212	.probe		= ena_probe,4213	.remove		= ena_remove,4214	.shutdown	= ena_shutdown,4215	.driver.pm	= &ena_pm_ops,4216	.sriov_configure = pci_sriov_configure_simple,4217};4218 4219static int __init ena_init(void)4220{4221	int ret;4222 4223	ena_wq = create_singlethread_workqueue(DRV_MODULE_NAME);4224	if (!ena_wq) {4225		pr_err("Failed to create workqueue\n");4226		return -ENOMEM;4227	}4228 4229	ret = pci_register_driver(&ena_pci_driver);4230	if (ret)4231		destroy_workqueue(ena_wq);4232 4233	return ret;4234}4235 4236static void __exit ena_cleanup(void)4237{4238	pci_unregister_driver(&ena_pci_driver);4239 4240	if (ena_wq) {4241		destroy_workqueue(ena_wq);4242		ena_wq = NULL;4243	}4244}4245 4246/******************************************************************************4247 ******************************** AENQ Handlers *******************************4248 *****************************************************************************/4249/* ena_update_on_link_change:4250 * Notify the network interface about the change in link status4251 */4252static void ena_update_on_link_change(void *adapter_data,4253				      struct ena_admin_aenq_entry *aenq_e)4254{4255	struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;4256	struct ena_admin_aenq_link_change_desc *aenq_desc =4257		(struct ena_admin_aenq_link_change_desc *)aenq_e;4258	int status = aenq_desc->flags &4259		ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK;4260 4261	if (status) {4262		netif_dbg(adapter, ifup, adapter->netdev, "%s\n", __func__);4263		set_bit(ENA_FLAG_LINK_UP, &adapter->flags);4264		if (!test_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags))4265			netif_carrier_on(adapter->netdev);4266	} else {4267		clear_bit(ENA_FLAG_LINK_UP, &adapter->flags);4268		netif_carrier_off(adapter->netdev);4269	}4270}4271 4272static void ena_keep_alive_wd(void *adapter_data,4273			      struct ena_admin_aenq_entry *aenq_e)4274{4275	struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;4276	struct ena_admin_aenq_keep_alive_desc *desc;4277	u64 rx_drops;4278	u64 tx_drops;4279 4280	desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e;4281	adapter->last_keep_alive_jiffies = jiffies;4282 4283	rx_drops = ((u64)desc->rx_drops_high << 32) | desc->rx_drops_low;4284	tx_drops = ((u64)desc->tx_drops_high << 32) | desc->tx_drops_low;4285 4286	u64_stats_update_begin(&adapter->syncp);4287	/* These stats are accumulated by the device, so the counters indicate4288	 * all drops since last reset.4289	 */4290	adapter->dev_stats.rx_drops = rx_drops;4291	adapter->dev_stats.tx_drops = tx_drops;4292	u64_stats_update_end(&adapter->syncp);4293}4294 4295static void ena_notification(void *adapter_data,4296			     struct ena_admin_aenq_entry *aenq_e)4297{4298	struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;4299	struct ena_admin_ena_hw_hints *hints;4300 4301	WARN(aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION,4302	     "Invalid group(%x) expected %x\n",4303	     aenq_e->aenq_common_desc.group,4304	     ENA_ADMIN_NOTIFICATION);4305 4306	switch (aenq_e->aenq_common_desc.syndrome) {4307	case ENA_ADMIN_UPDATE_HINTS:4308		hints = (struct ena_admin_ena_hw_hints *)4309			(&aenq_e->inline_data_w4);4310		ena_update_hints(adapter, hints);4311		break;4312	default:4313		netif_err(adapter, drv, adapter->netdev,4314			  "Invalid aenq notification link state %d\n",4315			  aenq_e->aenq_common_desc.syndrome);4316	}4317}4318 4319/* This handler will called for unknown event group or unimplemented handlers*/4320static void unimplemented_aenq_handler(void *data,4321				       struct ena_admin_aenq_entry *aenq_e)4322{4323	struct ena_adapter *adapter = (struct ena_adapter *)data;4324 4325	netif_err(adapter, drv, adapter->netdev,4326		  "Unknown event was received or event with unimplemented handler\n");4327}4328 4329static struct ena_aenq_handlers aenq_handlers = {4330	.handlers = {4331		[ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change,4332		[ENA_ADMIN_NOTIFICATION] = ena_notification,4333		[ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive_wd,4334	},4335	.unimplemented_handler = unimplemented_aenq_handler4336};4337 4338module_init(ena_init);4339module_exit(ena_cleanup);4340