brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause2/* Copyright (c) 2021, Microsoft Corporation. */3 4#include <uapi/linux/bpf.h>5 6#include <linux/inetdevice.h>7#include <linux/etherdevice.h>8#include <linux/ethtool.h>9#include <linux/filter.h>10#include <linux/mm.h>11#include <linux/pci.h>12 13#include <net/checksum.h>14#include <net/ip6_checksum.h>15#include <net/page_pool/helpers.h>16#include <net/xdp.h>17 18#include <net/mana/mana.h>19#include <net/mana/mana_auxiliary.h>20 21static DEFINE_IDA(mana_adev_ida);22 23static int mana_adev_idx_alloc(void)24{25	return ida_alloc(&mana_adev_ida, GFP_KERNEL);26}27 28static void mana_adev_idx_free(int idx)29{30	ida_free(&mana_adev_ida, idx);31}32 33/* Microsoft Azure Network Adapter (MANA) functions */34 35static int mana_open(struct net_device *ndev)36{37	struct mana_port_context *apc = netdev_priv(ndev);38	int err;39 40	err = mana_alloc_queues(ndev);41	if (err)42		return err;43 44	apc->port_is_up = true;45 46	/* Ensure port state updated before txq state */47	smp_wmb();48 49	netif_carrier_on(ndev);50	netif_tx_wake_all_queues(ndev);51 52	return 0;53}54 55static int mana_close(struct net_device *ndev)56{57	struct mana_port_context *apc = netdev_priv(ndev);58 59	if (!apc->port_is_up)60		return 0;61 62	return mana_detach(ndev, true);63}64 65static bool mana_can_tx(struct gdma_queue *wq)66{67	return mana_gd_wq_avail_space(wq) >= MAX_TX_WQE_SIZE;68}69 70static unsigned int mana_checksum_info(struct sk_buff *skb)71{72	if (skb->protocol == htons(ETH_P_IP)) {73		struct iphdr *ip = ip_hdr(skb);74 75		if (ip->protocol == IPPROTO_TCP)76			return IPPROTO_TCP;77 78		if (ip->protocol == IPPROTO_UDP)79			return IPPROTO_UDP;80	} else if (skb->protocol == htons(ETH_P_IPV6)) {81		struct ipv6hdr *ip6 = ipv6_hdr(skb);82 83		if (ip6->nexthdr == IPPROTO_TCP)84			return IPPROTO_TCP;85 86		if (ip6->nexthdr == IPPROTO_UDP)87			return IPPROTO_UDP;88	}89 90	/* No csum offloading */91	return 0;92}93 94static void mana_add_sge(struct mana_tx_package *tp, struct mana_skb_head *ash,95			 int sg_i, dma_addr_t da, int sge_len, u32 gpa_mkey)96{97	ash->dma_handle[sg_i] = da;98	ash->size[sg_i] = sge_len;99 100	tp->wqe_req.sgl[sg_i].address = da;101	tp->wqe_req.sgl[sg_i].mem_key = gpa_mkey;102	tp->wqe_req.sgl[sg_i].size = sge_len;103}104 105static int mana_map_skb(struct sk_buff *skb, struct mana_port_context *apc,106			struct mana_tx_package *tp, int gso_hs)107{108	struct mana_skb_head *ash = (struct mana_skb_head *)skb->head;109	int hsg = 1; /* num of SGEs of linear part */110	struct gdma_dev *gd = apc->ac->gdma_dev;111	int skb_hlen = skb_headlen(skb);112	int sge0_len, sge1_len = 0;113	struct gdma_context *gc;114	struct device *dev;115	skb_frag_t *frag;116	dma_addr_t da;117	int sg_i;118	int i;119 120	gc = gd->gdma_context;121	dev = gc->dev;122 123	if (gso_hs && gso_hs < skb_hlen) {124		sge0_len = gso_hs;125		sge1_len = skb_hlen - gso_hs;126	} else {127		sge0_len = skb_hlen;128	}129 130	da = dma_map_single(dev, skb->data, sge0_len, DMA_TO_DEVICE);131	if (dma_mapping_error(dev, da))132		return -ENOMEM;133 134	mana_add_sge(tp, ash, 0, da, sge0_len, gd->gpa_mkey);135 136	if (sge1_len) {137		sg_i = 1;138		da = dma_map_single(dev, skb->data + sge0_len, sge1_len,139				    DMA_TO_DEVICE);140		if (dma_mapping_error(dev, da))141			goto frag_err;142 143		mana_add_sge(tp, ash, sg_i, da, sge1_len, gd->gpa_mkey);144		hsg = 2;145	}146 147	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {148		sg_i = hsg + i;149 150		frag = &skb_shinfo(skb)->frags[i];151		da = skb_frag_dma_map(dev, frag, 0, skb_frag_size(frag),152				      DMA_TO_DEVICE);153		if (dma_mapping_error(dev, da))154			goto frag_err;155 156		mana_add_sge(tp, ash, sg_i, da, skb_frag_size(frag),157			     gd->gpa_mkey);158	}159 160	return 0;161 162frag_err:163	for (i = sg_i - 1; i >= hsg; i--)164		dma_unmap_page(dev, ash->dma_handle[i], ash->size[i],165			       DMA_TO_DEVICE);166 167	for (i = hsg - 1; i >= 0; i--)168		dma_unmap_single(dev, ash->dma_handle[i], ash->size[i],169				 DMA_TO_DEVICE);170 171	return -ENOMEM;172}173 174/* Handle the case when GSO SKB linear length is too large.175 * MANA NIC requires GSO packets to put only the packet header to SGE0.176 * So, we need 2 SGEs for the skb linear part which contains more than the177 * header.178 * Return a positive value for the number of SGEs, or a negative value179 * for an error.180 */181static int mana_fix_skb_head(struct net_device *ndev, struct sk_buff *skb,182			     int gso_hs)183{184	int num_sge = 1 + skb_shinfo(skb)->nr_frags;185	int skb_hlen = skb_headlen(skb);186 187	if (gso_hs < skb_hlen) {188		num_sge++;189	} else if (gso_hs > skb_hlen) {190		if (net_ratelimit())191			netdev_err(ndev,192				   "TX nonlinear head: hs:%d, skb_hlen:%d\n",193				   gso_hs, skb_hlen);194 195		return -EINVAL;196	}197 198	return num_sge;199}200 201/* Get the GSO packet's header size */202static int mana_get_gso_hs(struct sk_buff *skb)203{204	int gso_hs;205 206	if (skb->encapsulation) {207		gso_hs = skb_inner_tcp_all_headers(skb);208	} else {209		if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {210			gso_hs = skb_transport_offset(skb) +211				 sizeof(struct udphdr);212		} else {213			gso_hs = skb_tcp_all_headers(skb);214		}215	}216 217	return gso_hs;218}219 220netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev)221{222	enum mana_tx_pkt_format pkt_fmt = MANA_SHORT_PKT_FMT;223	struct mana_port_context *apc = netdev_priv(ndev);224	int gso_hs = 0; /* zero for non-GSO pkts */225	u16 txq_idx = skb_get_queue_mapping(skb);226	struct gdma_dev *gd = apc->ac->gdma_dev;227	bool ipv4 = false, ipv6 = false;228	struct mana_tx_package pkg = {};229	struct netdev_queue *net_txq;230	struct mana_stats_tx *tx_stats;231	struct gdma_queue *gdma_sq;232	unsigned int csum_type;233	struct mana_txq *txq;234	struct mana_cq *cq;235	int err, len;236 237	if (unlikely(!apc->port_is_up))238		goto tx_drop;239 240	if (skb_cow_head(skb, MANA_HEADROOM))241		goto tx_drop_count;242 243	txq = &apc->tx_qp[txq_idx].txq;244	gdma_sq = txq->gdma_sq;245	cq = &apc->tx_qp[txq_idx].tx_cq;246	tx_stats = &txq->stats;247 248	pkg.tx_oob.s_oob.vcq_num = cq->gdma_id;249	pkg.tx_oob.s_oob.vsq_frame = txq->vsq_frame;250 251	if (txq->vp_offset > MANA_SHORT_VPORT_OFFSET_MAX) {252		pkg.tx_oob.l_oob.long_vp_offset = txq->vp_offset;253		pkt_fmt = MANA_LONG_PKT_FMT;254	} else {255		pkg.tx_oob.s_oob.short_vp_offset = txq->vp_offset;256	}257 258	if (skb_vlan_tag_present(skb)) {259		pkt_fmt = MANA_LONG_PKT_FMT;260		pkg.tx_oob.l_oob.inject_vlan_pri_tag = 1;261		pkg.tx_oob.l_oob.pcp = skb_vlan_tag_get_prio(skb);262		pkg.tx_oob.l_oob.dei = skb_vlan_tag_get_cfi(skb);263		pkg.tx_oob.l_oob.vlan_id = skb_vlan_tag_get_id(skb);264	}265 266	pkg.tx_oob.s_oob.pkt_fmt = pkt_fmt;267 268	if (pkt_fmt == MANA_SHORT_PKT_FMT) {269		pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_short_oob);270		u64_stats_update_begin(&tx_stats->syncp);271		tx_stats->short_pkt_fmt++;272		u64_stats_update_end(&tx_stats->syncp);273	} else {274		pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_oob);275		u64_stats_update_begin(&tx_stats->syncp);276		tx_stats->long_pkt_fmt++;277		u64_stats_update_end(&tx_stats->syncp);278	}279 280	pkg.wqe_req.inline_oob_data = &pkg.tx_oob;281	pkg.wqe_req.flags = 0;282	pkg.wqe_req.client_data_unit = 0;283 284	pkg.wqe_req.num_sge = 1 + skb_shinfo(skb)->nr_frags;285 286	if (skb->protocol == htons(ETH_P_IP))287		ipv4 = true;288	else if (skb->protocol == htons(ETH_P_IPV6))289		ipv6 = true;290 291	if (skb_is_gso(skb)) {292		int num_sge;293 294		gso_hs = mana_get_gso_hs(skb);295 296		num_sge = mana_fix_skb_head(ndev, skb, gso_hs);297		if (num_sge > 0)298			pkg.wqe_req.num_sge = num_sge;299		else300			goto tx_drop_count;301 302		u64_stats_update_begin(&tx_stats->syncp);303		if (skb->encapsulation) {304			tx_stats->tso_inner_packets++;305			tx_stats->tso_inner_bytes += skb->len - gso_hs;306		} else {307			tx_stats->tso_packets++;308			tx_stats->tso_bytes += skb->len - gso_hs;309		}310		u64_stats_update_end(&tx_stats->syncp);311 312		pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4;313		pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6;314 315		pkg.tx_oob.s_oob.comp_iphdr_csum = 1;316		pkg.tx_oob.s_oob.comp_tcp_csum = 1;317		pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb);318 319		pkg.wqe_req.client_data_unit = skb_shinfo(skb)->gso_size;320		pkg.wqe_req.flags = GDMA_WR_OOB_IN_SGL | GDMA_WR_PAD_BY_SGE0;321		if (ipv4) {322			ip_hdr(skb)->tot_len = 0;323			ip_hdr(skb)->check = 0;324			tcp_hdr(skb)->check =325				~csum_tcpudp_magic(ip_hdr(skb)->saddr,326						   ip_hdr(skb)->daddr, 0,327						   IPPROTO_TCP, 0);328		} else {329			ipv6_hdr(skb)->payload_len = 0;330			tcp_hdr(skb)->check =331				~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,332						 &ipv6_hdr(skb)->daddr, 0,333						 IPPROTO_TCP, 0);334		}335	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {336		csum_type = mana_checksum_info(skb);337 338		u64_stats_update_begin(&tx_stats->syncp);339		tx_stats->csum_partial++;340		u64_stats_update_end(&tx_stats->syncp);341 342		if (csum_type == IPPROTO_TCP) {343			pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4;344			pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6;345 346			pkg.tx_oob.s_oob.comp_tcp_csum = 1;347			pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb);348 349		} else if (csum_type == IPPROTO_UDP) {350			pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4;351			pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6;352 353			pkg.tx_oob.s_oob.comp_udp_csum = 1;354		} else {355			/* Can't do offload of this type of checksum */356			if (skb_checksum_help(skb))357				goto tx_drop_count;358		}359	}360 361	WARN_ON_ONCE(pkg.wqe_req.num_sge > MAX_TX_WQE_SGL_ENTRIES);362 363	if (pkg.wqe_req.num_sge <= ARRAY_SIZE(pkg.sgl_array)) {364		pkg.wqe_req.sgl = pkg.sgl_array;365	} else {366		pkg.sgl_ptr = kmalloc_array(pkg.wqe_req.num_sge,367					    sizeof(struct gdma_sge),368					    GFP_ATOMIC);369		if (!pkg.sgl_ptr)370			goto tx_drop_count;371 372		pkg.wqe_req.sgl = pkg.sgl_ptr;373	}374 375	if (mana_map_skb(skb, apc, &pkg, gso_hs)) {376		u64_stats_update_begin(&tx_stats->syncp);377		tx_stats->mana_map_err++;378		u64_stats_update_end(&tx_stats->syncp);379		goto free_sgl_ptr;380	}381 382	skb_queue_tail(&txq->pending_skbs, skb);383 384	len = skb->len;385	net_txq = netdev_get_tx_queue(ndev, txq_idx);386 387	err = mana_gd_post_work_request(gdma_sq, &pkg.wqe_req,388					(struct gdma_posted_wqe_info *)skb->cb);389	if (!mana_can_tx(gdma_sq)) {390		netif_tx_stop_queue(net_txq);391		apc->eth_stats.stop_queue++;392	}393 394	if (err) {395		(void)skb_dequeue_tail(&txq->pending_skbs);396		netdev_warn(ndev, "Failed to post TX OOB: %d\n", err);397		err = NETDEV_TX_BUSY;398		goto tx_busy;399	}400 401	err = NETDEV_TX_OK;402	atomic_inc(&txq->pending_sends);403 404	mana_gd_wq_ring_doorbell(gd->gdma_context, gdma_sq);405 406	/* skb may be freed after mana_gd_post_work_request. Do not use it. */407	skb = NULL;408 409	tx_stats = &txq->stats;410	u64_stats_update_begin(&tx_stats->syncp);411	tx_stats->packets++;412	tx_stats->bytes += len;413	u64_stats_update_end(&tx_stats->syncp);414 415tx_busy:416	if (netif_tx_queue_stopped(net_txq) && mana_can_tx(gdma_sq)) {417		netif_tx_wake_queue(net_txq);418		apc->eth_stats.wake_queue++;419	}420 421	kfree(pkg.sgl_ptr);422	return err;423 424free_sgl_ptr:425	kfree(pkg.sgl_ptr);426tx_drop_count:427	ndev->stats.tx_dropped++;428tx_drop:429	dev_kfree_skb_any(skb);430	return NETDEV_TX_OK;431}432 433static void mana_get_stats64(struct net_device *ndev,434			     struct rtnl_link_stats64 *st)435{436	struct mana_port_context *apc = netdev_priv(ndev);437	unsigned int num_queues = apc->num_queues;438	struct mana_stats_rx *rx_stats;439	struct mana_stats_tx *tx_stats;440	unsigned int start;441	u64 packets, bytes;442	int q;443 444	if (!apc->port_is_up)445		return;446 447	netdev_stats_to_stats64(st, &ndev->stats);448 449	for (q = 0; q < num_queues; q++) {450		rx_stats = &apc->rxqs[q]->stats;451 452		do {453			start = u64_stats_fetch_begin(&rx_stats->syncp);454			packets = rx_stats->packets;455			bytes = rx_stats->bytes;456		} while (u64_stats_fetch_retry(&rx_stats->syncp, start));457 458		st->rx_packets += packets;459		st->rx_bytes += bytes;460	}461 462	for (q = 0; q < num_queues; q++) {463		tx_stats = &apc->tx_qp[q].txq.stats;464 465		do {466			start = u64_stats_fetch_begin(&tx_stats->syncp);467			packets = tx_stats->packets;468			bytes = tx_stats->bytes;469		} while (u64_stats_fetch_retry(&tx_stats->syncp, start));470 471		st->tx_packets += packets;472		st->tx_bytes += bytes;473	}474}475 476static int mana_get_tx_queue(struct net_device *ndev, struct sk_buff *skb,477			     int old_q)478{479	struct mana_port_context *apc = netdev_priv(ndev);480	u32 hash = skb_get_hash(skb);481	struct sock *sk = skb->sk;482	int txq;483 484	txq = apc->indir_table[hash & (apc->indir_table_sz - 1)];485 486	if (txq != old_q && sk && sk_fullsock(sk) &&487	    rcu_access_pointer(sk->sk_dst_cache))488		sk_tx_queue_set(sk, txq);489 490	return txq;491}492 493static u16 mana_select_queue(struct net_device *ndev, struct sk_buff *skb,494			     struct net_device *sb_dev)495{496	int txq;497 498	if (ndev->real_num_tx_queues == 1)499		return 0;500 501	txq = sk_tx_queue_get(skb->sk);502 503	if (txq < 0 || skb->ooo_okay || txq >= ndev->real_num_tx_queues) {504		if (skb_rx_queue_recorded(skb))505			txq = skb_get_rx_queue(skb);506		else507			txq = mana_get_tx_queue(ndev, skb, txq);508	}509 510	return txq;511}512 513/* Release pre-allocated RX buffers */514void mana_pre_dealloc_rxbufs(struct mana_port_context *mpc)515{516	struct device *dev;517	int i;518 519	dev = mpc->ac->gdma_dev->gdma_context->dev;520 521	if (!mpc->rxbufs_pre)522		goto out1;523 524	if (!mpc->das_pre)525		goto out2;526 527	while (mpc->rxbpre_total) {528		i = --mpc->rxbpre_total;529		dma_unmap_single(dev, mpc->das_pre[i], mpc->rxbpre_datasize,530				 DMA_FROM_DEVICE);531		put_page(virt_to_head_page(mpc->rxbufs_pre[i]));532	}533 534	kfree(mpc->das_pre);535	mpc->das_pre = NULL;536 537out2:538	kfree(mpc->rxbufs_pre);539	mpc->rxbufs_pre = NULL;540 541out1:542	mpc->rxbpre_datasize = 0;543	mpc->rxbpre_alloc_size = 0;544	mpc->rxbpre_headroom = 0;545}546 547/* Get a buffer from the pre-allocated RX buffers */548static void *mana_get_rxbuf_pre(struct mana_rxq *rxq, dma_addr_t *da)549{550	struct net_device *ndev = rxq->ndev;551	struct mana_port_context *mpc;552	void *va;553 554	mpc = netdev_priv(ndev);555 556	if (!mpc->rxbufs_pre || !mpc->das_pre || !mpc->rxbpre_total) {557		netdev_err(ndev, "No RX pre-allocated bufs\n");558		return NULL;559	}560 561	/* Check sizes to catch unexpected coding error */562	if (mpc->rxbpre_datasize != rxq->datasize) {563		netdev_err(ndev, "rxbpre_datasize mismatch: %u: %u\n",564			   mpc->rxbpre_datasize, rxq->datasize);565		return NULL;566	}567 568	if (mpc->rxbpre_alloc_size != rxq->alloc_size) {569		netdev_err(ndev, "rxbpre_alloc_size mismatch: %u: %u\n",570			   mpc->rxbpre_alloc_size, rxq->alloc_size);571		return NULL;572	}573 574	if (mpc->rxbpre_headroom != rxq->headroom) {575		netdev_err(ndev, "rxbpre_headroom mismatch: %u: %u\n",576			   mpc->rxbpre_headroom, rxq->headroom);577		return NULL;578	}579 580	mpc->rxbpre_total--;581 582	*da = mpc->das_pre[mpc->rxbpre_total];583	va = mpc->rxbufs_pre[mpc->rxbpre_total];584	mpc->rxbufs_pre[mpc->rxbpre_total] = NULL;585 586	/* Deallocate the array after all buffers are gone */587	if (!mpc->rxbpre_total)588		mana_pre_dealloc_rxbufs(mpc);589 590	return va;591}592 593/* Get RX buffer's data size, alloc size, XDP headroom based on MTU */594static void mana_get_rxbuf_cfg(int mtu, u32 *datasize, u32 *alloc_size,595			       u32 *headroom)596{597	if (mtu > MANA_XDP_MTU_MAX)598		*headroom = 0; /* no support for XDP */599	else600		*headroom = XDP_PACKET_HEADROOM;601 602	*alloc_size = SKB_DATA_ALIGN(mtu + MANA_RXBUF_PAD + *headroom);603 604	/* Using page pool in this case, so alloc_size is PAGE_SIZE */605	if (*alloc_size < PAGE_SIZE)606		*alloc_size = PAGE_SIZE;607 608	*datasize = mtu + ETH_HLEN;609}610 611int mana_pre_alloc_rxbufs(struct mana_port_context *mpc, int new_mtu, int num_queues)612{613	struct device *dev;614	struct page *page;615	dma_addr_t da;616	int num_rxb;617	void *va;618	int i;619 620	mana_get_rxbuf_cfg(new_mtu, &mpc->rxbpre_datasize,621			   &mpc->rxbpre_alloc_size, &mpc->rxbpre_headroom);622 623	dev = mpc->ac->gdma_dev->gdma_context->dev;624 625	num_rxb = num_queues * mpc->rx_queue_size;626 627	WARN(mpc->rxbufs_pre, "mana rxbufs_pre exists\n");628	mpc->rxbufs_pre = kmalloc_array(num_rxb, sizeof(void *), GFP_KERNEL);629	if (!mpc->rxbufs_pre)630		goto error;631 632	mpc->das_pre = kmalloc_array(num_rxb, sizeof(dma_addr_t), GFP_KERNEL);633	if (!mpc->das_pre)634		goto error;635 636	mpc->rxbpre_total = 0;637 638	for (i = 0; i < num_rxb; i++) {639		if (mpc->rxbpre_alloc_size > PAGE_SIZE) {640			va = netdev_alloc_frag(mpc->rxbpre_alloc_size);641			if (!va)642				goto error;643 644			page = virt_to_head_page(va);645			/* Check if the frag falls back to single page */646			if (compound_order(page) <647			    get_order(mpc->rxbpre_alloc_size)) {648				put_page(page);649				goto error;650			}651		} else {652			page = dev_alloc_page();653			if (!page)654				goto error;655 656			va = page_to_virt(page);657		}658 659		da = dma_map_single(dev, va + mpc->rxbpre_headroom,660				    mpc->rxbpre_datasize, DMA_FROM_DEVICE);661		if (dma_mapping_error(dev, da)) {662			put_page(virt_to_head_page(va));663			goto error;664		}665 666		mpc->rxbufs_pre[i] = va;667		mpc->das_pre[i] = da;668		mpc->rxbpre_total = i + 1;669	}670 671	return 0;672 673error:674	mana_pre_dealloc_rxbufs(mpc);675	return -ENOMEM;676}677 678static int mana_change_mtu(struct net_device *ndev, int new_mtu)679{680	struct mana_port_context *mpc = netdev_priv(ndev);681	unsigned int old_mtu = ndev->mtu;682	int err;683 684	/* Pre-allocate buffers to prevent failure in mana_attach later */685	err = mana_pre_alloc_rxbufs(mpc, new_mtu, mpc->num_queues);686	if (err) {687		netdev_err(ndev, "Insufficient memory for new MTU\n");688		return err;689	}690 691	err = mana_detach(ndev, false);692	if (err) {693		netdev_err(ndev, "mana_detach failed: %d\n", err);694		goto out;695	}696 697	WRITE_ONCE(ndev->mtu, new_mtu);698 699	err = mana_attach(ndev);700	if (err) {701		netdev_err(ndev, "mana_attach failed: %d\n", err);702		WRITE_ONCE(ndev->mtu, old_mtu);703	}704 705out:706	mana_pre_dealloc_rxbufs(mpc);707	return err;708}709 710static const struct net_device_ops mana_devops = {711	.ndo_open		= mana_open,712	.ndo_stop		= mana_close,713	.ndo_select_queue	= mana_select_queue,714	.ndo_start_xmit		= mana_start_xmit,715	.ndo_validate_addr	= eth_validate_addr,716	.ndo_get_stats64	= mana_get_stats64,717	.ndo_bpf		= mana_bpf,718	.ndo_xdp_xmit		= mana_xdp_xmit,719	.ndo_change_mtu		= mana_change_mtu,720};721 722static void mana_cleanup_port_context(struct mana_port_context *apc)723{724	kfree(apc->rxqs);725	apc->rxqs = NULL;726}727 728static void mana_cleanup_indir_table(struct mana_port_context *apc)729{730	apc->indir_table_sz = 0;731	kfree(apc->indir_table);732	kfree(apc->rxobj_table);733}734 735static int mana_init_port_context(struct mana_port_context *apc)736{737	apc->rxqs = kcalloc(apc->num_queues, sizeof(struct mana_rxq *),738			    GFP_KERNEL);739 740	return !apc->rxqs ? -ENOMEM : 0;741}742 743static int mana_send_request(struct mana_context *ac, void *in_buf,744			     u32 in_len, void *out_buf, u32 out_len)745{746	struct gdma_context *gc = ac->gdma_dev->gdma_context;747	struct gdma_resp_hdr *resp = out_buf;748	struct gdma_req_hdr *req = in_buf;749	struct device *dev = gc->dev;750	static atomic_t activity_id;751	int err;752 753	req->dev_id = gc->mana.dev_id;754	req->activity_id = atomic_inc_return(&activity_id);755 756	err = mana_gd_send_request(gc, in_len, in_buf, out_len,757				   out_buf);758	if (err || resp->status) {759		dev_err(dev, "Failed to send mana message: %d, 0x%x\n",760			err, resp->status);761		return err ? err : -EPROTO;762	}763 764	if (req->dev_id.as_uint32 != resp->dev_id.as_uint32 ||765	    req->activity_id != resp->activity_id) {766		dev_err(dev, "Unexpected mana message response: %x,%x,%x,%x\n",767			req->dev_id.as_uint32, resp->dev_id.as_uint32,768			req->activity_id, resp->activity_id);769		return -EPROTO;770	}771 772	return 0;773}774 775static int mana_verify_resp_hdr(const struct gdma_resp_hdr *resp_hdr,776				const enum mana_command_code expected_code,777				const u32 min_size)778{779	if (resp_hdr->response.msg_type != expected_code)780		return -EPROTO;781 782	if (resp_hdr->response.msg_version < GDMA_MESSAGE_V1)783		return -EPROTO;784 785	if (resp_hdr->response.msg_size < min_size)786		return -EPROTO;787 788	return 0;789}790 791static int mana_pf_register_hw_vport(struct mana_port_context *apc)792{793	struct mana_register_hw_vport_resp resp = {};794	struct mana_register_hw_vport_req req = {};795	int err;796 797	mana_gd_init_req_hdr(&req.hdr, MANA_REGISTER_HW_PORT,798			     sizeof(req), sizeof(resp));799	req.attached_gfid = 1;800	req.is_pf_default_vport = 1;801	req.allow_all_ether_types = 1;802 803	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,804				sizeof(resp));805	if (err) {806		netdev_err(apc->ndev, "Failed to register hw vPort: %d\n", err);807		return err;808	}809 810	err = mana_verify_resp_hdr(&resp.hdr, MANA_REGISTER_HW_PORT,811				   sizeof(resp));812	if (err || resp.hdr.status) {813		netdev_err(apc->ndev, "Failed to register hw vPort: %d, 0x%x\n",814			   err, resp.hdr.status);815		return err ? err : -EPROTO;816	}817 818	apc->port_handle = resp.hw_vport_handle;819	return 0;820}821 822static void mana_pf_deregister_hw_vport(struct mana_port_context *apc)823{824	struct mana_deregister_hw_vport_resp resp = {};825	struct mana_deregister_hw_vport_req req = {};826	int err;827 828	mana_gd_init_req_hdr(&req.hdr, MANA_DEREGISTER_HW_PORT,829			     sizeof(req), sizeof(resp));830	req.hw_vport_handle = apc->port_handle;831 832	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,833				sizeof(resp));834	if (err) {835		netdev_err(apc->ndev, "Failed to unregister hw vPort: %d\n",836			   err);837		return;838	}839 840	err = mana_verify_resp_hdr(&resp.hdr, MANA_DEREGISTER_HW_PORT,841				   sizeof(resp));842	if (err || resp.hdr.status)843		netdev_err(apc->ndev,844			   "Failed to deregister hw vPort: %d, 0x%x\n",845			   err, resp.hdr.status);846}847 848static int mana_pf_register_filter(struct mana_port_context *apc)849{850	struct mana_register_filter_resp resp = {};851	struct mana_register_filter_req req = {};852	int err;853 854	mana_gd_init_req_hdr(&req.hdr, MANA_REGISTER_FILTER,855			     sizeof(req), sizeof(resp));856	req.vport = apc->port_handle;857	memcpy(req.mac_addr, apc->mac_addr, ETH_ALEN);858 859	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,860				sizeof(resp));861	if (err) {862		netdev_err(apc->ndev, "Failed to register filter: %d\n", err);863		return err;864	}865 866	err = mana_verify_resp_hdr(&resp.hdr, MANA_REGISTER_FILTER,867				   sizeof(resp));868	if (err || resp.hdr.status) {869		netdev_err(apc->ndev, "Failed to register filter: %d, 0x%x\n",870			   err, resp.hdr.status);871		return err ? err : -EPROTO;872	}873 874	apc->pf_filter_handle = resp.filter_handle;875	return 0;876}877 878static void mana_pf_deregister_filter(struct mana_port_context *apc)879{880	struct mana_deregister_filter_resp resp = {};881	struct mana_deregister_filter_req req = {};882	int err;883 884	mana_gd_init_req_hdr(&req.hdr, MANA_DEREGISTER_FILTER,885			     sizeof(req), sizeof(resp));886	req.filter_handle = apc->pf_filter_handle;887 888	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,889				sizeof(resp));890	if (err) {891		netdev_err(apc->ndev, "Failed to unregister filter: %d\n",892			   err);893		return;894	}895 896	err = mana_verify_resp_hdr(&resp.hdr, MANA_DEREGISTER_FILTER,897				   sizeof(resp));898	if (err || resp.hdr.status)899		netdev_err(apc->ndev,900			   "Failed to deregister filter: %d, 0x%x\n",901			   err, resp.hdr.status);902}903 904static int mana_query_device_cfg(struct mana_context *ac, u32 proto_major_ver,905				 u32 proto_minor_ver, u32 proto_micro_ver,906				 u16 *max_num_vports)907{908	struct gdma_context *gc = ac->gdma_dev->gdma_context;909	struct mana_query_device_cfg_resp resp = {};910	struct mana_query_device_cfg_req req = {};911	struct device *dev = gc->dev;912	int err = 0;913 914	mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_DEV_CONFIG,915			     sizeof(req), sizeof(resp));916 917	req.hdr.resp.msg_version = GDMA_MESSAGE_V2;918 919	req.proto_major_ver = proto_major_ver;920	req.proto_minor_ver = proto_minor_ver;921	req.proto_micro_ver = proto_micro_ver;922 923	err = mana_send_request(ac, &req, sizeof(req), &resp, sizeof(resp));924	if (err) {925		dev_err(dev, "Failed to query config: %d", err);926		return err;927	}928 929	err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_DEV_CONFIG,930				   sizeof(resp));931	if (err || resp.hdr.status) {932		dev_err(dev, "Invalid query result: %d, 0x%x\n", err,933			resp.hdr.status);934		if (!err)935			err = -EPROTO;936		return err;937	}938 939	*max_num_vports = resp.max_num_vports;940 941	if (resp.hdr.response.msg_version == GDMA_MESSAGE_V2)942		gc->adapter_mtu = resp.adapter_mtu;943	else944		gc->adapter_mtu = ETH_FRAME_LEN;945 946	return 0;947}948 949static int mana_query_vport_cfg(struct mana_port_context *apc, u32 vport_index,950				u32 *max_sq, u32 *max_rq, u32 *num_indir_entry)951{952	struct mana_query_vport_cfg_resp resp = {};953	struct mana_query_vport_cfg_req req = {};954	int err;955 956	mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_VPORT_CONFIG,957			     sizeof(req), sizeof(resp));958 959	req.vport_index = vport_index;960 961	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,962				sizeof(resp));963	if (err)964		return err;965 966	err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_VPORT_CONFIG,967				   sizeof(resp));968	if (err)969		return err;970 971	if (resp.hdr.status)972		return -EPROTO;973 974	*max_sq = resp.max_num_sq;975	*max_rq = resp.max_num_rq;976	if (resp.num_indirection_ent > 0 &&977	    resp.num_indirection_ent <= MANA_INDIRECT_TABLE_MAX_SIZE &&978	    is_power_of_2(resp.num_indirection_ent)) {979		*num_indir_entry = resp.num_indirection_ent;980	} else {981		netdev_warn(apc->ndev,982			    "Setting indirection table size to default %d for vPort %d\n",983			    MANA_INDIRECT_TABLE_DEF_SIZE, apc->port_idx);984		*num_indir_entry = MANA_INDIRECT_TABLE_DEF_SIZE;985	}986 987	apc->port_handle = resp.vport;988	ether_addr_copy(apc->mac_addr, resp.mac_addr);989 990	return 0;991}992 993void mana_uncfg_vport(struct mana_port_context *apc)994{995	mutex_lock(&apc->vport_mutex);996	apc->vport_use_count--;997	WARN_ON(apc->vport_use_count < 0);998	mutex_unlock(&apc->vport_mutex);999}1000EXPORT_SYMBOL_NS(mana_uncfg_vport, NET_MANA);1001 1002int mana_cfg_vport(struct mana_port_context *apc, u32 protection_dom_id,1003		   u32 doorbell_pg_id)1004{1005	struct mana_config_vport_resp resp = {};1006	struct mana_config_vport_req req = {};1007	int err;1008 1009	/* This function is used to program the Ethernet port in the hardware1010	 * table. It can be called from the Ethernet driver or the RDMA driver.1011	 *1012	 * For Ethernet usage, the hardware supports only one active user on a1013	 * physical port. The driver checks on the port usage before programming1014	 * the hardware when creating the RAW QP (RDMA driver) or exposing the1015	 * device to kernel NET layer (Ethernet driver).1016	 *1017	 * Because the RDMA driver doesn't know in advance which QP type the1018	 * user will create, it exposes the device with all its ports. The user1019	 * may not be able to create RAW QP on a port if this port is already1020	 * in used by the Ethernet driver from the kernel.1021	 *1022	 * This physical port limitation only applies to the RAW QP. For RC QP,1023	 * the hardware doesn't have this limitation. The user can create RC1024	 * QPs on a physical port up to the hardware limits independent of the1025	 * Ethernet usage on the same port.1026	 */1027	mutex_lock(&apc->vport_mutex);1028	if (apc->vport_use_count > 0) {1029		mutex_unlock(&apc->vport_mutex);1030		return -EBUSY;1031	}1032	apc->vport_use_count++;1033	mutex_unlock(&apc->vport_mutex);1034 1035	mana_gd_init_req_hdr(&req.hdr, MANA_CONFIG_VPORT_TX,1036			     sizeof(req), sizeof(resp));1037	req.vport = apc->port_handle;1038	req.pdid = protection_dom_id;1039	req.doorbell_pageid = doorbell_pg_id;1040 1041	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,1042				sizeof(resp));1043	if (err) {1044		netdev_err(apc->ndev, "Failed to configure vPort: %d\n", err);1045		goto out;1046	}1047 1048	err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_TX,1049				   sizeof(resp));1050	if (err || resp.hdr.status) {1051		netdev_err(apc->ndev, "Failed to configure vPort: %d, 0x%x\n",1052			   err, resp.hdr.status);1053		if (!err)1054			err = -EPROTO;1055 1056		goto out;1057	}1058 1059	apc->tx_shortform_allowed = resp.short_form_allowed;1060	apc->tx_vp_offset = resp.tx_vport_offset;1061 1062	netdev_info(apc->ndev, "Configured vPort %llu PD %u DB %u\n",1063		    apc->port_handle, protection_dom_id, doorbell_pg_id);1064out:1065	if (err)1066		mana_uncfg_vport(apc);1067 1068	return err;1069}1070EXPORT_SYMBOL_NS(mana_cfg_vport, NET_MANA);1071 1072static int mana_cfg_vport_steering(struct mana_port_context *apc,1073				   enum TRI_STATE rx,1074				   bool update_default_rxobj, bool update_key,1075				   bool update_tab)1076{1077	struct mana_cfg_rx_steer_req_v2 *req;1078	struct mana_cfg_rx_steer_resp resp = {};1079	struct net_device *ndev = apc->ndev;1080	u32 req_buf_size;1081	int err;1082 1083	req_buf_size = struct_size(req, indir_tab, apc->indir_table_sz);1084	req = kzalloc(req_buf_size, GFP_KERNEL);1085	if (!req)1086		return -ENOMEM;1087 1088	mana_gd_init_req_hdr(&req->hdr, MANA_CONFIG_VPORT_RX, req_buf_size,1089			     sizeof(resp));1090 1091	req->hdr.req.msg_version = GDMA_MESSAGE_V2;1092 1093	req->vport = apc->port_handle;1094	req->num_indir_entries = apc->indir_table_sz;1095	req->indir_tab_offset = offsetof(struct mana_cfg_rx_steer_req_v2,1096					 indir_tab);1097	req->rx_enable = rx;1098	req->rss_enable = apc->rss_state;1099	req->update_default_rxobj = update_default_rxobj;1100	req->update_hashkey = update_key;1101	req->update_indir_tab = update_tab;1102	req->default_rxobj = apc->default_rxobj;1103	req->cqe_coalescing_enable = 0;1104 1105	if (update_key)1106		memcpy(&req->hashkey, apc->hashkey, MANA_HASH_KEY_SIZE);1107 1108	if (update_tab)1109		memcpy(req->indir_tab, apc->rxobj_table,1110		       flex_array_size(req, indir_tab, req->num_indir_entries));1111 1112	err = mana_send_request(apc->ac, req, req_buf_size, &resp,1113				sizeof(resp));1114	if (err) {1115		netdev_err(ndev, "Failed to configure vPort RX: %d\n", err);1116		goto out;1117	}1118 1119	err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_RX,1120				   sizeof(resp));1121	if (err) {1122		netdev_err(ndev, "vPort RX configuration failed: %d\n", err);1123		goto out;1124	}1125 1126	if (resp.hdr.status) {1127		netdev_err(ndev, "vPort RX configuration failed: 0x%x\n",1128			   resp.hdr.status);1129		err = -EPROTO;1130	}1131 1132	netdev_info(ndev, "Configured steering vPort %llu entries %u\n",1133		    apc->port_handle, apc->indir_table_sz);1134out:1135	kfree(req);1136	return err;1137}1138 1139int mana_create_wq_obj(struct mana_port_context *apc,1140		       mana_handle_t vport,1141		       u32 wq_type, struct mana_obj_spec *wq_spec,1142		       struct mana_obj_spec *cq_spec,1143		       mana_handle_t *wq_obj)1144{1145	struct mana_create_wqobj_resp resp = {};1146	struct mana_create_wqobj_req req = {};1147	struct net_device *ndev = apc->ndev;1148	int err;1149 1150	mana_gd_init_req_hdr(&req.hdr, MANA_CREATE_WQ_OBJ,1151			     sizeof(req), sizeof(resp));1152	req.vport = vport;1153	req.wq_type = wq_type;1154	req.wq_gdma_region = wq_spec->gdma_region;1155	req.cq_gdma_region = cq_spec->gdma_region;1156	req.wq_size = wq_spec->queue_size;1157	req.cq_size = cq_spec->queue_size;1158	req.cq_moderation_ctx_id = cq_spec->modr_ctx_id;1159	req.cq_parent_qid = cq_spec->attached_eq;1160 1161	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,1162				sizeof(resp));1163	if (err) {1164		netdev_err(ndev, "Failed to create WQ object: %d\n", err);1165		goto out;1166	}1167 1168	err = mana_verify_resp_hdr(&resp.hdr, MANA_CREATE_WQ_OBJ,1169				   sizeof(resp));1170	if (err || resp.hdr.status) {1171		netdev_err(ndev, "Failed to create WQ object: %d, 0x%x\n", err,1172			   resp.hdr.status);1173		if (!err)1174			err = -EPROTO;1175		goto out;1176	}1177 1178	if (resp.wq_obj == INVALID_MANA_HANDLE) {1179		netdev_err(ndev, "Got an invalid WQ object handle\n");1180		err = -EPROTO;1181		goto out;1182	}1183 1184	*wq_obj = resp.wq_obj;1185	wq_spec->queue_index = resp.wq_id;1186	cq_spec->queue_index = resp.cq_id;1187 1188	return 0;1189out:1190	return err;1191}1192EXPORT_SYMBOL_NS(mana_create_wq_obj, NET_MANA);1193 1194void mana_destroy_wq_obj(struct mana_port_context *apc, u32 wq_type,1195			 mana_handle_t wq_obj)1196{1197	struct mana_destroy_wqobj_resp resp = {};1198	struct mana_destroy_wqobj_req req = {};1199	struct net_device *ndev = apc->ndev;1200	int err;1201 1202	mana_gd_init_req_hdr(&req.hdr, MANA_DESTROY_WQ_OBJ,1203			     sizeof(req), sizeof(resp));1204	req.wq_type = wq_type;1205	req.wq_obj_handle = wq_obj;1206 1207	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,1208				sizeof(resp));1209	if (err) {1210		netdev_err(ndev, "Failed to destroy WQ object: %d\n", err);1211		return;1212	}1213 1214	err = mana_verify_resp_hdr(&resp.hdr, MANA_DESTROY_WQ_OBJ,1215				   sizeof(resp));1216	if (err || resp.hdr.status)1217		netdev_err(ndev, "Failed to destroy WQ object: %d, 0x%x\n", err,1218			   resp.hdr.status);1219}1220EXPORT_SYMBOL_NS(mana_destroy_wq_obj, NET_MANA);1221 1222static void mana_destroy_eq(struct mana_context *ac)1223{1224	struct gdma_context *gc = ac->gdma_dev->gdma_context;1225	struct gdma_queue *eq;1226	int i;1227 1228	if (!ac->eqs)1229		return;1230 1231	for (i = 0; i < gc->max_num_queues; i++) {1232		eq = ac->eqs[i].eq;1233		if (!eq)1234			continue;1235 1236		mana_gd_destroy_queue(gc, eq);1237	}1238 1239	kfree(ac->eqs);1240	ac->eqs = NULL;1241}1242 1243static int mana_create_eq(struct mana_context *ac)1244{1245	struct gdma_dev *gd = ac->gdma_dev;1246	struct gdma_context *gc = gd->gdma_context;1247	struct gdma_queue_spec spec = {};1248	int err;1249	int i;1250 1251	ac->eqs = kcalloc(gc->max_num_queues, sizeof(struct mana_eq),1252			  GFP_KERNEL);1253	if (!ac->eqs)1254		return -ENOMEM;1255 1256	spec.type = GDMA_EQ;1257	spec.monitor_avl_buf = false;1258	spec.queue_size = EQ_SIZE;1259	spec.eq.callback = NULL;1260	spec.eq.context = ac->eqs;1261	spec.eq.log2_throttle_limit = LOG2_EQ_THROTTLE;1262 1263	for (i = 0; i < gc->max_num_queues; i++) {1264		spec.eq.msix_index = (i + 1) % gc->num_msix_usable;1265		err = mana_gd_create_mana_eq(gd, &spec, &ac->eqs[i].eq);1266		if (err)1267			goto out;1268	}1269 1270	return 0;1271out:1272	mana_destroy_eq(ac);1273	return err;1274}1275 1276static int mana_fence_rq(struct mana_port_context *apc, struct mana_rxq *rxq)1277{1278	struct mana_fence_rq_resp resp = {};1279	struct mana_fence_rq_req req = {};1280	int err;1281 1282	init_completion(&rxq->fence_event);1283 1284	mana_gd_init_req_hdr(&req.hdr, MANA_FENCE_RQ,1285			     sizeof(req), sizeof(resp));1286	req.wq_obj_handle =  rxq->rxobj;1287 1288	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,1289				sizeof(resp));1290	if (err) {1291		netdev_err(apc->ndev, "Failed to fence RQ %u: %d\n",1292			   rxq->rxq_idx, err);1293		return err;1294	}1295 1296	err = mana_verify_resp_hdr(&resp.hdr, MANA_FENCE_RQ, sizeof(resp));1297	if (err || resp.hdr.status) {1298		netdev_err(apc->ndev, "Failed to fence RQ %u: %d, 0x%x\n",1299			   rxq->rxq_idx, err, resp.hdr.status);1300		if (!err)1301			err = -EPROTO;1302 1303		return err;1304	}1305 1306	if (wait_for_completion_timeout(&rxq->fence_event, 10 * HZ) == 0) {1307		netdev_err(apc->ndev, "Failed to fence RQ %u: timed out\n",1308			   rxq->rxq_idx);1309		return -ETIMEDOUT;1310	}1311 1312	return 0;1313}1314 1315static void mana_fence_rqs(struct mana_port_context *apc)1316{1317	unsigned int rxq_idx;1318	struct mana_rxq *rxq;1319	int err;1320 1321	for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) {1322		rxq = apc->rxqs[rxq_idx];1323		err = mana_fence_rq(apc, rxq);1324 1325		/* In case of any error, use sleep instead. */1326		if (err)1327			msleep(100);1328	}1329}1330 1331static int mana_move_wq_tail(struct gdma_queue *wq, u32 num_units)1332{1333	u32 used_space_old;1334	u32 used_space_new;1335 1336	used_space_old = wq->head - wq->tail;1337	used_space_new = wq->head - (wq->tail + num_units);1338 1339	if (WARN_ON_ONCE(used_space_new > used_space_old))1340		return -ERANGE;1341 1342	wq->tail += num_units;1343	return 0;1344}1345 1346static void mana_unmap_skb(struct sk_buff *skb, struct mana_port_context *apc)1347{1348	struct mana_skb_head *ash = (struct mana_skb_head *)skb->head;1349	struct gdma_context *gc = apc->ac->gdma_dev->gdma_context;1350	struct device *dev = gc->dev;1351	int hsg, i;1352 1353	/* Number of SGEs of linear part */1354	hsg = (skb_is_gso(skb) && skb_headlen(skb) > ash->size[0]) ? 2 : 1;1355 1356	for (i = 0; i < hsg; i++)1357		dma_unmap_single(dev, ash->dma_handle[i], ash->size[i],1358				 DMA_TO_DEVICE);1359 1360	for (i = hsg; i < skb_shinfo(skb)->nr_frags + hsg; i++)1361		dma_unmap_page(dev, ash->dma_handle[i], ash->size[i],1362			       DMA_TO_DEVICE);1363}1364 1365static void mana_poll_tx_cq(struct mana_cq *cq)1366{1367	struct gdma_comp *completions = cq->gdma_comp_buf;1368	struct gdma_posted_wqe_info *wqe_info;1369	unsigned int pkt_transmitted = 0;1370	unsigned int wqe_unit_cnt = 0;1371	struct mana_txq *txq = cq->txq;1372	struct mana_port_context *apc;1373	struct netdev_queue *net_txq;1374	struct gdma_queue *gdma_wq;1375	unsigned int avail_space;1376	struct net_device *ndev;1377	struct sk_buff *skb;1378	bool txq_stopped;1379	int comp_read;1380	int i;1381 1382	ndev = txq->ndev;1383	apc = netdev_priv(ndev);1384 1385	comp_read = mana_gd_poll_cq(cq->gdma_cq, completions,1386				    CQE_POLLING_BUFFER);1387 1388	if (comp_read < 1)1389		return;1390 1391	for (i = 0; i < comp_read; i++) {1392		struct mana_tx_comp_oob *cqe_oob;1393 1394		if (WARN_ON_ONCE(!completions[i].is_sq))1395			return;1396 1397		cqe_oob = (struct mana_tx_comp_oob *)completions[i].cqe_data;1398		if (WARN_ON_ONCE(cqe_oob->cqe_hdr.client_type !=1399				 MANA_CQE_COMPLETION))1400			return;1401 1402		switch (cqe_oob->cqe_hdr.cqe_type) {1403		case CQE_TX_OKAY:1404			break;1405 1406		case CQE_TX_SA_DROP:1407		case CQE_TX_MTU_DROP:1408		case CQE_TX_INVALID_OOB:1409		case CQE_TX_INVALID_ETH_TYPE:1410		case CQE_TX_HDR_PROCESSING_ERROR:1411		case CQE_TX_VF_DISABLED:1412		case CQE_TX_VPORT_IDX_OUT_OF_RANGE:1413		case CQE_TX_VPORT_DISABLED:1414		case CQE_TX_VLAN_TAGGING_VIOLATION:1415			if (net_ratelimit())1416				netdev_err(ndev, "TX: CQE error %d\n",1417					   cqe_oob->cqe_hdr.cqe_type);1418 1419			apc->eth_stats.tx_cqe_err++;1420			break;1421 1422		default:1423			/* If the CQE type is unknown, log an error,1424			 * and still free the SKB, update tail, etc.1425			 */1426			if (net_ratelimit())1427				netdev_err(ndev, "TX: unknown CQE type %d\n",1428					   cqe_oob->cqe_hdr.cqe_type);1429 1430			apc->eth_stats.tx_cqe_unknown_type++;1431			break;1432		}1433 1434		if (WARN_ON_ONCE(txq->gdma_txq_id != completions[i].wq_num))1435			return;1436 1437		skb = skb_dequeue(&txq->pending_skbs);1438		if (WARN_ON_ONCE(!skb))1439			return;1440 1441		wqe_info = (struct gdma_posted_wqe_info *)skb->cb;1442		wqe_unit_cnt += wqe_info->wqe_size_in_bu;1443 1444		mana_unmap_skb(skb, apc);1445 1446		napi_consume_skb(skb, cq->budget);1447 1448		pkt_transmitted++;1449	}1450 1451	if (WARN_ON_ONCE(wqe_unit_cnt == 0))1452		return;1453 1454	mana_move_wq_tail(txq->gdma_sq, wqe_unit_cnt);1455 1456	gdma_wq = txq->gdma_sq;1457	avail_space = mana_gd_wq_avail_space(gdma_wq);1458 1459	/* Ensure tail updated before checking q stop */1460	smp_mb();1461 1462	net_txq = txq->net_txq;1463	txq_stopped = netif_tx_queue_stopped(net_txq);1464 1465	/* Ensure checking txq_stopped before apc->port_is_up. */1466	smp_rmb();1467 1468	if (txq_stopped && apc->port_is_up && avail_space >= MAX_TX_WQE_SIZE) {1469		netif_tx_wake_queue(net_txq);1470		apc->eth_stats.wake_queue++;1471	}1472 1473	if (atomic_sub_return(pkt_transmitted, &txq->pending_sends) < 0)1474		WARN_ON_ONCE(1);1475 1476	cq->work_done = pkt_transmitted;1477}1478 1479static void mana_post_pkt_rxq(struct mana_rxq *rxq)1480{1481	struct mana_recv_buf_oob *recv_buf_oob;1482	u32 curr_index;1483	int err;1484 1485	curr_index = rxq->buf_index++;1486	if (rxq->buf_index == rxq->num_rx_buf)1487		rxq->buf_index = 0;1488 1489	recv_buf_oob = &rxq->rx_oobs[curr_index];1490 1491	err = mana_gd_post_work_request(rxq->gdma_rq, &recv_buf_oob->wqe_req,1492					&recv_buf_oob->wqe_inf);1493	if (WARN_ON_ONCE(err))1494		return;1495 1496	WARN_ON_ONCE(recv_buf_oob->wqe_inf.wqe_size_in_bu != 1);1497}1498 1499static struct sk_buff *mana_build_skb(struct mana_rxq *rxq, void *buf_va,1500				      uint pkt_len, struct xdp_buff *xdp)1501{1502	struct sk_buff *skb = napi_build_skb(buf_va, rxq->alloc_size);1503 1504	if (!skb)1505		return NULL;1506 1507	if (xdp->data_hard_start) {1508		skb_reserve(skb, xdp->data - xdp->data_hard_start);1509		skb_put(skb, xdp->data_end - xdp->data);1510		return skb;1511	}1512 1513	skb_reserve(skb, rxq->headroom);1514	skb_put(skb, pkt_len);1515 1516	return skb;1517}1518 1519static void mana_rx_skb(void *buf_va, bool from_pool,1520			struct mana_rxcomp_oob *cqe, struct mana_rxq *rxq)1521{1522	struct mana_stats_rx *rx_stats = &rxq->stats;1523	struct net_device *ndev = rxq->ndev;1524	uint pkt_len = cqe->ppi[0].pkt_len;1525	u16 rxq_idx = rxq->rxq_idx;1526	struct napi_struct *napi;1527	struct xdp_buff xdp = {};1528	struct sk_buff *skb;1529	u32 hash_value;1530	u32 act;1531 1532	rxq->rx_cq.work_done++;1533	napi = &rxq->rx_cq.napi;1534 1535	if (!buf_va) {1536		++ndev->stats.rx_dropped;1537		return;1538	}1539 1540	act = mana_run_xdp(ndev, rxq, &xdp, buf_va, pkt_len);1541 1542	if (act == XDP_REDIRECT && !rxq->xdp_rc)1543		return;1544 1545	if (act != XDP_PASS && act != XDP_TX)1546		goto drop_xdp;1547 1548	skb = mana_build_skb(rxq, buf_va, pkt_len, &xdp);1549 1550	if (!skb)1551		goto drop;1552 1553	if (from_pool)1554		skb_mark_for_recycle(skb);1555 1556	skb->dev = napi->dev;1557 1558	skb->protocol = eth_type_trans(skb, ndev);1559	skb_checksum_none_assert(skb);1560	skb_record_rx_queue(skb, rxq_idx);1561 1562	if ((ndev->features & NETIF_F_RXCSUM) && cqe->rx_iphdr_csum_succeed) {1563		if (cqe->rx_tcp_csum_succeed || cqe->rx_udp_csum_succeed)1564			skb->ip_summed = CHECKSUM_UNNECESSARY;1565	}1566 1567	if (cqe->rx_hashtype != 0 && (ndev->features & NETIF_F_RXHASH)) {1568		hash_value = cqe->ppi[0].pkt_hash;1569 1570		if (cqe->rx_hashtype & MANA_HASH_L4)1571			skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L4);1572		else1573			skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L3);1574	}1575 1576	if (cqe->rx_vlantag_present) {1577		u16 vlan_tci = cqe->rx_vlan_id;1578 1579		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci);1580	}1581 1582	u64_stats_update_begin(&rx_stats->syncp);1583	rx_stats->packets++;1584	rx_stats->bytes += pkt_len;1585 1586	if (act == XDP_TX)1587		rx_stats->xdp_tx++;1588	u64_stats_update_end(&rx_stats->syncp);1589 1590	if (act == XDP_TX) {1591		skb_set_queue_mapping(skb, rxq_idx);1592		mana_xdp_tx(skb, ndev);1593		return;1594	}1595 1596	napi_gro_receive(napi, skb);1597 1598	return;1599 1600drop_xdp:1601	u64_stats_update_begin(&rx_stats->syncp);1602	rx_stats->xdp_drop++;1603	u64_stats_update_end(&rx_stats->syncp);1604 1605drop:1606	if (from_pool) {1607		page_pool_recycle_direct(rxq->page_pool,1608					 virt_to_head_page(buf_va));1609	} else {1610		WARN_ON_ONCE(rxq->xdp_save_va);1611		/* Save for reuse */1612		rxq->xdp_save_va = buf_va;1613	}1614 1615	++ndev->stats.rx_dropped;1616 1617	return;1618}1619 1620static void *mana_get_rxfrag(struct mana_rxq *rxq, struct device *dev,1621			     dma_addr_t *da, bool *from_pool, bool is_napi)1622{1623	struct page *page;1624	void *va;1625 1626	*from_pool = false;1627 1628	/* Reuse XDP dropped page if available */1629	if (rxq->xdp_save_va) {1630		va = rxq->xdp_save_va;1631		rxq->xdp_save_va = NULL;1632	} else if (rxq->alloc_size > PAGE_SIZE) {1633		if (is_napi)1634			va = napi_alloc_frag(rxq->alloc_size);1635		else1636			va = netdev_alloc_frag(rxq->alloc_size);1637 1638		if (!va)1639			return NULL;1640 1641		page = virt_to_head_page(va);1642		/* Check if the frag falls back to single page */1643		if (compound_order(page) < get_order(rxq->alloc_size)) {1644			put_page(page);1645			return NULL;1646		}1647	} else {1648		page = page_pool_dev_alloc_pages(rxq->page_pool);1649		if (!page)1650			return NULL;1651 1652		*from_pool = true;1653		va = page_to_virt(page);1654	}1655 1656	*da = dma_map_single(dev, va + rxq->headroom, rxq->datasize,1657			     DMA_FROM_DEVICE);1658	if (dma_mapping_error(dev, *da)) {1659		if (*from_pool)1660			page_pool_put_full_page(rxq->page_pool, page, false);1661		else1662			put_page(virt_to_head_page(va));1663 1664		return NULL;1665	}1666 1667	return va;1668}1669 1670/* Allocate frag for rx buffer, and save the old buf */1671static void mana_refill_rx_oob(struct device *dev, struct mana_rxq *rxq,1672			       struct mana_recv_buf_oob *rxoob, void **old_buf,1673			       bool *old_fp)1674{1675	bool from_pool;1676	dma_addr_t da;1677	void *va;1678 1679	va = mana_get_rxfrag(rxq, dev, &da, &from_pool, true);1680	if (!va)1681		return;1682 1683	dma_unmap_single(dev, rxoob->sgl[0].address, rxq->datasize,1684			 DMA_FROM_DEVICE);1685	*old_buf = rxoob->buf_va;1686	*old_fp = rxoob->from_pool;1687 1688	rxoob->buf_va = va;1689	rxoob->sgl[0].address = da;1690	rxoob->from_pool = from_pool;1691}1692 1693static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,1694				struct gdma_comp *cqe)1695{1696	struct mana_rxcomp_oob *oob = (struct mana_rxcomp_oob *)cqe->cqe_data;1697	struct gdma_context *gc = rxq->gdma_rq->gdma_dev->gdma_context;1698	struct net_device *ndev = rxq->ndev;1699	struct mana_recv_buf_oob *rxbuf_oob;1700	struct mana_port_context *apc;1701	struct device *dev = gc->dev;1702	void *old_buf = NULL;1703	u32 curr, pktlen;1704	bool old_fp;1705 1706	apc = netdev_priv(ndev);1707 1708	switch (oob->cqe_hdr.cqe_type) {1709	case CQE_RX_OKAY:1710		break;1711 1712	case CQE_RX_TRUNCATED:1713		++ndev->stats.rx_dropped;1714		rxbuf_oob = &rxq->rx_oobs[rxq->buf_index];1715		netdev_warn_once(ndev, "Dropped a truncated packet\n");1716		goto drop;1717 1718	case CQE_RX_COALESCED_4:1719		netdev_err(ndev, "RX coalescing is unsupported\n");1720		apc->eth_stats.rx_coalesced_err++;1721		return;1722 1723	case CQE_RX_OBJECT_FENCE:1724		complete(&rxq->fence_event);1725		return;1726 1727	default:1728		netdev_err(ndev, "Unknown RX CQE type = %d\n",1729			   oob->cqe_hdr.cqe_type);1730		apc->eth_stats.rx_cqe_unknown_type++;1731		return;1732	}1733 1734	pktlen = oob->ppi[0].pkt_len;1735 1736	if (pktlen == 0) {1737		/* data packets should never have packetlength of zero */1738		netdev_err(ndev, "RX pkt len=0, rq=%u, cq=%u, rxobj=0x%llx\n",1739			   rxq->gdma_id, cq->gdma_id, rxq->rxobj);1740		return;1741	}1742 1743	curr = rxq->buf_index;1744	rxbuf_oob = &rxq->rx_oobs[curr];1745	WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != 1);1746 1747	mana_refill_rx_oob(dev, rxq, rxbuf_oob, &old_buf, &old_fp);1748 1749	/* Unsuccessful refill will have old_buf == NULL.1750	 * In this case, mana_rx_skb() will drop the packet.1751	 */1752	mana_rx_skb(old_buf, old_fp, oob, rxq);1753 1754drop:1755	mana_move_wq_tail(rxq->gdma_rq, rxbuf_oob->wqe_inf.wqe_size_in_bu);1756 1757	mana_post_pkt_rxq(rxq);1758}1759 1760static void mana_poll_rx_cq(struct mana_cq *cq)1761{1762	struct gdma_comp *comp = cq->gdma_comp_buf;1763	struct mana_rxq *rxq = cq->rxq;1764	int comp_read, i;1765 1766	comp_read = mana_gd_poll_cq(cq->gdma_cq, comp, CQE_POLLING_BUFFER);1767	WARN_ON_ONCE(comp_read > CQE_POLLING_BUFFER);1768 1769	rxq->xdp_flush = false;1770 1771	for (i = 0; i < comp_read; i++) {1772		if (WARN_ON_ONCE(comp[i].is_sq))1773			return;1774 1775		/* verify recv cqe references the right rxq */1776		if (WARN_ON_ONCE(comp[i].wq_num != cq->rxq->gdma_id))1777			return;1778 1779		mana_process_rx_cqe(rxq, cq, &comp[i]);1780	}1781 1782	if (comp_read > 0) {1783		struct gdma_context *gc = rxq->gdma_rq->gdma_dev->gdma_context;1784 1785		mana_gd_wq_ring_doorbell(gc, rxq->gdma_rq);1786	}1787 1788	if (rxq->xdp_flush)1789		xdp_do_flush();1790}1791 1792static int mana_cq_handler(void *context, struct gdma_queue *gdma_queue)1793{1794	struct mana_cq *cq = context;1795	int w;1796 1797	WARN_ON_ONCE(cq->gdma_cq != gdma_queue);1798 1799	if (cq->type == MANA_CQ_TYPE_RX)1800		mana_poll_rx_cq(cq);1801	else1802		mana_poll_tx_cq(cq);1803 1804	w = cq->work_done;1805	cq->work_done_since_doorbell += w;1806 1807	if (w < cq->budget) {1808		mana_gd_ring_cq(gdma_queue, SET_ARM_BIT);1809		cq->work_done_since_doorbell = 0;1810		napi_complete_done(&cq->napi, w);1811	} else if (cq->work_done_since_doorbell >1812		   cq->gdma_cq->queue_size / COMP_ENTRY_SIZE * 4) {1813		/* MANA hardware requires at least one doorbell ring every 81814		 * wraparounds of CQ even if there is no need to arm the CQ.1815		 * This driver rings the doorbell as soon as we have exceeded1816		 * 4 wraparounds.1817		 */1818		mana_gd_ring_cq(gdma_queue, 0);1819		cq->work_done_since_doorbell = 0;1820	}1821 1822	return w;1823}1824 1825static int mana_poll(struct napi_struct *napi, int budget)1826{1827	struct mana_cq *cq = container_of(napi, struct mana_cq, napi);1828	int w;1829 1830	cq->work_done = 0;1831	cq->budget = budget;1832 1833	w = mana_cq_handler(cq, cq->gdma_cq);1834 1835	return min(w, budget);1836}1837 1838static void mana_schedule_napi(void *context, struct gdma_queue *gdma_queue)1839{1840	struct mana_cq *cq = context;1841 1842	napi_schedule_irqoff(&cq->napi);1843}1844 1845static void mana_deinit_cq(struct mana_port_context *apc, struct mana_cq *cq)1846{1847	struct gdma_dev *gd = apc->ac->gdma_dev;1848 1849	if (!cq->gdma_cq)1850		return;1851 1852	mana_gd_destroy_queue(gd->gdma_context, cq->gdma_cq);1853}1854 1855static void mana_deinit_txq(struct mana_port_context *apc, struct mana_txq *txq)1856{1857	struct gdma_dev *gd = apc->ac->gdma_dev;1858 1859	if (!txq->gdma_sq)1860		return;1861 1862	mana_gd_destroy_queue(gd->gdma_context, txq->gdma_sq);1863}1864 1865static void mana_destroy_txq(struct mana_port_context *apc)1866{1867	struct napi_struct *napi;1868	int i;1869 1870	if (!apc->tx_qp)1871		return;1872 1873	for (i = 0; i < apc->num_queues; i++) {1874		napi = &apc->tx_qp[i].tx_cq.napi;1875		if (apc->tx_qp[i].txq.napi_initialized) {1876			napi_synchronize(napi);1877			napi_disable(napi);1878			netif_napi_del(napi);1879			apc->tx_qp[i].txq.napi_initialized = false;1880		}1881		mana_destroy_wq_obj(apc, GDMA_SQ, apc->tx_qp[i].tx_object);1882 1883		mana_deinit_cq(apc, &apc->tx_qp[i].tx_cq);1884 1885		mana_deinit_txq(apc, &apc->tx_qp[i].txq);1886	}1887 1888	kfree(apc->tx_qp);1889	apc->tx_qp = NULL;1890}1891 1892static int mana_create_txq(struct mana_port_context *apc,1893			   struct net_device *net)1894{1895	struct mana_context *ac = apc->ac;1896	struct gdma_dev *gd = ac->gdma_dev;1897	struct mana_obj_spec wq_spec;1898	struct mana_obj_spec cq_spec;1899	struct gdma_queue_spec spec;1900	struct gdma_context *gc;1901	struct mana_txq *txq;1902	struct mana_cq *cq;1903	u32 txq_size;1904	u32 cq_size;1905	int err;1906	int i;1907 1908	apc->tx_qp = kcalloc(apc->num_queues, sizeof(struct mana_tx_qp),1909			     GFP_KERNEL);1910	if (!apc->tx_qp)1911		return -ENOMEM;1912 1913	/*  The minimum size of the WQE is 32 bytes, hence1914	 *  apc->tx_queue_size represents the maximum number of WQEs1915	 *  the SQ can store. This value is then used to size other queues1916	 *  to prevent overflow.1917	 *  Also note that the txq_size is always going to be MANA_PAGE_ALIGNED,1918	 *  as min val of apc->tx_queue_size is 128 and that would make1919	 *  txq_size 128*32 = 4096 and the other higher values of apc->tx_queue_size1920	 *  are always power of two1921	 */1922	txq_size = apc->tx_queue_size * 32;1923 1924	cq_size = apc->tx_queue_size * COMP_ENTRY_SIZE;1925 1926	gc = gd->gdma_context;1927 1928	for (i = 0; i < apc->num_queues; i++) {1929		apc->tx_qp[i].tx_object = INVALID_MANA_HANDLE;1930 1931		/* Create SQ */1932		txq = &apc->tx_qp[i].txq;1933 1934		u64_stats_init(&txq->stats.syncp);1935		txq->ndev = net;1936		txq->net_txq = netdev_get_tx_queue(net, i);1937		txq->vp_offset = apc->tx_vp_offset;1938		txq->napi_initialized = false;1939		skb_queue_head_init(&txq->pending_skbs);1940 1941		memset(&spec, 0, sizeof(spec));1942		spec.type = GDMA_SQ;1943		spec.monitor_avl_buf = true;1944		spec.queue_size = txq_size;1945		err = mana_gd_create_mana_wq_cq(gd, &spec, &txq->gdma_sq);1946		if (err)1947			goto out;1948 1949		/* Create SQ's CQ */1950		cq = &apc->tx_qp[i].tx_cq;1951		cq->type = MANA_CQ_TYPE_TX;1952 1953		cq->txq = txq;1954 1955		memset(&spec, 0, sizeof(spec));1956		spec.type = GDMA_CQ;1957		spec.monitor_avl_buf = false;1958		spec.queue_size = cq_size;1959		spec.cq.callback = mana_schedule_napi;1960		spec.cq.parent_eq = ac->eqs[i].eq;1961		spec.cq.context = cq;1962		err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq);1963		if (err)1964			goto out;1965 1966		memset(&wq_spec, 0, sizeof(wq_spec));1967		memset(&cq_spec, 0, sizeof(cq_spec));1968 1969		wq_spec.gdma_region = txq->gdma_sq->mem_info.dma_region_handle;1970		wq_spec.queue_size = txq->gdma_sq->queue_size;1971 1972		cq_spec.gdma_region = cq->gdma_cq->mem_info.dma_region_handle;1973		cq_spec.queue_size = cq->gdma_cq->queue_size;1974		cq_spec.modr_ctx_id = 0;1975		cq_spec.attached_eq = cq->gdma_cq->cq.parent->id;1976 1977		err = mana_create_wq_obj(apc, apc->port_handle, GDMA_SQ,1978					 &wq_spec, &cq_spec,1979					 &apc->tx_qp[i].tx_object);1980 1981		if (err)1982			goto out;1983 1984		txq->gdma_sq->id = wq_spec.queue_index;1985		cq->gdma_cq->id = cq_spec.queue_index;1986 1987		txq->gdma_sq->mem_info.dma_region_handle =1988			GDMA_INVALID_DMA_REGION;1989		cq->gdma_cq->mem_info.dma_region_handle =1990			GDMA_INVALID_DMA_REGION;1991 1992		txq->gdma_txq_id = txq->gdma_sq->id;1993 1994		cq->gdma_id = cq->gdma_cq->id;1995 1996		if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) {1997			err = -EINVAL;1998			goto out;1999		}2000 2001		gc->cq_table[cq->gdma_id] = cq->gdma_cq;2002 2003		netif_napi_add_tx(net, &cq->napi, mana_poll);2004		napi_enable(&cq->napi);2005		txq->napi_initialized = true;2006 2007		mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT);2008	}2009 2010	return 0;2011out:2012	mana_destroy_txq(apc);2013	return err;2014}2015 2016static void mana_destroy_rxq(struct mana_port_context *apc,2017			     struct mana_rxq *rxq, bool napi_initialized)2018 2019{2020	struct gdma_context *gc = apc->ac->gdma_dev->gdma_context;2021	struct mana_recv_buf_oob *rx_oob;2022	struct device *dev = gc->dev;2023	struct napi_struct *napi;2024	struct page *page;2025	int i;2026 2027	if (!rxq)2028		return;2029 2030	napi = &rxq->rx_cq.napi;2031 2032	if (napi_initialized) {2033		napi_synchronize(napi);2034 2035		napi_disable(napi);2036 2037		netif_napi_del(napi);2038	}2039	xdp_rxq_info_unreg(&rxq->xdp_rxq);2040 2041	mana_destroy_wq_obj(apc, GDMA_RQ, rxq->rxobj);2042 2043	mana_deinit_cq(apc, &rxq->rx_cq);2044 2045	if (rxq->xdp_save_va)2046		put_page(virt_to_head_page(rxq->xdp_save_va));2047 2048	for (i = 0; i < rxq->num_rx_buf; i++) {2049		rx_oob = &rxq->rx_oobs[i];2050 2051		if (!rx_oob->buf_va)2052			continue;2053 2054		dma_unmap_single(dev, rx_oob->sgl[0].address,2055				 rx_oob->sgl[0].size, DMA_FROM_DEVICE);2056 2057		page = virt_to_head_page(rx_oob->buf_va);2058 2059		if (rx_oob->from_pool)2060			page_pool_put_full_page(rxq->page_pool, page, false);2061		else2062			put_page(page);2063 2064		rx_oob->buf_va = NULL;2065	}2066 2067	page_pool_destroy(rxq->page_pool);2068 2069	if (rxq->gdma_rq)2070		mana_gd_destroy_queue(gc, rxq->gdma_rq);2071 2072	kfree(rxq);2073}2074 2075static int mana_fill_rx_oob(struct mana_recv_buf_oob *rx_oob, u32 mem_key,2076			    struct mana_rxq *rxq, struct device *dev)2077{2078	struct mana_port_context *mpc = netdev_priv(rxq->ndev);2079	bool from_pool = false;2080	dma_addr_t da;2081	void *va;2082 2083	if (mpc->rxbufs_pre)2084		va = mana_get_rxbuf_pre(rxq, &da);2085	else2086		va = mana_get_rxfrag(rxq, dev, &da, &from_pool, false);2087 2088	if (!va)2089		return -ENOMEM;2090 2091	rx_oob->buf_va = va;2092	rx_oob->from_pool = from_pool;2093 2094	rx_oob->sgl[0].address = da;2095	rx_oob->sgl[0].size = rxq->datasize;2096	rx_oob->sgl[0].mem_key = mem_key;2097 2098	return 0;2099}2100 2101#define MANA_WQE_HEADER_SIZE 162102#define MANA_WQE_SGE_SIZE 162103 2104static int mana_alloc_rx_wqe(struct mana_port_context *apc,2105			     struct mana_rxq *rxq, u32 *rxq_size, u32 *cq_size)2106{2107	struct gdma_context *gc = apc->ac->gdma_dev->gdma_context;2108	struct mana_recv_buf_oob *rx_oob;2109	struct device *dev = gc->dev;2110	u32 buf_idx;2111	int ret;2112 2113	WARN_ON(rxq->datasize == 0);2114 2115	*rxq_size = 0;2116	*cq_size = 0;2117 2118	for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) {2119		rx_oob = &rxq->rx_oobs[buf_idx];2120		memset(rx_oob, 0, sizeof(*rx_oob));2121 2122		rx_oob->num_sge = 1;2123 2124		ret = mana_fill_rx_oob(rx_oob, apc->ac->gdma_dev->gpa_mkey, rxq,2125				       dev);2126		if (ret)2127			return ret;2128 2129		rx_oob->wqe_req.sgl = rx_oob->sgl;2130		rx_oob->wqe_req.num_sge = rx_oob->num_sge;2131		rx_oob->wqe_req.inline_oob_size = 0;2132		rx_oob->wqe_req.inline_oob_data = NULL;2133		rx_oob->wqe_req.flags = 0;2134		rx_oob->wqe_req.client_data_unit = 0;2135 2136		*rxq_size += ALIGN(MANA_WQE_HEADER_SIZE +2137				   MANA_WQE_SGE_SIZE * rx_oob->num_sge, 32);2138		*cq_size += COMP_ENTRY_SIZE;2139	}2140 2141	return 0;2142}2143 2144static int mana_push_wqe(struct mana_rxq *rxq)2145{2146	struct mana_recv_buf_oob *rx_oob;2147	u32 buf_idx;2148	int err;2149 2150	for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) {2151		rx_oob = &rxq->rx_oobs[buf_idx];2152 2153		err = mana_gd_post_and_ring(rxq->gdma_rq, &rx_oob->wqe_req,2154					    &rx_oob->wqe_inf);2155		if (err)2156			return -ENOSPC;2157	}2158 2159	return 0;2160}2161 2162static int mana_create_page_pool(struct mana_rxq *rxq, struct gdma_context *gc)2163{2164	struct mana_port_context *mpc = netdev_priv(rxq->ndev);2165	struct page_pool_params pprm = {};2166	int ret;2167 2168	pprm.pool_size = mpc->rx_queue_size;2169	pprm.nid = gc->numa_node;2170	pprm.napi = &rxq->rx_cq.napi;2171	pprm.netdev = rxq->ndev;2172 2173	rxq->page_pool = page_pool_create(&pprm);2174 2175	if (IS_ERR(rxq->page_pool)) {2176		ret = PTR_ERR(rxq->page_pool);2177		rxq->page_pool = NULL;2178		return ret;2179	}2180 2181	return 0;2182}2183 2184static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc,2185					u32 rxq_idx, struct mana_eq *eq,2186					struct net_device *ndev)2187{2188	struct gdma_dev *gd = apc->ac->gdma_dev;2189	struct mana_obj_spec wq_spec;2190	struct mana_obj_spec cq_spec;2191	struct gdma_queue_spec spec;2192	struct mana_cq *cq = NULL;2193	struct gdma_context *gc;2194	u32 cq_size, rq_size;2195	struct mana_rxq *rxq;2196	int err;2197 2198	gc = gd->gdma_context;2199 2200	rxq = kzalloc(struct_size(rxq, rx_oobs, apc->rx_queue_size),2201		      GFP_KERNEL);2202	if (!rxq)2203		return NULL;2204 2205	rxq->ndev = ndev;2206	rxq->num_rx_buf = apc->rx_queue_size;2207	rxq->rxq_idx = rxq_idx;2208	rxq->rxobj = INVALID_MANA_HANDLE;2209 2210	mana_get_rxbuf_cfg(ndev->mtu, &rxq->datasize, &rxq->alloc_size,2211			   &rxq->headroom);2212 2213	/* Create page pool for RX queue */2214	err = mana_create_page_pool(rxq, gc);2215	if (err) {2216		netdev_err(ndev, "Create page pool err:%d\n", err);2217		goto out;2218	}2219 2220	err = mana_alloc_rx_wqe(apc, rxq, &rq_size, &cq_size);2221	if (err)2222		goto out;2223 2224	rq_size = MANA_PAGE_ALIGN(rq_size);2225	cq_size = MANA_PAGE_ALIGN(cq_size);2226 2227	/* Create RQ */2228	memset(&spec, 0, sizeof(spec));2229	spec.type = GDMA_RQ;2230	spec.monitor_avl_buf = true;2231	spec.queue_size = rq_size;2232	err = mana_gd_create_mana_wq_cq(gd, &spec, &rxq->gdma_rq);2233	if (err)2234		goto out;2235 2236	/* Create RQ's CQ */2237	cq = &rxq->rx_cq;2238	cq->type = MANA_CQ_TYPE_RX;2239	cq->rxq = rxq;2240 2241	memset(&spec, 0, sizeof(spec));2242	spec.type = GDMA_CQ;2243	spec.monitor_avl_buf = false;2244	spec.queue_size = cq_size;2245	spec.cq.callback = mana_schedule_napi;2246	spec.cq.parent_eq = eq->eq;2247	spec.cq.context = cq;2248	err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq);2249	if (err)2250		goto out;2251 2252	memset(&wq_spec, 0, sizeof(wq_spec));2253	memset(&cq_spec, 0, sizeof(cq_spec));2254	wq_spec.gdma_region = rxq->gdma_rq->mem_info.dma_region_handle;2255	wq_spec.queue_size = rxq->gdma_rq->queue_size;2256 2257	cq_spec.gdma_region = cq->gdma_cq->mem_info.dma_region_handle;2258	cq_spec.queue_size = cq->gdma_cq->queue_size;2259	cq_spec.modr_ctx_id = 0;2260	cq_spec.attached_eq = cq->gdma_cq->cq.parent->id;2261 2262	err = mana_create_wq_obj(apc, apc->port_handle, GDMA_RQ,2263				 &wq_spec, &cq_spec, &rxq->rxobj);2264	if (err)2265		goto out;2266 2267	rxq->gdma_rq->id = wq_spec.queue_index;2268	cq->gdma_cq->id = cq_spec.queue_index;2269 2270	rxq->gdma_rq->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION;2271	cq->gdma_cq->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION;2272 2273	rxq->gdma_id = rxq->gdma_rq->id;2274	cq->gdma_id = cq->gdma_cq->id;2275 2276	err = mana_push_wqe(rxq);2277	if (err)2278		goto out;2279 2280	if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) {2281		err = -EINVAL;2282		goto out;2283	}2284 2285	gc->cq_table[cq->gdma_id] = cq->gdma_cq;2286 2287	netif_napi_add_weight(ndev, &cq->napi, mana_poll, 1);2288 2289	WARN_ON(xdp_rxq_info_reg(&rxq->xdp_rxq, ndev, rxq_idx,2290				 cq->napi.napi_id));2291	WARN_ON(xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq, MEM_TYPE_PAGE_POOL,2292					   rxq->page_pool));2293 2294	napi_enable(&cq->napi);2295 2296	mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT);2297out:2298	if (!err)2299		return rxq;2300 2301	netdev_err(ndev, "Failed to create RXQ: err = %d\n", err);2302 2303	mana_destroy_rxq(apc, rxq, false);2304 2305	if (cq)2306		mana_deinit_cq(apc, cq);2307 2308	return NULL;2309}2310 2311static int mana_add_rx_queues(struct mana_port_context *apc,2312			      struct net_device *ndev)2313{2314	struct mana_context *ac = apc->ac;2315	struct mana_rxq *rxq;2316	int err = 0;2317	int i;2318 2319	for (i = 0; i < apc->num_queues; i++) {2320		rxq = mana_create_rxq(apc, i, &ac->eqs[i], ndev);2321		if (!rxq) {2322			err = -ENOMEM;2323			goto out;2324		}2325 2326		u64_stats_init(&rxq->stats.syncp);2327 2328		apc->rxqs[i] = rxq;2329	}2330 2331	apc->default_rxobj = apc->rxqs[0]->rxobj;2332out:2333	return err;2334}2335 2336static void mana_destroy_vport(struct mana_port_context *apc)2337{2338	struct gdma_dev *gd = apc->ac->gdma_dev;2339	struct mana_rxq *rxq;2340	u32 rxq_idx;2341 2342	for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) {2343		rxq = apc->rxqs[rxq_idx];2344		if (!rxq)2345			continue;2346 2347		mana_destroy_rxq(apc, rxq, true);2348		apc->rxqs[rxq_idx] = NULL;2349	}2350 2351	mana_destroy_txq(apc);2352	mana_uncfg_vport(apc);2353 2354	if (gd->gdma_context->is_pf)2355		mana_pf_deregister_hw_vport(apc);2356}2357 2358static int mana_create_vport(struct mana_port_context *apc,2359			     struct net_device *net)2360{2361	struct gdma_dev *gd = apc->ac->gdma_dev;2362	int err;2363 2364	apc->default_rxobj = INVALID_MANA_HANDLE;2365 2366	if (gd->gdma_context->is_pf) {2367		err = mana_pf_register_hw_vport(apc);2368		if (err)2369			return err;2370	}2371 2372	err = mana_cfg_vport(apc, gd->pdid, gd->doorbell);2373	if (err)2374		return err;2375 2376	return mana_create_txq(apc, net);2377}2378 2379static int mana_rss_table_alloc(struct mana_port_context *apc)2380{2381	if (!apc->indir_table_sz) {2382		netdev_err(apc->ndev,2383			   "Indirection table size not set for vPort %d\n",2384			   apc->port_idx);2385		return -EINVAL;2386	}2387 2388	apc->indir_table = kcalloc(apc->indir_table_sz, sizeof(u32), GFP_KERNEL);2389	if (!apc->indir_table)2390		return -ENOMEM;2391 2392	apc->rxobj_table = kcalloc(apc->indir_table_sz, sizeof(mana_handle_t), GFP_KERNEL);2393	if (!apc->rxobj_table) {2394		kfree(apc->indir_table);2395		return -ENOMEM;2396	}2397 2398	return 0;2399}2400 2401static void mana_rss_table_init(struct mana_port_context *apc)2402{2403	int i;2404 2405	for (i = 0; i < apc->indir_table_sz; i++)2406		apc->indir_table[i] =2407			ethtool_rxfh_indir_default(i, apc->num_queues);2408}2409 2410int mana_config_rss(struct mana_port_context *apc, enum TRI_STATE rx,2411		    bool update_hash, bool update_tab)2412{2413	u32 queue_idx;2414	int err;2415	int i;2416 2417	if (update_tab) {2418		for (i = 0; i < apc->indir_table_sz; i++) {2419			queue_idx = apc->indir_table[i];2420			apc->rxobj_table[i] = apc->rxqs[queue_idx]->rxobj;2421		}2422	}2423 2424	err = mana_cfg_vport_steering(apc, rx, true, update_hash, update_tab);2425	if (err)2426		return err;2427 2428	mana_fence_rqs(apc);2429 2430	return 0;2431}2432 2433void mana_query_gf_stats(struct mana_port_context *apc)2434{2435	struct mana_query_gf_stat_resp resp = {};2436	struct mana_query_gf_stat_req req = {};2437	struct net_device *ndev = apc->ndev;2438	int err;2439 2440	mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_GF_STAT,2441			     sizeof(req), sizeof(resp));2442	req.req_stats = STATISTICS_FLAGS_RX_DISCARDS_NO_WQE |2443			STATISTICS_FLAGS_RX_ERRORS_VPORT_DISABLED |2444			STATISTICS_FLAGS_HC_RX_BYTES |2445			STATISTICS_FLAGS_HC_RX_UCAST_PACKETS |2446			STATISTICS_FLAGS_HC_RX_UCAST_BYTES |2447			STATISTICS_FLAGS_HC_RX_MCAST_PACKETS |2448			STATISTICS_FLAGS_HC_RX_MCAST_BYTES |2449			STATISTICS_FLAGS_HC_RX_BCAST_PACKETS |2450			STATISTICS_FLAGS_HC_RX_BCAST_BYTES |2451			STATISTICS_FLAGS_TX_ERRORS_GF_DISABLED |2452			STATISTICS_FLAGS_TX_ERRORS_VPORT_DISABLED |2453			STATISTICS_FLAGS_TX_ERRORS_INVAL_VPORT_OFFSET_PACKETS |2454			STATISTICS_FLAGS_TX_ERRORS_VLAN_ENFORCEMENT |2455			STATISTICS_FLAGS_TX_ERRORS_ETH_TYPE_ENFORCEMENT |2456			STATISTICS_FLAGS_TX_ERRORS_SA_ENFORCEMENT |2457			STATISTICS_FLAGS_TX_ERRORS_SQPDID_ENFORCEMENT |2458			STATISTICS_FLAGS_TX_ERRORS_CQPDID_ENFORCEMENT |2459			STATISTICS_FLAGS_TX_ERRORS_MTU_VIOLATION |2460			STATISTICS_FLAGS_TX_ERRORS_INVALID_OOB |2461			STATISTICS_FLAGS_HC_TX_BYTES |2462			STATISTICS_FLAGS_HC_TX_UCAST_PACKETS |2463			STATISTICS_FLAGS_HC_TX_UCAST_BYTES |2464			STATISTICS_FLAGS_HC_TX_MCAST_PACKETS |2465			STATISTICS_FLAGS_HC_TX_MCAST_BYTES |2466			STATISTICS_FLAGS_HC_TX_BCAST_PACKETS |2467			STATISTICS_FLAGS_HC_TX_BCAST_BYTES |2468			STATISTICS_FLAGS_TX_ERRORS_GDMA_ERROR;2469 2470	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,2471				sizeof(resp));2472	if (err) {2473		netdev_err(ndev, "Failed to query GF stats: %d\n", err);2474		return;2475	}2476	err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_GF_STAT,2477				   sizeof(resp));2478	if (err || resp.hdr.status) {2479		netdev_err(ndev, "Failed to query GF stats: %d, 0x%x\n", err,2480			   resp.hdr.status);2481		return;2482	}2483 2484	apc->eth_stats.hc_rx_discards_no_wqe = resp.rx_discards_nowqe;2485	apc->eth_stats.hc_rx_err_vport_disabled = resp.rx_err_vport_disabled;2486	apc->eth_stats.hc_rx_bytes = resp.hc_rx_bytes;2487	apc->eth_stats.hc_rx_ucast_pkts = resp.hc_rx_ucast_pkts;2488	apc->eth_stats.hc_rx_ucast_bytes = resp.hc_rx_ucast_bytes;2489	apc->eth_stats.hc_rx_bcast_pkts = resp.hc_rx_bcast_pkts;2490	apc->eth_stats.hc_rx_bcast_bytes = resp.hc_rx_bcast_bytes;2491	apc->eth_stats.hc_rx_mcast_pkts = resp.hc_rx_mcast_pkts;2492	apc->eth_stats.hc_rx_mcast_bytes = resp.hc_rx_mcast_bytes;2493	apc->eth_stats.hc_tx_err_gf_disabled = resp.tx_err_gf_disabled;2494	apc->eth_stats.hc_tx_err_vport_disabled = resp.tx_err_vport_disabled;2495	apc->eth_stats.hc_tx_err_inval_vportoffset_pkt =2496					     resp.tx_err_inval_vport_offset_pkt;2497	apc->eth_stats.hc_tx_err_vlan_enforcement =2498					     resp.tx_err_vlan_enforcement;2499	apc->eth_stats.hc_tx_err_eth_type_enforcement =2500					     resp.tx_err_ethtype_enforcement;2501	apc->eth_stats.hc_tx_err_sa_enforcement = resp.tx_err_SA_enforcement;2502	apc->eth_stats.hc_tx_err_sqpdid_enforcement =2503					     resp.tx_err_SQPDID_enforcement;2504	apc->eth_stats.hc_tx_err_cqpdid_enforcement =2505					     resp.tx_err_CQPDID_enforcement;2506	apc->eth_stats.hc_tx_err_mtu_violation = resp.tx_err_mtu_violation;2507	apc->eth_stats.hc_tx_err_inval_oob = resp.tx_err_inval_oob;2508	apc->eth_stats.hc_tx_bytes = resp.hc_tx_bytes;2509	apc->eth_stats.hc_tx_ucast_pkts = resp.hc_tx_ucast_pkts;2510	apc->eth_stats.hc_tx_ucast_bytes = resp.hc_tx_ucast_bytes;2511	apc->eth_stats.hc_tx_bcast_pkts = resp.hc_tx_bcast_pkts;2512	apc->eth_stats.hc_tx_bcast_bytes = resp.hc_tx_bcast_bytes;2513	apc->eth_stats.hc_tx_mcast_pkts = resp.hc_tx_mcast_pkts;2514	apc->eth_stats.hc_tx_mcast_bytes = resp.hc_tx_mcast_bytes;2515	apc->eth_stats.hc_tx_err_gdma = resp.tx_err_gdma;2516}2517 2518static int mana_init_port(struct net_device *ndev)2519{2520	struct mana_port_context *apc = netdev_priv(ndev);2521	u32 max_txq, max_rxq, max_queues;2522	int port_idx = apc->port_idx;2523	int err;2524 2525	err = mana_init_port_context(apc);2526	if (err)2527		return err;2528 2529	err = mana_query_vport_cfg(apc, port_idx, &max_txq, &max_rxq,2530				   &apc->indir_table_sz);2531	if (err) {2532		netdev_err(ndev, "Failed to query info for vPort %d\n",2533			   port_idx);2534		goto reset_apc;2535	}2536 2537	max_queues = min_t(u32, max_txq, max_rxq);2538	if (apc->max_queues > max_queues)2539		apc->max_queues = max_queues;2540 2541	if (apc->num_queues > apc->max_queues)2542		apc->num_queues = apc->max_queues;2543 2544	eth_hw_addr_set(ndev, apc->mac_addr);2545 2546	return 0;2547 2548reset_apc:2549	mana_cleanup_port_context(apc);2550	return err;2551}2552 2553int mana_alloc_queues(struct net_device *ndev)2554{2555	struct mana_port_context *apc = netdev_priv(ndev);2556	struct gdma_dev *gd = apc->ac->gdma_dev;2557	int err;2558 2559	err = mana_create_vport(apc, ndev);2560	if (err)2561		return err;2562 2563	err = netif_set_real_num_tx_queues(ndev, apc->num_queues);2564	if (err)2565		goto destroy_vport;2566 2567	err = mana_add_rx_queues(apc, ndev);2568	if (err)2569		goto destroy_vport;2570 2571	apc->rss_state = apc->num_queues > 1 ? TRI_STATE_TRUE : TRI_STATE_FALSE;2572 2573	err = netif_set_real_num_rx_queues(ndev, apc->num_queues);2574	if (err)2575		goto destroy_vport;2576 2577	mana_rss_table_init(apc);2578 2579	err = mana_config_rss(apc, TRI_STATE_TRUE, true, true);2580	if (err)2581		goto destroy_vport;2582 2583	if (gd->gdma_context->is_pf) {2584		err = mana_pf_register_filter(apc);2585		if (err)2586			goto destroy_vport;2587	}2588 2589	mana_chn_setxdp(apc, mana_xdp_get(apc));2590 2591	return 0;2592 2593destroy_vport:2594	mana_destroy_vport(apc);2595	return err;2596}2597 2598int mana_attach(struct net_device *ndev)2599{2600	struct mana_port_context *apc = netdev_priv(ndev);2601	int err;2602 2603	ASSERT_RTNL();2604 2605	err = mana_init_port(ndev);2606	if (err)2607		return err;2608 2609	if (apc->port_st_save) {2610		err = mana_alloc_queues(ndev);2611		if (err) {2612			mana_cleanup_port_context(apc);2613			return err;2614		}2615	}2616 2617	apc->port_is_up = apc->port_st_save;2618 2619	/* Ensure port state updated before txq state */2620	smp_wmb();2621 2622	if (apc->port_is_up)2623		netif_carrier_on(ndev);2624 2625	netif_device_attach(ndev);2626 2627	return 0;2628}2629 2630static int mana_dealloc_queues(struct net_device *ndev)2631{2632	struct mana_port_context *apc = netdev_priv(ndev);2633	unsigned long timeout = jiffies + 120 * HZ;2634	struct gdma_dev *gd = apc->ac->gdma_dev;2635	struct mana_txq *txq;2636	struct sk_buff *skb;2637	int i, err;2638	u32 tsleep;2639 2640	if (apc->port_is_up)2641		return -EINVAL;2642 2643	mana_chn_setxdp(apc, NULL);2644 2645	if (gd->gdma_context->is_pf)2646		mana_pf_deregister_filter(apc);2647 2648	/* No packet can be transmitted now since apc->port_is_up is false.2649	 * There is still a tiny chance that mana_poll_tx_cq() can re-enable2650	 * a txq because it may not timely see apc->port_is_up being cleared2651	 * to false, but it doesn't matter since mana_start_xmit() drops any2652	 * new packets due to apc->port_is_up being false.2653	 *2654	 * Drain all the in-flight TX packets.2655	 * A timeout of 120 seconds for all the queues is used.2656	 * This will break the while loop when h/w is not responding.2657	 * This value of 120 has been decided here considering max2658	 * number of queues.2659	 */2660 2661	for (i = 0; i < apc->num_queues; i++) {2662		txq = &apc->tx_qp[i].txq;2663		tsleep = 1000;2664		while (atomic_read(&txq->pending_sends) > 0 &&2665		       time_before(jiffies, timeout)) {2666			usleep_range(tsleep, tsleep + 1000);2667			tsleep <<= 1;2668		}2669		if (atomic_read(&txq->pending_sends)) {2670			err = pcie_flr(to_pci_dev(gd->gdma_context->dev));2671			if (err) {2672				netdev_err(ndev, "flr failed %d with %d pkts pending in txq %u\n",2673					   err, atomic_read(&txq->pending_sends),2674					   txq->gdma_txq_id);2675			}2676			break;2677		}2678	}2679 2680	for (i = 0; i < apc->num_queues; i++) {2681		txq = &apc->tx_qp[i].txq;2682		while ((skb = skb_dequeue(&txq->pending_skbs))) {2683			mana_unmap_skb(skb, apc);2684			dev_kfree_skb_any(skb);2685		}2686		atomic_set(&txq->pending_sends, 0);2687	}2688	/* We're 100% sure the queues can no longer be woken up, because2689	 * we're sure now mana_poll_tx_cq() can't be running.2690	 */2691 2692	apc->rss_state = TRI_STATE_FALSE;2693	err = mana_config_rss(apc, TRI_STATE_FALSE, false, false);2694	if (err) {2695		netdev_err(ndev, "Failed to disable vPort: %d\n", err);2696		return err;2697	}2698 2699	mana_destroy_vport(apc);2700 2701	return 0;2702}2703 2704int mana_detach(struct net_device *ndev, bool from_close)2705{2706	struct mana_port_context *apc = netdev_priv(ndev);2707	int err;2708 2709	ASSERT_RTNL();2710 2711	apc->port_st_save = apc->port_is_up;2712	apc->port_is_up = false;2713 2714	/* Ensure port state updated before txq state */2715	smp_wmb();2716 2717	netif_tx_disable(ndev);2718	netif_carrier_off(ndev);2719 2720	if (apc->port_st_save) {2721		err = mana_dealloc_queues(ndev);2722		if (err)2723			return err;2724	}2725 2726	if (!from_close) {2727		netif_device_detach(ndev);2728		mana_cleanup_port_context(apc);2729	}2730 2731	return 0;2732}2733 2734static int mana_probe_port(struct mana_context *ac, int port_idx,2735			   struct net_device **ndev_storage)2736{2737	struct gdma_context *gc = ac->gdma_dev->gdma_context;2738	struct mana_port_context *apc;2739	struct net_device *ndev;2740	int err;2741 2742	ndev = alloc_etherdev_mq(sizeof(struct mana_port_context),2743				 gc->max_num_queues);2744	if (!ndev)2745		return -ENOMEM;2746 2747	*ndev_storage = ndev;2748 2749	apc = netdev_priv(ndev);2750	apc->ac = ac;2751	apc->ndev = ndev;2752	apc->max_queues = gc->max_num_queues;2753	apc->num_queues = gc->max_num_queues;2754	apc->tx_queue_size = DEF_TX_BUFFERS_PER_QUEUE;2755	apc->rx_queue_size = DEF_RX_BUFFERS_PER_QUEUE;2756	apc->port_handle = INVALID_MANA_HANDLE;2757	apc->pf_filter_handle = INVALID_MANA_HANDLE;2758	apc->port_idx = port_idx;2759 2760	mutex_init(&apc->vport_mutex);2761	apc->vport_use_count = 0;2762 2763	ndev->netdev_ops = &mana_devops;2764	ndev->ethtool_ops = &mana_ethtool_ops;2765	ndev->mtu = ETH_DATA_LEN;2766	ndev->max_mtu = gc->adapter_mtu - ETH_HLEN;2767	ndev->min_mtu = ETH_MIN_MTU;2768	ndev->needed_headroom = MANA_HEADROOM;2769	ndev->dev_port = port_idx;2770	SET_NETDEV_DEV(ndev, gc->dev);2771 2772	netif_carrier_off(ndev);2773 2774	netdev_rss_key_fill(apc->hashkey, MANA_HASH_KEY_SIZE);2775 2776	err = mana_init_port(ndev);2777	if (err)2778		goto free_net;2779 2780	err = mana_rss_table_alloc(apc);2781	if (err)2782		goto reset_apc;2783 2784	netdev_lockdep_set_classes(ndev);2785 2786	ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;2787	ndev->hw_features |= NETIF_F_RXCSUM;2788	ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;2789	ndev->hw_features |= NETIF_F_RXHASH;2790	ndev->features = ndev->hw_features | NETIF_F_HW_VLAN_CTAG_TX |2791			 NETIF_F_HW_VLAN_CTAG_RX;2792	ndev->vlan_features = ndev->features;2793	xdp_set_features_flag(ndev, NETDEV_XDP_ACT_BASIC |2794			      NETDEV_XDP_ACT_REDIRECT |2795			      NETDEV_XDP_ACT_NDO_XMIT);2796 2797	err = register_netdev(ndev);2798	if (err) {2799		netdev_err(ndev, "Unable to register netdev.\n");2800		goto free_indir;2801	}2802 2803	return 0;2804 2805free_indir:2806	mana_cleanup_indir_table(apc);2807reset_apc:2808	mana_cleanup_port_context(apc);2809free_net:2810	*ndev_storage = NULL;2811	netdev_err(ndev, "Failed to probe vPort %d: %d\n", port_idx, err);2812	free_netdev(ndev);2813	return err;2814}2815 2816static void adev_release(struct device *dev)2817{2818	struct mana_adev *madev = container_of(dev, struct mana_adev, adev.dev);2819 2820	kfree(madev);2821}2822 2823static void remove_adev(struct gdma_dev *gd)2824{2825	struct auxiliary_device *adev = gd->adev;2826	int id = adev->id;2827 2828	auxiliary_device_delete(adev);2829	auxiliary_device_uninit(adev);2830 2831	mana_adev_idx_free(id);2832	gd->adev = NULL;2833}2834 2835static int add_adev(struct gdma_dev *gd)2836{2837	struct auxiliary_device *adev;2838	struct mana_adev *madev;2839	int ret;2840 2841	madev = kzalloc(sizeof(*madev), GFP_KERNEL);2842	if (!madev)2843		return -ENOMEM;2844 2845	adev = &madev->adev;2846	ret = mana_adev_idx_alloc();2847	if (ret < 0)2848		goto idx_fail;2849	adev->id = ret;2850 2851	adev->name = "rdma";2852	adev->dev.parent = gd->gdma_context->dev;2853	adev->dev.release = adev_release;2854	madev->mdev = gd;2855 2856	ret = auxiliary_device_init(adev);2857	if (ret)2858		goto init_fail;2859 2860	/* madev is owned by the auxiliary device */2861	madev = NULL;2862	ret = auxiliary_device_add(adev);2863	if (ret)2864		goto add_fail;2865 2866	gd->adev = adev;2867	return 0;2868 2869add_fail:2870	auxiliary_device_uninit(adev);2871 2872init_fail:2873	mana_adev_idx_free(adev->id);2874 2875idx_fail:2876	kfree(madev);2877 2878	return ret;2879}2880 2881int mana_probe(struct gdma_dev *gd, bool resuming)2882{2883	struct gdma_context *gc = gd->gdma_context;2884	struct mana_context *ac = gd->driver_data;2885	struct device *dev = gc->dev;2886	u16 num_ports = 0;2887	int err;2888	int i;2889 2890	dev_info(dev,2891		 "Microsoft Azure Network Adapter protocol version: %d.%d.%d\n",2892		 MANA_MAJOR_VERSION, MANA_MINOR_VERSION, MANA_MICRO_VERSION);2893 2894	err = mana_gd_register_device(gd);2895	if (err)2896		return err;2897 2898	if (!resuming) {2899		ac = kzalloc(sizeof(*ac), GFP_KERNEL);2900		if (!ac)2901			return -ENOMEM;2902 2903		ac->gdma_dev = gd;2904		gd->driver_data = ac;2905	}2906 2907	err = mana_create_eq(ac);2908	if (err)2909		goto out;2910 2911	err = mana_query_device_cfg(ac, MANA_MAJOR_VERSION, MANA_MINOR_VERSION,2912				    MANA_MICRO_VERSION, &num_ports);2913	if (err)2914		goto out;2915 2916	if (!resuming) {2917		ac->num_ports = num_ports;2918	} else {2919		if (ac->num_ports != num_ports) {2920			dev_err(dev, "The number of vPorts changed: %d->%d\n",2921				ac->num_ports, num_ports);2922			err = -EPROTO;2923			goto out;2924		}2925	}2926 2927	if (ac->num_ports == 0)2928		dev_err(dev, "Failed to detect any vPort\n");2929 2930	if (ac->num_ports > MAX_PORTS_IN_MANA_DEV)2931		ac->num_ports = MAX_PORTS_IN_MANA_DEV;2932 2933	if (!resuming) {2934		for (i = 0; i < ac->num_ports; i++) {2935			err = mana_probe_port(ac, i, &ac->ports[i]);2936			/* we log the port for which the probe failed and stop2937			 * probes for subsequent ports.2938			 * Note that we keep running ports, for which the probes2939			 * were successful, unless add_adev fails too2940			 */2941			if (err) {2942				dev_err(dev, "Probe Failed for port %d\n", i);2943				break;2944			}2945		}2946	} else {2947		for (i = 0; i < ac->num_ports; i++) {2948			rtnl_lock();2949			err = mana_attach(ac->ports[i]);2950			rtnl_unlock();2951			/* we log the port for which the attach failed and stop2952			 * attach for subsequent ports2953			 * Note that we keep running ports, for which the attach2954			 * were successful, unless add_adev fails too2955			 */2956			if (err) {2957				dev_err(dev, "Attach Failed for port %d\n", i);2958				break;2959			}2960		}2961	}2962 2963	err = add_adev(gd);2964out:2965	if (err)2966		mana_remove(gd, false);2967 2968	return err;2969}2970 2971void mana_remove(struct gdma_dev *gd, bool suspending)2972{2973	struct gdma_context *gc = gd->gdma_context;2974	struct mana_context *ac = gd->driver_data;2975	struct mana_port_context *apc;2976	struct device *dev = gc->dev;2977	struct net_device *ndev;2978	int err;2979	int i;2980 2981	/* adev currently doesn't support suspending, always remove it */2982	if (gd->adev)2983		remove_adev(gd);2984 2985	for (i = 0; i < ac->num_ports; i++) {2986		ndev = ac->ports[i];2987		apc = netdev_priv(ndev);2988		if (!ndev) {2989			if (i == 0)2990				dev_err(dev, "No net device to remove\n");2991			goto out;2992		}2993 2994		/* All cleanup actions should stay after rtnl_lock(), otherwise2995		 * other functions may access partially cleaned up data.2996		 */2997		rtnl_lock();2998 2999		err = mana_detach(ndev, false);3000		if (err)3001			netdev_err(ndev, "Failed to detach vPort %d: %d\n",3002				   i, err);3003 3004		if (suspending) {3005			/* No need to unregister the ndev. */3006			rtnl_unlock();3007			continue;3008		}3009 3010		unregister_netdevice(ndev);3011		mana_cleanup_indir_table(apc);3012 3013		rtnl_unlock();3014 3015		free_netdev(ndev);3016	}3017 3018	mana_destroy_eq(ac);3019out:3020	mana_gd_deregister_device(gd);3021 3022	if (suspending)3023		return;3024 3025	gd->driver_data = NULL;3026	gd->gdma_context = NULL;3027	kfree(ac);3028}3029 3030struct net_device *mana_get_primary_netdev_rcu(struct mana_context *ac, u32 port_index)3031{3032	struct net_device *ndev;3033 3034	RCU_LOCKDEP_WARN(!rcu_read_lock_held(),3035			 "Taking primary netdev without holding the RCU read lock");3036	if (port_index >= ac->num_ports)3037		return NULL;3038 3039	/* When mana is used in netvsc, the upper netdevice should be returned. */3040	if (ac->ports[port_index]->flags & IFF_SLAVE)3041		ndev = netdev_master_upper_dev_get_rcu(ac->ports[port_index]);3042	else3043		ndev = ac->ports[port_index];3044 3045	return ndev;3046}3047EXPORT_SYMBOL_NS(mana_get_primary_netdev_rcu, NET_MANA);3048