brintos / linux-shallow public Read only

1// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)2/* Copyright 2017-2019 NXP */3 4#include "enetc.h"5#include <linux/bpf_trace.h>6#include <linux/tcp.h>7#include <linux/udp.h>8#include <linux/vmalloc.h>9#include <linux/ptp_classify.h>10#include <net/ip6_checksum.h>11#include <net/pkt_sched.h>12#include <net/tso.h>13 14u32 enetc_port_mac_rd(struct enetc_si *si, u32 reg)15{16	return enetc_port_rd(&si->hw, reg);17}18EXPORT_SYMBOL_GPL(enetc_port_mac_rd);19 20void enetc_port_mac_wr(struct enetc_si *si, u32 reg, u32 val)21{22	enetc_port_wr(&si->hw, reg, val);23	if (si->hw_features & ENETC_SI_F_QBU)24		enetc_port_wr(&si->hw, reg + ENETC_PMAC_OFFSET, val);25}26EXPORT_SYMBOL_GPL(enetc_port_mac_wr);27 28static void enetc_change_preemptible_tcs(struct enetc_ndev_priv *priv,29					 u8 preemptible_tcs)30{31	priv->preemptible_tcs = preemptible_tcs;32	enetc_mm_commit_preemptible_tcs(priv);33}34 35static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv)36{37	int num_tx_rings = priv->num_tx_rings;38 39	if (priv->xdp_prog)40		return num_tx_rings - num_possible_cpus();41 42	return num_tx_rings;43}44 45static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv,46							struct enetc_bdr *tx_ring)47{48	int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring;49 50	return priv->rx_ring[index];51}52 53static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd)54{55	if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect)56		return NULL;57 58	return tx_swbd->skb;59}60 61static struct xdp_frame *62enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd)63{64	if (tx_swbd->is_xdp_redirect)65		return tx_swbd->xdp_frame;66 67	return NULL;68}69 70static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,71				struct enetc_tx_swbd *tx_swbd)72{73	/* For XDP_TX, pages come from RX, whereas for the other contexts where74	 * we have is_dma_page_set, those come from skb_frag_dma_map. We need75	 * to match the DMA mapping length, so we need to differentiate those.76	 */77	if (tx_swbd->is_dma_page)78		dma_unmap_page(tx_ring->dev, tx_swbd->dma,79			       tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len,80			       tx_swbd->dir);81	else82		dma_unmap_single(tx_ring->dev, tx_swbd->dma,83				 tx_swbd->len, tx_swbd->dir);84	tx_swbd->dma = 0;85}86 87static void enetc_free_tx_frame(struct enetc_bdr *tx_ring,88				struct enetc_tx_swbd *tx_swbd)89{90	struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);91	struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);92 93	if (tx_swbd->dma)94		enetc_unmap_tx_buff(tx_ring, tx_swbd);95 96	if (xdp_frame) {97		xdp_return_frame(tx_swbd->xdp_frame);98		tx_swbd->xdp_frame = NULL;99	} else if (skb) {100		dev_kfree_skb_any(skb);101		tx_swbd->skb = NULL;102	}103}104 105/* Let H/W know BD ring has been updated */106static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring)107{108	/* includes wmb() */109	enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use);110}111 112static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp,113			   u8 *msgtype, u8 *twostep,114			   u16 *correction_offset, u16 *body_offset)115{116	unsigned int ptp_class;117	struct ptp_header *hdr;118	unsigned int type;119	u8 *base;120 121	ptp_class = ptp_classify_raw(skb);122	if (ptp_class == PTP_CLASS_NONE)123		return -EINVAL;124 125	hdr = ptp_parse_header(skb, ptp_class);126	if (!hdr)127		return -EINVAL;128 129	type = ptp_class & PTP_CLASS_PMASK;130	if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6)131		*udp = 1;132	else133		*udp = 0;134 135	*msgtype = ptp_get_msgtype(hdr, ptp_class);136	*twostep = hdr->flag_field[0] & 0x2;137 138	base = skb_mac_header(skb);139	*correction_offset = (u8 *)&hdr->correction - base;140	*body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base;141 142	return 0;143}144 145static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)146{147	bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false;148	struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);149	struct enetc_hw *hw = &priv->si->hw;150	struct enetc_tx_swbd *tx_swbd;151	int len = skb_headlen(skb);152	union enetc_tx_bd temp_bd;153	u8 msgtype, twostep, udp;154	union enetc_tx_bd *txbd;155	u16 offset1, offset2;156	int i, count = 0;157	skb_frag_t *frag;158	unsigned int f;159	dma_addr_t dma;160	u8 flags = 0;161 162	i = tx_ring->next_to_use;163	txbd = ENETC_TXBD(*tx_ring, i);164	prefetchw(txbd);165 166	dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE);167	if (unlikely(dma_mapping_error(tx_ring->dev, dma)))168		goto dma_err;169 170	temp_bd.addr = cpu_to_le64(dma);171	temp_bd.buf_len = cpu_to_le16(len);172	temp_bd.lstatus = 0;173 174	tx_swbd = &tx_ring->tx_swbd[i];175	tx_swbd->dma = dma;176	tx_swbd->len = len;177	tx_swbd->is_dma_page = 0;178	tx_swbd->dir = DMA_TO_DEVICE;179	count++;180 181	do_vlan = skb_vlan_tag_present(skb);182	if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {183		if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, &offset1,184				    &offset2) ||185		    msgtype != PTP_MSGTYPE_SYNC || twostep)186			WARN_ONCE(1, "Bad packet for one-step timestamping\n");187		else188			do_onestep_tstamp = true;189	} else if (skb->cb[0] & ENETC_F_TX_TSTAMP) {190		do_twostep_tstamp = true;191	}192 193	tx_swbd->do_twostep_tstamp = do_twostep_tstamp;194	tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV);195	tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en;196 197	if (do_vlan || do_onestep_tstamp || do_twostep_tstamp)198		flags |= ENETC_TXBD_FLAGS_EX;199 200	if (tx_ring->tsd_enable)201		flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART;202 203	/* first BD needs frm_len and offload flags set */204	temp_bd.frm_len = cpu_to_le16(skb->len);205	temp_bd.flags = flags;206 207	if (flags & ENETC_TXBD_FLAGS_TSE)208		temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns,209							  flags);210 211	if (flags & ENETC_TXBD_FLAGS_EX) {212		u8 e_flags = 0;213		*txbd = temp_bd;214		enetc_clear_tx_bd(&temp_bd);215 216		/* add extension BD for VLAN and/or timestamping */217		flags = 0;218		tx_swbd++;219		txbd++;220		i++;221		if (unlikely(i == tx_ring->bd_count)) {222			i = 0;223			tx_swbd = tx_ring->tx_swbd;224			txbd = ENETC_TXBD(*tx_ring, 0);225		}226		prefetchw(txbd);227 228		if (do_vlan) {229			temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));230			temp_bd.ext.tpid = 0; /* < C-TAG */231			e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;232		}233 234		if (do_onestep_tstamp) {235			u32 lo, hi, val;236			u64 sec, nsec;237			u8 *data;238 239			lo = enetc_rd_hot(hw, ENETC_SICTR0);240			hi = enetc_rd_hot(hw, ENETC_SICTR1);241			sec = (u64)hi << 32 | lo;242			nsec = do_div(sec, 1000000000);243 244			/* Configure extension BD */245			temp_bd.ext.tstamp = cpu_to_le32(lo & 0x3fffffff);246			e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP;247 248			/* Update originTimestamp field of Sync packet249			 * - 48 bits seconds field250			 * - 32 bits nanseconds field251			 */252			data = skb_mac_header(skb);253			*(__be16 *)(data + offset2) =254				htons((sec >> 32) & 0xffff);255			*(__be32 *)(data + offset2 + 2) =256				htonl(sec & 0xffffffff);257			*(__be32 *)(data + offset2 + 6) = htonl(nsec);258 259			/* Configure single-step register */260			val = ENETC_PM0_SINGLE_STEP_EN;261			val |= ENETC_SET_SINGLE_STEP_OFFSET(offset1);262			if (udp)263				val |= ENETC_PM0_SINGLE_STEP_CH;264 265			enetc_port_mac_wr(priv->si, ENETC_PM0_SINGLE_STEP,266					  val);267		} else if (do_twostep_tstamp) {268			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;269			e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP;270		}271 272		temp_bd.ext.e_flags = e_flags;273		count++;274	}275 276	frag = &skb_shinfo(skb)->frags[0];277	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {278		len = skb_frag_size(frag);279		dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,280				       DMA_TO_DEVICE);281		if (dma_mapping_error(tx_ring->dev, dma))282			goto dma_err;283 284		*txbd = temp_bd;285		enetc_clear_tx_bd(&temp_bd);286 287		flags = 0;288		tx_swbd++;289		txbd++;290		i++;291		if (unlikely(i == tx_ring->bd_count)) {292			i = 0;293			tx_swbd = tx_ring->tx_swbd;294			txbd = ENETC_TXBD(*tx_ring, 0);295		}296		prefetchw(txbd);297 298		temp_bd.addr = cpu_to_le64(dma);299		temp_bd.buf_len = cpu_to_le16(len);300 301		tx_swbd->dma = dma;302		tx_swbd->len = len;303		tx_swbd->is_dma_page = 1;304		tx_swbd->dir = DMA_TO_DEVICE;305		count++;306	}307 308	/* last BD needs 'F' bit set */309	flags |= ENETC_TXBD_FLAGS_F;310	temp_bd.flags = flags;311	*txbd = temp_bd;312 313	tx_ring->tx_swbd[i].is_eof = true;314	tx_ring->tx_swbd[i].skb = skb;315 316	enetc_bdr_idx_inc(tx_ring, &i);317	tx_ring->next_to_use = i;318 319	skb_tx_timestamp(skb);320 321	enetc_update_tx_ring_tail(tx_ring);322 323	return count;324 325dma_err:326	dev_err(tx_ring->dev, "DMA map error");327 328	do {329		tx_swbd = &tx_ring->tx_swbd[i];330		enetc_free_tx_frame(tx_ring, tx_swbd);331		if (i == 0)332			i = tx_ring->bd_count;333		i--;334	} while (count--);335 336	return 0;337}338 339static void enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb,340				 struct enetc_tx_swbd *tx_swbd,341				 union enetc_tx_bd *txbd, int *i, int hdr_len,342				 int data_len)343{344	union enetc_tx_bd txbd_tmp;345	u8 flags = 0, e_flags = 0;346	dma_addr_t addr;347 348	enetc_clear_tx_bd(&txbd_tmp);349	addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE;350 351	if (skb_vlan_tag_present(skb))352		flags |= ENETC_TXBD_FLAGS_EX;353 354	txbd_tmp.addr = cpu_to_le64(addr);355	txbd_tmp.buf_len = cpu_to_le16(hdr_len);356 357	/* first BD needs frm_len and offload flags set */358	txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len);359	txbd_tmp.flags = flags;360 361	/* For the TSO header we do not set the dma address since we do not362	 * want it unmapped when we do cleanup. We still set len so that we363	 * count the bytes sent.364	 */365	tx_swbd->len = hdr_len;366	tx_swbd->do_twostep_tstamp = false;367	tx_swbd->check_wb = false;368 369	/* Actually write the header in the BD */370	*txbd = txbd_tmp;371 372	/* Add extension BD for VLAN */373	if (flags & ENETC_TXBD_FLAGS_EX) {374		/* Get the next BD */375		enetc_bdr_idx_inc(tx_ring, i);376		txbd = ENETC_TXBD(*tx_ring, *i);377		tx_swbd = &tx_ring->tx_swbd[*i];378		prefetchw(txbd);379 380		/* Setup the VLAN fields */381		enetc_clear_tx_bd(&txbd_tmp);382		txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));383		txbd_tmp.ext.tpid = 0; /* < C-TAG */384		e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;385 386		/* Write the BD */387		txbd_tmp.ext.e_flags = e_flags;388		*txbd = txbd_tmp;389	}390}391 392static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb,393				 struct enetc_tx_swbd *tx_swbd,394				 union enetc_tx_bd *txbd, char *data,395				 int size, bool last_bd)396{397	union enetc_tx_bd txbd_tmp;398	dma_addr_t addr;399	u8 flags = 0;400 401	enetc_clear_tx_bd(&txbd_tmp);402 403	addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE);404	if (unlikely(dma_mapping_error(tx_ring->dev, addr))) {405		netdev_err(tx_ring->ndev, "DMA map error\n");406		return -ENOMEM;407	}408 409	if (last_bd) {410		flags |= ENETC_TXBD_FLAGS_F;411		tx_swbd->is_eof = 1;412	}413 414	txbd_tmp.addr = cpu_to_le64(addr);415	txbd_tmp.buf_len = cpu_to_le16(size);416	txbd_tmp.flags = flags;417 418	tx_swbd->dma = addr;419	tx_swbd->len = size;420	tx_swbd->dir = DMA_TO_DEVICE;421 422	*txbd = txbd_tmp;423 424	return 0;425}426 427static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb,428				 char *hdr, int hdr_len, int *l4_hdr_len)429{430	char *l4_hdr = hdr + skb_transport_offset(skb);431	int mac_hdr_len = skb_network_offset(skb);432 433	if (tso->tlen != sizeof(struct udphdr)) {434		struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);435 436		tcph->check = 0;437	} else {438		struct udphdr *udph = (struct udphdr *)(l4_hdr);439 440		udph->check = 0;441	}442 443	/* Compute the IP checksum. This is necessary since tso_build_hdr()444	 * already incremented the IP ID field.445	 */446	if (!tso->ipv6) {447		struct iphdr *iph = (void *)(hdr + mac_hdr_len);448 449		iph->check = 0;450		iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);451	}452 453	/* Compute the checksum over the L4 header. */454	*l4_hdr_len = hdr_len - skb_transport_offset(skb);455	return csum_partial(l4_hdr, *l4_hdr_len, 0);456}457 458static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso,459				    struct sk_buff *skb, char *hdr, int len,460				    __wsum sum)461{462	char *l4_hdr = hdr + skb_transport_offset(skb);463	__sum16 csum_final;464 465	/* Complete the L4 checksum by appending the pseudo-header to the466	 * already computed checksum.467	 */468	if (!tso->ipv6)469		csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr,470					       ip_hdr(skb)->daddr,471					       len, ip_hdr(skb)->protocol, sum);472	else473		csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr,474					     &ipv6_hdr(skb)->daddr,475					     len, ipv6_hdr(skb)->nexthdr, sum);476 477	if (tso->tlen != sizeof(struct udphdr)) {478		struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);479 480		tcph->check = csum_final;481	} else {482		struct udphdr *udph = (struct udphdr *)(l4_hdr);483 484		udph->check = csum_final;485	}486}487 488static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)489{490	int hdr_len, total_len, data_len;491	struct enetc_tx_swbd *tx_swbd;492	union enetc_tx_bd *txbd;493	struct tso_t tso;494	__wsum csum, csum2;495	int count = 0, pos;496	int err, i, bd_data_num;497 498	/* Initialize the TSO handler, and prepare the first payload */499	hdr_len = tso_start(skb, &tso);500	total_len = skb->len - hdr_len;501	i = tx_ring->next_to_use;502 503	while (total_len > 0) {504		char *hdr;505 506		/* Get the BD */507		txbd = ENETC_TXBD(*tx_ring, i);508		tx_swbd = &tx_ring->tx_swbd[i];509		prefetchw(txbd);510 511		/* Determine the length of this packet */512		data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len);513		total_len -= data_len;514 515		/* prepare packet headers: MAC + IP + TCP */516		hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE;517		tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0);518 519		/* compute the csum over the L4 header */520		csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos);521		enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, &i, hdr_len, data_len);522		bd_data_num = 0;523		count++;524 525		while (data_len > 0) {526			int size;527 528			size = min_t(int, tso.size, data_len);529 530			/* Advance the index in the BDR */531			enetc_bdr_idx_inc(tx_ring, &i);532			txbd = ENETC_TXBD(*tx_ring, i);533			tx_swbd = &tx_ring->tx_swbd[i];534			prefetchw(txbd);535 536			/* Compute the checksum over this segment of data and537			 * add it to the csum already computed (over the L4538			 * header and possible other data segments).539			 */540			csum2 = csum_partial(tso.data, size, 0);541			csum = csum_block_add(csum, csum2, pos);542			pos += size;543 544			err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd,545						    tso.data, size,546						    size == data_len);547			if (err)548				goto err_map_data;549 550			data_len -= size;551			count++;552			bd_data_num++;553			tso_build_data(skb, &tso, size);554 555			if (unlikely(bd_data_num >= ENETC_MAX_SKB_FRAGS && data_len))556				goto err_chained_bd;557		}558 559		enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum);560 561		if (total_len == 0)562			tx_swbd->skb = skb;563 564		/* Go to the next BD */565		enetc_bdr_idx_inc(tx_ring, &i);566	}567 568	tx_ring->next_to_use = i;569	enetc_update_tx_ring_tail(tx_ring);570 571	return count;572 573err_map_data:574	dev_err(tx_ring->dev, "DMA map error");575 576err_chained_bd:577	do {578		tx_swbd = &tx_ring->tx_swbd[i];579		enetc_free_tx_frame(tx_ring, tx_swbd);580		if (i == 0)581			i = tx_ring->bd_count;582		i--;583	} while (count--);584 585	return 0;586}587 588static netdev_tx_t enetc_start_xmit(struct sk_buff *skb,589				    struct net_device *ndev)590{591	struct enetc_ndev_priv *priv = netdev_priv(ndev);592	struct enetc_bdr *tx_ring;593	int count, err;594 595	/* Queue one-step Sync packet if already locked */596	if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {597		if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS,598					  &priv->flags)) {599			skb_queue_tail(&priv->tx_skbs, skb);600			return NETDEV_TX_OK;601		}602	}603 604	tx_ring = priv->tx_ring[skb->queue_mapping];605 606	if (skb_is_gso(skb)) {607		if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) {608			netif_stop_subqueue(ndev, tx_ring->index);609			return NETDEV_TX_BUSY;610		}611 612		enetc_lock_mdio();613		count = enetc_map_tx_tso_buffs(tx_ring, skb);614		enetc_unlock_mdio();615	} else {616		if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS))617			if (unlikely(skb_linearize(skb)))618				goto drop_packet_err;619 620		count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */621		if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) {622			netif_stop_subqueue(ndev, tx_ring->index);623			return NETDEV_TX_BUSY;624		}625 626		if (skb->ip_summed == CHECKSUM_PARTIAL) {627			err = skb_checksum_help(skb);628			if (err)629				goto drop_packet_err;630		}631		enetc_lock_mdio();632		count = enetc_map_tx_buffs(tx_ring, skb);633		enetc_unlock_mdio();634	}635 636	if (unlikely(!count))637		goto drop_packet_err;638 639	if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED)640		netif_stop_subqueue(ndev, tx_ring->index);641 642	return NETDEV_TX_OK;643 644drop_packet_err:645	dev_kfree_skb_any(skb);646	return NETDEV_TX_OK;647}648 649netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)650{651	struct enetc_ndev_priv *priv = netdev_priv(ndev);652	u8 udp, msgtype, twostep;653	u16 offset1, offset2;654 655	/* Mark tx timestamp type on skb->cb[0] if requires */656	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&657	    (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) {658		skb->cb[0] = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK;659	} else {660		skb->cb[0] = 0;661	}662 663	/* Fall back to two-step timestamp if not one-step Sync packet */664	if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {665		if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep,666				    &offset1, &offset2) ||667		    msgtype != PTP_MSGTYPE_SYNC || twostep != 0)668			skb->cb[0] = ENETC_F_TX_TSTAMP;669	}670 671	return enetc_start_xmit(skb, ndev);672}673EXPORT_SYMBOL_GPL(enetc_xmit);674 675static irqreturn_t enetc_msix(int irq, void *data)676{677	struct enetc_int_vector	*v = data;678	int i;679 680	enetc_lock_mdio();681 682	/* disable interrupts */683	enetc_wr_reg_hot(v->rbier, 0);684	enetc_wr_reg_hot(v->ricr1, v->rx_ictt);685 686	for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)687		enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0);688 689	enetc_unlock_mdio();690 691	napi_schedule(&v->napi);692 693	return IRQ_HANDLED;694}695 696static void enetc_rx_dim_work(struct work_struct *w)697{698	struct dim *dim = container_of(w, struct dim, work);699	struct dim_cq_moder moder =700		net_dim_get_rx_moderation(dim->mode, dim->profile_ix);701	struct enetc_int_vector	*v =702		container_of(dim, struct enetc_int_vector, rx_dim);703 704	v->rx_ictt = enetc_usecs_to_cycles(moder.usec);705	dim->state = DIM_START_MEASURE;706}707 708static void enetc_rx_net_dim(struct enetc_int_vector *v)709{710	struct dim_sample dim_sample = {};711 712	v->comp_cnt++;713 714	if (!v->rx_napi_work)715		return;716 717	dim_update_sample(v->comp_cnt,718			  v->rx_ring.stats.packets,719			  v->rx_ring.stats.bytes,720			  &dim_sample);721	net_dim(&v->rx_dim, dim_sample);722}723 724static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)725{726	int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;727 728	return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;729}730 731static bool enetc_page_reusable(struct page *page)732{733	return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1);734}735 736static void enetc_reuse_page(struct enetc_bdr *rx_ring,737			     struct enetc_rx_swbd *old)738{739	struct enetc_rx_swbd *new;740 741	new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];742 743	/* next buf that may reuse a page */744	enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);745 746	/* copy page reference */747	*new = *old;748}749 750static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,751				u64 *tstamp)752{753	u32 lo, hi, tstamp_lo;754 755	lo = enetc_rd_hot(hw, ENETC_SICTR0);756	hi = enetc_rd_hot(hw, ENETC_SICTR1);757	tstamp_lo = le32_to_cpu(txbd->wb.tstamp);758	if (lo <= tstamp_lo)759		hi -= 1;760	*tstamp = (u64)hi << 32 | tstamp_lo;761}762 763static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)764{765	struct skb_shared_hwtstamps shhwtstamps;766 767	if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {768		memset(&shhwtstamps, 0, sizeof(shhwtstamps));769		shhwtstamps.hwtstamp = ns_to_ktime(tstamp);770		skb_txtime_consumed(skb);771		skb_tstamp_tx(skb, &shhwtstamps);772	}773}774 775static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring,776				      struct enetc_tx_swbd *tx_swbd)777{778	struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);779	struct enetc_rx_swbd rx_swbd = {780		.dma = tx_swbd->dma,781		.page = tx_swbd->page,782		.page_offset = tx_swbd->page_offset,783		.dir = tx_swbd->dir,784		.len = tx_swbd->len,785	};786	struct enetc_bdr *rx_ring;787 788	rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring);789 790	if (likely(enetc_swbd_unused(rx_ring))) {791		enetc_reuse_page(rx_ring, &rx_swbd);792 793		/* sync for use by the device */794		dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma,795						 rx_swbd.page_offset,796						 ENETC_RXB_DMA_SIZE_XDP,797						 rx_swbd.dir);798 799		rx_ring->stats.recycles++;800	} else {801		/* RX ring is already full, we need to unmap and free the802		 * page, since there's nothing useful we can do with it.803		 */804		rx_ring->stats.recycle_failures++;805 806		dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE,807			       rx_swbd.dir);808		__free_page(rx_swbd.page);809	}810 811	rx_ring->xdp.xdp_tx_in_flight--;812}813 814static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)815{816	int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0;817	struct net_device *ndev = tx_ring->ndev;818	struct enetc_ndev_priv *priv = netdev_priv(ndev);819	struct enetc_tx_swbd *tx_swbd;820	int i, bds_to_clean;821	bool do_twostep_tstamp;822	u64 tstamp = 0;823 824	i = tx_ring->next_to_clean;825	tx_swbd = &tx_ring->tx_swbd[i];826 827	bds_to_clean = enetc_bd_ready_count(tx_ring, i);828 829	do_twostep_tstamp = false;830 831	while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) {832		struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);833		struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);834		bool is_eof = tx_swbd->is_eof;835 836		if (unlikely(tx_swbd->check_wb)) {837			union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);838 839			if (txbd->flags & ENETC_TXBD_FLAGS_W &&840			    tx_swbd->do_twostep_tstamp) {841				enetc_get_tx_tstamp(&priv->si->hw, txbd,842						    &tstamp);843				do_twostep_tstamp = true;844			}845 846			if (tx_swbd->qbv_en &&847			    txbd->wb.status & ENETC_TXBD_STATS_WIN)848				tx_win_drop++;849		}850 851		if (tx_swbd->is_xdp_tx)852			enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd);853		else if (likely(tx_swbd->dma))854			enetc_unmap_tx_buff(tx_ring, tx_swbd);855 856		if (xdp_frame) {857			xdp_return_frame(xdp_frame);858		} else if (skb) {859			if (unlikely(skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) {860				/* Start work to release lock for next one-step861				 * timestamping packet. And send one skb in862				 * tx_skbs queue if has.863				 */864				schedule_work(&priv->tx_onestep_tstamp);865			} else if (unlikely(do_twostep_tstamp)) {866				enetc_tstamp_tx(skb, tstamp);867				do_twostep_tstamp = false;868			}869			napi_consume_skb(skb, napi_budget);870		}871 872		tx_byte_cnt += tx_swbd->len;873		/* Scrub the swbd here so we don't have to do that874		 * when we reuse it during xmit875		 */876		memset(tx_swbd, 0, sizeof(*tx_swbd));877 878		bds_to_clean--;879		tx_swbd++;880		i++;881		if (unlikely(i == tx_ring->bd_count)) {882			i = 0;883			tx_swbd = tx_ring->tx_swbd;884		}885 886		/* BD iteration loop end */887		if (is_eof) {888			tx_frm_cnt++;889			/* re-arm interrupt source */890			enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) |891					 BIT(16 + tx_ring->index));892		}893 894		if (unlikely(!bds_to_clean))895			bds_to_clean = enetc_bd_ready_count(tx_ring, i);896	}897 898	tx_ring->next_to_clean = i;899	tx_ring->stats.packets += tx_frm_cnt;900	tx_ring->stats.bytes += tx_byte_cnt;901	tx_ring->stats.win_drop += tx_win_drop;902 903	if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) &&904		     __netif_subqueue_stopped(ndev, tx_ring->index) &&905		     !test_bit(ENETC_TX_DOWN, &priv->flags) &&906		     (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) {907		netif_wake_subqueue(ndev, tx_ring->index);908	}909 910	return tx_frm_cnt != ENETC_DEFAULT_TX_WORK;911}912 913static bool enetc_new_page(struct enetc_bdr *rx_ring,914			   struct enetc_rx_swbd *rx_swbd)915{916	bool xdp = !!(rx_ring->xdp.prog);917	struct page *page;918	dma_addr_t addr;919 920	page = dev_alloc_page();921	if (unlikely(!page))922		return false;923 924	/* For XDP_TX, we forgo dma_unmap -> dma_map */925	rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;926 927	addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir);928	if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {929		__free_page(page);930 931		return false;932	}933 934	rx_swbd->dma = addr;935	rx_swbd->page = page;936	rx_swbd->page_offset = rx_ring->buffer_offset;937 938	return true;939}940 941static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)942{943	struct enetc_rx_swbd *rx_swbd;944	union enetc_rx_bd *rxbd;945	int i, j;946 947	i = rx_ring->next_to_use;948	rx_swbd = &rx_ring->rx_swbd[i];949	rxbd = enetc_rxbd(rx_ring, i);950 951	for (j = 0; j < buff_cnt; j++) {952		/* try reuse page */953		if (unlikely(!rx_swbd->page)) {954			if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {955				rx_ring->stats.rx_alloc_errs++;956				break;957			}958		}959 960		/* update RxBD */961		rxbd->w.addr = cpu_to_le64(rx_swbd->dma +962					   rx_swbd->page_offset);963		/* clear 'R" as well */964		rxbd->r.lstatus = 0;965 966		enetc_rxbd_next(rx_ring, &rxbd, &i);967		rx_swbd = &rx_ring->rx_swbd[i];968	}969 970	if (likely(j)) {971		rx_ring->next_to_alloc = i; /* keep track from page reuse */972		rx_ring->next_to_use = i;973 974		/* update ENETC's consumer index */975		enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use);976	}977 978	return j;979}980 981static void enetc_get_rx_tstamp(struct net_device *ndev,982				union enetc_rx_bd *rxbd,983				struct sk_buff *skb)984{985	struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);986	struct enetc_ndev_priv *priv = netdev_priv(ndev);987	struct enetc_hw *hw = &priv->si->hw;988	u32 lo, hi, tstamp_lo;989	u64 tstamp;990 991	if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {992		lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0);993		hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1);994		rxbd = enetc_rxbd_ext(rxbd);995		tstamp_lo = le32_to_cpu(rxbd->ext.tstamp);996		if (lo <= tstamp_lo)997			hi -= 1;998 999		tstamp = (u64)hi << 32 | tstamp_lo;1000		memset(shhwtstamps, 0, sizeof(*shhwtstamps));1001		shhwtstamps->hwtstamp = ns_to_ktime(tstamp);1002	}1003}1004 1005static void enetc_get_offloads(struct enetc_bdr *rx_ring,1006			       union enetc_rx_bd *rxbd, struct sk_buff *skb)1007{1008	struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);1009 1010	/* TODO: hashing */1011	if (rx_ring->ndev->features & NETIF_F_RXCSUM) {1012		u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum);1013 1014		skb->csum = csum_unfold((__force __sum16)~htons(inet_csum));1015		skb->ip_summed = CHECKSUM_COMPLETE;1016	}1017 1018	if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) {1019		__be16 tpid = 0;1020 1021		switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) {1022		case 0:1023			tpid = htons(ETH_P_8021Q);1024			break;1025		case 1:1026			tpid = htons(ETH_P_8021AD);1027			break;1028		case 2:1029			tpid = htons(enetc_port_rd(&priv->si->hw,1030						   ENETC_PCVLANR1));1031			break;1032		case 3:1033			tpid = htons(enetc_port_rd(&priv->si->hw,1034						   ENETC_PCVLANR2));1035			break;1036		default:1037			break;1038		}1039 1040		__vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt));1041	}1042 1043	if (IS_ENABLED(CONFIG_FSL_ENETC_PTP_CLOCK) &&1044	    (priv->active_offloads & ENETC_F_RX_TSTAMP))1045		enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);1046}1047 1048/* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS,1049 * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL1050 * mapped buffers.1051 */1052static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,1053					       int i, u16 size)1054{1055	struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];1056 1057	dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,1058				      rx_swbd->page_offset,1059				      size, rx_swbd->dir);1060	return rx_swbd;1061}1062 1063/* Reuse the current page without performing half-page buffer flipping */1064static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,1065			      struct enetc_rx_swbd *rx_swbd)1066{1067	size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset;1068 1069	enetc_reuse_page(rx_ring, rx_swbd);1070 1071	dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,1072					 rx_swbd->page_offset,1073					 buffer_size, rx_swbd->dir);1074 1075	rx_swbd->page = NULL;1076}1077 1078/* Reuse the current page by performing half-page buffer flipping */1079static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring,1080			       struct enetc_rx_swbd *rx_swbd)1081{1082	if (likely(enetc_page_reusable(rx_swbd->page))) {1083		rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE;1084		page_ref_inc(rx_swbd->page);1085 1086		enetc_put_rx_buff(rx_ring, rx_swbd);1087	} else {1088		dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,1089			       rx_swbd->dir);1090		rx_swbd->page = NULL;1091	}1092}1093 1094static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,1095						int i, u16 size)1096{1097	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);1098	struct sk_buff *skb;1099	void *ba;1100 1101	ba = page_address(rx_swbd->page) + rx_swbd->page_offset;1102	skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE);1103	if (unlikely(!skb)) {1104		rx_ring->stats.rx_alloc_errs++;1105		return NULL;1106	}1107 1108	skb_reserve(skb, rx_ring->buffer_offset);1109	__skb_put(skb, size);1110 1111	enetc_flip_rx_buff(rx_ring, rx_swbd);1112 1113	return skb;1114}1115 1116static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,1117				     u16 size, struct sk_buff *skb)1118{1119	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);1120 1121	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page,1122			rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE);1123 1124	enetc_flip_rx_buff(rx_ring, rx_swbd);1125}1126 1127static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring,1128					      u32 bd_status,1129					      union enetc_rx_bd **rxbd, int *i)1130{1131	if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))))1132		return false;1133 1134	enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);1135	enetc_rxbd_next(rx_ring, rxbd, i);1136 1137	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {1138		dma_rmb();1139		bd_status = le32_to_cpu((*rxbd)->r.lstatus);1140 1141		enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);1142		enetc_rxbd_next(rx_ring, rxbd, i);1143	}1144 1145	rx_ring->ndev->stats.rx_dropped++;1146	rx_ring->ndev->stats.rx_errors++;1147 1148	return true;1149}1150 1151static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring,1152				       u32 bd_status, union enetc_rx_bd **rxbd,1153				       int *i, int *cleaned_cnt, int buffer_size)1154{1155	struct sk_buff *skb;1156	u16 size;1157 1158	size = le16_to_cpu((*rxbd)->r.buf_len);1159	skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size);1160	if (!skb)1161		return NULL;1162 1163	enetc_get_offloads(rx_ring, *rxbd, skb);1164 1165	(*cleaned_cnt)++;1166 1167	enetc_rxbd_next(rx_ring, rxbd, i);1168 1169	/* not last BD in frame? */1170	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {1171		bd_status = le32_to_cpu((*rxbd)->r.lstatus);1172		size = buffer_size;1173 1174		if (bd_status & ENETC_RXBD_LSTATUS_F) {1175			dma_rmb();1176			size = le16_to_cpu((*rxbd)->r.buf_len);1177		}1178 1179		enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb);1180 1181		(*cleaned_cnt)++;1182 1183		enetc_rxbd_next(rx_ring, rxbd, i);1184	}1185 1186	skb_record_rx_queue(skb, rx_ring->index);1187	skb->protocol = eth_type_trans(skb, rx_ring->ndev);1188 1189	return skb;1190}1191 1192#define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */1193 1194static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,1195			       struct napi_struct *napi, int work_limit)1196{1197	int rx_frm_cnt = 0, rx_byte_cnt = 0;1198	int cleaned_cnt, i;1199 1200	cleaned_cnt = enetc_bd_unused(rx_ring);1201	/* next descriptor to process */1202	i = rx_ring->next_to_clean;1203 1204	while (likely(rx_frm_cnt < work_limit)) {1205		union enetc_rx_bd *rxbd;1206		struct sk_buff *skb;1207		u32 bd_status;1208 1209		if (cleaned_cnt >= ENETC_RXBD_BUNDLE)1210			cleaned_cnt -= enetc_refill_rx_ring(rx_ring,1211							    cleaned_cnt);1212 1213		rxbd = enetc_rxbd(rx_ring, i);1214		bd_status = le32_to_cpu(rxbd->r.lstatus);1215		if (!bd_status)1216			break;1217 1218		enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));1219		dma_rmb(); /* for reading other rxbd fields */1220 1221		if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,1222						      &rxbd, &i))1223			break;1224 1225		skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i,1226				      &cleaned_cnt, ENETC_RXB_DMA_SIZE);1227		if (!skb)1228			break;1229 1230		/* When set, the outer VLAN header is extracted and reported1231		 * in the receive buffer descriptor. So rx_byte_cnt should1232		 * add the length of the extracted VLAN header.1233		 */1234		if (bd_status & ENETC_RXBD_FLAG_VLAN)1235			rx_byte_cnt += VLAN_HLEN;1236		rx_byte_cnt += skb->len + ETH_HLEN;1237		rx_frm_cnt++;1238 1239		napi_gro_receive(napi, skb);1240	}1241 1242	rx_ring->next_to_clean = i;1243 1244	rx_ring->stats.packets += rx_frm_cnt;1245	rx_ring->stats.bytes += rx_byte_cnt;1246 1247	return rx_frm_cnt;1248}1249 1250static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i,1251				  struct enetc_tx_swbd *tx_swbd,1252				  int frm_len)1253{1254	union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);1255 1256	prefetchw(txbd);1257 1258	enetc_clear_tx_bd(txbd);1259	txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset);1260	txbd->buf_len = cpu_to_le16(tx_swbd->len);1261	txbd->frm_len = cpu_to_le16(frm_len);1262 1263	memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd));1264}1265 1266/* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer1267 * descriptors.1268 */1269static bool enetc_xdp_tx(struct enetc_bdr *tx_ring,1270			 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd)1271{1272	struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr;1273	int i, k, frm_len = tmp_tx_swbd->len;1274 1275	if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd)))1276		return false;1277 1278	while (unlikely(!tmp_tx_swbd->is_eof)) {1279		tmp_tx_swbd++;1280		frm_len += tmp_tx_swbd->len;1281	}1282 1283	i = tx_ring->next_to_use;1284 1285	for (k = 0; k < num_tx_swbd; k++) {1286		struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k];1287 1288		enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len);1289 1290		/* last BD needs 'F' bit set */1291		if (xdp_tx_swbd->is_eof) {1292			union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);1293 1294			txbd->flags = ENETC_TXBD_FLAGS_F;1295		}1296 1297		enetc_bdr_idx_inc(tx_ring, &i);1298	}1299 1300	tx_ring->next_to_use = i;1301 1302	return true;1303}1304 1305static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring,1306					  struct enetc_tx_swbd *xdp_tx_arr,1307					  struct xdp_frame *xdp_frame)1308{1309	struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0];1310	struct skb_shared_info *shinfo;1311	void *data = xdp_frame->data;1312	int len = xdp_frame->len;1313	skb_frag_t *frag;1314	dma_addr_t dma;1315	unsigned int f;1316	int n = 0;1317 1318	dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);1319	if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {1320		netdev_err(tx_ring->ndev, "DMA map error\n");1321		return -1;1322	}1323 1324	xdp_tx_swbd->dma = dma;1325	xdp_tx_swbd->dir = DMA_TO_DEVICE;1326	xdp_tx_swbd->len = len;1327	xdp_tx_swbd->is_xdp_redirect = true;1328	xdp_tx_swbd->is_eof = false;1329	xdp_tx_swbd->xdp_frame = NULL;1330 1331	n++;1332 1333	if (!xdp_frame_has_frags(xdp_frame))1334		goto out;1335 1336	xdp_tx_swbd = &xdp_tx_arr[n];1337 1338	shinfo = xdp_get_shared_info_from_frame(xdp_frame);1339 1340	for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags;1341	     f++, frag++) {1342		data = skb_frag_address(frag);1343		len = skb_frag_size(frag);1344 1345		dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);1346		if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {1347			/* Undo the DMA mapping for all fragments */1348			while (--n >= 0)1349				enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]);1350 1351			netdev_err(tx_ring->ndev, "DMA map error\n");1352			return -1;1353		}1354 1355		xdp_tx_swbd->dma = dma;1356		xdp_tx_swbd->dir = DMA_TO_DEVICE;1357		xdp_tx_swbd->len = len;1358		xdp_tx_swbd->is_xdp_redirect = true;1359		xdp_tx_swbd->is_eof = false;1360		xdp_tx_swbd->xdp_frame = NULL;1361 1362		n++;1363		xdp_tx_swbd = &xdp_tx_arr[n];1364	}1365out:1366	xdp_tx_arr[n - 1].is_eof = true;1367	xdp_tx_arr[n - 1].xdp_frame = xdp_frame;1368 1369	return n;1370}1371 1372int enetc_xdp_xmit(struct net_device *ndev, int num_frames,1373		   struct xdp_frame **frames, u32 flags)1374{1375	struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0};1376	struct enetc_ndev_priv *priv = netdev_priv(ndev);1377	struct enetc_bdr *tx_ring;1378	int xdp_tx_bd_cnt, i, k;1379	int xdp_tx_frm_cnt = 0;1380 1381	if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags)))1382		return -ENETDOWN;1383 1384	enetc_lock_mdio();1385 1386	tx_ring = priv->xdp_tx_ring[smp_processor_id()];1387 1388	prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use));1389 1390	for (k = 0; k < num_frames; k++) {1391		xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring,1392							       xdp_redirect_arr,1393							       frames[k]);1394		if (unlikely(xdp_tx_bd_cnt < 0))1395			break;1396 1397		if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr,1398					   xdp_tx_bd_cnt))) {1399			for (i = 0; i < xdp_tx_bd_cnt; i++)1400				enetc_unmap_tx_buff(tx_ring,1401						    &xdp_redirect_arr[i]);1402			tx_ring->stats.xdp_tx_drops++;1403			break;1404		}1405 1406		xdp_tx_frm_cnt++;1407	}1408 1409	if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt))1410		enetc_update_tx_ring_tail(tx_ring);1411 1412	tx_ring->stats.xdp_tx += xdp_tx_frm_cnt;1413 1414	enetc_unlock_mdio();1415 1416	return xdp_tx_frm_cnt;1417}1418EXPORT_SYMBOL_GPL(enetc_xdp_xmit);1419 1420static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,1421				     struct xdp_buff *xdp_buff, u16 size)1422{1423	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);1424	void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset;1425 1426	/* To be used for XDP_TX */1427	rx_swbd->len = size;1428 1429	xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset,1430			 rx_ring->buffer_offset, size, false);1431}1432 1433static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,1434				     u16 size, struct xdp_buff *xdp_buff)1435{1436	struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff);1437	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);1438	skb_frag_t *frag;1439 1440	/* To be used for XDP_TX */1441	rx_swbd->len = size;1442 1443	if (!xdp_buff_has_frags(xdp_buff)) {1444		xdp_buff_set_frags_flag(xdp_buff);1445		shinfo->xdp_frags_size = size;1446		shinfo->nr_frags = 0;1447	} else {1448		shinfo->xdp_frags_size += size;1449	}1450 1451	if (page_is_pfmemalloc(rx_swbd->page))1452		xdp_buff_set_frag_pfmemalloc(xdp_buff);1453 1454	frag = &shinfo->frags[shinfo->nr_frags];1455	skb_frag_fill_page_desc(frag, rx_swbd->page, rx_swbd->page_offset,1456				size);1457 1458	shinfo->nr_frags++;1459}1460 1461static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status,1462				 union enetc_rx_bd **rxbd, int *i,1463				 int *cleaned_cnt, struct xdp_buff *xdp_buff)1464{1465	u16 size = le16_to_cpu((*rxbd)->r.buf_len);1466 1467	xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq);1468 1469	enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size);1470	(*cleaned_cnt)++;1471	enetc_rxbd_next(rx_ring, rxbd, i);1472 1473	/* not last BD in frame? */1474	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {1475		bd_status = le32_to_cpu((*rxbd)->r.lstatus);1476		size = ENETC_RXB_DMA_SIZE_XDP;1477 1478		if (bd_status & ENETC_RXBD_LSTATUS_F) {1479			dma_rmb();1480			size = le16_to_cpu((*rxbd)->r.buf_len);1481		}1482 1483		enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff);1484		(*cleaned_cnt)++;1485		enetc_rxbd_next(rx_ring, rxbd, i);1486	}1487}1488 1489/* Convert RX buffer descriptors to TX buffer descriptors. These will be1490 * recycled back into the RX ring in enetc_clean_tx_ring.1491 */1492static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr,1493					struct enetc_bdr *rx_ring,1494					int rx_ring_first, int rx_ring_last)1495{1496	int n = 0;1497 1498	for (; rx_ring_first != rx_ring_last;1499	     n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) {1500		struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first];1501		struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n];1502 1503		/* No need to dma_map, we already have DMA_BIDIRECTIONAL */1504		tx_swbd->dma = rx_swbd->dma;1505		tx_swbd->dir = rx_swbd->dir;1506		tx_swbd->page = rx_swbd->page;1507		tx_swbd->page_offset = rx_swbd->page_offset;1508		tx_swbd->len = rx_swbd->len;1509		tx_swbd->is_dma_page = true;1510		tx_swbd->is_xdp_tx = true;1511		tx_swbd->is_eof = false;1512	}1513 1514	/* We rely on caller providing an rx_ring_last > rx_ring_first */1515	xdp_tx_arr[n - 1].is_eof = true;1516 1517	return n;1518}1519 1520static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first,1521			   int rx_ring_last)1522{1523	while (rx_ring_first != rx_ring_last) {1524		enetc_put_rx_buff(rx_ring,1525				  &rx_ring->rx_swbd[rx_ring_first]);1526		enetc_bdr_idx_inc(rx_ring, &rx_ring_first);1527	}1528}1529 1530static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring,1531				   struct napi_struct *napi, int work_limit,1532				   struct bpf_prog *prog)1533{1534	int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0;1535	struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0};1536	struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);1537	int rx_frm_cnt = 0, rx_byte_cnt = 0;1538	struct enetc_bdr *tx_ring;1539	int cleaned_cnt, i;1540	u32 xdp_act;1541 1542	cleaned_cnt = enetc_bd_unused(rx_ring);1543	/* next descriptor to process */1544	i = rx_ring->next_to_clean;1545 1546	while (likely(rx_frm_cnt < work_limit)) {1547		union enetc_rx_bd *rxbd, *orig_rxbd;1548		int orig_i, orig_cleaned_cnt;1549		struct xdp_buff xdp_buff;1550		struct sk_buff *skb;1551		u32 bd_status;1552		int err;1553 1554		rxbd = enetc_rxbd(rx_ring, i);1555		bd_status = le32_to_cpu(rxbd->r.lstatus);1556		if (!bd_status)1557			break;1558 1559		enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));1560		dma_rmb(); /* for reading other rxbd fields */1561 1562		if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,1563						      &rxbd, &i))1564			break;1565 1566		orig_rxbd = rxbd;1567		orig_cleaned_cnt = cleaned_cnt;1568		orig_i = i;1569 1570		enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i,1571				     &cleaned_cnt, &xdp_buff);1572 1573		/* When set, the outer VLAN header is extracted and reported1574		 * in the receive buffer descriptor. So rx_byte_cnt should1575		 * add the length of the extracted VLAN header.1576		 */1577		if (bd_status & ENETC_RXBD_FLAG_VLAN)1578			rx_byte_cnt += VLAN_HLEN;1579		rx_byte_cnt += xdp_get_buff_len(&xdp_buff);1580 1581		xdp_act = bpf_prog_run_xdp(prog, &xdp_buff);1582 1583		switch (xdp_act) {1584		default:1585			bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act);1586			fallthrough;1587		case XDP_ABORTED:1588			trace_xdp_exception(rx_ring->ndev, prog, xdp_act);1589			fallthrough;1590		case XDP_DROP:1591			enetc_xdp_drop(rx_ring, orig_i, i);1592			rx_ring->stats.xdp_drops++;1593			break;1594		case XDP_PASS:1595			rxbd = orig_rxbd;1596			cleaned_cnt = orig_cleaned_cnt;1597			i = orig_i;1598 1599			skb = enetc_build_skb(rx_ring, bd_status, &rxbd,1600					      &i, &cleaned_cnt,1601					      ENETC_RXB_DMA_SIZE_XDP);1602			if (unlikely(!skb))1603				goto out;1604 1605			napi_gro_receive(napi, skb);1606			break;1607		case XDP_TX:1608			tx_ring = priv->xdp_tx_ring[rx_ring->index];1609			if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags))) {1610				enetc_xdp_drop(rx_ring, orig_i, i);1611				tx_ring->stats.xdp_tx_drops++;1612				break;1613			}1614 1615			xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr,1616								     rx_ring,1617								     orig_i, i);1618 1619			if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) {1620				enetc_xdp_drop(rx_ring, orig_i, i);1621				tx_ring->stats.xdp_tx_drops++;1622			} else {1623				tx_ring->stats.xdp_tx += xdp_tx_bd_cnt;1624				rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt;1625				xdp_tx_frm_cnt++;1626				/* The XDP_TX enqueue was successful, so we1627				 * need to scrub the RX software BDs because1628				 * the ownership of the buffers no longer1629				 * belongs to the RX ring, and we must prevent1630				 * enetc_refill_rx_ring() from reusing1631				 * rx_swbd->page.1632				 */1633				while (orig_i != i) {1634					rx_ring->rx_swbd[orig_i].page = NULL;1635					enetc_bdr_idx_inc(rx_ring, &orig_i);1636				}1637			}1638			break;1639		case XDP_REDIRECT:1640			err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog);1641			if (unlikely(err)) {1642				enetc_xdp_drop(rx_ring, orig_i, i);1643				rx_ring->stats.xdp_redirect_failures++;1644			} else {1645				while (orig_i != i) {1646					enetc_flip_rx_buff(rx_ring,1647							   &rx_ring->rx_swbd[orig_i]);1648					enetc_bdr_idx_inc(rx_ring, &orig_i);1649				}1650				xdp_redirect_frm_cnt++;1651				rx_ring->stats.xdp_redirect++;1652			}1653		}1654 1655		rx_frm_cnt++;1656	}1657 1658out:1659	rx_ring->next_to_clean = i;1660 1661	rx_ring->stats.packets += rx_frm_cnt;1662	rx_ring->stats.bytes += rx_byte_cnt;1663 1664	if (xdp_redirect_frm_cnt)1665		xdp_do_flush();1666 1667	if (xdp_tx_frm_cnt)1668		enetc_update_tx_ring_tail(tx_ring);1669 1670	if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight)1671		enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) -1672				     rx_ring->xdp.xdp_tx_in_flight);1673 1674	return rx_frm_cnt;1675}1676 1677static int enetc_poll(struct napi_struct *napi, int budget)1678{1679	struct enetc_int_vector1680		*v = container_of(napi, struct enetc_int_vector, napi);1681	struct enetc_bdr *rx_ring = &v->rx_ring;1682	struct bpf_prog *prog;1683	bool complete = true;1684	int work_done;1685	int i;1686 1687	enetc_lock_mdio();1688 1689	for (i = 0; i < v->count_tx_rings; i++)1690		if (!enetc_clean_tx_ring(&v->tx_ring[i], budget))1691			complete = false;1692 1693	prog = rx_ring->xdp.prog;1694	if (prog)1695		work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog);1696	else1697		work_done = enetc_clean_rx_ring(rx_ring, napi, budget);1698	if (work_done == budget)1699		complete = false;1700	if (work_done)1701		v->rx_napi_work = true;1702 1703	if (!complete) {1704		enetc_unlock_mdio();1705		return budget;1706	}1707 1708	napi_complete_done(napi, work_done);1709 1710	if (likely(v->rx_dim_en))1711		enetc_rx_net_dim(v);1712 1713	v->rx_napi_work = false;1714 1715	/* enable interrupts */1716	enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE);1717 1718	for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)1719		enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i),1720				 ENETC_TBIER_TXTIE);1721 1722	enetc_unlock_mdio();1723 1724	return work_done;1725}1726 1727/* Probing and Init */1728#define ENETC_MAX_RFS_SIZE 641729void enetc_get_si_caps(struct enetc_si *si)1730{1731	struct enetc_hw *hw = &si->hw;1732	u32 val;1733 1734	/* find out how many of various resources we have to work with */1735	val = enetc_rd(hw, ENETC_SICAPR0);1736	si->num_rx_rings = (val >> 16) & 0xff;1737	si->num_tx_rings = val & 0xff;1738 1739	val = enetc_rd(hw, ENETC_SIRFSCAPR);1740	si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val);1741	si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE);1742 1743	si->num_rss = 0;1744	val = enetc_rd(hw, ENETC_SIPCAPR0);1745	if (val & ENETC_SIPCAPR0_RSS) {1746		u32 rss;1747 1748		rss = enetc_rd(hw, ENETC_SIRSSCAPR);1749		si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss);1750	}1751 1752	if (val & ENETC_SIPCAPR0_QBV)1753		si->hw_features |= ENETC_SI_F_QBV;1754 1755	if (val & ENETC_SIPCAPR0_QBU)1756		si->hw_features |= ENETC_SI_F_QBU;1757 1758	if (val & ENETC_SIPCAPR0_PSFP)1759		si->hw_features |= ENETC_SI_F_PSFP;1760}1761EXPORT_SYMBOL_GPL(enetc_get_si_caps);1762 1763static int enetc_dma_alloc_bdr(struct enetc_bdr_resource *res)1764{1765	size_t bd_base_size = res->bd_count * res->bd_size;1766 1767	res->bd_base = dma_alloc_coherent(res->dev, bd_base_size,1768					  &res->bd_dma_base, GFP_KERNEL);1769	if (!res->bd_base)1770		return -ENOMEM;1771 1772	/* h/w requires 128B alignment */1773	if (!IS_ALIGNED(res->bd_dma_base, 128)) {1774		dma_free_coherent(res->dev, bd_base_size, res->bd_base,1775				  res->bd_dma_base);1776		return -EINVAL;1777	}1778 1779	return 0;1780}1781 1782static void enetc_dma_free_bdr(const struct enetc_bdr_resource *res)1783{1784	size_t bd_base_size = res->bd_count * res->bd_size;1785 1786	dma_free_coherent(res->dev, bd_base_size, res->bd_base,1787			  res->bd_dma_base);1788}1789 1790static int enetc_alloc_tx_resource(struct enetc_bdr_resource *res,1791				   struct device *dev, size_t bd_count)1792{1793	int err;1794 1795	res->dev = dev;1796	res->bd_count = bd_count;1797	res->bd_size = sizeof(union enetc_tx_bd);1798 1799	res->tx_swbd = vcalloc(bd_count, sizeof(*res->tx_swbd));1800	if (!res->tx_swbd)1801		return -ENOMEM;1802 1803	err = enetc_dma_alloc_bdr(res);1804	if (err)1805		goto err_alloc_bdr;1806 1807	res->tso_headers = dma_alloc_coherent(dev, bd_count * TSO_HEADER_SIZE,1808					      &res->tso_headers_dma,1809					      GFP_KERNEL);1810	if (!res->tso_headers) {1811		err = -ENOMEM;1812		goto err_alloc_tso;1813	}1814 1815	return 0;1816 1817err_alloc_tso:1818	enetc_dma_free_bdr(res);1819err_alloc_bdr:1820	vfree(res->tx_swbd);1821	res->tx_swbd = NULL;1822 1823	return err;1824}1825 1826static void enetc_free_tx_resource(const struct enetc_bdr_resource *res)1827{1828	dma_free_coherent(res->dev, res->bd_count * TSO_HEADER_SIZE,1829			  res->tso_headers, res->tso_headers_dma);1830	enetc_dma_free_bdr(res);1831	vfree(res->tx_swbd);1832}1833 1834static struct enetc_bdr_resource *1835enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)1836{1837	struct enetc_bdr_resource *tx_res;1838	int i, err;1839 1840	tx_res = kcalloc(priv->num_tx_rings, sizeof(*tx_res), GFP_KERNEL);1841	if (!tx_res)1842		return ERR_PTR(-ENOMEM);1843 1844	for (i = 0; i < priv->num_tx_rings; i++) {1845		struct enetc_bdr *tx_ring = priv->tx_ring[i];1846 1847		err = enetc_alloc_tx_resource(&tx_res[i], tx_ring->dev,1848					      tx_ring->bd_count);1849		if (err)1850			goto fail;1851	}1852 1853	return tx_res;1854 1855fail:1856	while (i-- > 0)1857		enetc_free_tx_resource(&tx_res[i]);1858 1859	kfree(tx_res);1860 1861	return ERR_PTR(err);1862}1863 1864static void enetc_free_tx_resources(const struct enetc_bdr_resource *tx_res,1865				    size_t num_resources)1866{1867	size_t i;1868 1869	for (i = 0; i < num_resources; i++)1870		enetc_free_tx_resource(&tx_res[i]);1871 1872	kfree(tx_res);1873}1874 1875static int enetc_alloc_rx_resource(struct enetc_bdr_resource *res,1876				   struct device *dev, size_t bd_count,1877				   bool extended)1878{1879	int err;1880 1881	res->dev = dev;1882	res->bd_count = bd_count;1883	res->bd_size = sizeof(union enetc_rx_bd);1884	if (extended)1885		res->bd_size *= 2;1886 1887	res->rx_swbd = vcalloc(bd_count, sizeof(struct enetc_rx_swbd));1888	if (!res->rx_swbd)1889		return -ENOMEM;1890 1891	err = enetc_dma_alloc_bdr(res);1892	if (err) {1893		vfree(res->rx_swbd);1894		return err;1895	}1896 1897	return 0;1898}1899 1900static void enetc_free_rx_resource(const struct enetc_bdr_resource *res)1901{1902	enetc_dma_free_bdr(res);1903	vfree(res->rx_swbd);1904}1905 1906static struct enetc_bdr_resource *1907enetc_alloc_rx_resources(struct enetc_ndev_priv *priv, bool extended)1908{1909	struct enetc_bdr_resource *rx_res;1910	int i, err;1911 1912	rx_res = kcalloc(priv->num_rx_rings, sizeof(*rx_res), GFP_KERNEL);1913	if (!rx_res)1914		return ERR_PTR(-ENOMEM);1915 1916	for (i = 0; i < priv->num_rx_rings; i++) {1917		struct enetc_bdr *rx_ring = priv->rx_ring[i];1918 1919		err = enetc_alloc_rx_resource(&rx_res[i], rx_ring->dev,1920					      rx_ring->bd_count, extended);1921		if (err)1922			goto fail;1923	}1924 1925	return rx_res;1926 1927fail:1928	while (i-- > 0)1929		enetc_free_rx_resource(&rx_res[i]);1930 1931	kfree(rx_res);1932 1933	return ERR_PTR(err);1934}1935 1936static void enetc_free_rx_resources(const struct enetc_bdr_resource *rx_res,1937				    size_t num_resources)1938{1939	size_t i;1940 1941	for (i = 0; i < num_resources; i++)1942		enetc_free_rx_resource(&rx_res[i]);1943 1944	kfree(rx_res);1945}1946 1947static void enetc_assign_tx_resource(struct enetc_bdr *tx_ring,1948				     const struct enetc_bdr_resource *res)1949{1950	tx_ring->bd_base = res ? res->bd_base : NULL;1951	tx_ring->bd_dma_base = res ? res->bd_dma_base : 0;1952	tx_ring->tx_swbd = res ? res->tx_swbd : NULL;1953	tx_ring->tso_headers = res ? res->tso_headers : NULL;1954	tx_ring->tso_headers_dma = res ? res->tso_headers_dma : 0;1955}1956 1957static void enetc_assign_rx_resource(struct enetc_bdr *rx_ring,1958				     const struct enetc_bdr_resource *res)1959{1960	rx_ring->bd_base = res ? res->bd_base : NULL;1961	rx_ring->bd_dma_base = res ? res->bd_dma_base : 0;1962	rx_ring->rx_swbd = res ? res->rx_swbd : NULL;1963}1964 1965static void enetc_assign_tx_resources(struct enetc_ndev_priv *priv,1966				      const struct enetc_bdr_resource *res)1967{1968	int i;1969 1970	if (priv->tx_res)1971		enetc_free_tx_resources(priv->tx_res, priv->num_tx_rings);1972 1973	for (i = 0; i < priv->num_tx_rings; i++) {1974		enetc_assign_tx_resource(priv->tx_ring[i],1975					 res ? &res[i] : NULL);1976	}1977 1978	priv->tx_res = res;1979}1980 1981static void enetc_assign_rx_resources(struct enetc_ndev_priv *priv,1982				      const struct enetc_bdr_resource *res)1983{1984	int i;1985 1986	if (priv->rx_res)1987		enetc_free_rx_resources(priv->rx_res, priv->num_rx_rings);1988 1989	for (i = 0; i < priv->num_rx_rings; i++) {1990		enetc_assign_rx_resource(priv->rx_ring[i],1991					 res ? &res[i] : NULL);1992	}1993 1994	priv->rx_res = res;1995}1996 1997static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)1998{1999	int i;2000 2001	for (i = 0; i < tx_ring->bd_count; i++) {2002		struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];2003 2004		enetc_free_tx_frame(tx_ring, tx_swbd);2005	}2006}2007 2008static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)2009{2010	int i;2011 2012	for (i = 0; i < rx_ring->bd_count; i++) {2013		struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];2014 2015		if (!rx_swbd->page)2016			continue;2017 2018		dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,2019			       rx_swbd->dir);2020		__free_page(rx_swbd->page);2021		rx_swbd->page = NULL;2022	}2023}2024 2025static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)2026{2027	int i;2028 2029	for (i = 0; i < priv->num_rx_rings; i++)2030		enetc_free_rx_ring(priv->rx_ring[i]);2031 2032	for (i = 0; i < priv->num_tx_rings; i++)2033		enetc_free_tx_ring(priv->tx_ring[i]);2034}2035 2036static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)2037{2038	int *rss_table;2039	int i;2040 2041	rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL);2042	if (!rss_table)2043		return -ENOMEM;2044 2045	/* Set up RSS table defaults */2046	for (i = 0; i < si->num_rss; i++)2047		rss_table[i] = i % num_groups;2048 2049	enetc_set_rss_table(si, rss_table, si->num_rss);2050 2051	kfree(rss_table);2052 2053	return 0;2054}2055 2056int enetc_configure_si(struct enetc_ndev_priv *priv)2057{2058	struct enetc_si *si = priv->si;2059	struct enetc_hw *hw = &si->hw;2060	int err;2061 2062	/* set SI cache attributes */2063	enetc_wr(hw, ENETC_SICAR0,2064		 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);2065	enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI);2066	/* enable SI */2067	enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN);2068 2069	if (si->num_rss) {2070		err = enetc_setup_default_rss_table(si, priv->num_rx_rings);2071		if (err)2072			return err;2073	}2074 2075	return 0;2076}2077EXPORT_SYMBOL_GPL(enetc_configure_si);2078 2079void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)2080{2081	struct enetc_si *si = priv->si;2082	int cpus = num_online_cpus();2083 2084	priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE;2085	priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE;2086 2087	/* Enable all available TX rings in order to configure as many2088	 * priorities as possible, when needed.2089	 * TODO: Make # of TX rings run-time configurable2090	 */2091	priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);2092	priv->num_tx_rings = si->num_tx_rings;2093	priv->bdr_int_num = cpus;2094	priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL;2095	priv->tx_ictt = ENETC_TXIC_TIMETHR;2096}2097EXPORT_SYMBOL_GPL(enetc_init_si_rings_params);2098 2099int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)2100{2101	struct enetc_si *si = priv->si;2102 2103	priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules),2104				  GFP_KERNEL);2105	if (!priv->cls_rules)2106		return -ENOMEM;2107 2108	return 0;2109}2110EXPORT_SYMBOL_GPL(enetc_alloc_si_resources);2111 2112void enetc_free_si_resources(struct enetc_ndev_priv *priv)2113{2114	kfree(priv->cls_rules);2115}2116EXPORT_SYMBOL_GPL(enetc_free_si_resources);2117 2118static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)2119{2120	int idx = tx_ring->index;2121	u32 tbmr;2122 2123	enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,2124		       lower_32_bits(tx_ring->bd_dma_base));2125 2126	enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,2127		       upper_32_bits(tx_ring->bd_dma_base));2128 2129	WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */2130	enetc_txbdr_wr(hw, idx, ENETC_TBLENR,2131		       ENETC_RTBLENR_LEN(tx_ring->bd_count));2132 2133	/* clearing PI/CI registers for Tx not supported, adjust sw indexes */2134	tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR);2135	tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);2136 2137	/* enable Tx ints by setting pkt thr to 1 */2138	enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1);2139 2140	tbmr = ENETC_TBMR_SET_PRIO(tx_ring->prio);2141	if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)2142		tbmr |= ENETC_TBMR_VIH;2143 2144	/* enable ring */2145	enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);2146 2147	tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR);2148	tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR);2149	tx_ring->idr = hw->reg + ENETC_SITXIDR;2150}2151 2152static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring,2153			      bool extended)2154{2155	int idx = rx_ring->index;2156	u32 rbmr = 0;2157 2158	enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,2159		       lower_32_bits(rx_ring->bd_dma_base));2160 2161	enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,2162		       upper_32_bits(rx_ring->bd_dma_base));2163 2164	WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */2165	enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,2166		       ENETC_RTBLENR_LEN(rx_ring->bd_count));2167 2168	if (rx_ring->xdp.prog)2169		enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP);2170	else2171		enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE);2172 2173	/* Also prepare the consumer index in case page allocation never2174	 * succeeds. In that case, hardware will never advance producer index2175	 * to match consumer index, and will drop all frames.2176	 */2177	enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);2178	enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1);2179 2180	/* enable Rx ints by setting pkt thr to 1 */2181	enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1);2182 2183	rx_ring->ext_en = extended;2184	if (rx_ring->ext_en)2185		rbmr |= ENETC_RBMR_BDS;2186 2187	if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)2188		rbmr |= ENETC_RBMR_VTE;2189 2190	rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);2191	rx_ring->idr = hw->reg + ENETC_SIRXIDR;2192 2193	rx_ring->next_to_clean = 0;2194	rx_ring->next_to_use = 0;2195	rx_ring->next_to_alloc = 0;2196 2197	enetc_lock_mdio();2198	enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));2199	enetc_unlock_mdio();2200 2201	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);2202}2203 2204static void enetc_setup_bdrs(struct enetc_ndev_priv *priv, bool extended)2205{2206	struct enetc_hw *hw = &priv->si->hw;2207	int i;2208 2209	for (i = 0; i < priv->num_tx_rings; i++)2210		enetc_setup_txbdr(hw, priv->tx_ring[i]);2211 2212	for (i = 0; i < priv->num_rx_rings; i++)2213		enetc_setup_rxbdr(hw, priv->rx_ring[i], extended);2214}2215 2216static void enetc_enable_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)2217{2218	int idx = tx_ring->index;2219	u32 tbmr;2220 2221	tbmr = enetc_txbdr_rd(hw, idx, ENETC_TBMR);2222	tbmr |= ENETC_TBMR_EN;2223	enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);2224}2225 2226static void enetc_enable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)2227{2228	int idx = rx_ring->index;2229	u32 rbmr;2230 2231	rbmr = enetc_rxbdr_rd(hw, idx, ENETC_RBMR);2232	rbmr |= ENETC_RBMR_EN;2233	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);2234}2235 2236static void enetc_enable_rx_bdrs(struct enetc_ndev_priv *priv)2237{2238	struct enetc_hw *hw = &priv->si->hw;2239	int i;2240 2241	for (i = 0; i < priv->num_rx_rings; i++)2242		enetc_enable_rxbdr(hw, priv->rx_ring[i]);2243}2244 2245static void enetc_enable_tx_bdrs(struct enetc_ndev_priv *priv)2246{2247	struct enetc_hw *hw = &priv->si->hw;2248	int i;2249 2250	for (i = 0; i < priv->num_tx_rings; i++)2251		enetc_enable_txbdr(hw, priv->tx_ring[i]);2252}2253 2254static void enetc_disable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)2255{2256	int idx = rx_ring->index;2257 2258	/* disable EN bit on ring */2259	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0);2260}2261 2262static void enetc_disable_txbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)2263{2264	int idx = rx_ring->index;2265 2266	/* disable EN bit on ring */2267	enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0);2268}2269 2270static void enetc_disable_rx_bdrs(struct enetc_ndev_priv *priv)2271{2272	struct enetc_hw *hw = &priv->si->hw;2273	int i;2274 2275	for (i = 0; i < priv->num_rx_rings; i++)2276		enetc_disable_rxbdr(hw, priv->rx_ring[i]);2277}2278 2279static void enetc_disable_tx_bdrs(struct enetc_ndev_priv *priv)2280{2281	struct enetc_hw *hw = &priv->si->hw;2282	int i;2283 2284	for (i = 0; i < priv->num_tx_rings; i++)2285		enetc_disable_txbdr(hw, priv->tx_ring[i]);2286}2287 2288static void enetc_wait_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)2289{2290	int delay = 8, timeout = 100;2291	int idx = tx_ring->index;2292 2293	/* wait for busy to clear */2294	while (delay < timeout &&2295	       enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) {2296		msleep(delay);2297		delay *= 2;2298	}2299 2300	if (delay >= timeout)2301		netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n",2302			    idx);2303}2304 2305static void enetc_wait_bdrs(struct enetc_ndev_priv *priv)2306{2307	struct enetc_hw *hw = &priv->si->hw;2308	int i;2309 2310	for (i = 0; i < priv->num_tx_rings; i++)2311		enetc_wait_txbdr(hw, priv->tx_ring[i]);2312}2313 2314static int enetc_setup_irqs(struct enetc_ndev_priv *priv)2315{2316	struct pci_dev *pdev = priv->si->pdev;2317	struct enetc_hw *hw = &priv->si->hw;2318	int i, j, err;2319 2320	for (i = 0; i < priv->bdr_int_num; i++) {2321		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);2322		struct enetc_int_vector *v = priv->int_vector[i];2323		int entry = ENETC_BDR_INT_BASE_IDX + i;2324 2325		snprintf(v->name, sizeof(v->name), "%s-rxtx%d",2326			 priv->ndev->name, i);2327		err = request_irq(irq, enetc_msix, IRQF_NO_AUTOEN, v->name, v);2328		if (err) {2329			dev_err(priv->dev, "request_irq() failed!\n");2330			goto irq_err;2331		}2332 2333		v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);2334		v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);2335		v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1);2336 2337		enetc_wr(hw, ENETC_SIMSIRRV(i), entry);2338 2339		for (j = 0; j < v->count_tx_rings; j++) {2340			int idx = v->tx_ring[j].index;2341 2342			enetc_wr(hw, ENETC_SIMSITRV(idx), entry);2343		}2344		irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus()));2345	}2346 2347	return 0;2348 2349irq_err:2350	while (i--) {2351		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);2352 2353		irq_set_affinity_hint(irq, NULL);2354		free_irq(irq, priv->int_vector[i]);2355	}2356 2357	return err;2358}2359 2360static void enetc_free_irqs(struct enetc_ndev_priv *priv)2361{2362	struct pci_dev *pdev = priv->si->pdev;2363	int i;2364 2365	for (i = 0; i < priv->bdr_int_num; i++) {2366		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);2367 2368		irq_set_affinity_hint(irq, NULL);2369		free_irq(irq, priv->int_vector[i]);2370	}2371}2372 2373static void enetc_setup_interrupts(struct enetc_ndev_priv *priv)2374{2375	struct enetc_hw *hw = &priv->si->hw;2376	u32 icpt, ictt;2377	int i;2378 2379	/* enable Tx & Rx event indication */2380	if (priv->ic_mode &2381	    (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) {2382		icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR);2383		/* init to non-0 minimum, will be adjusted later */2384		ictt = 0x1;2385	} else {2386		icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */2387		ictt = 0;2388	}2389 2390	for (i = 0; i < priv->num_rx_rings; i++) {2391		enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt);2392		enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt);2393		enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE);2394	}2395 2396	if (priv->ic_mode & ENETC_IC_TX_MANUAL)2397		icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR);2398	else2399		icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */2400 2401	for (i = 0; i < priv->num_tx_rings; i++) {2402		enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt);2403		enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt);2404		enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE);2405	}2406}2407 2408static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)2409{2410	struct enetc_hw *hw = &priv->si->hw;2411	int i;2412 2413	for (i = 0; i < priv->num_tx_rings; i++)2414		enetc_txbdr_wr(hw, i, ENETC_TBIER, 0);2415 2416	for (i = 0; i < priv->num_rx_rings; i++)2417		enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0);2418}2419 2420static int enetc_phylink_connect(struct net_device *ndev)2421{2422	struct enetc_ndev_priv *priv = netdev_priv(ndev);2423	struct ethtool_keee edata;2424	int err;2425 2426	if (!priv->phylink) {2427		/* phy-less mode */2428		netif_carrier_on(ndev);2429		return 0;2430	}2431 2432	err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0);2433	if (err) {2434		dev_err(&ndev->dev, "could not attach to PHY\n");2435		return err;2436	}2437 2438	/* disable EEE autoneg, until ENETC driver supports it */2439	memset(&edata, 0, sizeof(struct ethtool_keee));2440	phylink_ethtool_set_eee(priv->phylink, &edata);2441 2442	phylink_start(priv->phylink);2443 2444	return 0;2445}2446 2447static void enetc_tx_onestep_tstamp(struct work_struct *work)2448{2449	struct enetc_ndev_priv *priv;2450	struct sk_buff *skb;2451 2452	priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp);2453 2454	netif_tx_lock_bh(priv->ndev);2455 2456	clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags);2457	skb = skb_dequeue(&priv->tx_skbs);2458	if (skb)2459		enetc_start_xmit(skb, priv->ndev);2460 2461	netif_tx_unlock_bh(priv->ndev);2462}2463 2464static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv)2465{2466	INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp);2467	skb_queue_head_init(&priv->tx_skbs);2468}2469 2470void enetc_start(struct net_device *ndev)2471{2472	struct enetc_ndev_priv *priv = netdev_priv(ndev);2473	int i;2474 2475	enetc_setup_interrupts(priv);2476 2477	for (i = 0; i < priv->bdr_int_num; i++) {2478		int irq = pci_irq_vector(priv->si->pdev,2479					 ENETC_BDR_INT_BASE_IDX + i);2480 2481		napi_enable(&priv->int_vector[i]->napi);2482		enable_irq(irq);2483	}2484 2485	enetc_enable_tx_bdrs(priv);2486 2487	enetc_enable_rx_bdrs(priv);2488 2489	netif_tx_start_all_queues(ndev);2490 2491	clear_bit(ENETC_TX_DOWN, &priv->flags);2492}2493EXPORT_SYMBOL_GPL(enetc_start);2494 2495int enetc_open(struct net_device *ndev)2496{2497	struct enetc_ndev_priv *priv = netdev_priv(ndev);2498	struct enetc_bdr_resource *tx_res, *rx_res;2499	bool extended;2500	int err;2501 2502	extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);2503 2504	err = enetc_setup_irqs(priv);2505	if (err)2506		return err;2507 2508	err = enetc_phylink_connect(ndev);2509	if (err)2510		goto err_phy_connect;2511 2512	tx_res = enetc_alloc_tx_resources(priv);2513	if (IS_ERR(tx_res)) {2514		err = PTR_ERR(tx_res);2515		goto err_alloc_tx;2516	}2517 2518	rx_res = enetc_alloc_rx_resources(priv, extended);2519	if (IS_ERR(rx_res)) {2520		err = PTR_ERR(rx_res);2521		goto err_alloc_rx;2522	}2523 2524	enetc_tx_onestep_tstamp_init(priv);2525	enetc_assign_tx_resources(priv, tx_res);2526	enetc_assign_rx_resources(priv, rx_res);2527	enetc_setup_bdrs(priv, extended);2528	enetc_start(ndev);2529 2530	return 0;2531 2532err_alloc_rx:2533	enetc_free_tx_resources(tx_res, priv->num_tx_rings);2534err_alloc_tx:2535	if (priv->phylink)2536		phylink_disconnect_phy(priv->phylink);2537err_phy_connect:2538	enetc_free_irqs(priv);2539 2540	return err;2541}2542EXPORT_SYMBOL_GPL(enetc_open);2543 2544void enetc_stop(struct net_device *ndev)2545{2546	struct enetc_ndev_priv *priv = netdev_priv(ndev);2547	int i;2548 2549	set_bit(ENETC_TX_DOWN, &priv->flags);2550 2551	netif_tx_stop_all_queues(ndev);2552 2553	enetc_disable_rx_bdrs(priv);2554 2555	enetc_wait_bdrs(priv);2556 2557	enetc_disable_tx_bdrs(priv);2558 2559	for (i = 0; i < priv->bdr_int_num; i++) {2560		int irq = pci_irq_vector(priv->si->pdev,2561					 ENETC_BDR_INT_BASE_IDX + i);2562 2563		disable_irq(irq);2564		napi_synchronize(&priv->int_vector[i]->napi);2565		napi_disable(&priv->int_vector[i]->napi);2566	}2567 2568	enetc_clear_interrupts(priv);2569}2570EXPORT_SYMBOL_GPL(enetc_stop);2571 2572int enetc_close(struct net_device *ndev)2573{2574	struct enetc_ndev_priv *priv = netdev_priv(ndev);2575 2576	enetc_stop(ndev);2577 2578	if (priv->phylink) {2579		phylink_stop(priv->phylink);2580		phylink_disconnect_phy(priv->phylink);2581	} else {2582		netif_carrier_off(ndev);2583	}2584 2585	enetc_free_rxtx_rings(priv);2586 2587	/* Avoids dangling pointers and also frees old resources */2588	enetc_assign_rx_resources(priv, NULL);2589	enetc_assign_tx_resources(priv, NULL);2590 2591	enetc_free_irqs(priv);2592 2593	return 0;2594}2595EXPORT_SYMBOL_GPL(enetc_close);2596 2597static int enetc_reconfigure(struct enetc_ndev_priv *priv, bool extended,2598			     int (*cb)(struct enetc_ndev_priv *priv, void *ctx),2599			     void *ctx)2600{2601	struct enetc_bdr_resource *tx_res, *rx_res;2602	int err;2603 2604	ASSERT_RTNL();2605 2606	/* If the interface is down, run the callback right away,2607	 * without reconfiguration.2608	 */2609	if (!netif_running(priv->ndev)) {2610		if (cb) {2611			err = cb(priv, ctx);2612			if (err)2613				return err;2614		}2615 2616		return 0;2617	}2618 2619	tx_res = enetc_alloc_tx_resources(priv);2620	if (IS_ERR(tx_res)) {2621		err = PTR_ERR(tx_res);2622		goto out;2623	}2624 2625	rx_res = enetc_alloc_rx_resources(priv, extended);2626	if (IS_ERR(rx_res)) {2627		err = PTR_ERR(rx_res);2628		goto out_free_tx_res;2629	}2630 2631	enetc_stop(priv->ndev);2632	enetc_free_rxtx_rings(priv);2633 2634	/* Interface is down, run optional callback now */2635	if (cb) {2636		err = cb(priv, ctx);2637		if (err)2638			goto out_restart;2639	}2640 2641	enetc_assign_tx_resources(priv, tx_res);2642	enetc_assign_rx_resources(priv, rx_res);2643	enetc_setup_bdrs(priv, extended);2644	enetc_start(priv->ndev);2645 2646	return 0;2647 2648out_restart:2649	enetc_setup_bdrs(priv, extended);2650	enetc_start(priv->ndev);2651	enetc_free_rx_resources(rx_res, priv->num_rx_rings);2652out_free_tx_res:2653	enetc_free_tx_resources(tx_res, priv->num_tx_rings);2654out:2655	return err;2656}2657 2658static void enetc_debug_tx_ring_prios(struct enetc_ndev_priv *priv)2659{2660	int i;2661 2662	for (i = 0; i < priv->num_tx_rings; i++)2663		netdev_dbg(priv->ndev, "TX ring %d prio %d\n", i,2664			   priv->tx_ring[i]->prio);2665}2666 2667void enetc_reset_tc_mqprio(struct net_device *ndev)2668{2669	struct enetc_ndev_priv *priv = netdev_priv(ndev);2670	struct enetc_hw *hw = &priv->si->hw;2671	struct enetc_bdr *tx_ring;2672	int num_stack_tx_queues;2673	int i;2674 2675	num_stack_tx_queues = enetc_num_stack_tx_queues(priv);2676 2677	netdev_reset_tc(ndev);2678	netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);2679	priv->min_num_stack_tx_queues = num_possible_cpus();2680 2681	/* Reset all ring priorities to 0 */2682	for (i = 0; i < priv->num_tx_rings; i++) {2683		tx_ring = priv->tx_ring[i];2684		tx_ring->prio = 0;2685		enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);2686	}2687 2688	enetc_debug_tx_ring_prios(priv);2689 2690	enetc_change_preemptible_tcs(priv, 0);2691}2692EXPORT_SYMBOL_GPL(enetc_reset_tc_mqprio);2693 2694int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)2695{2696	struct tc_mqprio_qopt_offload *mqprio = type_data;2697	struct enetc_ndev_priv *priv = netdev_priv(ndev);2698	struct tc_mqprio_qopt *qopt = &mqprio->qopt;2699	struct enetc_hw *hw = &priv->si->hw;2700	int num_stack_tx_queues = 0;2701	struct enetc_bdr *tx_ring;2702	u8 num_tc = qopt->num_tc;2703	int offset, count;2704	int err, tc, q;2705 2706	if (!num_tc) {2707		enetc_reset_tc_mqprio(ndev);2708		return 0;2709	}2710 2711	err = netdev_set_num_tc(ndev, num_tc);2712	if (err)2713		return err;2714 2715	for (tc = 0; tc < num_tc; tc++) {2716		offset = qopt->offset[tc];2717		count = qopt->count[tc];2718		num_stack_tx_queues += count;2719 2720		err = netdev_set_tc_queue(ndev, tc, count, offset);2721		if (err)2722			goto err_reset_tc;2723 2724		for (q = offset; q < offset + count; q++) {2725			tx_ring = priv->tx_ring[q];2726			/* The prio_tc_map is skb_tx_hash()'s way of selecting2727			 * between TX queues based on skb->priority. As such,2728			 * there's nothing to offload based on it.2729			 * Make the mqprio "traffic class" be the priority of2730			 * this ring group, and leave the Tx IPV to traffic2731			 * class mapping as its default mapping value of 1:1.2732			 */2733			tx_ring->prio = tc;2734			enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);2735		}2736	}2737 2738	err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);2739	if (err)2740		goto err_reset_tc;2741 2742	priv->min_num_stack_tx_queues = num_stack_tx_queues;2743 2744	enetc_debug_tx_ring_prios(priv);2745 2746	enetc_change_preemptible_tcs(priv, mqprio->preemptible_tcs);2747 2748	return 0;2749 2750err_reset_tc:2751	enetc_reset_tc_mqprio(ndev);2752	return err;2753}2754EXPORT_SYMBOL_GPL(enetc_setup_tc_mqprio);2755 2756static int enetc_reconfigure_xdp_cb(struct enetc_ndev_priv *priv, void *ctx)2757{2758	struct bpf_prog *old_prog, *prog = ctx;2759	int num_stack_tx_queues;2760	int err, i;2761 2762	old_prog = xchg(&priv->xdp_prog, prog);2763 2764	num_stack_tx_queues = enetc_num_stack_tx_queues(priv);2765	err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues);2766	if (err) {2767		xchg(&priv->xdp_prog, old_prog);2768		return err;2769	}2770 2771	if (old_prog)2772		bpf_prog_put(old_prog);2773 2774	for (i = 0; i < priv->num_rx_rings; i++) {2775		struct enetc_bdr *rx_ring = priv->rx_ring[i];2776 2777		rx_ring->xdp.prog = prog;2778 2779		if (prog)2780			rx_ring->buffer_offset = XDP_PACKET_HEADROOM;2781		else2782			rx_ring->buffer_offset = ENETC_RXB_PAD;2783	}2784 2785	return 0;2786}2787 2788static int enetc_setup_xdp_prog(struct net_device *ndev, struct bpf_prog *prog,2789				struct netlink_ext_ack *extack)2790{2791	int num_xdp_tx_queues = prog ? num_possible_cpus() : 0;2792	struct enetc_ndev_priv *priv = netdev_priv(ndev);2793	bool extended;2794 2795	if (priv->min_num_stack_tx_queues + num_xdp_tx_queues >2796	    priv->num_tx_rings) {2797		NL_SET_ERR_MSG_FMT_MOD(extack,2798				       "Reserving %d XDP TXQs leaves under %d for stack (total %d)",2799				       num_xdp_tx_queues,2800				       priv->min_num_stack_tx_queues,2801				       priv->num_tx_rings);2802		return -EBUSY;2803	}2804 2805	extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);2806 2807	/* The buffer layout is changing, so we need to drain the old2808	 * RX buffers and seed new ones.2809	 */2810	return enetc_reconfigure(priv, extended, enetc_reconfigure_xdp_cb, prog);2811}2812 2813int enetc_setup_bpf(struct net_device *ndev, struct netdev_bpf *bpf)2814{2815	switch (bpf->command) {2816	case XDP_SETUP_PROG:2817		return enetc_setup_xdp_prog(ndev, bpf->prog, bpf->extack);2818	default:2819		return -EINVAL;2820	}2821 2822	return 0;2823}2824EXPORT_SYMBOL_GPL(enetc_setup_bpf);2825 2826struct net_device_stats *enetc_get_stats(struct net_device *ndev)2827{2828	struct enetc_ndev_priv *priv = netdev_priv(ndev);2829	struct net_device_stats *stats = &ndev->stats;2830	unsigned long packets = 0, bytes = 0;2831	unsigned long tx_dropped = 0;2832	int i;2833 2834	for (i = 0; i < priv->num_rx_rings; i++) {2835		packets += priv->rx_ring[i]->stats.packets;2836		bytes	+= priv->rx_ring[i]->stats.bytes;2837	}2838 2839	stats->rx_packets = packets;2840	stats->rx_bytes = bytes;2841	bytes = 0;2842	packets = 0;2843 2844	for (i = 0; i < priv->num_tx_rings; i++) {2845		packets += priv->tx_ring[i]->stats.packets;2846		bytes	+= priv->tx_ring[i]->stats.bytes;2847		tx_dropped += priv->tx_ring[i]->stats.win_drop;2848	}2849 2850	stats->tx_packets = packets;2851	stats->tx_bytes = bytes;2852	stats->tx_dropped = tx_dropped;2853 2854	return stats;2855}2856EXPORT_SYMBOL_GPL(enetc_get_stats);2857 2858static int enetc_set_rss(struct net_device *ndev, int en)2859{2860	struct enetc_ndev_priv *priv = netdev_priv(ndev);2861	struct enetc_hw *hw = &priv->si->hw;2862	u32 reg;2863 2864	enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings);2865 2866	reg = enetc_rd(hw, ENETC_SIMR);2867	reg &= ~ENETC_SIMR_RSSE;2868	reg |= (en) ? ENETC_SIMR_RSSE : 0;2869	enetc_wr(hw, ENETC_SIMR, reg);2870 2871	return 0;2872}2873 2874static void enetc_enable_rxvlan(struct net_device *ndev, bool en)2875{2876	struct enetc_ndev_priv *priv = netdev_priv(ndev);2877	struct enetc_hw *hw = &priv->si->hw;2878	int i;2879 2880	for (i = 0; i < priv->num_rx_rings; i++)2881		enetc_bdr_enable_rxvlan(hw, i, en);2882}2883 2884static void enetc_enable_txvlan(struct net_device *ndev, bool en)2885{2886	struct enetc_ndev_priv *priv = netdev_priv(ndev);2887	struct enetc_hw *hw = &priv->si->hw;2888	int i;2889 2890	for (i = 0; i < priv->num_tx_rings; i++)2891		enetc_bdr_enable_txvlan(hw, i, en);2892}2893 2894void enetc_set_features(struct net_device *ndev, netdev_features_t features)2895{2896	netdev_features_t changed = ndev->features ^ features;2897 2898	if (changed & NETIF_F_RXHASH)2899		enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH));2900 2901	if (changed & NETIF_F_HW_VLAN_CTAG_RX)2902		enetc_enable_rxvlan(ndev,2903				    !!(features & NETIF_F_HW_VLAN_CTAG_RX));2904 2905	if (changed & NETIF_F_HW_VLAN_CTAG_TX)2906		enetc_enable_txvlan(ndev,2907				    !!(features & NETIF_F_HW_VLAN_CTAG_TX));2908}2909EXPORT_SYMBOL_GPL(enetc_set_features);2910 2911static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr)2912{2913	struct enetc_ndev_priv *priv = netdev_priv(ndev);2914	int err, new_offloads = priv->active_offloads;2915	struct hwtstamp_config config;2916 2917	if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))2918		return -EFAULT;2919 2920	switch (config.tx_type) {2921	case HWTSTAMP_TX_OFF:2922		new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;2923		break;2924	case HWTSTAMP_TX_ON:2925		new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;2926		new_offloads |= ENETC_F_TX_TSTAMP;2927		break;2928	case HWTSTAMP_TX_ONESTEP_SYNC:2929		new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;2930		new_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP;2931		break;2932	default:2933		return -ERANGE;2934	}2935 2936	switch (config.rx_filter) {2937	case HWTSTAMP_FILTER_NONE:2938		new_offloads &= ~ENETC_F_RX_TSTAMP;2939		break;2940	default:2941		new_offloads |= ENETC_F_RX_TSTAMP;2942		config.rx_filter = HWTSTAMP_FILTER_ALL;2943	}2944 2945	if ((new_offloads ^ priv->active_offloads) & ENETC_F_RX_TSTAMP) {2946		bool extended = !!(new_offloads & ENETC_F_RX_TSTAMP);2947 2948		err = enetc_reconfigure(priv, extended, NULL, NULL);2949		if (err)2950			return err;2951	}2952 2953	priv->active_offloads = new_offloads;2954 2955	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?2956	       -EFAULT : 0;2957}2958 2959static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr)2960{2961	struct enetc_ndev_priv *priv = netdev_priv(ndev);2962	struct hwtstamp_config config;2963 2964	config.flags = 0;2965 2966	if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)2967		config.tx_type = HWTSTAMP_TX_ONESTEP_SYNC;2968	else if (priv->active_offloads & ENETC_F_TX_TSTAMP)2969		config.tx_type = HWTSTAMP_TX_ON;2970	else2971		config.tx_type = HWTSTAMP_TX_OFF;2972 2973	config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ?2974			    HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;2975 2976	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?2977	       -EFAULT : 0;2978}2979 2980int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)2981{2982	struct enetc_ndev_priv *priv = netdev_priv(ndev);2983 2984	if (IS_ENABLED(CONFIG_FSL_ENETC_PTP_CLOCK)) {2985		if (cmd == SIOCSHWTSTAMP)2986			return enetc_hwtstamp_set(ndev, rq);2987		if (cmd == SIOCGHWTSTAMP)2988			return enetc_hwtstamp_get(ndev, rq);2989	}2990 2991	if (!priv->phylink)2992		return -EOPNOTSUPP;2993 2994	return phylink_mii_ioctl(priv->phylink, rq, cmd);2995}2996EXPORT_SYMBOL_GPL(enetc_ioctl);2997 2998int enetc_alloc_msix(struct enetc_ndev_priv *priv)2999{3000	struct pci_dev *pdev = priv->si->pdev;3001	int num_stack_tx_queues;3002	int first_xdp_tx_ring;3003	int i, n, err, nvec;3004	int v_tx_rings;3005 3006	nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num;3007	/* allocate MSIX for both messaging and Rx/Tx interrupts */3008	n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);3009 3010	if (n < 0)3011		return n;3012 3013	if (n != nvec)3014		return -EPERM;3015 3016	/* # of tx rings per int vector */3017	v_tx_rings = priv->num_tx_rings / priv->bdr_int_num;3018 3019	for (i = 0; i < priv->bdr_int_num; i++) {3020		struct enetc_int_vector *v;3021		struct enetc_bdr *bdr;3022		int j;3023 3024		v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL);3025		if (!v) {3026			err = -ENOMEM;3027			goto fail;3028		}3029 3030		priv->int_vector[i] = v;3031 3032		bdr = &v->rx_ring;3033		bdr->index = i;3034		bdr->ndev = priv->ndev;3035		bdr->dev = priv->dev;3036		bdr->bd_count = priv->rx_bd_count;3037		bdr->buffer_offset = ENETC_RXB_PAD;3038		priv->rx_ring[i] = bdr;3039 3040		err = xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0);3041		if (err) {3042			kfree(v);3043			goto fail;3044		}3045 3046		err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq,3047						 MEM_TYPE_PAGE_SHARED, NULL);3048		if (err) {3049			xdp_rxq_info_unreg(&bdr->xdp.rxq);3050			kfree(v);3051			goto fail;3052		}3053 3054		/* init defaults for adaptive IC */3055		if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) {3056			v->rx_ictt = 0x1;3057			v->rx_dim_en = true;3058		}3059		INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work);3060		netif_napi_add(priv->ndev, &v->napi, enetc_poll);3061		v->count_tx_rings = v_tx_rings;3062 3063		for (j = 0; j < v_tx_rings; j++) {3064			int idx;3065 3066			/* default tx ring mapping policy */3067			idx = priv->bdr_int_num * j + i;3068			__set_bit(idx, &v->tx_rings_map);3069			bdr = &v->tx_ring[j];3070			bdr->index = idx;3071			bdr->ndev = priv->ndev;3072			bdr->dev = priv->dev;3073			bdr->bd_count = priv->tx_bd_count;3074			priv->tx_ring[idx] = bdr;3075		}3076	}3077 3078	num_stack_tx_queues = enetc_num_stack_tx_queues(priv);3079 3080	err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues);3081	if (err)3082		goto fail;3083 3084	err = netif_set_real_num_rx_queues(priv->ndev, priv->num_rx_rings);3085	if (err)3086		goto fail;3087 3088	priv->min_num_stack_tx_queues = num_possible_cpus();3089	first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus();3090	priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring];3091 3092	return 0;3093 3094fail:3095	while (i--) {3096		struct enetc_int_vector *v = priv->int_vector[i];3097		struct enetc_bdr *rx_ring = &v->rx_ring;3098 3099		xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);3100		xdp_rxq_info_unreg(&rx_ring->xdp.rxq);3101		netif_napi_del(&v->napi);3102		cancel_work_sync(&v->rx_dim.work);3103		kfree(v);3104	}3105 3106	pci_free_irq_vectors(pdev);3107 3108	return err;3109}3110EXPORT_SYMBOL_GPL(enetc_alloc_msix);3111 3112void enetc_free_msix(struct enetc_ndev_priv *priv)3113{3114	int i;3115 3116	for (i = 0; i < priv->bdr_int_num; i++) {3117		struct enetc_int_vector *v = priv->int_vector[i];3118		struct enetc_bdr *rx_ring = &v->rx_ring;3119 3120		xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);3121		xdp_rxq_info_unreg(&rx_ring->xdp.rxq);3122		netif_napi_del(&v->napi);3123		cancel_work_sync(&v->rx_dim.work);3124	}3125 3126	for (i = 0; i < priv->num_rx_rings; i++)3127		priv->rx_ring[i] = NULL;3128 3129	for (i = 0; i < priv->num_tx_rings; i++)3130		priv->tx_ring[i] = NULL;3131 3132	for (i = 0; i < priv->bdr_int_num; i++) {3133		kfree(priv->int_vector[i]);3134		priv->int_vector[i] = NULL;3135	}3136 3137	/* disable all MSIX for this device */3138	pci_free_irq_vectors(priv->si->pdev);3139}3140EXPORT_SYMBOL_GPL(enetc_free_msix);3141 3142static void enetc_kfree_si(struct enetc_si *si)3143{3144	char *p = (char *)si - si->pad;3145 3146	kfree(p);3147}3148 3149static void enetc_detect_errata(struct enetc_si *si)3150{3151	if (si->pdev->revision == ENETC_REV1)3152		si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP;3153}3154 3155int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)3156{3157	struct enetc_si *si, *p;3158	struct enetc_hw *hw;3159	size_t alloc_size;3160	int err, len;3161 3162	pcie_flr(pdev);3163	err = pci_enable_device_mem(pdev);3164	if (err)3165		return dev_err_probe(&pdev->dev, err, "device enable failed\n");3166 3167	/* set up for high or low dma */3168	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));3169	if (err) {3170		dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err);3171		goto err_dma;3172	}3173 3174	err = pci_request_mem_regions(pdev, name);3175	if (err) {3176		dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err);3177		goto err_pci_mem_reg;3178	}3179 3180	pci_set_master(pdev);3181 3182	alloc_size = sizeof(struct enetc_si);3183	if (sizeof_priv) {3184		/* align priv to 32B */3185		alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN);3186		alloc_size += sizeof_priv;3187	}3188	/* force 32B alignment for enetc_si */3189	alloc_size += ENETC_SI_ALIGN - 1;3190 3191	p = kzalloc(alloc_size, GFP_KERNEL);3192	if (!p) {3193		err = -ENOMEM;3194		goto err_alloc_si;3195	}3196 3197	si = PTR_ALIGN(p, ENETC_SI_ALIGN);3198	si->pad = (char *)si - (char *)p;3199 3200	pci_set_drvdata(pdev, si);3201	si->pdev = pdev;3202	hw = &si->hw;3203 3204	len = pci_resource_len(pdev, ENETC_BAR_REGS);3205	hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len);3206	if (!hw->reg) {3207		err = -ENXIO;3208		dev_err(&pdev->dev, "ioremap() failed\n");3209		goto err_ioremap;3210	}3211	if (len > ENETC_PORT_BASE)3212		hw->port = hw->reg + ENETC_PORT_BASE;3213	if (len > ENETC_GLOBAL_BASE)3214		hw->global = hw->reg + ENETC_GLOBAL_BASE;3215 3216	enetc_detect_errata(si);3217 3218	return 0;3219 3220err_ioremap:3221	enetc_kfree_si(si);3222err_alloc_si:3223	pci_release_mem_regions(pdev);3224err_pci_mem_reg:3225err_dma:3226	pci_disable_device(pdev);3227 3228	return err;3229}3230EXPORT_SYMBOL_GPL(enetc_pci_probe);3231 3232void enetc_pci_remove(struct pci_dev *pdev)3233{3234	struct enetc_si *si = pci_get_drvdata(pdev);3235	struct enetc_hw *hw = &si->hw;3236 3237	iounmap(hw->reg);3238	enetc_kfree_si(si);3239	pci_release_mem_regions(pdev);3240	pci_disable_device(pdev);3241}3242EXPORT_SYMBOL_GPL(enetc_pci_remove);3243 3244MODULE_DESCRIPTION("NXP ENETC Ethernet driver");3245MODULE_LICENSE("Dual BSD/GPL");3246