brintos / linux-shallow public Read only

1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)2/* Copyright (C) 2015-2019 Netronome Systems, Inc. */3 4/*5 * nfp_net_common.c6 * Netronome network device driver: Common functions between PF and VF7 * Authors: Jakub Kicinski <jakub.kicinski@netronome.com>8 *          Jason McMullan <jason.mcmullan@netronome.com>9 *          Rolf Neugebauer <rolf.neugebauer@netronome.com>10 *          Brad Petrus <brad.petrus@netronome.com>11 *          Chris Telfer <chris.telfer@netronome.com>12 */13 14#include <linux/bitfield.h>15#include <linux/bpf.h>16#include <linux/module.h>17#include <linux/kernel.h>18#include <linux/init.h>19#include <linux/fs.h>20#include <linux/netdevice.h>21#include <linux/etherdevice.h>22#include <linux/interrupt.h>23#include <linux/ip.h>24#include <linux/ipv6.h>25#include <linux/mm.h>26#include <linux/overflow.h>27#include <linux/page_ref.h>28#include <linux/pci.h>29#include <linux/pci_regs.h>30#include <linux/ethtool.h>31#include <linux/log2.h>32#include <linux/if_vlan.h>33#include <linux/if_bridge.h>34#include <linux/random.h>35#include <linux/vmalloc.h>36#include <linux/ktime.h>37 38#include <net/tls.h>39#include <net/vxlan.h>40#include <net/xdp_sock_drv.h>41#include <net/xfrm.h>42 43#include "nfpcore/nfp_dev.h"44#include "nfpcore/nfp_nsp.h"45#include "ccm.h"46#include "nfp_app.h"47#include "nfp_net_ctrl.h"48#include "nfp_net.h"49#include "nfp_net_dp.h"50#include "nfp_net_sriov.h"51#include "nfp_net_xsk.h"52#include "nfp_port.h"53#include "crypto/crypto.h"54#include "crypto/fw.h"55 56static int nfp_net_mc_unsync(struct net_device *netdev, const unsigned char *addr);57 58/**59 * nfp_net_get_fw_version() - Read and parse the FW version60 * @fw_ver:	Output fw_version structure to read to61 * @ctrl_bar:	Mapped address of the control BAR62 */63void nfp_net_get_fw_version(struct nfp_net_fw_version *fw_ver,64			    void __iomem *ctrl_bar)65{66	u32 reg;67 68	reg = readl(ctrl_bar + NFP_NET_CFG_VERSION);69	put_unaligned_le32(reg, fw_ver);70}71 72u32 nfp_qcp_queue_offset(const struct nfp_dev_info *dev_info, u16 queue)73{74	queue &= dev_info->qc_idx_mask;75	return dev_info->qc_addr_offset + NFP_QCP_QUEUE_ADDR_SZ * queue;76}77 78/* Firmware reconfig79 *80 * Firmware reconfig may take a while so we have two versions of it -81 * synchronous and asynchronous (posted).  All synchronous callers are holding82 * RTNL so we don't have to worry about serializing them.83 */84static void nfp_net_reconfig_start(struct nfp_net *nn, u32 update)85{86	nn_writel(nn, NFP_NET_CFG_UPDATE, update);87	/* ensure update is written before pinging HW */88	nn_pci_flush(nn);89	nfp_qcp_wr_ptr_add(nn->qcp_cfg, 1);90	nn->reconfig_in_progress_update = update;91}92 93/* Pass 0 as update to run posted reconfigs. */94static void nfp_net_reconfig_start_async(struct nfp_net *nn, u32 update)95{96	update |= nn->reconfig_posted;97	nn->reconfig_posted = 0;98 99	nfp_net_reconfig_start(nn, update);100 101	nn->reconfig_timer_active = true;102	mod_timer(&nn->reconfig_timer, jiffies + NFP_NET_POLL_TIMEOUT * HZ);103}104 105static bool nfp_net_reconfig_check_done(struct nfp_net *nn, bool last_check)106{107	u32 reg;108 109	reg = nn_readl(nn, NFP_NET_CFG_UPDATE);110	if (reg == 0)111		return true;112	if (reg & NFP_NET_CFG_UPDATE_ERR) {113		nn_err(nn, "Reconfig error (status: 0x%08x update: 0x%08x ctrl: 0x%08x)\n",114		       reg, nn->reconfig_in_progress_update,115		       nn_readl(nn, NFP_NET_CFG_CTRL));116		return true;117	} else if (last_check) {118		nn_err(nn, "Reconfig timeout (status: 0x%08x update: 0x%08x ctrl: 0x%08x)\n",119		       reg, nn->reconfig_in_progress_update,120		       nn_readl(nn, NFP_NET_CFG_CTRL));121		return true;122	}123 124	return false;125}126 127static bool __nfp_net_reconfig_wait(struct nfp_net *nn, unsigned long deadline)128{129	bool timed_out = false;130	int i;131 132	/* Poll update field, waiting for NFP to ack the config.133	 * Do an opportunistic wait-busy loop, afterward sleep.134	 */135	for (i = 0; i < 50; i++) {136		if (nfp_net_reconfig_check_done(nn, false))137			return false;138		udelay(4);139	}140 141	while (!nfp_net_reconfig_check_done(nn, timed_out)) {142		usleep_range(250, 500);143		timed_out = time_is_before_eq_jiffies(deadline);144	}145 146	return timed_out;147}148 149static int nfp_net_reconfig_wait(struct nfp_net *nn, unsigned long deadline)150{151	if (__nfp_net_reconfig_wait(nn, deadline))152		return -EIO;153 154	if (nn_readl(nn, NFP_NET_CFG_UPDATE) & NFP_NET_CFG_UPDATE_ERR)155		return -EIO;156 157	return 0;158}159 160static void nfp_net_reconfig_timer(struct timer_list *t)161{162	struct nfp_net *nn = from_timer(nn, t, reconfig_timer);163 164	spin_lock_bh(&nn->reconfig_lock);165 166	nn->reconfig_timer_active = false;167 168	/* If sync caller is present it will take over from us */169	if (nn->reconfig_sync_present)170		goto done;171 172	/* Read reconfig status and report errors */173	nfp_net_reconfig_check_done(nn, true);174 175	if (nn->reconfig_posted)176		nfp_net_reconfig_start_async(nn, 0);177done:178	spin_unlock_bh(&nn->reconfig_lock);179}180 181/**182 * nfp_net_reconfig_post() - Post async reconfig request183 * @nn:      NFP Net device to reconfigure184 * @update:  The value for the update field in the BAR config185 *186 * Record FW reconfiguration request.  Reconfiguration will be kicked off187 * whenever reconfiguration machinery is idle.  Multiple requests can be188 * merged together!189 */190static void nfp_net_reconfig_post(struct nfp_net *nn, u32 update)191{192	spin_lock_bh(&nn->reconfig_lock);193 194	/* Sync caller will kick off async reconf when it's done, just post */195	if (nn->reconfig_sync_present) {196		nn->reconfig_posted |= update;197		goto done;198	}199 200	/* Opportunistically check if the previous command is done */201	if (!nn->reconfig_timer_active ||202	    nfp_net_reconfig_check_done(nn, false))203		nfp_net_reconfig_start_async(nn, update);204	else205		nn->reconfig_posted |= update;206done:207	spin_unlock_bh(&nn->reconfig_lock);208}209 210static void nfp_net_reconfig_sync_enter(struct nfp_net *nn)211{212	bool cancelled_timer = false;213	u32 pre_posted_requests;214 215	spin_lock_bh(&nn->reconfig_lock);216 217	WARN_ON(nn->reconfig_sync_present);218	nn->reconfig_sync_present = true;219 220	if (nn->reconfig_timer_active) {221		nn->reconfig_timer_active = false;222		cancelled_timer = true;223	}224	pre_posted_requests = nn->reconfig_posted;225	nn->reconfig_posted = 0;226 227	spin_unlock_bh(&nn->reconfig_lock);228 229	if (cancelled_timer) {230		del_timer_sync(&nn->reconfig_timer);231		nfp_net_reconfig_wait(nn, nn->reconfig_timer.expires);232	}233 234	/* Run the posted reconfigs which were issued before we started */235	if (pre_posted_requests) {236		nfp_net_reconfig_start(nn, pre_posted_requests);237		nfp_net_reconfig_wait(nn, jiffies + HZ * NFP_NET_POLL_TIMEOUT);238	}239}240 241static void nfp_net_reconfig_wait_posted(struct nfp_net *nn)242{243	nfp_net_reconfig_sync_enter(nn);244 245	spin_lock_bh(&nn->reconfig_lock);246	nn->reconfig_sync_present = false;247	spin_unlock_bh(&nn->reconfig_lock);248}249 250/**251 * __nfp_net_reconfig() - Reconfigure the firmware252 * @nn:      NFP Net device to reconfigure253 * @update:  The value for the update field in the BAR config254 *255 * Write the update word to the BAR and ping the reconfig queue.  The256 * poll until the firmware has acknowledged the update by zeroing the257 * update word.258 *259 * Return: Negative errno on error, 0 on success260 */261int __nfp_net_reconfig(struct nfp_net *nn, u32 update)262{263	int ret;264 265	nfp_net_reconfig_sync_enter(nn);266 267	nfp_net_reconfig_start(nn, update);268	ret = nfp_net_reconfig_wait(nn, jiffies + HZ * NFP_NET_POLL_TIMEOUT);269 270	spin_lock_bh(&nn->reconfig_lock);271 272	if (nn->reconfig_posted)273		nfp_net_reconfig_start_async(nn, 0);274 275	nn->reconfig_sync_present = false;276 277	spin_unlock_bh(&nn->reconfig_lock);278 279	return ret;280}281 282int nfp_net_reconfig(struct nfp_net *nn, u32 update)283{284	int ret;285 286	nn_ctrl_bar_lock(nn);287	ret = __nfp_net_reconfig(nn, update);288	nn_ctrl_bar_unlock(nn);289 290	return ret;291}292 293int nfp_net_mbox_lock(struct nfp_net *nn, unsigned int data_size)294{295	if (nn->tlv_caps.mbox_len < NFP_NET_CFG_MBOX_SIMPLE_VAL + data_size) {296		nn_err(nn, "mailbox too small for %u of data (%u)\n",297		       data_size, nn->tlv_caps.mbox_len);298		return -EIO;299	}300 301	nn_ctrl_bar_lock(nn);302	return 0;303}304 305/**306 * nfp_net_mbox_reconfig() - Reconfigure the firmware via the mailbox307 * @nn:        NFP Net device to reconfigure308 * @mbox_cmd:  The value for the mailbox command309 *310 * Helper function for mailbox updates311 *312 * Return: Negative errno on error, 0 on success313 */314int nfp_net_mbox_reconfig(struct nfp_net *nn, u32 mbox_cmd)315{316	u32 mbox = nn->tlv_caps.mbox_off;317	int ret;318 319	nn_writeq(nn, mbox + NFP_NET_CFG_MBOX_SIMPLE_CMD, mbox_cmd);320 321	ret = __nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_MBOX);322	if (ret) {323		nn_err(nn, "Mailbox update error\n");324		return ret;325	}326 327	return -nn_readl(nn, mbox + NFP_NET_CFG_MBOX_SIMPLE_RET);328}329 330void nfp_net_mbox_reconfig_post(struct nfp_net *nn, u32 mbox_cmd)331{332	u32 mbox = nn->tlv_caps.mbox_off;333 334	nn_writeq(nn, mbox + NFP_NET_CFG_MBOX_SIMPLE_CMD, mbox_cmd);335 336	nfp_net_reconfig_post(nn, NFP_NET_CFG_UPDATE_MBOX);337}338 339int nfp_net_mbox_reconfig_wait_posted(struct nfp_net *nn)340{341	u32 mbox = nn->tlv_caps.mbox_off;342 343	nfp_net_reconfig_wait_posted(nn);344 345	return -nn_readl(nn, mbox + NFP_NET_CFG_MBOX_SIMPLE_RET);346}347 348int nfp_net_mbox_reconfig_and_unlock(struct nfp_net *nn, u32 mbox_cmd)349{350	int ret;351 352	ret = nfp_net_mbox_reconfig(nn, mbox_cmd);353	nn_ctrl_bar_unlock(nn);354	return ret;355}356 357/* Interrupt configuration and handling358 */359 360/**361 * nfp_net_irqs_alloc() - allocates MSI-X irqs362 * @pdev:        PCI device structure363 * @irq_entries: Array to be initialized and used to hold the irq entries364 * @min_irqs:    Minimal acceptable number of interrupts365 * @wanted_irqs: Target number of interrupts to allocate366 *367 * Return: Number of irqs obtained or 0 on error.368 */369unsigned int370nfp_net_irqs_alloc(struct pci_dev *pdev, struct msix_entry *irq_entries,371		   unsigned int min_irqs, unsigned int wanted_irqs)372{373	unsigned int i;374	int got_irqs;375 376	for (i = 0; i < wanted_irqs; i++)377		irq_entries[i].entry = i;378 379	got_irqs = pci_enable_msix_range(pdev, irq_entries,380					 min_irqs, wanted_irqs);381	if (got_irqs < 0) {382		dev_err(&pdev->dev, "Failed to enable %d-%d MSI-X (err=%d)\n",383			min_irqs, wanted_irqs, got_irqs);384		return 0;385	}386 387	if (got_irqs < wanted_irqs)388		dev_warn(&pdev->dev, "Unable to allocate %d IRQs got only %d\n",389			 wanted_irqs, got_irqs);390 391	return got_irqs;392}393 394/**395 * nfp_net_irqs_assign() - Assign interrupts allocated externally to netdev396 * @nn:		 NFP Network structure397 * @irq_entries: Table of allocated interrupts398 * @n:		 Size of @irq_entries (number of entries to grab)399 *400 * After interrupts are allocated with nfp_net_irqs_alloc() this function401 * should be called to assign them to a specific netdev (port).402 */403void404nfp_net_irqs_assign(struct nfp_net *nn, struct msix_entry *irq_entries,405		    unsigned int n)406{407	struct nfp_net_dp *dp = &nn->dp;408 409	nn->max_r_vecs = n - NFP_NET_NON_Q_VECTORS;410	dp->num_r_vecs = nn->max_r_vecs;411 412	memcpy(nn->irq_entries, irq_entries, sizeof(*irq_entries) * n);413 414	if (dp->num_rx_rings > dp->num_r_vecs ||415	    dp->num_tx_rings > dp->num_r_vecs)416		dev_warn(nn->dp.dev, "More rings (%d,%d) than vectors (%d).\n",417			 dp->num_rx_rings, dp->num_tx_rings,418			 dp->num_r_vecs);419 420	dp->num_rx_rings = min(dp->num_r_vecs, dp->num_rx_rings);421	dp->num_tx_rings = min(dp->num_r_vecs, dp->num_tx_rings);422	dp->num_stack_tx_rings = dp->num_tx_rings;423}424 425/**426 * nfp_net_irqs_disable() - Disable interrupts427 * @pdev:        PCI device structure428 *429 * Undoes what @nfp_net_irqs_alloc() does.430 */431void nfp_net_irqs_disable(struct pci_dev *pdev)432{433	pci_disable_msix(pdev);434}435 436/**437 * nfp_net_irq_rxtx() - Interrupt service routine for RX/TX rings.438 * @irq:      Interrupt439 * @data:     Opaque data structure440 *441 * Return: Indicate if the interrupt has been handled.442 */443static irqreturn_t nfp_net_irq_rxtx(int irq, void *data)444{445	struct nfp_net_r_vector *r_vec = data;446 447	/* Currently we cannot tell if it's a rx or tx interrupt,448	 * since dim does not need accurate event_ctr to calculate,449	 * we just use this counter for both rx and tx dim.450	 */451	r_vec->event_ctr++;452 453	napi_schedule_irqoff(&r_vec->napi);454 455	/* The FW auto-masks any interrupt, either via the MASK bit in456	 * the MSI-X table or via the per entry ICR field.  So there457	 * is no need to disable interrupts here.458	 */459	return IRQ_HANDLED;460}461 462static irqreturn_t nfp_ctrl_irq_rxtx(int irq, void *data)463{464	struct nfp_net_r_vector *r_vec = data;465 466	tasklet_schedule(&r_vec->tasklet);467 468	return IRQ_HANDLED;469}470 471/**472 * nfp_net_read_link_status() - Reread link status from control BAR473 * @nn:       NFP Network structure474 */475static void nfp_net_read_link_status(struct nfp_net *nn)476{477	unsigned long flags;478	bool link_up;479	u16 sts;480 481	spin_lock_irqsave(&nn->link_status_lock, flags);482 483	sts = nn_readw(nn, NFP_NET_CFG_STS);484	link_up = !!(sts & NFP_NET_CFG_STS_LINK);485 486	if (nn->link_up == link_up)487		goto out;488 489	nn->link_up = link_up;490	if (nn->port) {491		set_bit(NFP_PORT_CHANGED, &nn->port->flags);492		if (nn->port->link_cb)493			nn->port->link_cb(nn->port);494	}495 496	if (nn->link_up) {497		netif_carrier_on(nn->dp.netdev);498		netdev_info(nn->dp.netdev, "NIC Link is Up\n");499	} else {500		netif_carrier_off(nn->dp.netdev);501		netdev_info(nn->dp.netdev, "NIC Link is Down\n");502	}503out:504	spin_unlock_irqrestore(&nn->link_status_lock, flags);505}506 507/**508 * nfp_net_irq_lsc() - Interrupt service routine for link state changes509 * @irq:      Interrupt510 * @data:     Opaque data structure511 *512 * Return: Indicate if the interrupt has been handled.513 */514static irqreturn_t nfp_net_irq_lsc(int irq, void *data)515{516	struct nfp_net *nn = data;517	struct msix_entry *entry;518 519	entry = &nn->irq_entries[NFP_NET_IRQ_LSC_IDX];520 521	nfp_net_read_link_status(nn);522 523	nfp_net_irq_unmask(nn, entry->entry);524 525	return IRQ_HANDLED;526}527 528/**529 * nfp_net_irq_exn() - Interrupt service routine for exceptions530 * @irq:      Interrupt531 * @data:     Opaque data structure532 *533 * Return: Indicate if the interrupt has been handled.534 */535static irqreturn_t nfp_net_irq_exn(int irq, void *data)536{537	struct nfp_net *nn = data;538 539	nn_err(nn, "%s: UNIMPLEMENTED.\n", __func__);540	/* XXX TO BE IMPLEMENTED */541	return IRQ_HANDLED;542}543 544/**545 * nfp_net_aux_irq_request() - Request an auxiliary interrupt (LSC or EXN)546 * @nn:		NFP Network structure547 * @ctrl_offset: Control BAR offset where IRQ configuration should be written548 * @format:	printf-style format to construct the interrupt name549 * @name:	Pointer to allocated space for interrupt name550 * @name_sz:	Size of space for interrupt name551 * @vector_idx:	Index of MSI-X vector used for this interrupt552 * @handler:	IRQ handler to register for this interrupt553 */554static int555nfp_net_aux_irq_request(struct nfp_net *nn, u32 ctrl_offset,556			const char *format, char *name, size_t name_sz,557			unsigned int vector_idx, irq_handler_t handler)558{559	struct msix_entry *entry;560	int err;561 562	entry = &nn->irq_entries[vector_idx];563 564	snprintf(name, name_sz, format, nfp_net_name(nn));565	err = request_irq(entry->vector, handler, 0, name, nn);566	if (err) {567		nn_err(nn, "Failed to request IRQ %d (err=%d).\n",568		       entry->vector, err);569		return err;570	}571	nn_writeb(nn, ctrl_offset, entry->entry);572	nfp_net_irq_unmask(nn, entry->entry);573 574	return 0;575}576 577/**578 * nfp_net_aux_irq_free() - Free an auxiliary interrupt (LSC or EXN)579 * @nn:		NFP Network structure580 * @ctrl_offset: Control BAR offset where IRQ configuration should be written581 * @vector_idx:	Index of MSI-X vector used for this interrupt582 */583static void nfp_net_aux_irq_free(struct nfp_net *nn, u32 ctrl_offset,584				 unsigned int vector_idx)585{586	nn_writeb(nn, ctrl_offset, 0xff);587	nn_pci_flush(nn);588	free_irq(nn->irq_entries[vector_idx].vector, nn);589}590 591struct sk_buff *592nfp_net_tls_tx(struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec,593	       struct sk_buff *skb, u64 *tls_handle, int *nr_frags)594{595#ifdef CONFIG_TLS_DEVICE596	struct nfp_net_tls_offload_ctx *ntls;597	struct sk_buff *nskb;598	bool resync_pending;599	u32 datalen, seq;600 601	if (likely(!dp->ktls_tx))602		return skb;603	if (!tls_is_skb_tx_device_offloaded(skb))604		return skb;605 606	datalen = skb->len - skb_tcp_all_headers(skb);607	seq = ntohl(tcp_hdr(skb)->seq);608	ntls = tls_driver_ctx(skb->sk, TLS_OFFLOAD_CTX_DIR_TX);609	resync_pending = tls_offload_tx_resync_pending(skb->sk);610	if (unlikely(resync_pending || ntls->next_seq != seq)) {611		/* Pure ACK out of order already */612		if (!datalen)613			return skb;614 615		u64_stats_update_begin(&r_vec->tx_sync);616		r_vec->tls_tx_fallback++;617		u64_stats_update_end(&r_vec->tx_sync);618 619		nskb = tls_encrypt_skb(skb);620		if (!nskb) {621			u64_stats_update_begin(&r_vec->tx_sync);622			r_vec->tls_tx_no_fallback++;623			u64_stats_update_end(&r_vec->tx_sync);624			return NULL;625		}626		/* encryption wasn't necessary */627		if (nskb == skb)628			return skb;629		/* we don't re-check ring space */630		if (unlikely(skb_is_nonlinear(nskb))) {631			nn_dp_warn(dp, "tls_encrypt_skb() produced fragmented frame\n");632			u64_stats_update_begin(&r_vec->tx_sync);633			r_vec->tx_errors++;634			u64_stats_update_end(&r_vec->tx_sync);635			dev_kfree_skb_any(nskb);636			return NULL;637		}638 639		/* jump forward, a TX may have gotten lost, need to sync TX */640		if (!resync_pending && seq - ntls->next_seq < U32_MAX / 4)641			tls_offload_tx_resync_request(nskb->sk, seq,642						      ntls->next_seq);643 644		*nr_frags = 0;645		return nskb;646	}647 648	if (datalen) {649		u64_stats_update_begin(&r_vec->tx_sync);650		if (!skb_is_gso(skb))651			r_vec->hw_tls_tx++;652		else653			r_vec->hw_tls_tx += skb_shinfo(skb)->gso_segs;654		u64_stats_update_end(&r_vec->tx_sync);655	}656 657	memcpy(tls_handle, ntls->fw_handle, sizeof(ntls->fw_handle));658	ntls->next_seq += datalen;659#endif660	return skb;661}662 663void nfp_net_tls_tx_undo(struct sk_buff *skb, u64 tls_handle)664{665#ifdef CONFIG_TLS_DEVICE666	struct nfp_net_tls_offload_ctx *ntls;667	u32 datalen, seq;668 669	if (!tls_handle)670		return;671	if (WARN_ON_ONCE(!tls_is_skb_tx_device_offloaded(skb)))672		return;673 674	datalen = skb->len - skb_tcp_all_headers(skb);675	seq = ntohl(tcp_hdr(skb)->seq);676 677	ntls = tls_driver_ctx(skb->sk, TLS_OFFLOAD_CTX_DIR_TX);678	if (ntls->next_seq == seq + datalen)679		ntls->next_seq = seq;680	else681		WARN_ON_ONCE(1);682#endif683}684 685static void nfp_net_tx_timeout(struct net_device *netdev, unsigned int txqueue)686{687	struct nfp_net *nn = netdev_priv(netdev);688 689	nn_warn(nn, "TX watchdog timeout on ring: %u\n", txqueue);690}691 692/* Receive processing */693static unsigned int694nfp_net_calc_fl_bufsz_data(struct nfp_net_dp *dp)695{696	unsigned int fl_bufsz = 0;697 698	if (dp->rx_offset == NFP_NET_CFG_RX_OFFSET_DYNAMIC)699		fl_bufsz += NFP_NET_MAX_PREPEND;700	else701		fl_bufsz += dp->rx_offset;702	fl_bufsz += ETH_HLEN + VLAN_HLEN * 2 + dp->mtu;703 704	return fl_bufsz;705}706 707static unsigned int nfp_net_calc_fl_bufsz(struct nfp_net_dp *dp)708{709	unsigned int fl_bufsz;710 711	fl_bufsz = NFP_NET_RX_BUF_HEADROOM;712	fl_bufsz += dp->rx_dma_off;713	fl_bufsz += nfp_net_calc_fl_bufsz_data(dp);714 715	fl_bufsz = SKB_DATA_ALIGN(fl_bufsz);716	fl_bufsz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));717 718	return fl_bufsz;719}720 721static unsigned int nfp_net_calc_fl_bufsz_xsk(struct nfp_net_dp *dp)722{723	unsigned int fl_bufsz;724 725	fl_bufsz = XDP_PACKET_HEADROOM;726	fl_bufsz += nfp_net_calc_fl_bufsz_data(dp);727 728	return fl_bufsz;729}730 731/* Setup and Configuration732 */733 734/**735 * nfp_net_vecs_init() - Assign IRQs and setup rvecs.736 * @nn:		NFP Network structure737 */738static void nfp_net_vecs_init(struct nfp_net *nn)739{740	int numa_node = dev_to_node(&nn->pdev->dev);741	struct nfp_net_r_vector *r_vec;742	unsigned int r;743 744	nn->lsc_handler = nfp_net_irq_lsc;745	nn->exn_handler = nfp_net_irq_exn;746 747	for (r = 0; r < nn->max_r_vecs; r++) {748		struct msix_entry *entry;749 750		entry = &nn->irq_entries[NFP_NET_NON_Q_VECTORS + r];751 752		r_vec = &nn->r_vecs[r];753		r_vec->nfp_net = nn;754		r_vec->irq_entry = entry->entry;755		r_vec->irq_vector = entry->vector;756 757		if (nn->dp.netdev) {758			r_vec->handler = nfp_net_irq_rxtx;759		} else {760			r_vec->handler = nfp_ctrl_irq_rxtx;761 762			__skb_queue_head_init(&r_vec->queue);763			spin_lock_init(&r_vec->lock);764			tasklet_setup(&r_vec->tasklet, nn->dp.ops->ctrl_poll);765			tasklet_disable(&r_vec->tasklet);766		}767 768		cpumask_set_cpu(cpumask_local_spread(r, numa_node), &r_vec->affinity_mask);769	}770}771 772static void773nfp_net_napi_add(struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec, int idx)774{775	if (dp->netdev)776		netif_napi_add(dp->netdev, &r_vec->napi,777			       nfp_net_has_xsk_pool_slow(dp, idx) ? dp->ops->xsk_poll : dp->ops->poll);778	else779		tasklet_enable(&r_vec->tasklet);780}781 782static void783nfp_net_napi_del(struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec)784{785	if (dp->netdev)786		netif_napi_del(&r_vec->napi);787	else788		tasklet_disable(&r_vec->tasklet);789}790 791static void792nfp_net_vector_assign_rings(struct nfp_net_dp *dp,793			    struct nfp_net_r_vector *r_vec, int idx)794{795	r_vec->rx_ring = idx < dp->num_rx_rings ? &dp->rx_rings[idx] : NULL;796	r_vec->tx_ring =797		idx < dp->num_stack_tx_rings ? &dp->tx_rings[idx] : NULL;798 799	r_vec->xdp_ring = idx < dp->num_tx_rings - dp->num_stack_tx_rings ?800		&dp->tx_rings[dp->num_stack_tx_rings + idx] : NULL;801 802	if (nfp_net_has_xsk_pool_slow(dp, idx) || r_vec->xsk_pool) {803		r_vec->xsk_pool = dp->xdp_prog ? dp->xsk_pools[idx] : NULL;804 805		if (r_vec->xsk_pool)806			xsk_pool_set_rxq_info(r_vec->xsk_pool,807					      &r_vec->rx_ring->xdp_rxq);808 809		nfp_net_napi_del(dp, r_vec);810		nfp_net_napi_add(dp, r_vec, idx);811	}812}813 814static int815nfp_net_prepare_vector(struct nfp_net *nn, struct nfp_net_r_vector *r_vec,816		       int idx)817{818	int err;819 820	nfp_net_napi_add(&nn->dp, r_vec, idx);821 822	snprintf(r_vec->name, sizeof(r_vec->name),823		 "%s-rxtx-%d", nfp_net_name(nn), idx);824	err = request_irq(r_vec->irq_vector, r_vec->handler, IRQF_NO_AUTOEN,825			  r_vec->name, r_vec);826	if (err) {827		nfp_net_napi_del(&nn->dp, r_vec);828		nn_err(nn, "Error requesting IRQ %d\n", r_vec->irq_vector);829		return err;830	}831 832	irq_set_affinity_hint(r_vec->irq_vector, &r_vec->affinity_mask);833 834	nn_dbg(nn, "RV%02d: irq=%03d/%03d\n", idx, r_vec->irq_vector,835	       r_vec->irq_entry);836 837	return 0;838}839 840static void841nfp_net_cleanup_vector(struct nfp_net *nn, struct nfp_net_r_vector *r_vec)842{843	irq_set_affinity_hint(r_vec->irq_vector, NULL);844	nfp_net_napi_del(&nn->dp, r_vec);845	free_irq(r_vec->irq_vector, r_vec);846}847 848/**849 * nfp_net_rss_write_itbl() - Write RSS indirection table to device850 * @nn:      NFP Net device to reconfigure851 */852void nfp_net_rss_write_itbl(struct nfp_net *nn)853{854	int i;855 856	for (i = 0; i < NFP_NET_CFG_RSS_ITBL_SZ; i += 4)857		nn_writel(nn, NFP_NET_CFG_RSS_ITBL + i,858			  get_unaligned_le32(nn->rss_itbl + i));859}860 861/**862 * nfp_net_rss_write_key() - Write RSS hash key to device863 * @nn:      NFP Net device to reconfigure864 */865void nfp_net_rss_write_key(struct nfp_net *nn)866{867	int i;868 869	for (i = 0; i < nfp_net_rss_key_sz(nn); i += 4)870		nn_writel(nn, NFP_NET_CFG_RSS_KEY + i,871			  get_unaligned_le32(nn->rss_key + i));872}873 874/**875 * nfp_net_coalesce_write_cfg() - Write irq coalescence configuration to HW876 * @nn:      NFP Net device to reconfigure877 */878void nfp_net_coalesce_write_cfg(struct nfp_net *nn)879{880	u8 i;881	u32 factor;882	u32 value;883 884	/* Compute factor used to convert coalesce '_usecs' parameters to885	 * ME timestamp ticks.  There are 16 ME clock cycles for each timestamp886	 * count.887	 */888	factor = nn->tlv_caps.me_freq_mhz / 16;889 890	/* copy RX interrupt coalesce parameters */891	value = (nn->rx_coalesce_max_frames << 16) |892		(factor * nn->rx_coalesce_usecs);893	for (i = 0; i < nn->dp.num_rx_rings; i++)894		nn_writel(nn, NFP_NET_CFG_RXR_IRQ_MOD(i), value);895 896	/* copy TX interrupt coalesce parameters */897	value = (nn->tx_coalesce_max_frames << 16) |898		(factor * nn->tx_coalesce_usecs);899	for (i = 0; i < nn->dp.num_tx_rings; i++)900		nn_writel(nn, NFP_NET_CFG_TXR_IRQ_MOD(i), value);901}902 903/**904 * nfp_net_write_mac_addr() - Write mac address to the device control BAR905 * @nn:      NFP Net device to reconfigure906 * @addr:    MAC address to write907 *908 * Writes the MAC address from the netdev to the device control BAR.  Does not909 * perform the required reconfig.  We do a bit of byte swapping dance because910 * firmware is LE.911 */912static void nfp_net_write_mac_addr(struct nfp_net *nn, const u8 *addr)913{914	nn_writel(nn, NFP_NET_CFG_MACADDR + 0, get_unaligned_be32(addr));915	nn_writew(nn, NFP_NET_CFG_MACADDR + 6, get_unaligned_be16(addr + 4));916}917 918/**919 * nfp_net_clear_config_and_disable() - Clear control BAR and disable NFP920 * @nn:      NFP Net device to reconfigure921 *922 * Warning: must be fully idempotent.923 */924static void nfp_net_clear_config_and_disable(struct nfp_net *nn)925{926	u32 new_ctrl, new_ctrl_w1, update;927	unsigned int r;928	int err;929 930	new_ctrl = nn->dp.ctrl;931	new_ctrl &= ~NFP_NET_CFG_CTRL_ENABLE;932	update = NFP_NET_CFG_UPDATE_GEN;933	update |= NFP_NET_CFG_UPDATE_MSIX;934	update |= NFP_NET_CFG_UPDATE_RING;935 936	if (nn->cap & NFP_NET_CFG_CTRL_RINGCFG)937		new_ctrl &= ~NFP_NET_CFG_CTRL_RINGCFG;938 939	if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_FREELIST_EN)) {940		nn_writeq(nn, NFP_NET_CFG_TXRS_ENABLE, 0);941		nn_writeq(nn, NFP_NET_CFG_RXRS_ENABLE, 0);942	}943 944	nn_writel(nn, NFP_NET_CFG_CTRL, new_ctrl);945	err = nfp_net_reconfig(nn, update);946	if (err)947		nn_err(nn, "Could not disable device: %d\n", err);948 949	if (nn->cap_w1 & NFP_NET_CFG_CTRL_FREELIST_EN) {950		new_ctrl_w1 = nn->dp.ctrl_w1;951		new_ctrl_w1 &= ~NFP_NET_CFG_CTRL_FREELIST_EN;952		nn_writeq(nn, NFP_NET_CFG_TXRS_ENABLE, 0);953		nn_writeq(nn, NFP_NET_CFG_RXRS_ENABLE, 0);954 955		nn_writel(nn, NFP_NET_CFG_CTRL_WORD1, new_ctrl_w1);956		err = nfp_net_reconfig(nn, update);957		if (err)958			nn_err(nn, "Could not disable FREELIST_EN: %d\n", err);959		nn->dp.ctrl_w1 = new_ctrl_w1;960	}961 962	for (r = 0; r < nn->dp.num_rx_rings; r++) {963		nfp_net_rx_ring_reset(&nn->dp.rx_rings[r]);964		if (nfp_net_has_xsk_pool_slow(&nn->dp, nn->dp.rx_rings[r].idx))965			nfp_net_xsk_rx_bufs_free(&nn->dp.rx_rings[r]);966	}967	for (r = 0; r < nn->dp.num_tx_rings; r++)968		nfp_net_tx_ring_reset(&nn->dp, &nn->dp.tx_rings[r]);969	for (r = 0; r < nn->dp.num_r_vecs; r++)970		nfp_net_vec_clear_ring_data(nn, r);971 972	nn->dp.ctrl = new_ctrl;973}974 975/**976 * nfp_net_set_config_and_enable() - Write control BAR and enable NFP977 * @nn:      NFP Net device to reconfigure978 */979static int nfp_net_set_config_and_enable(struct nfp_net *nn)980{981	u32 bufsz, new_ctrl, new_ctrl_w1, update = 0;982	unsigned int r;983	int err;984 985	new_ctrl = nn->dp.ctrl;986	new_ctrl_w1 = nn->dp.ctrl_w1;987 988	if (nn->dp.ctrl & NFP_NET_CFG_CTRL_RSS_ANY) {989		nfp_net_rss_write_key(nn);990		nfp_net_rss_write_itbl(nn);991		nn_writel(nn, NFP_NET_CFG_RSS_CTRL, nn->rss_cfg);992		update |= NFP_NET_CFG_UPDATE_RSS;993	}994 995	if (nn->dp.ctrl & NFP_NET_CFG_CTRL_IRQMOD) {996		nfp_net_coalesce_write_cfg(nn);997		update |= NFP_NET_CFG_UPDATE_IRQMOD;998	}999 1000	for (r = 0; r < nn->dp.num_tx_rings; r++)1001		nfp_net_tx_ring_hw_cfg_write(nn, &nn->dp.tx_rings[r], r);1002	for (r = 0; r < nn->dp.num_rx_rings; r++)1003		nfp_net_rx_ring_hw_cfg_write(nn, &nn->dp.rx_rings[r], r);1004 1005	nn_writeq(nn, NFP_NET_CFG_TXRS_ENABLE,1006		  U64_MAX >> (64 - nn->dp.num_tx_rings));1007 1008	nn_writeq(nn, NFP_NET_CFG_RXRS_ENABLE,1009		  U64_MAX >> (64 - nn->dp.num_rx_rings));1010 1011	if (nn->dp.netdev)1012		nfp_net_write_mac_addr(nn, nn->dp.netdev->dev_addr);1013 1014	nn_writel(nn, NFP_NET_CFG_MTU, nn->dp.mtu);1015 1016	bufsz = nn->dp.fl_bufsz - nn->dp.rx_dma_off - NFP_NET_RX_BUF_NON_DATA;1017	nn_writel(nn, NFP_NET_CFG_FLBUFSZ, bufsz);1018 1019	/* Enable device1020	 * Step 1: Replace the CTRL_ENABLE by NFP_NET_CFG_CTRL_FREELIST_EN if1021	 * FREELIST_EN exits.1022	 */1023	if (nn->cap_w1 & NFP_NET_CFG_CTRL_FREELIST_EN)1024		new_ctrl_w1 |= NFP_NET_CFG_CTRL_FREELIST_EN;1025	else1026		new_ctrl |= NFP_NET_CFG_CTRL_ENABLE;1027	update |= NFP_NET_CFG_UPDATE_GEN;1028	update |= NFP_NET_CFG_UPDATE_MSIX;1029	update |= NFP_NET_CFG_UPDATE_RING;1030	if (nn->cap & NFP_NET_CFG_CTRL_RINGCFG)1031		new_ctrl |= NFP_NET_CFG_CTRL_RINGCFG;1032 1033	/* Step 2: Send the configuration and write the freelist.1034	 * - The freelist only need to be written once.1035	 */1036	nn_writel(nn, NFP_NET_CFG_CTRL, new_ctrl);1037	nn_writel(nn, NFP_NET_CFG_CTRL_WORD1, new_ctrl_w1);1038	err = nfp_net_reconfig(nn, update);1039	if (err) {1040		nfp_net_clear_config_and_disable(nn);1041		return err;1042	}1043 1044	nn->dp.ctrl = new_ctrl;1045	nn->dp.ctrl_w1 = new_ctrl_w1;1046 1047	for (r = 0; r < nn->dp.num_rx_rings; r++)1048		nfp_net_rx_ring_fill_freelist(&nn->dp, &nn->dp.rx_rings[r]);1049 1050	/* Step 3: Do the NFP_NET_CFG_CTRL_ENABLE. Send the configuration.1051	 */1052	if (nn->cap_w1 & NFP_NET_CFG_CTRL_FREELIST_EN) {1053		new_ctrl |= NFP_NET_CFG_CTRL_ENABLE;1054		nn_writel(nn, NFP_NET_CFG_CTRL, new_ctrl);1055 1056		err = nfp_net_reconfig(nn, update);1057		if (err) {1058			nfp_net_clear_config_and_disable(nn);1059			return err;1060		}1061		nn->dp.ctrl = new_ctrl;1062	}1063 1064	return 0;1065}1066 1067/**1068 * nfp_net_close_stack() - Quiesce the stack (part of close)1069 * @nn:	     NFP Net device to reconfigure1070 */1071static void nfp_net_close_stack(struct nfp_net *nn)1072{1073	struct nfp_net_r_vector *r_vec;1074	unsigned int r;1075 1076	disable_irq(nn->irq_entries[NFP_NET_IRQ_LSC_IDX].vector);1077	netif_carrier_off(nn->dp.netdev);1078	nn->link_up = false;1079 1080	for (r = 0; r < nn->dp.num_r_vecs; r++) {1081		r_vec = &nn->r_vecs[r];1082 1083		disable_irq(r_vec->irq_vector);1084		napi_disable(&r_vec->napi);1085 1086		if (r_vec->rx_ring)1087			cancel_work_sync(&r_vec->rx_dim.work);1088 1089		if (r_vec->tx_ring)1090			cancel_work_sync(&r_vec->tx_dim.work);1091	}1092 1093	netif_tx_disable(nn->dp.netdev);1094}1095 1096/**1097 * nfp_net_close_free_all() - Free all runtime resources1098 * @nn:      NFP Net device to reconfigure1099 */1100static void nfp_net_close_free_all(struct nfp_net *nn)1101{1102	unsigned int r;1103 1104	nfp_net_tx_rings_free(&nn->dp);1105	nfp_net_rx_rings_free(&nn->dp);1106 1107	for (r = 0; r < nn->dp.num_r_vecs; r++)1108		nfp_net_cleanup_vector(nn, &nn->r_vecs[r]);1109 1110	nfp_net_aux_irq_free(nn, NFP_NET_CFG_LSC, NFP_NET_IRQ_LSC_IDX);1111	nfp_net_aux_irq_free(nn, NFP_NET_CFG_EXN, NFP_NET_IRQ_EXN_IDX);1112}1113 1114/**1115 * nfp_net_netdev_close() - Called when the device is downed1116 * @netdev:      netdev structure1117 */1118static int nfp_net_netdev_close(struct net_device *netdev)1119{1120	struct nfp_net *nn = netdev_priv(netdev);1121 1122	/* Step 1: Disable RX and TX rings from the Linux kernel perspective1123	 */1124	nfp_net_close_stack(nn);1125 1126	/* Step 2: Tell NFP1127	 */1128	if (nn->cap_w1 & NFP_NET_CFG_CTRL_MCAST_FILTER)1129		__dev_mc_unsync(netdev, nfp_net_mc_unsync);1130 1131	nfp_net_clear_config_and_disable(nn);1132	nfp_port_configure(netdev, false);1133 1134	/* Step 3: Free resources1135	 */1136	nfp_net_close_free_all(nn);1137 1138	nn_dbg(nn, "%s down", netdev->name);1139	return 0;1140}1141 1142void nfp_ctrl_close(struct nfp_net *nn)1143{1144	int r;1145 1146	rtnl_lock();1147 1148	for (r = 0; r < nn->dp.num_r_vecs; r++) {1149		disable_irq(nn->r_vecs[r].irq_vector);1150		tasklet_disable(&nn->r_vecs[r].tasklet);1151	}1152 1153	nfp_net_clear_config_and_disable(nn);1154 1155	nfp_net_close_free_all(nn);1156 1157	rtnl_unlock();1158}1159 1160static void nfp_net_rx_dim_work(struct work_struct *work)1161{1162	struct nfp_net_r_vector *r_vec;1163	unsigned int factor, value;1164	struct dim_cq_moder moder;1165	struct nfp_net *nn;1166	struct dim *dim;1167 1168	dim = container_of(work, struct dim, work);1169	moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);1170	r_vec = container_of(dim, struct nfp_net_r_vector, rx_dim);1171	nn = r_vec->nfp_net;1172 1173	/* Compute factor used to convert coalesce '_usecs' parameters to1174	 * ME timestamp ticks.  There are 16 ME clock cycles for each timestamp1175	 * count.1176	 */1177	factor = nn->tlv_caps.me_freq_mhz / 16;1178	if (nfp_net_coalesce_para_check(factor * moder.usec) ||1179	    nfp_net_coalesce_para_check(moder.pkts))1180		return;1181 1182	/* copy RX interrupt coalesce parameters */1183	value = (moder.pkts << 16) | (factor * moder.usec);1184	nn_writel(nn, NFP_NET_CFG_RXR_IRQ_MOD(r_vec->rx_ring->idx), value);1185	(void)nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_IRQMOD);1186 1187	dim->state = DIM_START_MEASURE;1188}1189 1190static void nfp_net_tx_dim_work(struct work_struct *work)1191{1192	struct nfp_net_r_vector *r_vec;1193	unsigned int factor, value;1194	struct dim_cq_moder moder;1195	struct nfp_net *nn;1196	struct dim *dim;1197 1198	dim = container_of(work, struct dim, work);1199	moder = net_dim_get_tx_moderation(dim->mode, dim->profile_ix);1200	r_vec = container_of(dim, struct nfp_net_r_vector, tx_dim);1201	nn = r_vec->nfp_net;1202 1203	/* Compute factor used to convert coalesce '_usecs' parameters to1204	 * ME timestamp ticks.  There are 16 ME clock cycles for each timestamp1205	 * count.1206	 */1207	factor = nn->tlv_caps.me_freq_mhz / 16;1208	if (nfp_net_coalesce_para_check(factor * moder.usec) ||1209	    nfp_net_coalesce_para_check(moder.pkts))1210		return;1211 1212	/* copy TX interrupt coalesce parameters */1213	value = (moder.pkts << 16) | (factor * moder.usec);1214	nn_writel(nn, NFP_NET_CFG_TXR_IRQ_MOD(r_vec->tx_ring->idx), value);1215	(void)nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_IRQMOD);1216 1217	dim->state = DIM_START_MEASURE;1218}1219 1220/**1221 * nfp_net_open_stack() - Start the device from stack's perspective1222 * @nn:      NFP Net device to reconfigure1223 */1224static void nfp_net_open_stack(struct nfp_net *nn)1225{1226	struct nfp_net_r_vector *r_vec;1227	unsigned int r;1228 1229	for (r = 0; r < nn->dp.num_r_vecs; r++) {1230		r_vec = &nn->r_vecs[r];1231 1232		if (r_vec->rx_ring) {1233			INIT_WORK(&r_vec->rx_dim.work, nfp_net_rx_dim_work);1234			r_vec->rx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;1235		}1236 1237		if (r_vec->tx_ring) {1238			INIT_WORK(&r_vec->tx_dim.work, nfp_net_tx_dim_work);1239			r_vec->tx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;1240		}1241 1242		napi_enable(&r_vec->napi);1243		enable_irq(r_vec->irq_vector);1244	}1245 1246	netif_tx_wake_all_queues(nn->dp.netdev);1247 1248	enable_irq(nn->irq_entries[NFP_NET_IRQ_LSC_IDX].vector);1249	nfp_net_read_link_status(nn);1250}1251 1252static int nfp_net_open_alloc_all(struct nfp_net *nn)1253{1254	int err, r;1255 1256	err = nfp_net_aux_irq_request(nn, NFP_NET_CFG_EXN, "%s-exn",1257				      nn->exn_name, sizeof(nn->exn_name),1258				      NFP_NET_IRQ_EXN_IDX, nn->exn_handler);1259	if (err)1260		return err;1261	err = nfp_net_aux_irq_request(nn, NFP_NET_CFG_LSC, "%s-lsc",1262				      nn->lsc_name, sizeof(nn->lsc_name),1263				      NFP_NET_IRQ_LSC_IDX, nn->lsc_handler);1264	if (err)1265		goto err_free_exn;1266	disable_irq(nn->irq_entries[NFP_NET_IRQ_LSC_IDX].vector);1267 1268	for (r = 0; r < nn->dp.num_r_vecs; r++) {1269		err = nfp_net_prepare_vector(nn, &nn->r_vecs[r], r);1270		if (err)1271			goto err_cleanup_vec_p;1272	}1273 1274	err = nfp_net_rx_rings_prepare(nn, &nn->dp);1275	if (err)1276		goto err_cleanup_vec;1277 1278	err = nfp_net_tx_rings_prepare(nn, &nn->dp);1279	if (err)1280		goto err_free_rx_rings;1281 1282	for (r = 0; r < nn->max_r_vecs; r++)1283		nfp_net_vector_assign_rings(&nn->dp, &nn->r_vecs[r], r);1284 1285	return 0;1286 1287err_free_rx_rings:1288	nfp_net_rx_rings_free(&nn->dp);1289err_cleanup_vec:1290	r = nn->dp.num_r_vecs;1291err_cleanup_vec_p:1292	while (r--)1293		nfp_net_cleanup_vector(nn, &nn->r_vecs[r]);1294	nfp_net_aux_irq_free(nn, NFP_NET_CFG_LSC, NFP_NET_IRQ_LSC_IDX);1295err_free_exn:1296	nfp_net_aux_irq_free(nn, NFP_NET_CFG_EXN, NFP_NET_IRQ_EXN_IDX);1297	return err;1298}1299 1300static int nfp_net_netdev_open(struct net_device *netdev)1301{1302	struct nfp_net *nn = netdev_priv(netdev);1303	int err;1304 1305	/* Step 1: Allocate resources for rings and the like1306	 * - Request interrupts1307	 * - Allocate RX and TX ring resources1308	 * - Setup initial RSS table1309	 */1310	err = nfp_net_open_alloc_all(nn);1311	if (err)1312		return err;1313 1314	err = netif_set_real_num_tx_queues(netdev, nn->dp.num_stack_tx_rings);1315	if (err)1316		goto err_free_all;1317 1318	err = netif_set_real_num_rx_queues(netdev, nn->dp.num_rx_rings);1319	if (err)1320		goto err_free_all;1321 1322	/* Step 2: Configure the NFP1323	 * - Ifup the physical interface if it exists1324	 * - Enable rings from 0 to tx_rings/rx_rings - 1.1325	 * - Write MAC address (in case it changed)1326	 * - Set the MTU1327	 * - Set the Freelist buffer size1328	 * - Enable the FW1329	 */1330	err = nfp_port_configure(netdev, true);1331	if (err)1332		goto err_free_all;1333 1334	err = nfp_net_set_config_and_enable(nn);1335	if (err)1336		goto err_port_disable;1337 1338	/* Step 3: Enable for kernel1339	 * - put some freelist descriptors on each RX ring1340	 * - enable NAPI on each ring1341	 * - enable all TX queues1342	 * - set link state1343	 */1344	nfp_net_open_stack(nn);1345 1346	return 0;1347 1348err_port_disable:1349	nfp_port_configure(netdev, false);1350err_free_all:1351	nfp_net_close_free_all(nn);1352	return err;1353}1354 1355int nfp_ctrl_open(struct nfp_net *nn)1356{1357	int err, r;1358 1359	/* ring dumping depends on vNICs being opened/closed under rtnl */1360	rtnl_lock();1361 1362	err = nfp_net_open_alloc_all(nn);1363	if (err)1364		goto err_unlock;1365 1366	err = nfp_net_set_config_and_enable(nn);1367	if (err)1368		goto err_free_all;1369 1370	for (r = 0; r < nn->dp.num_r_vecs; r++)1371		enable_irq(nn->r_vecs[r].irq_vector);1372 1373	rtnl_unlock();1374 1375	return 0;1376 1377err_free_all:1378	nfp_net_close_free_all(nn);1379err_unlock:1380	rtnl_unlock();1381	return err;1382}1383 1384int nfp_net_sched_mbox_amsg_work(struct nfp_net *nn, u32 cmd, const void *data, size_t len,1385				 int (*cb)(struct nfp_net *, struct nfp_mbox_amsg_entry *))1386{1387	struct nfp_mbox_amsg_entry *entry;1388 1389	entry = kmalloc(sizeof(*entry) + len, GFP_ATOMIC);1390	if (!entry)1391		return -ENOMEM;1392 1393	memcpy(entry->msg, data, len);1394	entry->cmd = cmd;1395	entry->cfg = cb;1396 1397	spin_lock_bh(&nn->mbox_amsg.lock);1398	list_add_tail(&entry->list, &nn->mbox_amsg.list);1399	spin_unlock_bh(&nn->mbox_amsg.lock);1400 1401	schedule_work(&nn->mbox_amsg.work);1402 1403	return 0;1404}1405 1406static void nfp_net_mbox_amsg_work(struct work_struct *work)1407{1408	struct nfp_net *nn = container_of(work, struct nfp_net, mbox_amsg.work);1409	struct nfp_mbox_amsg_entry *entry, *tmp;1410	struct list_head tmp_list;1411 1412	INIT_LIST_HEAD(&tmp_list);1413 1414	spin_lock_bh(&nn->mbox_amsg.lock);1415	list_splice_init(&nn->mbox_amsg.list, &tmp_list);1416	spin_unlock_bh(&nn->mbox_amsg.lock);1417 1418	list_for_each_entry_safe(entry, tmp, &tmp_list, list) {1419		int err = entry->cfg(nn, entry);1420 1421		if (err)1422			nn_err(nn, "Config cmd %d to HW failed %d.\n", entry->cmd, err);1423 1424		list_del(&entry->list);1425		kfree(entry);1426	}1427}1428 1429static int nfp_net_mc_cfg(struct nfp_net *nn, struct nfp_mbox_amsg_entry *entry)1430{1431	unsigned char *addr = entry->msg;1432	int ret;1433 1434	ret = nfp_net_mbox_lock(nn, NFP_NET_CFG_MULTICAST_SZ);1435	if (ret)1436		return ret;1437 1438	nn_writel(nn, nn->tlv_caps.mbox_off + NFP_NET_CFG_MULTICAST_MAC_HI,1439		  get_unaligned_be32(addr));1440	nn_writew(nn, nn->tlv_caps.mbox_off + NFP_NET_CFG_MULTICAST_MAC_LO,1441		  get_unaligned_be16(addr + 4));1442 1443	return nfp_net_mbox_reconfig_and_unlock(nn, entry->cmd);1444}1445 1446static int nfp_net_mc_sync(struct net_device *netdev, const unsigned char *addr)1447{1448	struct nfp_net *nn = netdev_priv(netdev);1449 1450	if (netdev_mc_count(netdev) > NFP_NET_CFG_MAC_MC_MAX) {1451		nn_err(nn, "Requested number of MC addresses (%d) exceeds maximum (%d).\n",1452		       netdev_mc_count(netdev), NFP_NET_CFG_MAC_MC_MAX);1453		return -EINVAL;1454	}1455 1456	return nfp_net_sched_mbox_amsg_work(nn, NFP_NET_CFG_MBOX_CMD_MULTICAST_ADD, addr,1457					    NFP_NET_CFG_MULTICAST_SZ, nfp_net_mc_cfg);1458}1459 1460static int nfp_net_mc_unsync(struct net_device *netdev, const unsigned char *addr)1461{1462	struct nfp_net *nn = netdev_priv(netdev);1463 1464	return nfp_net_sched_mbox_amsg_work(nn, NFP_NET_CFG_MBOX_CMD_MULTICAST_DEL, addr,1465					    NFP_NET_CFG_MULTICAST_SZ, nfp_net_mc_cfg);1466}1467 1468static void nfp_net_set_rx_mode(struct net_device *netdev)1469{1470	struct nfp_net *nn = netdev_priv(netdev);1471	u32 new_ctrl, new_ctrl_w1;1472 1473	new_ctrl = nn->dp.ctrl;1474	new_ctrl_w1 = nn->dp.ctrl_w1;1475 1476	if (!netdev_mc_empty(netdev) || netdev->flags & IFF_ALLMULTI)1477		new_ctrl |= nn->cap & NFP_NET_CFG_CTRL_L2MC;1478	else1479		new_ctrl &= ~NFP_NET_CFG_CTRL_L2MC;1480 1481	if (netdev->flags & IFF_ALLMULTI)1482		new_ctrl_w1 &= ~NFP_NET_CFG_CTRL_MCAST_FILTER;1483	else1484		new_ctrl_w1 |= nn->cap_w1 & NFP_NET_CFG_CTRL_MCAST_FILTER;1485 1486	if (netdev->flags & IFF_PROMISC) {1487		if (nn->cap & NFP_NET_CFG_CTRL_PROMISC)1488			new_ctrl |= NFP_NET_CFG_CTRL_PROMISC;1489		else1490			nn_warn(nn, "FW does not support promiscuous mode\n");1491	} else {1492		new_ctrl &= ~NFP_NET_CFG_CTRL_PROMISC;1493	}1494 1495	if ((nn->cap_w1 & NFP_NET_CFG_CTRL_MCAST_FILTER) &&1496	    __dev_mc_sync(netdev, nfp_net_mc_sync, nfp_net_mc_unsync))1497		netdev_err(netdev, "Sync mc address failed\n");1498 1499	if (new_ctrl == nn->dp.ctrl && new_ctrl_w1 == nn->dp.ctrl_w1)1500		return;1501 1502	if (new_ctrl != nn->dp.ctrl)1503		nn_writel(nn, NFP_NET_CFG_CTRL, new_ctrl);1504	if (new_ctrl_w1 != nn->dp.ctrl_w1)1505		nn_writel(nn, NFP_NET_CFG_CTRL_WORD1, new_ctrl_w1);1506	nfp_net_reconfig_post(nn, NFP_NET_CFG_UPDATE_GEN);1507 1508	nn->dp.ctrl = new_ctrl;1509	nn->dp.ctrl_w1 = new_ctrl_w1;1510}1511 1512static void nfp_net_rss_init_itbl(struct nfp_net *nn)1513{1514	int i;1515 1516	for (i = 0; i < sizeof(nn->rss_itbl); i++)1517		nn->rss_itbl[i] =1518			ethtool_rxfh_indir_default(i, nn->dp.num_rx_rings);1519}1520 1521static void nfp_net_dp_swap(struct nfp_net *nn, struct nfp_net_dp *dp)1522{1523	struct nfp_net_dp new_dp = *dp;1524 1525	*dp = nn->dp;1526	nn->dp = new_dp;1527 1528	WRITE_ONCE(nn->dp.netdev->mtu, new_dp.mtu);1529 1530	if (!netif_is_rxfh_configured(nn->dp.netdev))1531		nfp_net_rss_init_itbl(nn);1532}1533 1534static int nfp_net_dp_swap_enable(struct nfp_net *nn, struct nfp_net_dp *dp)1535{1536	unsigned int r;1537	int err;1538 1539	nfp_net_dp_swap(nn, dp);1540 1541	for (r = 0; r <	nn->max_r_vecs; r++)1542		nfp_net_vector_assign_rings(&nn->dp, &nn->r_vecs[r], r);1543 1544	err = netif_set_real_num_queues(nn->dp.netdev,1545					nn->dp.num_stack_tx_rings,1546					nn->dp.num_rx_rings);1547	if (err)1548		return err;1549 1550	return nfp_net_set_config_and_enable(nn);1551}1552 1553struct nfp_net_dp *nfp_net_clone_dp(struct nfp_net *nn)1554{1555	struct nfp_net_dp *new;1556 1557	new = kmalloc(sizeof(*new), GFP_KERNEL);1558	if (!new)1559		return NULL;1560 1561	*new = nn->dp;1562 1563	new->xsk_pools = kmemdup(new->xsk_pools,1564				 array_size(nn->max_r_vecs,1565					    sizeof(new->xsk_pools)),1566				 GFP_KERNEL);1567	if (!new->xsk_pools) {1568		kfree(new);1569		return NULL;1570	}1571 1572	/* Clear things which need to be recomputed */1573	new->fl_bufsz = 0;1574	new->tx_rings = NULL;1575	new->rx_rings = NULL;1576	new->num_r_vecs = 0;1577	new->num_stack_tx_rings = 0;1578	new->txrwb = NULL;1579	new->txrwb_dma = 0;1580 1581	return new;1582}1583 1584static void nfp_net_free_dp(struct nfp_net_dp *dp)1585{1586	kfree(dp->xsk_pools);1587	kfree(dp);1588}1589 1590static int1591nfp_net_check_config(struct nfp_net *nn, struct nfp_net_dp *dp,1592		     struct netlink_ext_ack *extack)1593{1594	unsigned int r, xsk_min_fl_bufsz;1595 1596	/* XDP-enabled tests */1597	if (!dp->xdp_prog)1598		return 0;1599	if (dp->fl_bufsz > PAGE_SIZE) {1600		NL_SET_ERR_MSG_MOD(extack, "MTU too large w/ XDP enabled");1601		return -EINVAL;1602	}1603	if (dp->num_tx_rings > nn->max_tx_rings) {1604		NL_SET_ERR_MSG_MOD(extack, "Insufficient number of TX rings w/ XDP enabled");1605		return -EINVAL;1606	}1607 1608	xsk_min_fl_bufsz = nfp_net_calc_fl_bufsz_xsk(dp);1609	for (r = 0; r < nn->max_r_vecs; r++) {1610		if (!dp->xsk_pools[r])1611			continue;1612 1613		if (xsk_pool_get_rx_frame_size(dp->xsk_pools[r]) < xsk_min_fl_bufsz) {1614			NL_SET_ERR_MSG_MOD(extack,1615					   "XSK buffer pool chunk size too small");1616			return -EINVAL;1617		}1618	}1619 1620	return 0;1621}1622 1623int nfp_net_ring_reconfig(struct nfp_net *nn, struct nfp_net_dp *dp,1624			  struct netlink_ext_ack *extack)1625{1626	int r, err;1627 1628	dp->fl_bufsz = nfp_net_calc_fl_bufsz(dp);1629 1630	dp->num_stack_tx_rings = dp->num_tx_rings;1631	if (dp->xdp_prog)1632		dp->num_stack_tx_rings -= dp->num_rx_rings;1633 1634	dp->num_r_vecs = max(dp->num_rx_rings, dp->num_stack_tx_rings);1635 1636	err = nfp_net_check_config(nn, dp, extack);1637	if (err)1638		goto exit_free_dp;1639 1640	if (!netif_running(dp->netdev)) {1641		nfp_net_dp_swap(nn, dp);1642		err = 0;1643		goto exit_free_dp;1644	}1645 1646	/* Prepare new rings */1647	for (r = nn->dp.num_r_vecs; r < dp->num_r_vecs; r++) {1648		err = nfp_net_prepare_vector(nn, &nn->r_vecs[r], r);1649		if (err) {1650			dp->num_r_vecs = r;1651			goto err_cleanup_vecs;1652		}1653	}1654 1655	err = nfp_net_rx_rings_prepare(nn, dp);1656	if (err)1657		goto err_cleanup_vecs;1658 1659	err = nfp_net_tx_rings_prepare(nn, dp);1660	if (err)1661		goto err_free_rx;1662 1663	/* Stop device, swap in new rings, try to start the firmware */1664	nfp_net_close_stack(nn);1665	nfp_net_clear_config_and_disable(nn);1666 1667	err = nfp_net_dp_swap_enable(nn, dp);1668	if (err) {1669		int err2;1670 1671		nfp_net_clear_config_and_disable(nn);1672 1673		/* Try with old configuration and old rings */1674		err2 = nfp_net_dp_swap_enable(nn, dp);1675		if (err2)1676			nn_err(nn, "Can't restore ring config - FW communication failed (%d,%d)\n",1677			       err, err2);1678	}1679	for (r = dp->num_r_vecs - 1; r >= nn->dp.num_r_vecs; r--)1680		nfp_net_cleanup_vector(nn, &nn->r_vecs[r]);1681 1682	nfp_net_rx_rings_free(dp);1683	nfp_net_tx_rings_free(dp);1684 1685	nfp_net_open_stack(nn);1686exit_free_dp:1687	nfp_net_free_dp(dp);1688 1689	return err;1690 1691err_free_rx:1692	nfp_net_rx_rings_free(dp);1693err_cleanup_vecs:1694	for (r = dp->num_r_vecs - 1; r >= nn->dp.num_r_vecs; r--)1695		nfp_net_cleanup_vector(nn, &nn->r_vecs[r]);1696	nfp_net_free_dp(dp);1697	return err;1698}1699 1700static int nfp_net_change_mtu(struct net_device *netdev, int new_mtu)1701{1702	struct nfp_net *nn = netdev_priv(netdev);1703	struct nfp_net_dp *dp;1704	int err;1705 1706	err = nfp_app_check_mtu(nn->app, netdev, new_mtu);1707	if (err)1708		return err;1709 1710	dp = nfp_net_clone_dp(nn);1711	if (!dp)1712		return -ENOMEM;1713 1714	dp->mtu = new_mtu;1715 1716	return nfp_net_ring_reconfig(nn, dp, NULL);1717}1718 1719static int1720nfp_net_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)1721{1722	const u32 cmd = NFP_NET_CFG_MBOX_CMD_CTAG_FILTER_ADD;1723	struct nfp_net *nn = netdev_priv(netdev);1724	int err;1725 1726	/* Priority tagged packets with vlan id 0 are processed by the1727	 * NFP as untagged packets1728	 */1729	if (!vid)1730		return 0;1731 1732	err = nfp_net_mbox_lock(nn, NFP_NET_CFG_VLAN_FILTER_SZ);1733	if (err)1734		return err;1735 1736	nn_writew(nn, nn->tlv_caps.mbox_off + NFP_NET_CFG_VLAN_FILTER_VID, vid);1737	nn_writew(nn, nn->tlv_caps.mbox_off + NFP_NET_CFG_VLAN_FILTER_PROTO,1738		  ETH_P_8021Q);1739 1740	return nfp_net_mbox_reconfig_and_unlock(nn, cmd);1741}1742 1743static int1744nfp_net_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)1745{1746	const u32 cmd = NFP_NET_CFG_MBOX_CMD_CTAG_FILTER_KILL;1747	struct nfp_net *nn = netdev_priv(netdev);1748	int err;1749 1750	/* Priority tagged packets with vlan id 0 are processed by the1751	 * NFP as untagged packets1752	 */1753	if (!vid)1754		return 0;1755 1756	err = nfp_net_mbox_lock(nn, NFP_NET_CFG_VLAN_FILTER_SZ);1757	if (err)1758		return err;1759 1760	nn_writew(nn, nn->tlv_caps.mbox_off + NFP_NET_CFG_VLAN_FILTER_VID, vid);1761	nn_writew(nn, nn->tlv_caps.mbox_off + NFP_NET_CFG_VLAN_FILTER_PROTO,1762		  ETH_P_8021Q);1763 1764	return nfp_net_mbox_reconfig_and_unlock(nn, cmd);1765}1766 1767static void1768nfp_net_fs_fill_v4(struct nfp_net *nn, struct nfp_fs_entry *entry, u32 op, u32 *addr)1769{1770	unsigned int i;1771 1772	union {1773		struct {1774			__be16 loc;1775			u8 k_proto, m_proto;1776			__be32 k_sip, m_sip, k_dip, m_dip;1777			__be16 k_sport, m_sport, k_dport, m_dport;1778		};1779		__be32 val[7];1780	} v4_rule;1781 1782	nn_writel(nn, *addr, op);1783	*addr += sizeof(u32);1784 1785	v4_rule.loc     = cpu_to_be16(entry->loc);1786	v4_rule.k_proto = entry->key.l4_proto;1787	v4_rule.m_proto = entry->msk.l4_proto;1788	v4_rule.k_sip   = entry->key.sip4;1789	v4_rule.m_sip   = entry->msk.sip4;1790	v4_rule.k_dip   = entry->key.dip4;1791	v4_rule.m_dip   = entry->msk.dip4;1792	v4_rule.k_sport = entry->key.sport;1793	v4_rule.m_sport = entry->msk.sport;1794	v4_rule.k_dport = entry->key.dport;1795	v4_rule.m_dport = entry->msk.dport;1796 1797	for (i = 0; i < ARRAY_SIZE(v4_rule.val); i++, *addr += sizeof(__be32))1798		nn_writel(nn, *addr, be32_to_cpu(v4_rule.val[i]));1799}1800 1801static void1802nfp_net_fs_fill_v6(struct nfp_net *nn, struct nfp_fs_entry *entry, u32 op, u32 *addr)1803{1804	unsigned int i;1805 1806	union {1807		struct {1808			__be16 loc;1809			u8 k_proto, m_proto;1810			__be32 k_sip[4], m_sip[4], k_dip[4], m_dip[4];1811			__be16 k_sport, m_sport, k_dport, m_dport;1812		};1813		__be32 val[19];1814	} v6_rule;1815 1816	nn_writel(nn, *addr, op);1817	*addr += sizeof(u32);1818 1819	v6_rule.loc     = cpu_to_be16(entry->loc);1820	v6_rule.k_proto = entry->key.l4_proto;1821	v6_rule.m_proto = entry->msk.l4_proto;1822	for (i = 0; i < 4; i++) {1823		v6_rule.k_sip[i] = entry->key.sip6[i];1824		v6_rule.m_sip[i] = entry->msk.sip6[i];1825		v6_rule.k_dip[i] = entry->key.dip6[i];1826		v6_rule.m_dip[i] = entry->msk.dip6[i];1827	}1828	v6_rule.k_sport = entry->key.sport;1829	v6_rule.m_sport = entry->msk.sport;1830	v6_rule.k_dport = entry->key.dport;1831	v6_rule.m_dport = entry->msk.dport;1832 1833	for (i = 0; i < ARRAY_SIZE(v6_rule.val); i++, *addr += sizeof(__be32))1834		nn_writel(nn, *addr, be32_to_cpu(v6_rule.val[i]));1835}1836 1837#define NFP_FS_QUEUE_ID	GENMASK(22, 16)1838#define NFP_FS_ACT	GENMASK(15, 0)1839#define NFP_FS_ACT_DROP	BIT(0)1840#define NFP_FS_ACT_Q	BIT(1)1841static void1842nfp_net_fs_fill_act(struct nfp_net *nn, struct nfp_fs_entry *entry, u32 addr)1843{1844	u32 action = 0; /* 0 means default passthrough */1845 1846	if (entry->action == RX_CLS_FLOW_DISC)1847		action = NFP_FS_ACT_DROP;1848	else if (!(entry->flow_type & FLOW_RSS))1849		action = FIELD_PREP(NFP_FS_QUEUE_ID, entry->action) | NFP_FS_ACT_Q;1850 1851	nn_writel(nn, addr, action);1852}1853 1854int nfp_net_fs_add_hw(struct nfp_net *nn, struct nfp_fs_entry *entry)1855{1856	u32 addr = nn->tlv_caps.mbox_off + NFP_NET_CFG_MBOX_SIMPLE_VAL;1857	int err;1858 1859	err = nfp_net_mbox_lock(nn, NFP_NET_CFG_FS_SZ);1860	if (err)1861		return err;1862 1863	switch (entry->flow_type & ~FLOW_RSS) {1864	case TCP_V4_FLOW:1865	case UDP_V4_FLOW:1866	case SCTP_V4_FLOW:1867	case IPV4_USER_FLOW:1868		nfp_net_fs_fill_v4(nn, entry, NFP_NET_CFG_MBOX_CMD_FS_ADD_V4, &addr);1869		break;1870	case TCP_V6_FLOW:1871	case UDP_V6_FLOW:1872	case SCTP_V6_FLOW:1873	case IPV6_USER_FLOW:1874		nfp_net_fs_fill_v6(nn, entry, NFP_NET_CFG_MBOX_CMD_FS_ADD_V6, &addr);1875		break;1876	case ETHER_FLOW:1877		nn_writel(nn, addr, NFP_NET_CFG_MBOX_CMD_FS_ADD_ETHTYPE);1878		addr += sizeof(u32);1879		nn_writew(nn, addr, be16_to_cpu(entry->key.l3_proto));1880		addr += sizeof(u32);1881		break;1882	}1883 1884	nfp_net_fs_fill_act(nn, entry, addr);1885 1886	err = nfp_net_mbox_reconfig_and_unlock(nn, NFP_NET_CFG_MBOX_CMD_FLOW_STEER);1887	if (err) {1888		nn_err(nn, "Add new fs rule failed with %d\n", err);1889		return -EIO;1890	}1891 1892	return 0;1893}1894 1895int nfp_net_fs_del_hw(struct nfp_net *nn, struct nfp_fs_entry *entry)1896{1897	u32 addr = nn->tlv_caps.mbox_off + NFP_NET_CFG_MBOX_SIMPLE_VAL;1898	int err;1899 1900	err = nfp_net_mbox_lock(nn, NFP_NET_CFG_FS_SZ);1901	if (err)1902		return err;1903 1904	switch (entry->flow_type & ~FLOW_RSS) {1905	case TCP_V4_FLOW:1906	case UDP_V4_FLOW:1907	case SCTP_V4_FLOW:1908	case IPV4_USER_FLOW:1909		nfp_net_fs_fill_v4(nn, entry, NFP_NET_CFG_MBOX_CMD_FS_DEL_V4, &addr);1910		break;1911	case TCP_V6_FLOW:1912	case UDP_V6_FLOW:1913	case SCTP_V6_FLOW:1914	case IPV6_USER_FLOW:1915		nfp_net_fs_fill_v6(nn, entry, NFP_NET_CFG_MBOX_CMD_FS_DEL_V6, &addr);1916		break;1917	case ETHER_FLOW:1918		nn_writel(nn, addr, NFP_NET_CFG_MBOX_CMD_FS_DEL_ETHTYPE);1919		addr += sizeof(u32);1920		nn_writew(nn, addr, be16_to_cpu(entry->key.l3_proto));1921		addr += sizeof(u32);1922		break;1923	}1924 1925	nfp_net_fs_fill_act(nn, entry, addr);1926 1927	err = nfp_net_mbox_reconfig_and_unlock(nn, NFP_NET_CFG_MBOX_CMD_FLOW_STEER);1928	if (err) {1929		nn_err(nn, "Delete fs rule failed with %d\n", err);1930		return -EIO;1931	}1932 1933	return 0;1934}1935 1936static void nfp_net_fs_clean(struct nfp_net *nn)1937{1938	struct nfp_fs_entry *entry, *tmp;1939 1940	list_for_each_entry_safe(entry, tmp, &nn->fs.list, node) {1941		nfp_net_fs_del_hw(nn, entry);1942		list_del(&entry->node);1943		kfree(entry);1944	}1945}1946 1947static void nfp_net_stat64(struct net_device *netdev,1948			   struct rtnl_link_stats64 *stats)1949{1950	struct nfp_net *nn = netdev_priv(netdev);1951	int r;1952 1953	/* Collect software stats */1954	for (r = 0; r < nn->max_r_vecs; r++) {1955		struct nfp_net_r_vector *r_vec = &nn->r_vecs[r];1956		u64 data[3];1957		unsigned int start;1958 1959		do {1960			start = u64_stats_fetch_begin(&r_vec->rx_sync);1961			data[0] = r_vec->rx_pkts;1962			data[1] = r_vec->rx_bytes;1963			data[2] = r_vec->rx_drops;1964		} while (u64_stats_fetch_retry(&r_vec->rx_sync, start));1965		stats->rx_packets += data[0];1966		stats->rx_bytes += data[1];1967		stats->rx_dropped += data[2];1968 1969		do {1970			start = u64_stats_fetch_begin(&r_vec->tx_sync);1971			data[0] = r_vec->tx_pkts;1972			data[1] = r_vec->tx_bytes;1973			data[2] = r_vec->tx_errors;1974		} while (u64_stats_fetch_retry(&r_vec->tx_sync, start));1975		stats->tx_packets += data[0];1976		stats->tx_bytes += data[1];1977		stats->tx_errors += data[2];1978	}1979 1980	/* Add in device stats */1981	stats->multicast += nn_readq(nn, NFP_NET_CFG_STATS_RX_MC_FRAMES);1982	stats->rx_dropped += nn_readq(nn, NFP_NET_CFG_STATS_RX_DISCARDS);1983	stats->rx_errors += nn_readq(nn, NFP_NET_CFG_STATS_RX_ERRORS);1984 1985	stats->tx_dropped += nn_readq(nn, NFP_NET_CFG_STATS_TX_DISCARDS);1986	stats->tx_errors += nn_readq(nn, NFP_NET_CFG_STATS_TX_ERRORS);1987}1988 1989static int nfp_net_set_features(struct net_device *netdev,1990				netdev_features_t features)1991{1992	netdev_features_t changed = netdev->features ^ features;1993	struct nfp_net *nn = netdev_priv(netdev);1994	u32 new_ctrl;1995	int err;1996 1997	/* Assume this is not called with features we have not advertised */1998 1999	new_ctrl = nn->dp.ctrl;2000 2001	if (changed & NETIF_F_RXCSUM) {2002		if (features & NETIF_F_RXCSUM)2003			new_ctrl |= nn->cap & NFP_NET_CFG_CTRL_RXCSUM_ANY;2004		else2005			new_ctrl &= ~NFP_NET_CFG_CTRL_RXCSUM_ANY;2006	}2007 2008	if (changed & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {2009		if (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))2010			new_ctrl |= NFP_NET_CFG_CTRL_TXCSUM;2011		else2012			new_ctrl &= ~NFP_NET_CFG_CTRL_TXCSUM;2013	}2014 2015	if (changed & (NETIF_F_TSO | NETIF_F_TSO6)) {2016		if (features & (NETIF_F_TSO | NETIF_F_TSO6))2017			new_ctrl |= nn->cap & NFP_NET_CFG_CTRL_LSO2 ?:2018					      NFP_NET_CFG_CTRL_LSO;2019		else2020			new_ctrl &= ~NFP_NET_CFG_CTRL_LSO_ANY;2021	}2022 2023	if (changed & NETIF_F_HW_VLAN_CTAG_RX) {2024		if (features & NETIF_F_HW_VLAN_CTAG_RX)2025			new_ctrl |= nn->cap & NFP_NET_CFG_CTRL_RXVLAN_V2 ?:2026				    NFP_NET_CFG_CTRL_RXVLAN;2027		else2028			new_ctrl &= ~NFP_NET_CFG_CTRL_RXVLAN_ANY;2029	}2030 2031	if (changed & NETIF_F_HW_VLAN_CTAG_TX) {2032		if (features & NETIF_F_HW_VLAN_CTAG_TX)2033			new_ctrl |= nn->cap & NFP_NET_CFG_CTRL_TXVLAN_V2 ?:2034				    NFP_NET_CFG_CTRL_TXVLAN;2035		else2036			new_ctrl &= ~NFP_NET_CFG_CTRL_TXVLAN_ANY;2037	}2038 2039	if (changed & NETIF_F_HW_VLAN_CTAG_FILTER) {2040		if (features & NETIF_F_HW_VLAN_CTAG_FILTER)2041			new_ctrl |= NFP_NET_CFG_CTRL_CTAG_FILTER;2042		else2043			new_ctrl &= ~NFP_NET_CFG_CTRL_CTAG_FILTER;2044	}2045 2046	if (changed & NETIF_F_HW_VLAN_STAG_RX) {2047		if (features & NETIF_F_HW_VLAN_STAG_RX)2048			new_ctrl |= NFP_NET_CFG_CTRL_RXQINQ;2049		else2050			new_ctrl &= ~NFP_NET_CFG_CTRL_RXQINQ;2051	}2052 2053	if (changed & NETIF_F_SG) {2054		if (features & NETIF_F_SG)2055			new_ctrl |= NFP_NET_CFG_CTRL_GATHER;2056		else2057			new_ctrl &= ~NFP_NET_CFG_CTRL_GATHER;2058	}2059 2060	err = nfp_port_set_features(netdev, features);2061	if (err)2062		return err;2063 2064	nn_dbg(nn, "Feature change 0x%llx -> 0x%llx (changed=0x%llx)\n",2065	       netdev->features, features, changed);2066 2067	if (new_ctrl == nn->dp.ctrl)2068		return 0;2069 2070	nn_dbg(nn, "NIC ctrl: 0x%x -> 0x%x\n", nn->dp.ctrl, new_ctrl);2071	nn_writel(nn, NFP_NET_CFG_CTRL, new_ctrl);2072	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);2073	if (err)2074		return err;2075 2076	nn->dp.ctrl = new_ctrl;2077 2078	return 0;2079}2080 2081static netdev_features_t2082nfp_net_fix_features(struct net_device *netdev,2083		     netdev_features_t features)2084{2085	if ((features & NETIF_F_HW_VLAN_CTAG_RX) &&2086	    (features & NETIF_F_HW_VLAN_STAG_RX)) {2087		if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {2088			features &= ~NETIF_F_HW_VLAN_CTAG_RX;2089			netdev->wanted_features &= ~NETIF_F_HW_VLAN_CTAG_RX;2090			netdev_warn(netdev,2091				    "S-tag and C-tag stripping can't be enabled at the same time. Enabling S-tag stripping and disabling C-tag stripping\n");2092		} else if (netdev->features & NETIF_F_HW_VLAN_STAG_RX) {2093			features &= ~NETIF_F_HW_VLAN_STAG_RX;2094			netdev->wanted_features &= ~NETIF_F_HW_VLAN_STAG_RX;2095			netdev_warn(netdev,2096				    "S-tag and C-tag stripping can't be enabled at the same time. Enabling C-tag stripping and disabling S-tag stripping\n");2097		}2098	}2099	return features;2100}2101 2102static netdev_features_t2103nfp_net_features_check(struct sk_buff *skb, struct net_device *dev,2104		       netdev_features_t features)2105{2106	u8 l4_hdr;2107 2108	/* We can't do TSO over double tagged packets (802.1AD) */2109	features &= vlan_features_check(skb, features);2110 2111	if (!skb->encapsulation)2112		return features;2113 2114	/* Ensure that inner L4 header offset fits into TX descriptor field */2115	if (skb_is_gso(skb)) {2116		u32 hdrlen;2117 2118		if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4)2119			hdrlen = skb_inner_transport_offset(skb) + sizeof(struct udphdr);2120		else2121			hdrlen = skb_inner_tcp_all_headers(skb);2122 2123		/* Assume worst case scenario of having longest possible2124		 * metadata prepend - 8B2125		 */2126		if (unlikely(hdrlen > NFP_NET_LSO_MAX_HDR_SZ - 8))2127			features &= ~NETIF_F_GSO_MASK;2128	}2129 2130	if (xfrm_offload(skb))2131		return features;2132 2133	/* VXLAN/GRE check */2134	switch (vlan_get_protocol(skb)) {2135	case htons(ETH_P_IP):2136		l4_hdr = ip_hdr(skb)->protocol;2137		break;2138	case htons(ETH_P_IPV6):2139		l4_hdr = ipv6_hdr(skb)->nexthdr;2140		break;2141	default:2142		return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);2143	}2144 2145	if (skb->inner_protocol_type != ENCAP_TYPE_ETHER ||2146	    skb->inner_protocol != htons(ETH_P_TEB) ||2147	    (l4_hdr != IPPROTO_UDP && l4_hdr != IPPROTO_GRE) ||2148	    (l4_hdr == IPPROTO_UDP &&2149	     (skb_inner_mac_header(skb) - skb_transport_header(skb) !=2150	      sizeof(struct udphdr) + sizeof(struct vxlanhdr))))2151		return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);2152 2153	return features;2154}2155 2156static int2157nfp_net_get_phys_port_name(struct net_device *netdev, char *name, size_t len)2158{2159	struct nfp_net *nn = netdev_priv(netdev);2160	int n;2161 2162	/* If port is defined, devlink_port is registered and devlink core2163	 * is taking care of name formatting.2164	 */2165	if (nn->port)2166		return -EOPNOTSUPP;2167 2168	if (nn->dp.is_vf || nn->vnic_no_name)2169		return -EOPNOTSUPP;2170 2171	n = snprintf(name, len, "n%d", nn->id);2172	if (n >= len)2173		return -EINVAL;2174 2175	return 0;2176}2177 2178static int nfp_net_xdp_setup_drv(struct nfp_net *nn, struct netdev_bpf *bpf)2179{2180	struct bpf_prog *prog = bpf->prog;2181	struct nfp_net_dp *dp;2182	int err;2183 2184	if (!prog == !nn->dp.xdp_prog) {2185		WRITE_ONCE(nn->dp.xdp_prog, prog);2186		xdp_attachment_setup(&nn->xdp, bpf);2187		return 0;2188	}2189 2190	dp = nfp_net_clone_dp(nn);2191	if (!dp)2192		return -ENOMEM;2193 2194	dp->xdp_prog = prog;2195	dp->num_tx_rings += prog ? nn->dp.num_rx_rings : -nn->dp.num_rx_rings;2196	dp->rx_dma_dir = prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;2197	dp->rx_dma_off = prog ? XDP_PACKET_HEADROOM - nn->dp.rx_offset : 0;2198 2199	/* We need RX reconfig to remap the buffers (BIDIR vs FROM_DEV) */2200	err = nfp_net_ring_reconfig(nn, dp, bpf->extack);2201	if (err)2202		return err;2203 2204	xdp_attachment_setup(&nn->xdp, bpf);2205	return 0;2206}2207 2208static int nfp_net_xdp_setup_hw(struct nfp_net *nn, struct netdev_bpf *bpf)2209{2210	int err;2211 2212	err = nfp_app_xdp_offload(nn->app, nn, bpf->prog, bpf->extack);2213	if (err)2214		return err;2215 2216	xdp_attachment_setup(&nn->xdp_hw, bpf);2217	return 0;2218}2219 2220static int nfp_net_xdp(struct net_device *netdev, struct netdev_bpf *xdp)2221{2222	struct nfp_net *nn = netdev_priv(netdev);2223 2224	switch (xdp->command) {2225	case XDP_SETUP_PROG:2226		return nfp_net_xdp_setup_drv(nn, xdp);2227	case XDP_SETUP_PROG_HW:2228		return nfp_net_xdp_setup_hw(nn, xdp);2229	case XDP_SETUP_XSK_POOL:2230		return nfp_net_xsk_setup_pool(netdev, xdp->xsk.pool,2231					      xdp->xsk.queue_id);2232	default:2233		return nfp_app_bpf(nn->app, nn, xdp);2234	}2235}2236 2237static int nfp_net_set_mac_address(struct net_device *netdev, void *addr)2238{2239	struct nfp_net *nn = netdev_priv(netdev);2240	struct sockaddr *saddr = addr;2241	int err;2242 2243	err = eth_prepare_mac_addr_change(netdev, addr);2244	if (err)2245		return err;2246 2247	nfp_net_write_mac_addr(nn, saddr->sa_data);2248 2249	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_MACADDR);2250	if (err)2251		return err;2252 2253	eth_commit_mac_addr_change(netdev, addr);2254 2255	return 0;2256}2257 2258static int nfp_net_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,2259				  struct net_device *dev, u32 filter_mask,2260				  int nlflags)2261{2262	struct nfp_net *nn = netdev_priv(dev);2263	u16 mode;2264 2265	if (!(nn->cap & NFP_NET_CFG_CTRL_VEPA))2266		return -EOPNOTSUPP;2267 2268	mode = (nn->dp.ctrl & NFP_NET_CFG_CTRL_VEPA) ?2269	       BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB;2270 2271	return ndo_dflt_bridge_getlink(skb, pid, seq, dev, mode, 0, 0,2272				       nlflags, filter_mask, NULL);2273}2274 2275static int nfp_net_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,2276				  u16 flags, struct netlink_ext_ack *extack)2277{2278	struct nfp_net *nn = netdev_priv(dev);2279	struct nlattr *attr, *br_spec;2280	int rem, err;2281	u32 new_ctrl;2282	u16 mode;2283 2284	if (!(nn->cap & NFP_NET_CFG_CTRL_VEPA))2285		return -EOPNOTSUPP;2286 2287	br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);2288	if (!br_spec)2289		return -EINVAL;2290 2291	nla_for_each_nested_type(attr, IFLA_BRIDGE_MODE, br_spec, rem) {2292		new_ctrl = nn->dp.ctrl;2293		mode = nla_get_u16(attr);2294		if (mode == BRIDGE_MODE_VEPA)2295			new_ctrl |= NFP_NET_CFG_CTRL_VEPA;2296		else if (mode == BRIDGE_MODE_VEB)2297			new_ctrl &= ~NFP_NET_CFG_CTRL_VEPA;2298		else2299			return -EOPNOTSUPP;2300 2301		if (new_ctrl == nn->dp.ctrl)2302			return 0;2303 2304		nn_writel(nn, NFP_NET_CFG_CTRL, new_ctrl);2305		err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);2306		if (!err)2307			nn->dp.ctrl = new_ctrl;2308 2309		return err;2310	}2311 2312	return -EINVAL;2313}2314 2315const struct net_device_ops nfp_nfd3_netdev_ops = {2316	.ndo_init		= nfp_app_ndo_init,2317	.ndo_uninit		= nfp_app_ndo_uninit,2318	.ndo_open		= nfp_net_netdev_open,2319	.ndo_stop		= nfp_net_netdev_close,2320	.ndo_start_xmit		= nfp_net_tx,2321	.ndo_get_stats64	= nfp_net_stat64,2322	.ndo_vlan_rx_add_vid	= nfp_net_vlan_rx_add_vid,2323	.ndo_vlan_rx_kill_vid	= nfp_net_vlan_rx_kill_vid,2324	.ndo_set_vf_mac         = nfp_app_set_vf_mac,2325	.ndo_set_vf_vlan        = nfp_app_set_vf_vlan,2326	.ndo_set_vf_rate	= nfp_app_set_vf_rate,2327	.ndo_set_vf_spoofchk    = nfp_app_set_vf_spoofchk,2328	.ndo_set_vf_trust	= nfp_app_set_vf_trust,2329	.ndo_get_vf_config	= nfp_app_get_vf_config,2330	.ndo_set_vf_link_state  = nfp_app_set_vf_link_state,2331	.ndo_setup_tc		= nfp_port_setup_tc,2332	.ndo_tx_timeout		= nfp_net_tx_timeout,2333	.ndo_set_rx_mode	= nfp_net_set_rx_mode,2334	.ndo_change_mtu		= nfp_net_change_mtu,2335	.ndo_set_mac_address	= nfp_net_set_mac_address,2336	.ndo_set_features	= nfp_net_set_features,2337	.ndo_fix_features	= nfp_net_fix_features,2338	.ndo_features_check	= nfp_net_features_check,2339	.ndo_get_phys_port_name	= nfp_net_get_phys_port_name,2340	.ndo_bpf		= nfp_net_xdp,2341	.ndo_xsk_wakeup		= nfp_net_xsk_wakeup,2342	.ndo_bridge_getlink     = nfp_net_bridge_getlink,2343	.ndo_bridge_setlink     = nfp_net_bridge_setlink,2344};2345 2346const struct net_device_ops nfp_nfdk_netdev_ops = {2347	.ndo_init		= nfp_app_ndo_init,2348	.ndo_uninit		= nfp_app_ndo_uninit,2349	.ndo_open		= nfp_net_netdev_open,2350	.ndo_stop		= nfp_net_netdev_close,2351	.ndo_start_xmit		= nfp_net_tx,2352	.ndo_get_stats64	= nfp_net_stat64,2353	.ndo_vlan_rx_add_vid	= nfp_net_vlan_rx_add_vid,2354	.ndo_vlan_rx_kill_vid	= nfp_net_vlan_rx_kill_vid,2355	.ndo_set_vf_mac         = nfp_app_set_vf_mac,2356	.ndo_set_vf_vlan        = nfp_app_set_vf_vlan,2357	.ndo_set_vf_rate	= nfp_app_set_vf_rate,2358	.ndo_set_vf_spoofchk    = nfp_app_set_vf_spoofchk,2359	.ndo_set_vf_trust	= nfp_app_set_vf_trust,2360	.ndo_get_vf_config	= nfp_app_get_vf_config,2361	.ndo_set_vf_link_state  = nfp_app_set_vf_link_state,2362	.ndo_setup_tc		= nfp_port_setup_tc,2363	.ndo_tx_timeout		= nfp_net_tx_timeout,2364	.ndo_set_rx_mode	= nfp_net_set_rx_mode,2365	.ndo_change_mtu		= nfp_net_change_mtu,2366	.ndo_set_mac_address	= nfp_net_set_mac_address,2367	.ndo_set_features	= nfp_net_set_features,2368	.ndo_fix_features	= nfp_net_fix_features,2369	.ndo_features_check	= nfp_net_features_check,2370	.ndo_get_phys_port_name	= nfp_net_get_phys_port_name,2371	.ndo_bpf		= nfp_net_xdp,2372	.ndo_bridge_getlink     = nfp_net_bridge_getlink,2373	.ndo_bridge_setlink     = nfp_net_bridge_setlink,2374};2375 2376static int nfp_udp_tunnel_sync(struct net_device *netdev, unsigned int table)2377{2378	struct nfp_net *nn = netdev_priv(netdev);2379	int i;2380 2381	BUILD_BUG_ON(NFP_NET_N_VXLAN_PORTS & 1);2382	for (i = 0; i < NFP_NET_N_VXLAN_PORTS; i += 2) {2383		struct udp_tunnel_info ti0, ti1;2384 2385		udp_tunnel_nic_get_port(netdev, table, i, &ti0);2386		udp_tunnel_nic_get_port(netdev, table, i + 1, &ti1);2387 2388		nn_writel(nn, NFP_NET_CFG_VXLAN_PORT + i * sizeof(ti0.port),2389			  be16_to_cpu(ti1.port) << 16 | be16_to_cpu(ti0.port));2390	}2391 2392	return nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_VXLAN);2393}2394 2395static const struct udp_tunnel_nic_info nfp_udp_tunnels = {2396	.sync_table     = nfp_udp_tunnel_sync,2397	.flags          = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |2398			  UDP_TUNNEL_NIC_INFO_OPEN_ONLY,2399	.tables         = {2400		{2401			.n_entries      = NFP_NET_N_VXLAN_PORTS,2402			.tunnel_types   = UDP_TUNNEL_TYPE_VXLAN,2403		},2404	},2405};2406 2407/**2408 * nfp_net_info() - Print general info about the NIC2409 * @nn:      NFP Net device to reconfigure2410 */2411void nfp_net_info(struct nfp_net *nn)2412{2413	nn_info(nn, "NFP-6xxx %sNetdev: TxQs=%d/%d RxQs=%d/%d\n",2414		nn->dp.is_vf ? "VF " : "",2415		nn->dp.num_tx_rings, nn->max_tx_rings,2416		nn->dp.num_rx_rings, nn->max_rx_rings);2417	nn_info(nn, "VER: %d.%d.%d.%d, Maximum supported MTU: %d\n",2418		nn->fw_ver.extend, nn->fw_ver.class,2419		nn->fw_ver.major, nn->fw_ver.minor,2420		nn->max_mtu);2421	nn_info(nn, "CAP: %#x %s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",2422		nn->cap,2423		nn->cap & NFP_NET_CFG_CTRL_PROMISC  ? "PROMISC "  : "",2424		nn->cap & NFP_NET_CFG_CTRL_L2BC     ? "L2BCFILT " : "",2425		nn->cap & NFP_NET_CFG_CTRL_L2MC     ? "L2MCFILT " : "",2426		nn->cap & NFP_NET_CFG_CTRL_RXCSUM   ? "RXCSUM "   : "",2427		nn->cap & NFP_NET_CFG_CTRL_TXCSUM   ? "TXCSUM "   : "",2428		nn->cap & NFP_NET_CFG_CTRL_RXVLAN   ? "RXVLAN "   : "",2429		nn->cap & NFP_NET_CFG_CTRL_TXVLAN   ? "TXVLAN "   : "",2430		nn->cap & NFP_NET_CFG_CTRL_RXQINQ   ? "RXQINQ "   : "",2431		nn->cap & NFP_NET_CFG_CTRL_RXVLAN_V2 ? "RXVLANv2 "   : "",2432		nn->cap & NFP_NET_CFG_CTRL_TXVLAN_V2   ? "TXVLANv2 "   : "",2433		nn->cap & NFP_NET_CFG_CTRL_SCATTER  ? "SCATTER "  : "",2434		nn->cap & NFP_NET_CFG_CTRL_GATHER   ? "GATHER "   : "",2435		nn->cap & NFP_NET_CFG_CTRL_LSO      ? "TSO1 "     : "",2436		nn->cap & NFP_NET_CFG_CTRL_LSO2     ? "TSO2 "     : "",2437		nn->cap & NFP_NET_CFG_CTRL_RSS      ? "RSS1 "     : "",2438		nn->cap & NFP_NET_CFG_CTRL_RSS2     ? "RSS2 "     : "",2439		nn->cap & NFP_NET_CFG_CTRL_CTAG_FILTER ? "CTAG_FILTER " : "",2440		nn->cap & NFP_NET_CFG_CTRL_MSIXAUTO ? "AUTOMASK " : "",2441		nn->cap & NFP_NET_CFG_CTRL_IRQMOD   ? "IRQMOD "   : "",2442		nn->cap & NFP_NET_CFG_CTRL_TXRWB    ? "TXRWB "    : "",2443		nn->cap & NFP_NET_CFG_CTRL_VEPA     ? "VEPA "     : "",2444		nn->cap & NFP_NET_CFG_CTRL_VXLAN    ? "VXLAN "    : "",2445		nn->cap & NFP_NET_CFG_CTRL_NVGRE    ? "NVGRE "	  : "",2446		nn->cap & NFP_NET_CFG_CTRL_CSUM_COMPLETE ?2447						      "RXCSUM_COMPLETE " : "",2448		nn->cap & NFP_NET_CFG_CTRL_LIVE_ADDR ? "LIVE_ADDR " : "",2449		nn->cap_w1 & NFP_NET_CFG_CTRL_MCAST_FILTER ? "MULTICAST_FILTER " : "",2450		nn->cap_w1 & NFP_NET_CFG_CTRL_USO ? "USO " : "",2451		nfp_app_extra_cap(nn->app, nn));2452}2453 2454/**2455 * nfp_net_alloc() - Allocate netdev and related structure2456 * @pdev:         PCI device2457 * @dev_info:     NFP ASIC params2458 * @ctrl_bar:     PCI IOMEM with vNIC config memory2459 * @needs_netdev: Whether to allocate a netdev for this vNIC2460 * @max_tx_rings: Maximum number of TX rings supported by device2461 * @max_rx_rings: Maximum number of RX rings supported by device2462 *2463 * This function allocates a netdev device and fills in the initial2464 * part of the @struct nfp_net structure.  In case of control device2465 * nfp_net structure is allocated without the netdev.2466 *2467 * Return: NFP Net device structure, or ERR_PTR on error.2468 */2469struct nfp_net *2470nfp_net_alloc(struct pci_dev *pdev, const struct nfp_dev_info *dev_info,2471	      void __iomem *ctrl_bar, bool needs_netdev,2472	      unsigned int max_tx_rings, unsigned int max_rx_rings)2473{2474	u64 dma_mask = dma_get_mask(&pdev->dev);2475	struct nfp_net *nn;2476	int err;2477 2478	if (needs_netdev) {2479		struct net_device *netdev;2480 2481		netdev = alloc_etherdev_mqs(sizeof(struct nfp_net),2482					    max_tx_rings, max_rx_rings);2483		if (!netdev)2484			return ERR_PTR(-ENOMEM);2485 2486		SET_NETDEV_DEV(netdev, &pdev->dev);2487		nn = netdev_priv(netdev);2488		nn->dp.netdev = netdev;2489	} else {2490		nn = vzalloc(sizeof(*nn));2491		if (!nn)2492			return ERR_PTR(-ENOMEM);2493	}2494 2495	nn->dp.dev = &pdev->dev;2496	nn->dp.ctrl_bar = ctrl_bar;2497	nn->dev_info = dev_info;2498	nn->pdev = pdev;2499	nfp_net_get_fw_version(&nn->fw_ver, ctrl_bar);2500 2501	switch (FIELD_GET(NFP_NET_CFG_VERSION_DP_MASK, nn->fw_ver.extend)) {2502	case NFP_NET_CFG_VERSION_DP_NFD3:2503		nn->dp.ops = &nfp_nfd3_ops;2504		break;2505	case NFP_NET_CFG_VERSION_DP_NFDK:2506		if (nn->fw_ver.major < 5) {2507			dev_err(&pdev->dev,2508				"NFDK must use ABI 5 or newer, found: %d\n",2509				nn->fw_ver.major);2510			err = -EINVAL;2511			goto err_free_nn;2512		}2513		nn->dp.ops = &nfp_nfdk_ops;2514		break;2515	default:2516		err = -EINVAL;2517		goto err_free_nn;2518	}2519 2520	if ((dma_mask & nn->dp.ops->dma_mask) != dma_mask) {2521		dev_err(&pdev->dev,2522			"DMA mask of loaded firmware: %llx, required DMA mask: %llx\n",2523			nn->dp.ops->dma_mask, dma_mask);2524		err = -EINVAL;2525		goto err_free_nn;2526	}2527 2528	nn->max_tx_rings = max_tx_rings;2529	nn->max_rx_rings = max_rx_rings;2530 2531	nn->dp.num_tx_rings = min_t(unsigned int,2532				    max_tx_rings, num_online_cpus());2533	nn->dp.num_rx_rings = min_t(unsigned int, max_rx_rings,2534				 netif_get_num_default_rss_queues());2535 2536	nn->dp.num_r_vecs = max(nn->dp.num_tx_rings, nn->dp.num_rx_rings);2537	nn->dp.num_r_vecs = min_t(unsigned int,2538				  nn->dp.num_r_vecs, num_online_cpus());2539	nn->max_r_vecs = nn->dp.num_r_vecs;2540 2541	nn->dp.xsk_pools = kcalloc(nn->max_r_vecs, sizeof(nn->dp.xsk_pools),2542				   GFP_KERNEL);2543	if (!nn->dp.xsk_pools) {2544		err = -ENOMEM;2545		goto err_free_nn;2546	}2547 2548	nn->dp.txd_cnt = NFP_NET_TX_DESCS_DEFAULT;2549	nn->dp.rxd_cnt = NFP_NET_RX_DESCS_DEFAULT;2550 2551	sema_init(&nn->bar_lock, 1);2552 2553	spin_lock_init(&nn->reconfig_lock);2554	spin_lock_init(&nn->link_status_lock);2555 2556	timer_setup(&nn->reconfig_timer, nfp_net_reconfig_timer, 0);2557 2558	err = nfp_net_tlv_caps_parse(&nn->pdev->dev, nn->dp.ctrl_bar,2559				     &nn->tlv_caps);2560	if (err)2561		goto err_free_nn;2562 2563	err = nfp_ccm_mbox_alloc(nn);2564	if (err)2565		goto err_free_nn;2566 2567	return nn;2568 2569err_free_nn:2570	if (nn->dp.netdev)2571		free_netdev(nn->dp.netdev);2572	else2573		vfree(nn);2574	return ERR_PTR(err);2575}2576 2577/**2578 * nfp_net_free() - Undo what @nfp_net_alloc() did2579 * @nn:      NFP Net device to reconfigure2580 */2581void nfp_net_free(struct nfp_net *nn)2582{2583	WARN_ON(timer_pending(&nn->reconfig_timer) || nn->reconfig_posted);2584	nfp_ccm_mbox_free(nn);2585 2586	kfree(nn->dp.xsk_pools);2587	if (nn->dp.netdev)2588		free_netdev(nn->dp.netdev);2589	else2590		vfree(nn);2591}2592 2593/**2594 * nfp_net_rss_key_sz() - Get current size of the RSS key2595 * @nn:		NFP Net device instance2596 *2597 * Return: size of the RSS key for currently selected hash function.2598 */2599unsigned int nfp_net_rss_key_sz(struct nfp_net *nn)2600{2601	switch (nn->rss_hfunc) {2602	case ETH_RSS_HASH_TOP:2603		return NFP_NET_CFG_RSS_KEY_SZ;2604	case ETH_RSS_HASH_XOR:2605		return 0;2606	case ETH_RSS_HASH_CRC32:2607		return 4;2608	}2609 2610	nn_warn(nn, "Unknown hash function: %u\n", nn->rss_hfunc);2611	return 0;2612}2613 2614/**2615 * nfp_net_rss_init() - Set the initial RSS parameters2616 * @nn:	     NFP Net device to reconfigure2617 */2618static void nfp_net_rss_init(struct nfp_net *nn)2619{2620	unsigned long func_bit, rss_cap_hfunc;2621	u32 reg;2622 2623	/* Read the RSS function capability and select first supported func */2624	reg = nn_readl(nn, NFP_NET_CFG_RSS_CAP);2625	rss_cap_hfunc =	FIELD_GET(NFP_NET_CFG_RSS_CAP_HFUNC, reg);2626	if (!rss_cap_hfunc)2627		rss_cap_hfunc =	FIELD_GET(NFP_NET_CFG_RSS_CAP_HFUNC,2628					  NFP_NET_CFG_RSS_TOEPLITZ);2629 2630	func_bit = find_first_bit(&rss_cap_hfunc, NFP_NET_CFG_RSS_HFUNCS);2631	if (func_bit == NFP_NET_CFG_RSS_HFUNCS) {2632		dev_warn(nn->dp.dev,2633			 "Bad RSS config, defaulting to Toeplitz hash\n");2634		func_bit = ETH_RSS_HASH_TOP_BIT;2635	}2636	nn->rss_hfunc = 1 << func_bit;2637 2638	netdev_rss_key_fill(nn->rss_key, nfp_net_rss_key_sz(nn));2639 2640	nfp_net_rss_init_itbl(nn);2641 2642	/* Enable IPv4/IPv6 TCP by default */2643	nn->rss_cfg = NFP_NET_CFG_RSS_IPV4_TCP |2644		      NFP_NET_CFG_RSS_IPV6_TCP |2645		      NFP_NET_CFG_RSS_IPV4_UDP |2646		      NFP_NET_CFG_RSS_IPV6_UDP |2647		      FIELD_PREP(NFP_NET_CFG_RSS_HFUNC, nn->rss_hfunc) |2648		      NFP_NET_CFG_RSS_MASK;2649}2650 2651/**2652 * nfp_net_irqmod_init() - Set the initial IRQ moderation parameters2653 * @nn:	     NFP Net device to reconfigure2654 */2655static void nfp_net_irqmod_init(struct nfp_net *nn)2656{2657	nn->rx_coalesce_usecs      = 50;2658	nn->rx_coalesce_max_frames = 64;2659	nn->tx_coalesce_usecs      = 50;2660	nn->tx_coalesce_max_frames = 64;2661 2662	nn->rx_coalesce_adapt_on   = true;2663	nn->tx_coalesce_adapt_on   = true;2664}2665 2666static void nfp_net_netdev_init(struct nfp_net *nn)2667{2668	struct net_device *netdev = nn->dp.netdev;2669 2670	nfp_net_write_mac_addr(nn, nn->dp.netdev->dev_addr);2671 2672	netdev->mtu = nn->dp.mtu;2673 2674	/* Advertise/enable offloads based on capabilities2675	 *2676	 * Note: netdev->features show the currently enabled features2677	 * and netdev->hw_features advertises which features are2678	 * supported.  By default we enable most features.2679	 */2680	if (nn->cap & NFP_NET_CFG_CTRL_LIVE_ADDR)2681		netdev->priv_flags |= IFF_LIVE_ADDR_CHANGE;2682 2683	netdev->hw_features = NETIF_F_HIGHDMA;2684	if (nn->cap & NFP_NET_CFG_CTRL_RXCSUM_ANY) {2685		netdev->hw_features |= NETIF_F_RXCSUM;2686		nn->dp.ctrl |= nn->cap & NFP_NET_CFG_CTRL_RXCSUM_ANY;2687	}2688	if (nn->cap & NFP_NET_CFG_CTRL_TXCSUM) {2689		netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;2690		nn->dp.ctrl |= NFP_NET_CFG_CTRL_TXCSUM;2691	}2692	if (nn->cap & NFP_NET_CFG_CTRL_GATHER) {2693		netdev->hw_features |= NETIF_F_SG;2694		nn->dp.ctrl |= NFP_NET_CFG_CTRL_GATHER;2695	}2696	if ((nn->cap & NFP_NET_CFG_CTRL_LSO && nn->fw_ver.major > 2) ||2697	    nn->cap & NFP_NET_CFG_CTRL_LSO2) {2698		netdev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;2699		if (nn->cap_w1 & NFP_NET_CFG_CTRL_USO)2700			netdev->hw_features |= NETIF_F_GSO_UDP_L4;2701		nn->dp.ctrl |= nn->cap & NFP_NET_CFG_CTRL_LSO2 ?:2702					 NFP_NET_CFG_CTRL_LSO;2703	}2704	if (nn->cap & NFP_NET_CFG_CTRL_RSS_ANY)2705		netdev->hw_features |= NETIF_F_RXHASH;2706 2707#ifdef CONFIG_NFP_NET_IPSEC2708	if (nn->cap_w1 & NFP_NET_CFG_CTRL_IPSEC)2709		netdev->hw_features |= NETIF_F_HW_ESP | NETIF_F_HW_ESP_TX_CSUM;2710#endif2711 2712	if (nn->cap & NFP_NET_CFG_CTRL_VXLAN) {2713		if (nn->cap & NFP_NET_CFG_CTRL_LSO) {2714			netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL |2715					       NETIF_F_GSO_UDP_TUNNEL_CSUM |2716					       NETIF_F_GSO_PARTIAL;2717			netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM;2718		}2719		netdev->udp_tunnel_nic_info = &nfp_udp_tunnels;2720		nn->dp.ctrl |= NFP_NET_CFG_CTRL_VXLAN;2721	}2722	if (nn->cap & NFP_NET_CFG_CTRL_NVGRE) {2723		if (nn->cap & NFP_NET_CFG_CTRL_LSO)2724			netdev->hw_features |= NETIF_F_GSO_GRE;2725		nn->dp.ctrl |= NFP_NET_CFG_CTRL_NVGRE;2726	}2727	if (nn->cap & (NFP_NET_CFG_CTRL_VXLAN | NFP_NET_CFG_CTRL_NVGRE))2728		netdev->hw_enc_features = netdev->hw_features;2729 2730	netdev->vlan_features = netdev->hw_features;2731 2732	if (nn->cap & NFP_NET_CFG_CTRL_RXVLAN_ANY) {2733		netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;2734		nn->dp.ctrl |= nn->cap & NFP_NET_CFG_CTRL_RXVLAN_V2 ?:2735			       NFP_NET_CFG_CTRL_RXVLAN;2736	}2737	if (nn->cap & NFP_NET_CFG_CTRL_TXVLAN_ANY) {2738		if (nn->cap & NFP_NET_CFG_CTRL_LSO2) {2739			nn_warn(nn, "Device advertises both TSO2 and TXVLAN. Refusing to enable TXVLAN.\n");2740		} else {2741			netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;2742			nn->dp.ctrl |= nn->cap & NFP_NET_CFG_CTRL_TXVLAN_V2 ?:2743				       NFP_NET_CFG_CTRL_TXVLAN;2744		}2745	}2746	if (nn->cap & NFP_NET_CFG_CTRL_CTAG_FILTER) {2747		netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;2748		nn->dp.ctrl |= NFP_NET_CFG_CTRL_CTAG_FILTER;2749	}2750	if (nn->cap & NFP_NET_CFG_CTRL_RXQINQ) {2751		netdev->hw_features |= NETIF_F_HW_VLAN_STAG_RX;2752		nn->dp.ctrl |= NFP_NET_CFG_CTRL_RXQINQ;2753	}2754 2755	netdev->features = netdev->hw_features;2756 2757	if (nfp_app_has_tc(nn->app) && nn->port)2758		netdev->hw_features |= NETIF_F_HW_TC;2759 2760	/* C-Tag strip and S-Tag strip can't be supported simultaneously,2761	 * so enable C-Tag strip and disable S-Tag strip by default.2762	 */2763	netdev->features &= ~NETIF_F_HW_VLAN_STAG_RX;2764	nn->dp.ctrl &= ~NFP_NET_CFG_CTRL_RXQINQ;2765 2766	netdev->xdp_features = NETDEV_XDP_ACT_BASIC;2767	if (nn->app && nn->app->type->id == NFP_APP_BPF_NIC)2768		netdev->xdp_features |= NETDEV_XDP_ACT_HW_OFFLOAD;2769 2770	/* Finalise the netdev setup */2771	switch (nn->dp.ops->version) {2772	case NFP_NFD_VER_NFD3:2773		netdev->netdev_ops = &nfp_nfd3_netdev_ops;2774		netdev->xdp_features |= NETDEV_XDP_ACT_XSK_ZEROCOPY;2775		netdev->xdp_features |= NETDEV_XDP_ACT_REDIRECT;2776		break;2777	case NFP_NFD_VER_NFDK:2778		netdev->netdev_ops = &nfp_nfdk_netdev_ops;2779		break;2780	}2781 2782	netdev->watchdog_timeo = msecs_to_jiffies(5 * 1000);2783 2784	/* MTU range: 68 - hw-specific max */2785	netdev->min_mtu = ETH_MIN_MTU;2786	netdev->max_mtu = nn->max_mtu;2787 2788	netif_set_tso_max_segs(netdev, NFP_NET_LSO_MAX_SEGS);2789 2790	netif_carrier_off(netdev);2791 2792	nfp_net_set_ethtool_ops(netdev);2793}2794 2795static int nfp_net_read_caps(struct nfp_net *nn)2796{2797	/* Get some of the read-only fields from the BAR */2798	nn->cap = nn_readl(nn, NFP_NET_CFG_CAP);2799	nn->cap_w1 = nn_readl(nn, NFP_NET_CFG_CAP_WORD1);2800	nn->max_mtu = nn_readl(nn, NFP_NET_CFG_MAX_MTU);2801 2802	/* ABI 4.x and ctrl vNIC always use chained metadata, in other cases2803	 * we allow use of non-chained metadata if RSS(v1) is the only2804	 * advertised capability requiring metadata.2805	 */2806	nn->dp.chained_metadata_format = nn->fw_ver.major == 4 ||2807					 !nn->dp.netdev ||2808					 !(nn->cap & NFP_NET_CFG_CTRL_RSS) ||2809					 nn->cap & NFP_NET_CFG_CTRL_CHAIN_META;2810	/* RSS(v1) uses non-chained metadata format, except in ABI 4.x where2811	 * it has the same meaning as RSSv2.2812	 */2813	if (nn->dp.chained_metadata_format && nn->fw_ver.major != 4)2814		nn->cap &= ~NFP_NET_CFG_CTRL_RSS;2815 2816	/* Determine RX packet/metadata boundary offset */2817	if (nn->fw_ver.major >= 2) {2818		u32 reg;2819 2820		reg = nn_readl(nn, NFP_NET_CFG_RX_OFFSET);2821		if (reg > NFP_NET_MAX_PREPEND) {2822			nn_err(nn, "Invalid rx offset: %d\n", reg);2823			return -EINVAL;2824		}2825		nn->dp.rx_offset = reg;2826	} else {2827		nn->dp.rx_offset = NFP_NET_RX_OFFSET;2828	}2829 2830	/* Mask out NFD-version-specific features */2831	nn->cap &= nn->dp.ops->cap_mask;2832 2833	/* For control vNICs mask out the capabilities app doesn't want. */2834	if (!nn->dp.netdev)2835		nn->cap &= nn->app->type->ctrl_cap_mask;2836 2837	return 0;2838}2839 2840/**2841 * nfp_net_init() - Initialise/finalise the nfp_net structure2842 * @nn:		NFP Net device structure2843 *2844 * Return: 0 on success or negative errno on error.2845 */2846int nfp_net_init(struct nfp_net *nn)2847{2848	int err;2849 2850	nn->dp.rx_dma_dir = DMA_FROM_DEVICE;2851 2852	err = nfp_net_read_caps(nn);2853	if (err)2854		return err;2855 2856	/* Set default MTU and Freelist buffer size */2857	if (!nfp_net_is_data_vnic(nn) && nn->app->ctrl_mtu) {2858		nn->dp.mtu = min(nn->app->ctrl_mtu, nn->max_mtu);2859	} else if (nn->max_mtu < NFP_NET_DEFAULT_MTU) {2860		nn->dp.mtu = nn->max_mtu;2861	} else {2862		nn->dp.mtu = NFP_NET_DEFAULT_MTU;2863	}2864	nn->dp.fl_bufsz = nfp_net_calc_fl_bufsz(&nn->dp);2865 2866	if (nfp_app_ctrl_uses_data_vnics(nn->app))2867		nn->dp.ctrl |= nn->cap & NFP_NET_CFG_CTRL_CMSG_DATA;2868 2869	if (nn->cap & NFP_NET_CFG_CTRL_RSS_ANY) {2870		nfp_net_rss_init(nn);2871		nn->dp.ctrl |= nn->cap & NFP_NET_CFG_CTRL_RSS2 ?:2872					 NFP_NET_CFG_CTRL_RSS;2873	}2874 2875	/* Allow L2 Broadcast and Multicast through by default, if supported */2876	if (nn->cap & NFP_NET_CFG_CTRL_L2BC)2877		nn->dp.ctrl |= NFP_NET_CFG_CTRL_L2BC;2878 2879	/* Allow IRQ moderation, if supported */2880	if (nn->cap & NFP_NET_CFG_CTRL_IRQMOD) {2881		nfp_net_irqmod_init(nn);2882		nn->dp.ctrl |= NFP_NET_CFG_CTRL_IRQMOD;2883	}2884 2885	/* Enable TX pointer writeback, if supported */2886	if (nn->cap & NFP_NET_CFG_CTRL_TXRWB)2887		nn->dp.ctrl |= NFP_NET_CFG_CTRL_TXRWB;2888 2889	if (nn->cap_w1 & NFP_NET_CFG_CTRL_MCAST_FILTER)2890		nn->dp.ctrl_w1 |= NFP_NET_CFG_CTRL_MCAST_FILTER;2891 2892	/* Stash the re-configuration queue away.  First odd queue in TX Bar */2893	nn->qcp_cfg = nn->tx_bar + NFP_QCP_QUEUE_ADDR_SZ;2894 2895	/* Make sure the FW knows the netdev is supposed to be disabled here */2896	nn_writel(nn, NFP_NET_CFG_CTRL, 0);2897	nn_writeq(nn, NFP_NET_CFG_TXRS_ENABLE, 0);2898	nn_writeq(nn, NFP_NET_CFG_RXRS_ENABLE, 0);2899	nn_writel(nn, NFP_NET_CFG_CTRL_WORD1, 0);2900	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_RING |2901				   NFP_NET_CFG_UPDATE_GEN);2902	if (err)2903		return err;2904 2905	if (nn->dp.netdev) {2906		nfp_net_netdev_init(nn);2907 2908		err = nfp_ccm_mbox_init(nn);2909		if (err)2910			return err;2911 2912		err = nfp_net_tls_init(nn);2913		if (err)2914			goto err_clean_mbox;2915 2916		nfp_net_ipsec_init(nn);2917	}2918 2919	nfp_net_vecs_init(nn);2920 2921	if (!nn->dp.netdev)2922		return 0;2923 2924	spin_lock_init(&nn->mbox_amsg.lock);2925	INIT_LIST_HEAD(&nn->mbox_amsg.list);2926	INIT_WORK(&nn->mbox_amsg.work, nfp_net_mbox_amsg_work);2927 2928	INIT_LIST_HEAD(&nn->fs.list);2929 2930	return register_netdev(nn->dp.netdev);2931 2932err_clean_mbox:2933	nfp_ccm_mbox_clean(nn);2934	return err;2935}2936 2937/**2938 * nfp_net_clean() - Undo what nfp_net_init() did.2939 * @nn:		NFP Net device structure2940 */2941void nfp_net_clean(struct nfp_net *nn)2942{2943	if (!nn->dp.netdev)2944		return;2945 2946	unregister_netdev(nn->dp.netdev);2947	nfp_net_ipsec_clean(nn);2948	nfp_ccm_mbox_clean(nn);2949	nfp_net_fs_clean(nn);2950	flush_work(&nn->mbox_amsg.work);2951	nfp_net_reconfig_wait_posted(nn);2952}2953