brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0-only2/****************************************************************************3 * Driver for Solarflare network controllers and boards4 * Copyright 2012-2013 Solarflare Communications Inc.5 */6 7#include "net_driver.h"8#include "rx_common.h"9#include "tx_common.h"10#include "ef10_regs.h"11#include "io.h"12#include "mcdi.h"13#include "mcdi_pcol.h"14#include "mcdi_port.h"15#include "mcdi_port_common.h"16#include "mcdi_functions.h"17#include "nic.h"18#include "mcdi_filters.h"19#include "workarounds.h"20#include "selftest.h"21#include "ef10_sriov.h"22#include <linux/in.h>23#include <linux/jhash.h>24#include <linux/wait.h>25#include <linux/workqueue.h>26#include <net/udp_tunnel.h>27 28/* Hardware control for EF10 architecture including 'Huntington'. */29 30#define EFX_EF10_DRVGEN_EV		731enum {32	EFX_EF10_TEST = 1,33	EFX_EF10_REFILL,34};35 36/* VLAN list entry */37struct efx_ef10_vlan {38	struct list_head list;39	u16 vid;40};41 42static int efx_ef10_set_udp_tnl_ports(struct efx_nic *efx, bool unloading);43static const struct udp_tunnel_nic_info efx_ef10_udp_tunnels;44 45static int efx_ef10_get_warm_boot_count(struct efx_nic *efx)46{47	efx_dword_t reg;48 49	efx_readd(efx, &reg, ER_DZ_BIU_MC_SFT_STATUS);50	return EFX_DWORD_FIELD(reg, EFX_WORD_1) == 0xb007 ?51		EFX_DWORD_FIELD(reg, EFX_WORD_0) : -EIO;52}53 54/* On all EF10s up to and including SFC9220 (Medford1), all PFs use BAR 0 for55 * I/O space and BAR 2(&3) for memory.  On SFC9250 (Medford2), there is no I/O56 * bar; PFs use BAR 0/1 for memory.57 */58static unsigned int efx_ef10_pf_mem_bar(struct efx_nic *efx)59{60	switch (efx->pci_dev->device) {61	case 0x0b03: /* SFC9250 PF */62		return 0;63	default:64		return 2;65	}66}67 68/* All VFs use BAR 0/1 for memory */69static unsigned int efx_ef10_vf_mem_bar(struct efx_nic *efx)70{71	return 0;72}73 74static unsigned int efx_ef10_mem_map_size(struct efx_nic *efx)75{76	int bar;77 78	bar = efx->type->mem_bar(efx);79	return resource_size(&efx->pci_dev->resource[bar]);80}81 82static bool efx_ef10_is_vf(struct efx_nic *efx)83{84	return efx->type->is_vf;85}86 87#ifdef CONFIG_SFC_SRIOV88static int efx_ef10_get_vf_index(struct efx_nic *efx)89{90	MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_FUNCTION_INFO_OUT_LEN);91	struct efx_ef10_nic_data *nic_data = efx->nic_data;92	size_t outlen;93	int rc;94 95	rc = efx_mcdi_rpc(efx, MC_CMD_GET_FUNCTION_INFO, NULL, 0, outbuf,96			  sizeof(outbuf), &outlen);97	if (rc)98		return rc;99	if (outlen < sizeof(outbuf))100		return -EIO;101 102	nic_data->vf_index = MCDI_DWORD(outbuf, GET_FUNCTION_INFO_OUT_VF);103	return 0;104}105#endif106 107static int efx_ef10_init_datapath_caps(struct efx_nic *efx)108{109	MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_CAPABILITIES_V4_OUT_LEN);110	struct efx_ef10_nic_data *nic_data = efx->nic_data;111	size_t outlen;112	int rc;113 114	BUILD_BUG_ON(MC_CMD_GET_CAPABILITIES_IN_LEN != 0);115 116	rc = efx_mcdi_rpc(efx, MC_CMD_GET_CAPABILITIES, NULL, 0,117			  outbuf, sizeof(outbuf), &outlen);118	if (rc)119		return rc;120	if (outlen < MC_CMD_GET_CAPABILITIES_OUT_LEN) {121		netif_err(efx, drv, efx->net_dev,122			  "unable to read datapath firmware capabilities\n");123		return -EIO;124	}125 126	nic_data->datapath_caps =127		MCDI_DWORD(outbuf, GET_CAPABILITIES_OUT_FLAGS1);128 129	if (outlen >= MC_CMD_GET_CAPABILITIES_V2_OUT_LEN) {130		nic_data->datapath_caps2 = MCDI_DWORD(outbuf,131				GET_CAPABILITIES_V2_OUT_FLAGS2);132		nic_data->piobuf_size = MCDI_WORD(outbuf,133				GET_CAPABILITIES_V2_OUT_SIZE_PIO_BUFF);134	} else {135		nic_data->datapath_caps2 = 0;136		nic_data->piobuf_size = ER_DZ_TX_PIOBUF_SIZE;137	}138 139	/* record the DPCPU firmware IDs to determine VEB vswitching support.140	 */141	nic_data->rx_dpcpu_fw_id =142		MCDI_WORD(outbuf, GET_CAPABILITIES_OUT_RX_DPCPU_FW_ID);143	nic_data->tx_dpcpu_fw_id =144		MCDI_WORD(outbuf, GET_CAPABILITIES_OUT_TX_DPCPU_FW_ID);145 146	if (!(nic_data->datapath_caps &147	      (1 << MC_CMD_GET_CAPABILITIES_OUT_RX_PREFIX_LEN_14_LBN))) {148		netif_err(efx, probe, efx->net_dev,149			  "current firmware does not support an RX prefix\n");150		return -ENODEV;151	}152 153	if (outlen >= MC_CMD_GET_CAPABILITIES_V3_OUT_LEN) {154		u8 vi_window_mode = MCDI_BYTE(outbuf,155				GET_CAPABILITIES_V3_OUT_VI_WINDOW_MODE);156 157		rc = efx_mcdi_window_mode_to_stride(efx, vi_window_mode);158		if (rc)159			return rc;160	} else {161		/* keep default VI stride */162		netif_dbg(efx, probe, efx->net_dev,163			  "firmware did not report VI window mode, assuming vi_stride = %u\n",164			  efx->vi_stride);165	}166 167	if (outlen >= MC_CMD_GET_CAPABILITIES_V4_OUT_LEN) {168		efx->num_mac_stats = MCDI_WORD(outbuf,169				GET_CAPABILITIES_V4_OUT_MAC_STATS_NUM_STATS);170		netif_dbg(efx, probe, efx->net_dev,171			  "firmware reports num_mac_stats = %u\n",172			  efx->num_mac_stats);173	} else {174		/* leave num_mac_stats as the default value, MC_CMD_MAC_NSTATS */175		netif_dbg(efx, probe, efx->net_dev,176			  "firmware did not report num_mac_stats, assuming %u\n",177			  efx->num_mac_stats);178	}179 180	return 0;181}182 183static void efx_ef10_read_licensed_features(struct efx_nic *efx)184{185	MCDI_DECLARE_BUF(inbuf, MC_CMD_LICENSING_V3_IN_LEN);186	MCDI_DECLARE_BUF(outbuf, MC_CMD_LICENSING_V3_OUT_LEN);187	struct efx_ef10_nic_data *nic_data = efx->nic_data;188	size_t outlen;189	int rc;190 191	MCDI_SET_DWORD(inbuf, LICENSING_V3_IN_OP,192		       MC_CMD_LICENSING_V3_IN_OP_REPORT_LICENSE);193	rc = efx_mcdi_rpc_quiet(efx, MC_CMD_LICENSING_V3, inbuf, sizeof(inbuf),194				outbuf, sizeof(outbuf), &outlen);195	if (rc || (outlen < MC_CMD_LICENSING_V3_OUT_LEN))196		return;197 198	nic_data->licensed_features = MCDI_QWORD(outbuf,199					 LICENSING_V3_OUT_LICENSED_FEATURES);200}201 202static int efx_ef10_get_sysclk_freq(struct efx_nic *efx)203{204	MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_CLOCK_OUT_LEN);205	int rc;206 207	rc = efx_mcdi_rpc(efx, MC_CMD_GET_CLOCK, NULL, 0,208			  outbuf, sizeof(outbuf), NULL);209	if (rc)210		return rc;211	rc = MCDI_DWORD(outbuf, GET_CLOCK_OUT_SYS_FREQ);212	return rc > 0 ? rc : -ERANGE;213}214 215static int efx_ef10_get_timer_workarounds(struct efx_nic *efx)216{217	struct efx_ef10_nic_data *nic_data = efx->nic_data;218	unsigned int implemented;219	unsigned int enabled;220	int rc;221 222	nic_data->workaround_35388 = false;223	nic_data->workaround_61265 = false;224 225	rc = efx_mcdi_get_workarounds(efx, &implemented, &enabled);226 227	if (rc == -ENOSYS) {228		/* Firmware without GET_WORKAROUNDS - not a problem. */229		rc = 0;230	} else if (rc == 0) {231		/* Bug61265 workaround is always enabled if implemented. */232		if (enabled & MC_CMD_GET_WORKAROUNDS_OUT_BUG61265)233			nic_data->workaround_61265 = true;234 235		if (enabled & MC_CMD_GET_WORKAROUNDS_OUT_BUG35388) {236			nic_data->workaround_35388 = true;237		} else if (implemented & MC_CMD_GET_WORKAROUNDS_OUT_BUG35388) {238			/* Workaround is implemented but not enabled.239			 * Try to enable it.240			 */241			rc = efx_mcdi_set_workaround(efx,242						     MC_CMD_WORKAROUND_BUG35388,243						     true, NULL);244			if (rc == 0)245				nic_data->workaround_35388 = true;246			/* If we failed to set the workaround just carry on. */247			rc = 0;248		}249	}250 251	netif_dbg(efx, probe, efx->net_dev,252		  "workaround for bug 35388 is %sabled\n",253		  nic_data->workaround_35388 ? "en" : "dis");254	netif_dbg(efx, probe, efx->net_dev,255		  "workaround for bug 61265 is %sabled\n",256		  nic_data->workaround_61265 ? "en" : "dis");257 258	return rc;259}260 261static void efx_ef10_process_timer_config(struct efx_nic *efx,262					  const efx_dword_t *data)263{264	unsigned int max_count;265 266	if (EFX_EF10_WORKAROUND_61265(efx)) {267		efx->timer_quantum_ns = MCDI_DWORD(data,268			GET_EVQ_TMR_PROPERTIES_OUT_MCDI_TMR_STEP_NS);269		efx->timer_max_ns = MCDI_DWORD(data,270			GET_EVQ_TMR_PROPERTIES_OUT_MCDI_TMR_MAX_NS);271	} else if (EFX_EF10_WORKAROUND_35388(efx)) {272		efx->timer_quantum_ns = MCDI_DWORD(data,273			GET_EVQ_TMR_PROPERTIES_OUT_BUG35388_TMR_NS_PER_COUNT);274		max_count = MCDI_DWORD(data,275			GET_EVQ_TMR_PROPERTIES_OUT_BUG35388_TMR_MAX_COUNT);276		efx->timer_max_ns = max_count * efx->timer_quantum_ns;277	} else {278		efx->timer_quantum_ns = MCDI_DWORD(data,279			GET_EVQ_TMR_PROPERTIES_OUT_TMR_REG_NS_PER_COUNT);280		max_count = MCDI_DWORD(data,281			GET_EVQ_TMR_PROPERTIES_OUT_TMR_REG_MAX_COUNT);282		efx->timer_max_ns = max_count * efx->timer_quantum_ns;283	}284 285	netif_dbg(efx, probe, efx->net_dev,286		  "got timer properties from MC: quantum %u ns; max %u ns\n",287		  efx->timer_quantum_ns, efx->timer_max_ns);288}289 290static int efx_ef10_get_timer_config(struct efx_nic *efx)291{292	MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_LEN);293	int rc;294 295	rc = efx_ef10_get_timer_workarounds(efx);296	if (rc)297		return rc;298 299	rc = efx_mcdi_rpc_quiet(efx, MC_CMD_GET_EVQ_TMR_PROPERTIES, NULL, 0,300				outbuf, sizeof(outbuf), NULL);301 302	if (rc == 0) {303		efx_ef10_process_timer_config(efx, outbuf);304	} else if (rc == -ENOSYS || rc == -EPERM) {305		/* Not available - fall back to Huntington defaults. */306		unsigned int quantum;307 308		rc = efx_ef10_get_sysclk_freq(efx);309		if (rc < 0)310			return rc;311 312		quantum = 1536000 / rc; /* 1536 cycles */313		efx->timer_quantum_ns = quantum;314		efx->timer_max_ns = efx->type->timer_period_max * quantum;315		rc = 0;316	} else {317		efx_mcdi_display_error(efx, MC_CMD_GET_EVQ_TMR_PROPERTIES,318				       MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_LEN,319				       NULL, 0, rc);320	}321 322	return rc;323}324 325static int efx_ef10_get_mac_address_pf(struct efx_nic *efx, u8 *mac_address)326{327	MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_MAC_ADDRESSES_OUT_LEN);328	size_t outlen;329	int rc;330 331	BUILD_BUG_ON(MC_CMD_GET_MAC_ADDRESSES_IN_LEN != 0);332 333	rc = efx_mcdi_rpc(efx, MC_CMD_GET_MAC_ADDRESSES, NULL, 0,334			  outbuf, sizeof(outbuf), &outlen);335	if (rc)336		return rc;337	if (outlen < MC_CMD_GET_MAC_ADDRESSES_OUT_LEN)338		return -EIO;339 340	ether_addr_copy(mac_address,341			MCDI_PTR(outbuf, GET_MAC_ADDRESSES_OUT_MAC_ADDR_BASE));342	return 0;343}344 345static int efx_ef10_get_mac_address_vf(struct efx_nic *efx, u8 *mac_address)346{347	MCDI_DECLARE_BUF(inbuf, MC_CMD_VPORT_GET_MAC_ADDRESSES_IN_LEN);348	MCDI_DECLARE_BUF(outbuf, MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_LENMAX);349	size_t outlen;350	int num_addrs, rc;351 352	MCDI_SET_DWORD(inbuf, VPORT_GET_MAC_ADDRESSES_IN_VPORT_ID,353		       EVB_PORT_ID_ASSIGNED);354	rc = efx_mcdi_rpc(efx, MC_CMD_VPORT_GET_MAC_ADDRESSES, inbuf,355			  sizeof(inbuf), outbuf, sizeof(outbuf), &outlen);356 357	if (rc)358		return rc;359	if (outlen < MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_LENMIN)360		return -EIO;361 362	num_addrs = MCDI_DWORD(outbuf,363			       VPORT_GET_MAC_ADDRESSES_OUT_MACADDR_COUNT);364 365	WARN_ON(num_addrs != 1);366 367	ether_addr_copy(mac_address,368			MCDI_PTR(outbuf, VPORT_GET_MAC_ADDRESSES_OUT_MACADDR));369 370	return 0;371}372 373static ssize_t link_control_flag_show(struct device *dev,374				      struct device_attribute *attr,375				      char *buf)376{377	struct efx_nic *efx = dev_get_drvdata(dev);378 379	return sprintf(buf, "%d\n",380		       ((efx->mcdi->fn_flags) &381			(1 << MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_LINKCTRL))382		       ? 1 : 0);383}384 385static ssize_t primary_flag_show(struct device *dev,386				 struct device_attribute *attr,387				 char *buf)388{389	struct efx_nic *efx = dev_get_drvdata(dev);390 391	return sprintf(buf, "%d\n",392		       ((efx->mcdi->fn_flags) &393			(1 << MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_PRIMARY))394		       ? 1 : 0);395}396 397static struct efx_ef10_vlan *efx_ef10_find_vlan(struct efx_nic *efx, u16 vid)398{399	struct efx_ef10_nic_data *nic_data = efx->nic_data;400	struct efx_ef10_vlan *vlan;401 402	WARN_ON(!mutex_is_locked(&nic_data->vlan_lock));403 404	list_for_each_entry(vlan, &nic_data->vlan_list, list) {405		if (vlan->vid == vid)406			return vlan;407	}408 409	return NULL;410}411 412static int efx_ef10_add_vlan(struct efx_nic *efx, u16 vid)413{414	struct efx_ef10_nic_data *nic_data = efx->nic_data;415	struct efx_ef10_vlan *vlan;416	int rc;417 418	mutex_lock(&nic_data->vlan_lock);419 420	vlan = efx_ef10_find_vlan(efx, vid);421	if (vlan) {422		/* We add VID 0 on init. 8021q adds it on module init423		 * for all interfaces with VLAN filtring feature.424		 */425		if (vid == 0)426			goto done_unlock;427		netif_warn(efx, drv, efx->net_dev,428			   "VLAN %u already added\n", vid);429		rc = -EALREADY;430		goto fail_exist;431	}432 433	rc = -ENOMEM;434	vlan = kzalloc(sizeof(*vlan), GFP_KERNEL);435	if (!vlan)436		goto fail_alloc;437 438	vlan->vid = vid;439 440	list_add_tail(&vlan->list, &nic_data->vlan_list);441 442	if (efx->filter_state) {443		mutex_lock(&efx->mac_lock);444		down_write(&efx->filter_sem);445		rc = efx_mcdi_filter_add_vlan(efx, vlan->vid);446		up_write(&efx->filter_sem);447		mutex_unlock(&efx->mac_lock);448		if (rc)449			goto fail_filter_add_vlan;450	}451 452done_unlock:453	mutex_unlock(&nic_data->vlan_lock);454	return 0;455 456fail_filter_add_vlan:457	list_del(&vlan->list);458	kfree(vlan);459fail_alloc:460fail_exist:461	mutex_unlock(&nic_data->vlan_lock);462	return rc;463}464 465static void efx_ef10_del_vlan_internal(struct efx_nic *efx,466				       struct efx_ef10_vlan *vlan)467{468	struct efx_ef10_nic_data *nic_data = efx->nic_data;469 470	WARN_ON(!mutex_is_locked(&nic_data->vlan_lock));471 472	if (efx->filter_state) {473		down_write(&efx->filter_sem);474		efx_mcdi_filter_del_vlan(efx, vlan->vid);475		up_write(&efx->filter_sem);476	}477 478	list_del(&vlan->list);479	kfree(vlan);480}481 482static int efx_ef10_del_vlan(struct efx_nic *efx, u16 vid)483{484	struct efx_ef10_nic_data *nic_data = efx->nic_data;485	struct efx_ef10_vlan *vlan;486	int rc = 0;487 488	/* 8021q removes VID 0 on module unload for all interfaces489	 * with VLAN filtering feature. We need to keep it to receive490	 * untagged traffic.491	 */492	if (vid == 0)493		return 0;494 495	mutex_lock(&nic_data->vlan_lock);496 497	vlan = efx_ef10_find_vlan(efx, vid);498	if (!vlan) {499		netif_err(efx, drv, efx->net_dev,500			  "VLAN %u to be deleted not found\n", vid);501		rc = -ENOENT;502	} else {503		efx_ef10_del_vlan_internal(efx, vlan);504	}505 506	mutex_unlock(&nic_data->vlan_lock);507 508	return rc;509}510 511static void efx_ef10_cleanup_vlans(struct efx_nic *efx)512{513	struct efx_ef10_nic_data *nic_data = efx->nic_data;514	struct efx_ef10_vlan *vlan, *next_vlan;515 516	mutex_lock(&nic_data->vlan_lock);517	list_for_each_entry_safe(vlan, next_vlan, &nic_data->vlan_list, list)518		efx_ef10_del_vlan_internal(efx, vlan);519	mutex_unlock(&nic_data->vlan_lock);520}521 522static DEVICE_ATTR_RO(link_control_flag);523static DEVICE_ATTR_RO(primary_flag);524 525static int efx_ef10_probe(struct efx_nic *efx)526{527	struct efx_ef10_nic_data *nic_data;528	int i, rc;529 530	nic_data = kzalloc(sizeof(*nic_data), GFP_KERNEL);531	if (!nic_data)532		return -ENOMEM;533	efx->nic_data = nic_data;534 535	/* we assume later that we can copy from this buffer in dwords */536	BUILD_BUG_ON(MCDI_CTL_SDU_LEN_MAX_V2 % 4);537 538	rc = efx_nic_alloc_buffer(efx, &nic_data->mcdi_buf,539				  8 + MCDI_CTL_SDU_LEN_MAX_V2, GFP_KERNEL);540	if (rc)541		goto fail1;542 543	/* Get the MC's warm boot count.  In case it's rebooting right544	 * now, be prepared to retry.545	 */546	i = 0;547	for (;;) {548		rc = efx_ef10_get_warm_boot_count(efx);549		if (rc >= 0)550			break;551		if (++i == 5)552			goto fail2;553		ssleep(1);554	}555	nic_data->warm_boot_count = rc;556 557	/* In case we're recovering from a crash (kexec), we want to558	 * cancel any outstanding request by the previous user of this559	 * function.  We send a special message using the least560	 * significant bits of the 'high' (doorbell) register.561	 */562	_efx_writed(efx, cpu_to_le32(1), ER_DZ_MC_DB_HWRD);563 564	rc = efx_mcdi_init(efx);565	if (rc)566		goto fail2;567 568	mutex_init(&nic_data->udp_tunnels_lock);569	for (i = 0; i < ARRAY_SIZE(nic_data->udp_tunnels); ++i)570		nic_data->udp_tunnels[i].type =571			TUNNEL_ENCAP_UDP_PORT_ENTRY_INVALID;572 573	/* Reset (most) configuration for this function */574	rc = efx_mcdi_reset(efx, RESET_TYPE_ALL);575	if (rc)576		goto fail3;577 578	/* Enable event logging */579	rc = efx_mcdi_log_ctrl(efx, true, false, 0);580	if (rc)581		goto fail3;582 583	rc = device_create_file(&efx->pci_dev->dev,584				&dev_attr_link_control_flag);585	if (rc)586		goto fail3;587 588	rc = device_create_file(&efx->pci_dev->dev, &dev_attr_primary_flag);589	if (rc)590		goto fail4;591 592	rc = efx_get_pf_index(efx, &nic_data->pf_index);593	if (rc)594		goto fail5;595 596	rc = efx_ef10_init_datapath_caps(efx);597	if (rc < 0)598		goto fail5;599 600	efx_ef10_read_licensed_features(efx);601 602	/* We can have one VI for each vi_stride-byte region.603	 * However, until we use TX option descriptors we need up to four604	 * TX queues per channel for different checksumming combinations.605	 */606	if (nic_data->datapath_caps &607	    (1 << MC_CMD_GET_CAPABILITIES_OUT_VXLAN_NVGRE_LBN))608		efx->tx_queues_per_channel = 4;609	else610		efx->tx_queues_per_channel = 2;611	efx->max_vis = efx_ef10_mem_map_size(efx) / efx->vi_stride;612	if (!efx->max_vis) {613		netif_err(efx, drv, efx->net_dev, "error determining max VIs\n");614		rc = -EIO;615		goto fail5;616	}617	efx->max_channels = min_t(unsigned int, EFX_MAX_CHANNELS,618				  efx->max_vis / efx->tx_queues_per_channel);619	efx->max_tx_channels = efx->max_channels;620	if (WARN_ON(efx->max_channels == 0)) {621		rc = -EIO;622		goto fail5;623	}624 625	efx->rx_packet_len_offset =626		ES_DZ_RX_PREFIX_PKTLEN_OFST - ES_DZ_RX_PREFIX_SIZE;627 628	if (nic_data->datapath_caps &629	    (1 << MC_CMD_GET_CAPABILITIES_OUT_RX_INCLUDE_FCS_LBN))630		efx->net_dev->hw_features |= NETIF_F_RXFCS;631 632	rc = efx_mcdi_port_get_number(efx);633	if (rc < 0)634		goto fail5;635	efx->port_num = rc;636 637	rc = efx->type->get_mac_address(efx, efx->net_dev->perm_addr);638	if (rc)639		goto fail5;640 641	rc = efx_ef10_get_timer_config(efx);642	if (rc < 0)643		goto fail5;644 645	rc = efx_mcdi_mon_probe(efx);646	if (rc && rc != -EPERM)647		goto fail5;648 649	efx_ptp_defer_probe_with_channel(efx);650 651#ifdef CONFIG_SFC_SRIOV652	if ((efx->pci_dev->physfn) && (!efx->pci_dev->is_physfn)) {653		struct pci_dev *pci_dev_pf = efx->pci_dev->physfn;654		struct efx_nic *efx_pf = pci_get_drvdata(pci_dev_pf);655 656		efx_pf->type->get_mac_address(efx_pf, nic_data->port_id);657	} else658#endif659		ether_addr_copy(nic_data->port_id, efx->net_dev->perm_addr);660 661	INIT_LIST_HEAD(&nic_data->vlan_list);662	mutex_init(&nic_data->vlan_lock);663 664	/* Add unspecified VID to support VLAN filtering being disabled */665	rc = efx_ef10_add_vlan(efx, EFX_FILTER_VID_UNSPEC);666	if (rc)667		goto fail_add_vid_unspec;668 669	/* If VLAN filtering is enabled, we need VID 0 to get untagged670	 * traffic.  It is added automatically if 8021q module is loaded,671	 * but we can't rely on it since module may be not loaded.672	 */673	rc = efx_ef10_add_vlan(efx, 0);674	if (rc)675		goto fail_add_vid_0;676 677	if (nic_data->datapath_caps &678	    (1 << MC_CMD_GET_CAPABILITIES_OUT_VXLAN_NVGRE_LBN) &&679	    efx->mcdi->fn_flags &680	    (1 << MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_TRUSTED))681		efx->net_dev->udp_tunnel_nic_info = &efx_ef10_udp_tunnels;682 683	return 0;684 685fail_add_vid_0:686	efx_ef10_cleanup_vlans(efx);687fail_add_vid_unspec:688	mutex_destroy(&nic_data->vlan_lock);689	efx_ptp_remove(efx);690	efx_mcdi_mon_remove(efx);691fail5:692	device_remove_file(&efx->pci_dev->dev, &dev_attr_primary_flag);693fail4:694	device_remove_file(&efx->pci_dev->dev, &dev_attr_link_control_flag);695fail3:696	efx_mcdi_detach(efx);697 698	mutex_lock(&nic_data->udp_tunnels_lock);699	memset(nic_data->udp_tunnels, 0, sizeof(nic_data->udp_tunnels));700	(void)efx_ef10_set_udp_tnl_ports(efx, true);701	mutex_unlock(&nic_data->udp_tunnels_lock);702	mutex_destroy(&nic_data->udp_tunnels_lock);703 704	efx_mcdi_fini(efx);705fail2:706	efx_nic_free_buffer(efx, &nic_data->mcdi_buf);707fail1:708	kfree(nic_data);709	efx->nic_data = NULL;710	return rc;711}712 713#ifdef EFX_USE_PIO714 715static void efx_ef10_free_piobufs(struct efx_nic *efx)716{717	struct efx_ef10_nic_data *nic_data = efx->nic_data;718	MCDI_DECLARE_BUF(inbuf, MC_CMD_FREE_PIOBUF_IN_LEN);719	unsigned int i;720	int rc;721 722	BUILD_BUG_ON(MC_CMD_FREE_PIOBUF_OUT_LEN != 0);723 724	for (i = 0; i < nic_data->n_piobufs; i++) {725		MCDI_SET_DWORD(inbuf, FREE_PIOBUF_IN_PIOBUF_HANDLE,726			       nic_data->piobuf_handle[i]);727		rc = efx_mcdi_rpc(efx, MC_CMD_FREE_PIOBUF, inbuf, sizeof(inbuf),728				  NULL, 0, NULL);729		WARN_ON(rc);730	}731 732	nic_data->n_piobufs = 0;733}734 735static int efx_ef10_alloc_piobufs(struct efx_nic *efx, unsigned int n)736{737	struct efx_ef10_nic_data *nic_data = efx->nic_data;738	MCDI_DECLARE_BUF(outbuf, MC_CMD_ALLOC_PIOBUF_OUT_LEN);739	unsigned int i;740	size_t outlen;741	int rc = 0;742 743	BUILD_BUG_ON(MC_CMD_ALLOC_PIOBUF_IN_LEN != 0);744 745	for (i = 0; i < n; i++) {746		rc = efx_mcdi_rpc_quiet(efx, MC_CMD_ALLOC_PIOBUF, NULL, 0,747					outbuf, sizeof(outbuf), &outlen);748		if (rc) {749			/* Don't display the MC error if we didn't have space750			 * for a VF.751			 */752			if (!(efx_ef10_is_vf(efx) && rc == -ENOSPC))753				efx_mcdi_display_error(efx, MC_CMD_ALLOC_PIOBUF,754						       0, outbuf, outlen, rc);755			break;756		}757		if (outlen < MC_CMD_ALLOC_PIOBUF_OUT_LEN) {758			rc = -EIO;759			break;760		}761		nic_data->piobuf_handle[i] =762			MCDI_DWORD(outbuf, ALLOC_PIOBUF_OUT_PIOBUF_HANDLE);763		netif_dbg(efx, probe, efx->net_dev,764			  "allocated PIO buffer %u handle %x\n", i,765			  nic_data->piobuf_handle[i]);766	}767 768	nic_data->n_piobufs = i;769	if (rc)770		efx_ef10_free_piobufs(efx);771	return rc;772}773 774static int efx_ef10_link_piobufs(struct efx_nic *efx)775{776	struct efx_ef10_nic_data *nic_data = efx->nic_data;777	MCDI_DECLARE_BUF(inbuf, MC_CMD_LINK_PIOBUF_IN_LEN);778	struct efx_channel *channel;779	struct efx_tx_queue *tx_queue;780	unsigned int offset, index;781	int rc;782 783	BUILD_BUG_ON(MC_CMD_LINK_PIOBUF_OUT_LEN != 0);784	BUILD_BUG_ON(MC_CMD_UNLINK_PIOBUF_OUT_LEN != 0);785 786	/* Link a buffer to each VI in the write-combining mapping */787	for (index = 0; index < nic_data->n_piobufs; ++index) {788		MCDI_SET_DWORD(inbuf, LINK_PIOBUF_IN_PIOBUF_HANDLE,789			       nic_data->piobuf_handle[index]);790		MCDI_SET_DWORD(inbuf, LINK_PIOBUF_IN_TXQ_INSTANCE,791			       nic_data->pio_write_vi_base + index);792		rc = efx_mcdi_rpc(efx, MC_CMD_LINK_PIOBUF,793				  inbuf, MC_CMD_LINK_PIOBUF_IN_LEN,794				  NULL, 0, NULL);795		if (rc) {796			netif_err(efx, drv, efx->net_dev,797				  "failed to link VI %u to PIO buffer %u (%d)\n",798				  nic_data->pio_write_vi_base + index, index,799				  rc);800			goto fail;801		}802		netif_dbg(efx, probe, efx->net_dev,803			  "linked VI %u to PIO buffer %u\n",804			  nic_data->pio_write_vi_base + index, index);805	}806 807	/* Link a buffer to each TX queue */808	efx_for_each_channel(channel, efx) {809		/* Extra channels, even those with TXQs (PTP), do not require810		 * PIO resources.811		 */812		if (!channel->type->want_pio ||813		    channel->channel >= efx->xdp_channel_offset)814			continue;815 816		efx_for_each_channel_tx_queue(tx_queue, channel) {817			/* We assign the PIO buffers to queues in818			 * reverse order to allow for the following819			 * special case.820			 */821			offset = ((efx->tx_channel_offset + efx->n_tx_channels -822				   tx_queue->channel->channel - 1) *823				  efx_piobuf_size);824			index = offset / nic_data->piobuf_size;825			offset = offset % nic_data->piobuf_size;826 827			/* When the host page size is 4K, the first828			 * host page in the WC mapping may be within829			 * the same VI page as the last TX queue.  We830			 * can only link one buffer to each VI.831			 */832			if (tx_queue->queue == nic_data->pio_write_vi_base) {833				BUG_ON(index != 0);834				rc = 0;835			} else {836				MCDI_SET_DWORD(inbuf,837					       LINK_PIOBUF_IN_PIOBUF_HANDLE,838					       nic_data->piobuf_handle[index]);839				MCDI_SET_DWORD(inbuf,840					       LINK_PIOBUF_IN_TXQ_INSTANCE,841					       tx_queue->queue);842				rc = efx_mcdi_rpc(efx, MC_CMD_LINK_PIOBUF,843						  inbuf, MC_CMD_LINK_PIOBUF_IN_LEN,844						  NULL, 0, NULL);845			}846 847			if (rc) {848				/* This is non-fatal; the TX path just849				 * won't use PIO for this queue850				 */851				netif_err(efx, drv, efx->net_dev,852					  "failed to link VI %u to PIO buffer %u (%d)\n",853					  tx_queue->queue, index, rc);854				tx_queue->piobuf = NULL;855			} else {856				tx_queue->piobuf =857					nic_data->pio_write_base +858					index * efx->vi_stride + offset;859				tx_queue->piobuf_offset = offset;860				netif_dbg(efx, probe, efx->net_dev,861					  "linked VI %u to PIO buffer %u offset %x addr %p\n",862					  tx_queue->queue, index,863					  tx_queue->piobuf_offset,864					  tx_queue->piobuf);865			}866		}867	}868 869	return 0;870 871fail:872	/* inbuf was defined for MC_CMD_LINK_PIOBUF.  We can use the same873	 * buffer for MC_CMD_UNLINK_PIOBUF because it's shorter.874	 */875	BUILD_BUG_ON(MC_CMD_LINK_PIOBUF_IN_LEN < MC_CMD_UNLINK_PIOBUF_IN_LEN);876	while (index--) {877		MCDI_SET_DWORD(inbuf, UNLINK_PIOBUF_IN_TXQ_INSTANCE,878			       nic_data->pio_write_vi_base + index);879		efx_mcdi_rpc(efx, MC_CMD_UNLINK_PIOBUF,880			     inbuf, MC_CMD_UNLINK_PIOBUF_IN_LEN,881			     NULL, 0, NULL);882	}883	return rc;884}885 886static void efx_ef10_forget_old_piobufs(struct efx_nic *efx)887{888	struct efx_channel *channel;889	struct efx_tx_queue *tx_queue;890 891	/* All our existing PIO buffers went away */892	efx_for_each_channel(channel, efx)893		efx_for_each_channel_tx_queue(tx_queue, channel)894			tx_queue->piobuf = NULL;895}896 897#else /* !EFX_USE_PIO */898 899static int efx_ef10_alloc_piobufs(struct efx_nic *efx, unsigned int n)900{901	return n == 0 ? 0 : -ENOBUFS;902}903 904static int efx_ef10_link_piobufs(struct efx_nic *efx)905{906	return 0;907}908 909static void efx_ef10_free_piobufs(struct efx_nic *efx)910{911}912 913static void efx_ef10_forget_old_piobufs(struct efx_nic *efx)914{915}916 917#endif /* EFX_USE_PIO */918 919static void efx_ef10_remove(struct efx_nic *efx)920{921	struct efx_ef10_nic_data *nic_data = efx->nic_data;922	int rc;923 924#ifdef CONFIG_SFC_SRIOV925	struct efx_ef10_nic_data *nic_data_pf;926	struct pci_dev *pci_dev_pf;927	struct efx_nic *efx_pf;928	struct ef10_vf *vf;929 930	if (efx->pci_dev->is_virtfn) {931		pci_dev_pf = efx->pci_dev->physfn;932		if (pci_dev_pf) {933			efx_pf = pci_get_drvdata(pci_dev_pf);934			nic_data_pf = efx_pf->nic_data;935			vf = nic_data_pf->vf + nic_data->vf_index;936			vf->efx = NULL;937		} else938			netif_info(efx, drv, efx->net_dev,939				   "Could not get the PF id from VF\n");940	}941#endif942 943	efx_ef10_cleanup_vlans(efx);944	mutex_destroy(&nic_data->vlan_lock);945 946	efx_ptp_remove(efx);947 948	efx_mcdi_mon_remove(efx);949 950	efx_mcdi_rx_free_indir_table(efx);951 952	if (nic_data->wc_membase)953		iounmap(nic_data->wc_membase);954 955	rc = efx_mcdi_free_vis(efx);956	WARN_ON(rc != 0);957 958	if (!nic_data->must_restore_piobufs)959		efx_ef10_free_piobufs(efx);960 961	device_remove_file(&efx->pci_dev->dev, &dev_attr_primary_flag);962	device_remove_file(&efx->pci_dev->dev, &dev_attr_link_control_flag);963 964	efx_mcdi_detach(efx);965 966	memset(nic_data->udp_tunnels, 0, sizeof(nic_data->udp_tunnels));967	mutex_lock(&nic_data->udp_tunnels_lock);968	(void)efx_ef10_set_udp_tnl_ports(efx, true);969	mutex_unlock(&nic_data->udp_tunnels_lock);970 971	mutex_destroy(&nic_data->udp_tunnels_lock);972 973	efx_mcdi_fini(efx);974	efx_nic_free_buffer(efx, &nic_data->mcdi_buf);975	kfree(nic_data);976}977 978static int efx_ef10_probe_pf(struct efx_nic *efx)979{980	return efx_ef10_probe(efx);981}982 983int efx_ef10_vadaptor_query(struct efx_nic *efx, unsigned int port_id,984			    u32 *port_flags, u32 *vadaptor_flags,985			    unsigned int *vlan_tags)986{987	struct efx_ef10_nic_data *nic_data = efx->nic_data;988	MCDI_DECLARE_BUF(inbuf, MC_CMD_VADAPTOR_QUERY_IN_LEN);989	MCDI_DECLARE_BUF(outbuf, MC_CMD_VADAPTOR_QUERY_OUT_LEN);990	size_t outlen;991	int rc;992 993	if (nic_data->datapath_caps &994	    (1 << MC_CMD_GET_CAPABILITIES_OUT_VADAPTOR_QUERY_LBN)) {995		MCDI_SET_DWORD(inbuf, VADAPTOR_QUERY_IN_UPSTREAM_PORT_ID,996			       port_id);997 998		rc = efx_mcdi_rpc(efx, MC_CMD_VADAPTOR_QUERY, inbuf, sizeof(inbuf),999				  outbuf, sizeof(outbuf), &outlen);1000		if (rc)1001			return rc;1002 1003		if (outlen < sizeof(outbuf)) {1004			rc = -EIO;1005			return rc;1006		}1007	}1008 1009	if (port_flags)1010		*port_flags = MCDI_DWORD(outbuf, VADAPTOR_QUERY_OUT_PORT_FLAGS);1011	if (vadaptor_flags)1012		*vadaptor_flags =1013			MCDI_DWORD(outbuf, VADAPTOR_QUERY_OUT_VADAPTOR_FLAGS);1014	if (vlan_tags)1015		*vlan_tags =1016			MCDI_DWORD(outbuf,1017				   VADAPTOR_QUERY_OUT_NUM_AVAILABLE_VLAN_TAGS);1018 1019	return 0;1020}1021 1022int efx_ef10_vadaptor_alloc(struct efx_nic *efx, unsigned int port_id)1023{1024	MCDI_DECLARE_BUF(inbuf, MC_CMD_VADAPTOR_ALLOC_IN_LEN);1025 1026	MCDI_SET_DWORD(inbuf, VADAPTOR_ALLOC_IN_UPSTREAM_PORT_ID, port_id);1027	return efx_mcdi_rpc(efx, MC_CMD_VADAPTOR_ALLOC, inbuf, sizeof(inbuf),1028			    NULL, 0, NULL);1029}1030 1031int efx_ef10_vadaptor_free(struct efx_nic *efx, unsigned int port_id)1032{1033	MCDI_DECLARE_BUF(inbuf, MC_CMD_VADAPTOR_FREE_IN_LEN);1034 1035	MCDI_SET_DWORD(inbuf, VADAPTOR_FREE_IN_UPSTREAM_PORT_ID, port_id);1036	return efx_mcdi_rpc(efx, MC_CMD_VADAPTOR_FREE, inbuf, sizeof(inbuf),1037			    NULL, 0, NULL);1038}1039 1040int efx_ef10_vport_add_mac(struct efx_nic *efx,1041			   unsigned int port_id, const u8 *mac)1042{1043	MCDI_DECLARE_BUF(inbuf, MC_CMD_VPORT_ADD_MAC_ADDRESS_IN_LEN);1044 1045	MCDI_SET_DWORD(inbuf, VPORT_ADD_MAC_ADDRESS_IN_VPORT_ID, port_id);1046	ether_addr_copy(MCDI_PTR(inbuf, VPORT_ADD_MAC_ADDRESS_IN_MACADDR), mac);1047 1048	return efx_mcdi_rpc(efx, MC_CMD_VPORT_ADD_MAC_ADDRESS, inbuf,1049			    sizeof(inbuf), NULL, 0, NULL);1050}1051 1052int efx_ef10_vport_del_mac(struct efx_nic *efx,1053			   unsigned int port_id, const u8 *mac)1054{1055	MCDI_DECLARE_BUF(inbuf, MC_CMD_VPORT_DEL_MAC_ADDRESS_IN_LEN);1056 1057	MCDI_SET_DWORD(inbuf, VPORT_DEL_MAC_ADDRESS_IN_VPORT_ID, port_id);1058	ether_addr_copy(MCDI_PTR(inbuf, VPORT_DEL_MAC_ADDRESS_IN_MACADDR), mac);1059 1060	return efx_mcdi_rpc(efx, MC_CMD_VPORT_DEL_MAC_ADDRESS, inbuf,1061			    sizeof(inbuf), NULL, 0, NULL);1062}1063 1064#ifdef CONFIG_SFC_SRIOV1065static int efx_ef10_probe_vf(struct efx_nic *efx)1066{1067	int rc;1068	struct pci_dev *pci_dev_pf;1069 1070	/* If the parent PF has no VF data structure, it doesn't know about this1071	 * VF so fail probe.  The VF needs to be re-created.  This can happen1072	 * if the PF driver was unloaded while any VF was assigned to a guest1073	 * (using Xen, only).1074	 */1075	pci_dev_pf = efx->pci_dev->physfn;1076	if (pci_dev_pf) {1077		struct efx_nic *efx_pf = pci_get_drvdata(pci_dev_pf);1078		struct efx_ef10_nic_data *nic_data_pf = efx_pf->nic_data;1079 1080		if (!nic_data_pf->vf) {1081			netif_info(efx, drv, efx->net_dev,1082				   "The VF cannot link to its parent PF; "1083				   "please destroy and re-create the VF\n");1084			return -EBUSY;1085		}1086	}1087 1088	rc = efx_ef10_probe(efx);1089	if (rc)1090		return rc;1091 1092	rc = efx_ef10_get_vf_index(efx);1093	if (rc)1094		goto fail;1095 1096	if (efx->pci_dev->is_virtfn) {1097		if (efx->pci_dev->physfn) {1098			struct efx_nic *efx_pf =1099				pci_get_drvdata(efx->pci_dev->physfn);1100			struct efx_ef10_nic_data *nic_data_p = efx_pf->nic_data;1101			struct efx_ef10_nic_data *nic_data = efx->nic_data;1102 1103			nic_data_p->vf[nic_data->vf_index].efx = efx;1104			nic_data_p->vf[nic_data->vf_index].pci_dev =1105				efx->pci_dev;1106		} else1107			netif_info(efx, drv, efx->net_dev,1108				   "Could not get the PF id from VF\n");1109	}1110 1111	return 0;1112 1113fail:1114	efx_ef10_remove(efx);1115	return rc;1116}1117#else1118static int efx_ef10_probe_vf(struct efx_nic *efx __attribute__ ((unused)))1119{1120	return 0;1121}1122#endif1123 1124static int efx_ef10_alloc_vis(struct efx_nic *efx,1125			      unsigned int min_vis, unsigned int max_vis)1126{1127	struct efx_ef10_nic_data *nic_data = efx->nic_data;1128 1129	return efx_mcdi_alloc_vis(efx, min_vis, max_vis, &nic_data->vi_base,1130				  &nic_data->n_allocated_vis);1131}1132 1133/* Note that the failure path of this function does not free1134 * resources, as this will be done by efx_ef10_remove().1135 */1136static int efx_ef10_dimension_resources(struct efx_nic *efx)1137{1138	unsigned int min_vis = max_t(unsigned int, efx->tx_queues_per_channel,1139				     efx_separate_tx_channels ? 2 : 1);1140	unsigned int channel_vis, pio_write_vi_base, max_vis;1141	struct efx_ef10_nic_data *nic_data = efx->nic_data;1142	unsigned int uc_mem_map_size, wc_mem_map_size;1143	void __iomem *membase;1144	int rc;1145 1146	channel_vis = max(efx->n_channels,1147			  ((efx->n_tx_channels + efx->n_extra_tx_channels) *1148			   efx->tx_queues_per_channel) +1149			   efx->n_xdp_channels * efx->xdp_tx_per_channel);1150	if (efx->max_vis && efx->max_vis < channel_vis) {1151		netif_dbg(efx, drv, efx->net_dev,1152			  "Reducing channel VIs from %u to %u\n",1153			  channel_vis, efx->max_vis);1154		channel_vis = efx->max_vis;1155	}1156 1157#ifdef EFX_USE_PIO1158	/* Try to allocate PIO buffers if wanted and if the full1159	 * number of PIO buffers would be sufficient to allocate one1160	 * copy-buffer per TX channel.  Failure is non-fatal, as there1161	 * are only a small number of PIO buffers shared between all1162	 * functions of the controller.1163	 */1164	if (efx_piobuf_size != 0 &&1165	    nic_data->piobuf_size / efx_piobuf_size * EF10_TX_PIOBUF_COUNT >=1166	    efx->n_tx_channels) {1167		unsigned int n_piobufs =1168			DIV_ROUND_UP(efx->n_tx_channels,1169				     nic_data->piobuf_size / efx_piobuf_size);1170 1171		rc = efx_ef10_alloc_piobufs(efx, n_piobufs);1172		if (rc == -ENOSPC)1173			netif_dbg(efx, probe, efx->net_dev,1174				  "out of PIO buffers; cannot allocate more\n");1175		else if (rc == -EPERM)1176			netif_dbg(efx, probe, efx->net_dev,1177				  "not permitted to allocate PIO buffers\n");1178		else if (rc)1179			netif_err(efx, probe, efx->net_dev,1180				  "failed to allocate PIO buffers (%d)\n", rc);1181		else1182			netif_dbg(efx, probe, efx->net_dev,1183				  "allocated %u PIO buffers\n", n_piobufs);1184	}1185#else1186	nic_data->n_piobufs = 0;1187#endif1188 1189	/* PIO buffers should be mapped with write-combining enabled,1190	 * and we want to make single UC and WC mappings rather than1191	 * several of each (in fact that's the only option if host1192	 * page size is >4K).  So we may allocate some extra VIs just1193	 * for writing PIO buffers through.1194	 *1195	 * The UC mapping contains (channel_vis - 1) complete VIs and the1196	 * first 4K of the next VI.  Then the WC mapping begins with1197	 * the remainder of this last VI.1198	 */1199	uc_mem_map_size = PAGE_ALIGN((channel_vis - 1) * efx->vi_stride +1200				     ER_DZ_TX_PIOBUF);1201	if (nic_data->n_piobufs) {1202		/* pio_write_vi_base rounds down to give the number of complete1203		 * VIs inside the UC mapping.1204		 */1205		pio_write_vi_base = uc_mem_map_size / efx->vi_stride;1206		wc_mem_map_size = (PAGE_ALIGN((pio_write_vi_base +1207					       nic_data->n_piobufs) *1208					      efx->vi_stride) -1209				   uc_mem_map_size);1210		max_vis = pio_write_vi_base + nic_data->n_piobufs;1211	} else {1212		pio_write_vi_base = 0;1213		wc_mem_map_size = 0;1214		max_vis = channel_vis;1215	}1216 1217	/* In case the last attached driver failed to free VIs, do it now */1218	rc = efx_mcdi_free_vis(efx);1219	if (rc != 0)1220		return rc;1221 1222	rc = efx_ef10_alloc_vis(efx, min_vis, max_vis);1223	if (rc != 0)1224		return rc;1225 1226	if (nic_data->n_allocated_vis < channel_vis) {1227		netif_info(efx, drv, efx->net_dev,1228			   "Could not allocate enough VIs to satisfy RSS"1229			   " requirements. Performance may not be optimal.\n");1230		/* We didn't get the VIs to populate our channels.1231		 * We could keep what we got but then we'd have more1232		 * interrupts than we need.1233		 * Instead calculate new max_channels and restart1234		 */1235		efx->max_channels = nic_data->n_allocated_vis;1236		efx->max_tx_channels =1237			nic_data->n_allocated_vis / efx->tx_queues_per_channel;1238 1239		efx_mcdi_free_vis(efx);1240		return -EAGAIN;1241	}1242 1243	/* If we didn't get enough VIs to map all the PIO buffers, free the1244	 * PIO buffers1245	 */1246	if (nic_data->n_piobufs &&1247	    nic_data->n_allocated_vis <1248	    pio_write_vi_base + nic_data->n_piobufs) {1249		netif_dbg(efx, probe, efx->net_dev,1250			  "%u VIs are not sufficient to map %u PIO buffers\n",1251			  nic_data->n_allocated_vis, nic_data->n_piobufs);1252		efx_ef10_free_piobufs(efx);1253	}1254 1255	/* Shrink the original UC mapping of the memory BAR */1256	membase = ioremap(efx->membase_phys, uc_mem_map_size);1257	if (!membase) {1258		netif_err(efx, probe, efx->net_dev,1259			  "could not shrink memory BAR to %x\n",1260			  uc_mem_map_size);1261		return -ENOMEM;1262	}1263	iounmap(efx->membase);1264	efx->membase = membase;1265 1266	/* Set up the WC mapping if needed */1267	if (wc_mem_map_size) {1268		nic_data->wc_membase = ioremap_wc(efx->membase_phys +1269						  uc_mem_map_size,1270						  wc_mem_map_size);1271		if (!nic_data->wc_membase) {1272			netif_err(efx, probe, efx->net_dev,1273				  "could not allocate WC mapping of size %x\n",1274				  wc_mem_map_size);1275			return -ENOMEM;1276		}1277		nic_data->pio_write_vi_base = pio_write_vi_base;1278		nic_data->pio_write_base =1279			nic_data->wc_membase +1280			(pio_write_vi_base * efx->vi_stride + ER_DZ_TX_PIOBUF -1281			 uc_mem_map_size);1282 1283		rc = efx_ef10_link_piobufs(efx);1284		if (rc)1285			efx_ef10_free_piobufs(efx);1286	}1287 1288	netif_dbg(efx, probe, efx->net_dev,1289		  "memory BAR at %pa (virtual %p+%x UC, %p+%x WC)\n",1290		  &efx->membase_phys, efx->membase, uc_mem_map_size,1291		  nic_data->wc_membase, wc_mem_map_size);1292 1293	return 0;1294}1295 1296static void efx_ef10_fini_nic(struct efx_nic *efx)1297{1298	struct efx_ef10_nic_data *nic_data = efx->nic_data;1299 1300	spin_lock_bh(&efx->stats_lock);1301	kfree(nic_data->mc_stats);1302	nic_data->mc_stats = NULL;1303	spin_unlock_bh(&efx->stats_lock);1304}1305 1306static int efx_ef10_init_nic(struct efx_nic *efx)1307{1308	struct efx_ef10_nic_data *nic_data = efx->nic_data;1309	struct net_device *net_dev = efx->net_dev;1310	netdev_features_t tun_feats, tso_feats;1311	int rc;1312 1313	if (nic_data->must_check_datapath_caps) {1314		rc = efx_ef10_init_datapath_caps(efx);1315		if (rc)1316			return rc;1317		nic_data->must_check_datapath_caps = false;1318	}1319 1320	if (efx->must_realloc_vis) {1321		/* We cannot let the number of VIs change now */1322		rc = efx_ef10_alloc_vis(efx, nic_data->n_allocated_vis,1323					nic_data->n_allocated_vis);1324		if (rc)1325			return rc;1326		efx->must_realloc_vis = false;1327	}1328 1329	nic_data->mc_stats = kmalloc(efx->num_mac_stats * sizeof(__le64),1330				     GFP_KERNEL);1331	if (!nic_data->mc_stats)1332		return -ENOMEM;1333 1334	if (nic_data->must_restore_piobufs && nic_data->n_piobufs) {1335		rc = efx_ef10_alloc_piobufs(efx, nic_data->n_piobufs);1336		if (rc == 0) {1337			rc = efx_ef10_link_piobufs(efx);1338			if (rc)1339				efx_ef10_free_piobufs(efx);1340		}1341 1342		/* Log an error on failure, but this is non-fatal.1343		 * Permission errors are less important - we've presumably1344		 * had the PIO buffer licence removed.1345		 */1346		if (rc == -EPERM)1347			netif_dbg(efx, drv, efx->net_dev,1348				  "not permitted to restore PIO buffers\n");1349		else if (rc)1350			netif_err(efx, drv, efx->net_dev,1351				  "failed to restore PIO buffers (%d)\n", rc);1352		nic_data->must_restore_piobufs = false;1353	}1354 1355	/* encap features might change during reset if fw variant changed */1356	if (efx_has_cap(efx, VXLAN_NVGRE) && !efx_ef10_is_vf(efx))1357		net_dev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;1358	else1359		net_dev->hw_enc_features &= ~(NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);1360 1361	tun_feats = NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE |1362		    NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM;1363	tso_feats = NETIF_F_TSO | NETIF_F_TSO6;1364 1365	if (efx_has_cap(efx, TX_TSO_V2_ENCAP)) {1366		/* If this is first nic_init, or if it is a reset and a new fw1367		 * variant has added new features, enable them by default.1368		 * If the features are not new, maintain their current value.1369		 */1370		if (!(net_dev->hw_features & tun_feats))1371			net_dev->features |= tun_feats;1372		net_dev->hw_enc_features |= tun_feats | tso_feats;1373		net_dev->hw_features |= tun_feats;1374	} else {1375		net_dev->hw_enc_features &= ~(tun_feats | tso_feats);1376		net_dev->hw_features &= ~tun_feats;1377		net_dev->features &= ~tun_feats;1378	}1379 1380	/* don't fail init if RSS setup doesn't work */1381	rc = efx->type->rx_push_rss_config(efx, false,1382					   efx->rss_context.rx_indir_table, NULL);1383 1384	return 0;1385}1386 1387static void efx_ef10_table_reset_mc_allocations(struct efx_nic *efx)1388{1389	struct efx_ef10_nic_data *nic_data = efx->nic_data;1390#ifdef CONFIG_SFC_SRIOV1391	unsigned int i;1392#endif1393 1394	/* All our allocations have been reset */1395	efx->must_realloc_vis = true;1396	efx_mcdi_filter_table_reset_mc_allocations(efx);1397	nic_data->must_restore_piobufs = true;1398	efx_ef10_forget_old_piobufs(efx);1399	efx->rss_context.priv.context_id = EFX_MCDI_RSS_CONTEXT_INVALID;1400 1401	/* Driver-created vswitches and vports must be re-created */1402	nic_data->must_probe_vswitching = true;1403	efx->vport_id = EVB_PORT_ID_ASSIGNED;1404#ifdef CONFIG_SFC_SRIOV1405	if (nic_data->vf)1406		for (i = 0; i < efx->vf_count; i++)1407			nic_data->vf[i].vport_id = 0;1408#endif1409}1410 1411static enum reset_type efx_ef10_map_reset_reason(enum reset_type reason)1412{1413	if (reason == RESET_TYPE_MC_FAILURE)1414		return RESET_TYPE_DATAPATH;1415 1416	return efx_mcdi_map_reset_reason(reason);1417}1418 1419static int efx_ef10_map_reset_flags(u32 *flags)1420{1421	enum {1422		EF10_RESET_PORT = ((ETH_RESET_MAC | ETH_RESET_PHY) <<1423				   ETH_RESET_SHARED_SHIFT),1424		EF10_RESET_MC = ((ETH_RESET_DMA | ETH_RESET_FILTER |1425				  ETH_RESET_OFFLOAD | ETH_RESET_MAC |1426				  ETH_RESET_PHY | ETH_RESET_MGMT) <<1427				 ETH_RESET_SHARED_SHIFT)1428	};1429 1430	/* We assume for now that our PCI function is permitted to1431	 * reset everything.1432	 */1433 1434	if ((*flags & EF10_RESET_MC) == EF10_RESET_MC) {1435		*flags &= ~EF10_RESET_MC;1436		return RESET_TYPE_WORLD;1437	}1438 1439	if ((*flags & EF10_RESET_PORT) == EF10_RESET_PORT) {1440		*flags &= ~EF10_RESET_PORT;1441		return RESET_TYPE_ALL;1442	}1443 1444	/* no invisible reset implemented */1445 1446	return -EINVAL;1447}1448 1449static int efx_ef10_reset(struct efx_nic *efx, enum reset_type reset_type)1450{1451	int rc = efx_mcdi_reset(efx, reset_type);1452 1453	/* Unprivileged functions return -EPERM, but need to return success1454	 * here so that the datapath is brought back up.1455	 */1456	if (reset_type == RESET_TYPE_WORLD && rc == -EPERM)1457		rc = 0;1458 1459	/* If it was a port reset, trigger reallocation of MC resources.1460	 * Note that on an MC reset nothing needs to be done now because we'll1461	 * detect the MC reset later and handle it then.1462	 * For an FLR, we never get an MC reset event, but the MC has reset all1463	 * resources assigned to us, so we have to trigger reallocation now.1464	 */1465	if ((reset_type == RESET_TYPE_ALL ||1466	     reset_type == RESET_TYPE_MCDI_TIMEOUT) && !rc)1467		efx_ef10_table_reset_mc_allocations(efx);1468	return rc;1469}1470 1471#define EF10_DMA_STAT(ext_name, mcdi_name)			\1472	[EF10_STAT_ ## ext_name] =				\1473	{ #ext_name, 64, 8 * MC_CMD_MAC_ ## mcdi_name }1474#define EF10_DMA_INVIS_STAT(int_name, mcdi_name)		\1475	[EF10_STAT_ ## int_name] =				\1476	{ NULL, 64, 8 * MC_CMD_MAC_ ## mcdi_name }1477#define EF10_OTHER_STAT(ext_name)				\1478	[EF10_STAT_ ## ext_name] = { #ext_name, 0, 0 }1479 1480static const struct efx_hw_stat_desc efx_ef10_stat_desc[EF10_STAT_COUNT] = {1481	EF10_DMA_STAT(port_tx_bytes, TX_BYTES),1482	EF10_DMA_STAT(port_tx_packets, TX_PKTS),1483	EF10_DMA_STAT(port_tx_pause, TX_PAUSE_PKTS),1484	EF10_DMA_STAT(port_tx_control, TX_CONTROL_PKTS),1485	EF10_DMA_STAT(port_tx_unicast, TX_UNICAST_PKTS),1486	EF10_DMA_STAT(port_tx_multicast, TX_MULTICAST_PKTS),1487	EF10_DMA_STAT(port_tx_broadcast, TX_BROADCAST_PKTS),1488	EF10_DMA_STAT(port_tx_lt64, TX_LT64_PKTS),1489	EF10_DMA_STAT(port_tx_64, TX_64_PKTS),1490	EF10_DMA_STAT(port_tx_65_to_127, TX_65_TO_127_PKTS),1491	EF10_DMA_STAT(port_tx_128_to_255, TX_128_TO_255_PKTS),1492	EF10_DMA_STAT(port_tx_256_to_511, TX_256_TO_511_PKTS),1493	EF10_DMA_STAT(port_tx_512_to_1023, TX_512_TO_1023_PKTS),1494	EF10_DMA_STAT(port_tx_1024_to_15xx, TX_1024_TO_15XX_PKTS),1495	EF10_DMA_STAT(port_tx_15xx_to_jumbo, TX_15XX_TO_JUMBO_PKTS),1496	EF10_DMA_STAT(port_rx_bytes, RX_BYTES),1497	EF10_DMA_INVIS_STAT(port_rx_bytes_minus_good_bytes, RX_BAD_BYTES),1498	EF10_OTHER_STAT(port_rx_good_bytes),1499	EF10_OTHER_STAT(port_rx_bad_bytes),1500	EF10_DMA_STAT(port_rx_packets, RX_PKTS),1501	EF10_DMA_STAT(port_rx_good, RX_GOOD_PKTS),1502	EF10_DMA_STAT(port_rx_bad, RX_BAD_FCS_PKTS),1503	EF10_DMA_STAT(port_rx_pause, RX_PAUSE_PKTS),1504	EF10_DMA_STAT(port_rx_control, RX_CONTROL_PKTS),1505	EF10_DMA_STAT(port_rx_unicast, RX_UNICAST_PKTS),1506	EF10_DMA_STAT(port_rx_multicast, RX_MULTICAST_PKTS),1507	EF10_DMA_STAT(port_rx_broadcast, RX_BROADCAST_PKTS),1508	EF10_DMA_STAT(port_rx_lt64, RX_UNDERSIZE_PKTS),1509	EF10_DMA_STAT(port_rx_64, RX_64_PKTS),1510	EF10_DMA_STAT(port_rx_65_to_127, RX_65_TO_127_PKTS),1511	EF10_DMA_STAT(port_rx_128_to_255, RX_128_TO_255_PKTS),1512	EF10_DMA_STAT(port_rx_256_to_511, RX_256_TO_511_PKTS),1513	EF10_DMA_STAT(port_rx_512_to_1023, RX_512_TO_1023_PKTS),1514	EF10_DMA_STAT(port_rx_1024_to_15xx, RX_1024_TO_15XX_PKTS),1515	EF10_DMA_STAT(port_rx_15xx_to_jumbo, RX_15XX_TO_JUMBO_PKTS),1516	EF10_DMA_STAT(port_rx_gtjumbo, RX_GTJUMBO_PKTS),1517	EF10_DMA_STAT(port_rx_bad_gtjumbo, RX_JABBER_PKTS),1518	EF10_DMA_STAT(port_rx_overflow, RX_OVERFLOW_PKTS),1519	EF10_DMA_STAT(port_rx_align_error, RX_ALIGN_ERROR_PKTS),1520	EF10_DMA_STAT(port_rx_length_error, RX_LENGTH_ERROR_PKTS),1521	EF10_DMA_STAT(port_rx_nodesc_drops, RX_NODESC_DROPS),1522	EFX_GENERIC_SW_STAT(rx_nodesc_trunc),1523	EFX_GENERIC_SW_STAT(rx_noskb_drops),1524	EF10_DMA_STAT(port_rx_pm_trunc_bb_overflow, PM_TRUNC_BB_OVERFLOW),1525	EF10_DMA_STAT(port_rx_pm_discard_bb_overflow, PM_DISCARD_BB_OVERFLOW),1526	EF10_DMA_STAT(port_rx_pm_trunc_vfifo_full, PM_TRUNC_VFIFO_FULL),1527	EF10_DMA_STAT(port_rx_pm_discard_vfifo_full, PM_DISCARD_VFIFO_FULL),1528	EF10_DMA_STAT(port_rx_pm_trunc_qbb, PM_TRUNC_QBB),1529	EF10_DMA_STAT(port_rx_pm_discard_qbb, PM_DISCARD_QBB),1530	EF10_DMA_STAT(port_rx_pm_discard_mapping, PM_DISCARD_MAPPING),1531	EF10_DMA_STAT(port_rx_dp_q_disabled_packets, RXDP_Q_DISABLED_PKTS),1532	EF10_DMA_STAT(port_rx_dp_di_dropped_packets, RXDP_DI_DROPPED_PKTS),1533	EF10_DMA_STAT(port_rx_dp_streaming_packets, RXDP_STREAMING_PKTS),1534	EF10_DMA_STAT(port_rx_dp_hlb_fetch, RXDP_HLB_FETCH_CONDITIONS),1535	EF10_DMA_STAT(port_rx_dp_hlb_wait, RXDP_HLB_WAIT_CONDITIONS),1536	EF10_DMA_STAT(rx_unicast, VADAPTER_RX_UNICAST_PACKETS),1537	EF10_DMA_STAT(rx_unicast_bytes, VADAPTER_RX_UNICAST_BYTES),1538	EF10_DMA_STAT(rx_multicast, VADAPTER_RX_MULTICAST_PACKETS),1539	EF10_DMA_STAT(rx_multicast_bytes, VADAPTER_RX_MULTICAST_BYTES),1540	EF10_DMA_STAT(rx_broadcast, VADAPTER_RX_BROADCAST_PACKETS),1541	EF10_DMA_STAT(rx_broadcast_bytes, VADAPTER_RX_BROADCAST_BYTES),1542	EF10_DMA_STAT(rx_bad, VADAPTER_RX_BAD_PACKETS),1543	EF10_DMA_STAT(rx_bad_bytes, VADAPTER_RX_BAD_BYTES),1544	EF10_DMA_STAT(rx_overflow, VADAPTER_RX_OVERFLOW),1545	EF10_DMA_STAT(tx_unicast, VADAPTER_TX_UNICAST_PACKETS),1546	EF10_DMA_STAT(tx_unicast_bytes, VADAPTER_TX_UNICAST_BYTES),1547	EF10_DMA_STAT(tx_multicast, VADAPTER_TX_MULTICAST_PACKETS),1548	EF10_DMA_STAT(tx_multicast_bytes, VADAPTER_TX_MULTICAST_BYTES),1549	EF10_DMA_STAT(tx_broadcast, VADAPTER_TX_BROADCAST_PACKETS),1550	EF10_DMA_STAT(tx_broadcast_bytes, VADAPTER_TX_BROADCAST_BYTES),1551	EF10_DMA_STAT(tx_bad, VADAPTER_TX_BAD_PACKETS),1552	EF10_DMA_STAT(tx_bad_bytes, VADAPTER_TX_BAD_BYTES),1553	EF10_DMA_STAT(tx_overflow, VADAPTER_TX_OVERFLOW),1554	EF10_DMA_STAT(fec_uncorrected_errors, FEC_UNCORRECTED_ERRORS),1555	EF10_DMA_STAT(fec_corrected_errors, FEC_CORRECTED_ERRORS),1556	EF10_DMA_STAT(fec_corrected_symbols_lane0, FEC_CORRECTED_SYMBOLS_LANE0),1557	EF10_DMA_STAT(fec_corrected_symbols_lane1, FEC_CORRECTED_SYMBOLS_LANE1),1558	EF10_DMA_STAT(fec_corrected_symbols_lane2, FEC_CORRECTED_SYMBOLS_LANE2),1559	EF10_DMA_STAT(fec_corrected_symbols_lane3, FEC_CORRECTED_SYMBOLS_LANE3),1560	EF10_DMA_STAT(ctpio_vi_busy_fallback, CTPIO_VI_BUSY_FALLBACK),1561	EF10_DMA_STAT(ctpio_long_write_success, CTPIO_LONG_WRITE_SUCCESS),1562	EF10_DMA_STAT(ctpio_missing_dbell_fail, CTPIO_MISSING_DBELL_FAIL),1563	EF10_DMA_STAT(ctpio_overflow_fail, CTPIO_OVERFLOW_FAIL),1564	EF10_DMA_STAT(ctpio_underflow_fail, CTPIO_UNDERFLOW_FAIL),1565	EF10_DMA_STAT(ctpio_timeout_fail, CTPIO_TIMEOUT_FAIL),1566	EF10_DMA_STAT(ctpio_noncontig_wr_fail, CTPIO_NONCONTIG_WR_FAIL),1567	EF10_DMA_STAT(ctpio_frm_clobber_fail, CTPIO_FRM_CLOBBER_FAIL),1568	EF10_DMA_STAT(ctpio_invalid_wr_fail, CTPIO_INVALID_WR_FAIL),1569	EF10_DMA_STAT(ctpio_vi_clobber_fallback, CTPIO_VI_CLOBBER_FALLBACK),1570	EF10_DMA_STAT(ctpio_unqualified_fallback, CTPIO_UNQUALIFIED_FALLBACK),1571	EF10_DMA_STAT(ctpio_runt_fallback, CTPIO_RUNT_FALLBACK),1572	EF10_DMA_STAT(ctpio_success, CTPIO_SUCCESS),1573	EF10_DMA_STAT(ctpio_fallback, CTPIO_FALLBACK),1574	EF10_DMA_STAT(ctpio_poison, CTPIO_POISON),1575	EF10_DMA_STAT(ctpio_erase, CTPIO_ERASE),1576};1577 1578#define HUNT_COMMON_STAT_MASK ((1ULL << EF10_STAT_port_tx_bytes) |	\1579			       (1ULL << EF10_STAT_port_tx_packets) |	\1580			       (1ULL << EF10_STAT_port_tx_pause) |	\1581			       (1ULL << EF10_STAT_port_tx_unicast) |	\1582			       (1ULL << EF10_STAT_port_tx_multicast) |	\1583			       (1ULL << EF10_STAT_port_tx_broadcast) |	\1584			       (1ULL << EF10_STAT_port_rx_bytes) |	\1585			       (1ULL <<                                 \1586				EF10_STAT_port_rx_bytes_minus_good_bytes) | \1587			       (1ULL << EF10_STAT_port_rx_good_bytes) |	\1588			       (1ULL << EF10_STAT_port_rx_bad_bytes) |	\1589			       (1ULL << EF10_STAT_port_rx_packets) |	\1590			       (1ULL << EF10_STAT_port_rx_good) |	\1591			       (1ULL << EF10_STAT_port_rx_bad) |	\1592			       (1ULL << EF10_STAT_port_rx_pause) |	\1593			       (1ULL << EF10_STAT_port_rx_control) |	\1594			       (1ULL << EF10_STAT_port_rx_unicast) |	\1595			       (1ULL << EF10_STAT_port_rx_multicast) |	\1596			       (1ULL << EF10_STAT_port_rx_broadcast) |	\1597			       (1ULL << EF10_STAT_port_rx_lt64) |	\1598			       (1ULL << EF10_STAT_port_rx_64) |		\1599			       (1ULL << EF10_STAT_port_rx_65_to_127) |	\1600			       (1ULL << EF10_STAT_port_rx_128_to_255) |	\1601			       (1ULL << EF10_STAT_port_rx_256_to_511) |	\1602			       (1ULL << EF10_STAT_port_rx_512_to_1023) |\1603			       (1ULL << EF10_STAT_port_rx_1024_to_15xx) |\1604			       (1ULL << EF10_STAT_port_rx_15xx_to_jumbo) |\1605			       (1ULL << EF10_STAT_port_rx_gtjumbo) |	\1606			       (1ULL << EF10_STAT_port_rx_bad_gtjumbo) |\1607			       (1ULL << EF10_STAT_port_rx_overflow) |	\1608			       (1ULL << EF10_STAT_port_rx_nodesc_drops) |\1609			       (1ULL << GENERIC_STAT_rx_nodesc_trunc) |	\1610			       (1ULL << GENERIC_STAT_rx_noskb_drops))1611 1612/* On 7000 series NICs, these statistics are only provided by the 10G MAC.1613 * For a 10G/40G switchable port we do not expose these because they might1614 * not include all the packets they should.1615 * On 8000 series NICs these statistics are always provided.1616 */1617#define HUNT_10G_ONLY_STAT_MASK ((1ULL << EF10_STAT_port_tx_control) |	\1618				 (1ULL << EF10_STAT_port_tx_lt64) |	\1619				 (1ULL << EF10_STAT_port_tx_64) |	\1620				 (1ULL << EF10_STAT_port_tx_65_to_127) |\1621				 (1ULL << EF10_STAT_port_tx_128_to_255) |\1622				 (1ULL << EF10_STAT_port_tx_256_to_511) |\1623				 (1ULL << EF10_STAT_port_tx_512_to_1023) |\1624				 (1ULL << EF10_STAT_port_tx_1024_to_15xx) |\1625				 (1ULL << EF10_STAT_port_tx_15xx_to_jumbo))1626 1627/* These statistics are only provided by the 40G MAC.  For a 10G/40G1628 * switchable port we do expose these because the errors will otherwise1629 * be silent.1630 */1631#define HUNT_40G_EXTRA_STAT_MASK ((1ULL << EF10_STAT_port_rx_align_error) |\1632				  (1ULL << EF10_STAT_port_rx_length_error))1633 1634/* These statistics are only provided if the firmware supports the1635 * capability PM_AND_RXDP_COUNTERS.1636 */1637#define HUNT_PM_AND_RXDP_STAT_MASK (					\1638	(1ULL << EF10_STAT_port_rx_pm_trunc_bb_overflow) |		\1639	(1ULL << EF10_STAT_port_rx_pm_discard_bb_overflow) |		\1640	(1ULL << EF10_STAT_port_rx_pm_trunc_vfifo_full) |		\1641	(1ULL << EF10_STAT_port_rx_pm_discard_vfifo_full) |		\1642	(1ULL << EF10_STAT_port_rx_pm_trunc_qbb) |			\1643	(1ULL << EF10_STAT_port_rx_pm_discard_qbb) |			\1644	(1ULL << EF10_STAT_port_rx_pm_discard_mapping) |		\1645	(1ULL << EF10_STAT_port_rx_dp_q_disabled_packets) |		\1646	(1ULL << EF10_STAT_port_rx_dp_di_dropped_packets) |		\1647	(1ULL << EF10_STAT_port_rx_dp_streaming_packets) |		\1648	(1ULL << EF10_STAT_port_rx_dp_hlb_fetch) |			\1649	(1ULL << EF10_STAT_port_rx_dp_hlb_wait))1650 1651/* These statistics are only provided if the NIC supports MC_CMD_MAC_STATS_V2,1652 * indicated by returning a value >= MC_CMD_MAC_NSTATS_V2 in1653 * MC_CMD_GET_CAPABILITIES_V4_OUT_MAC_STATS_NUM_STATS.1654 * These bits are in the second u64 of the raw mask.1655 */1656#define EF10_FEC_STAT_MASK (						\1657	(1ULL << (EF10_STAT_fec_uncorrected_errors - 64)) |		\1658	(1ULL << (EF10_STAT_fec_corrected_errors - 64)) |		\1659	(1ULL << (EF10_STAT_fec_corrected_symbols_lane0 - 64)) |	\1660	(1ULL << (EF10_STAT_fec_corrected_symbols_lane1 - 64)) |	\1661	(1ULL << (EF10_STAT_fec_corrected_symbols_lane2 - 64)) |	\1662	(1ULL << (EF10_STAT_fec_corrected_symbols_lane3 - 64)))1663 1664/* These statistics are only provided if the NIC supports MC_CMD_MAC_STATS_V3,1665 * indicated by returning a value >= MC_CMD_MAC_NSTATS_V3 in1666 * MC_CMD_GET_CAPABILITIES_V4_OUT_MAC_STATS_NUM_STATS.1667 * These bits are in the second u64 of the raw mask.1668 */1669#define EF10_CTPIO_STAT_MASK (						\1670	(1ULL << (EF10_STAT_ctpio_vi_busy_fallback - 64)) |		\1671	(1ULL << (EF10_STAT_ctpio_long_write_success - 64)) |		\1672	(1ULL << (EF10_STAT_ctpio_missing_dbell_fail - 64)) |		\1673	(1ULL << (EF10_STAT_ctpio_overflow_fail - 64)) |		\1674	(1ULL << (EF10_STAT_ctpio_underflow_fail - 64)) |		\1675	(1ULL << (EF10_STAT_ctpio_timeout_fail - 64)) |			\1676	(1ULL << (EF10_STAT_ctpio_noncontig_wr_fail - 64)) |		\1677	(1ULL << (EF10_STAT_ctpio_frm_clobber_fail - 64)) |		\1678	(1ULL << (EF10_STAT_ctpio_invalid_wr_fail - 64)) |		\1679	(1ULL << (EF10_STAT_ctpio_vi_clobber_fallback - 64)) |		\1680	(1ULL << (EF10_STAT_ctpio_unqualified_fallback - 64)) |		\1681	(1ULL << (EF10_STAT_ctpio_runt_fallback - 64)) |		\1682	(1ULL << (EF10_STAT_ctpio_success - 64)) |			\1683	(1ULL << (EF10_STAT_ctpio_fallback - 64)) |			\1684	(1ULL << (EF10_STAT_ctpio_poison - 64)) |			\1685	(1ULL << (EF10_STAT_ctpio_erase - 64)))1686 1687static u64 efx_ef10_raw_stat_mask(struct efx_nic *efx)1688{1689	u64 raw_mask = HUNT_COMMON_STAT_MASK;1690	u32 port_caps = efx_mcdi_phy_get_caps(efx);1691	struct efx_ef10_nic_data *nic_data = efx->nic_data;1692 1693	if (!(efx->mcdi->fn_flags &1694	      1 << MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_LINKCTRL))1695		return 0;1696 1697	if (port_caps & (1 << MC_CMD_PHY_CAP_40000FDX_LBN)) {1698		raw_mask |= HUNT_40G_EXTRA_STAT_MASK;1699		/* 8000 series have everything even at 40G */1700		if (nic_data->datapath_caps2 &1701		    (1 << MC_CMD_GET_CAPABILITIES_V2_OUT_MAC_STATS_40G_TX_SIZE_BINS_LBN))1702			raw_mask |= HUNT_10G_ONLY_STAT_MASK;1703	} else {1704		raw_mask |= HUNT_10G_ONLY_STAT_MASK;1705	}1706 1707	if (nic_data->datapath_caps &1708	    (1 << MC_CMD_GET_CAPABILITIES_OUT_PM_AND_RXDP_COUNTERS_LBN))1709		raw_mask |= HUNT_PM_AND_RXDP_STAT_MASK;1710 1711	return raw_mask;1712}1713 1714static void efx_ef10_get_stat_mask(struct efx_nic *efx, unsigned long *mask)1715{1716	struct efx_ef10_nic_data *nic_data = efx->nic_data;1717	u64 raw_mask[2];1718 1719	raw_mask[0] = efx_ef10_raw_stat_mask(efx);1720 1721	/* Only show vadaptor stats when EVB capability is present */1722	if (nic_data->datapath_caps &1723	    (1 << MC_CMD_GET_CAPABILITIES_OUT_EVB_LBN)) {1724		raw_mask[0] |= ~((1ULL << EF10_STAT_rx_unicast) - 1);1725		raw_mask[1] = (1ULL << (EF10_STAT_V1_COUNT - 64)) - 1;1726	} else {1727		raw_mask[1] = 0;1728	}1729	/* Only show FEC stats when NIC supports MC_CMD_MAC_STATS_V2 */1730	if (efx->num_mac_stats >= MC_CMD_MAC_NSTATS_V2)1731		raw_mask[1] |= EF10_FEC_STAT_MASK;1732 1733	/* CTPIO stats appear in V3. Only show them on devices that actually1734	 * support CTPIO. Although this driver doesn't use CTPIO others might,1735	 * and we may be reporting the stats for the underlying port.1736	 */1737	if (efx->num_mac_stats >= MC_CMD_MAC_NSTATS_V3 &&1738	    (nic_data->datapath_caps2 &1739	     (1 << MC_CMD_GET_CAPABILITIES_V4_OUT_CTPIO_LBN)))1740		raw_mask[1] |= EF10_CTPIO_STAT_MASK;1741 1742#if BITS_PER_LONG == 641743	BUILD_BUG_ON(BITS_TO_LONGS(EF10_STAT_COUNT) != 2);1744	mask[0] = raw_mask[0];1745	mask[1] = raw_mask[1];1746#else1747	BUILD_BUG_ON(BITS_TO_LONGS(EF10_STAT_COUNT) != 3);1748	mask[0] = raw_mask[0] & 0xffffffff;1749	mask[1] = raw_mask[0] >> 32;1750	mask[2] = raw_mask[1] & 0xffffffff;1751#endif1752}1753 1754static size_t efx_ef10_describe_stats(struct efx_nic *efx, u8 *names)1755{1756	DECLARE_BITMAP(mask, EF10_STAT_COUNT);1757 1758	efx_ef10_get_stat_mask(efx, mask);1759	return efx_nic_describe_stats(efx_ef10_stat_desc, EF10_STAT_COUNT,1760				      mask, names);1761}1762 1763static void efx_ef10_get_fec_stats(struct efx_nic *efx,1764				   struct ethtool_fec_stats *fec_stats)1765{1766	DECLARE_BITMAP(mask, EF10_STAT_COUNT);1767	struct efx_ef10_nic_data *nic_data = efx->nic_data;1768	u64 *stats = nic_data->stats;1769 1770	efx_ef10_get_stat_mask(efx, mask);1771	if (test_bit(EF10_STAT_fec_corrected_errors, mask))1772		fec_stats->corrected_blocks.total =1773			stats[EF10_STAT_fec_corrected_errors];1774	if (test_bit(EF10_STAT_fec_uncorrected_errors, mask))1775		fec_stats->uncorrectable_blocks.total =1776			stats[EF10_STAT_fec_uncorrected_errors];1777}1778 1779static size_t efx_ef10_update_stats_common(struct efx_nic *efx, u64 *full_stats,1780					   struct rtnl_link_stats64 *core_stats)1781{1782	DECLARE_BITMAP(mask, EF10_STAT_COUNT);1783	struct efx_ef10_nic_data *nic_data = efx->nic_data;1784	u64 *stats = nic_data->stats;1785	size_t stats_count = 0, index;1786 1787	efx_ef10_get_stat_mask(efx, mask);1788 1789	if (full_stats) {1790		for_each_set_bit(index, mask, EF10_STAT_COUNT) {1791			if (efx_ef10_stat_desc[index].name) {1792				*full_stats++ = stats[index];1793				++stats_count;1794			}1795		}1796	}1797 1798	if (!core_stats)1799		return stats_count;1800 1801	if (nic_data->datapath_caps &1802			1 << MC_CMD_GET_CAPABILITIES_OUT_EVB_LBN) {1803		/* Use vadaptor stats. */1804		core_stats->rx_packets = stats[EF10_STAT_rx_unicast] +1805					 stats[EF10_STAT_rx_multicast] +1806					 stats[EF10_STAT_rx_broadcast];1807		core_stats->tx_packets = stats[EF10_STAT_tx_unicast] +1808					 stats[EF10_STAT_tx_multicast] +1809					 stats[EF10_STAT_tx_broadcast];1810		core_stats->rx_bytes = stats[EF10_STAT_rx_unicast_bytes] +1811				       stats[EF10_STAT_rx_multicast_bytes] +1812				       stats[EF10_STAT_rx_broadcast_bytes];1813		core_stats->tx_bytes = stats[EF10_STAT_tx_unicast_bytes] +1814				       stats[EF10_STAT_tx_multicast_bytes] +1815				       stats[EF10_STAT_tx_broadcast_bytes];1816		core_stats->rx_dropped = stats[GENERIC_STAT_rx_nodesc_trunc] +1817					 stats[GENERIC_STAT_rx_noskb_drops];1818		core_stats->multicast = stats[EF10_STAT_rx_multicast];1819		core_stats->rx_crc_errors = stats[EF10_STAT_rx_bad];1820		core_stats->rx_fifo_errors = stats[EF10_STAT_rx_overflow];1821		core_stats->rx_errors = core_stats->rx_crc_errors;1822		core_stats->tx_errors = stats[EF10_STAT_tx_bad];1823	} else {1824		/* Use port stats. */1825		core_stats->rx_packets = stats[EF10_STAT_port_rx_packets];1826		core_stats->tx_packets = stats[EF10_STAT_port_tx_packets];1827		core_stats->rx_bytes = stats[EF10_STAT_port_rx_bytes];1828		core_stats->tx_bytes = stats[EF10_STAT_port_tx_bytes];1829		core_stats->rx_dropped = stats[EF10_STAT_port_rx_nodesc_drops] +1830					 stats[GENERIC_STAT_rx_nodesc_trunc] +1831					 stats[GENERIC_STAT_rx_noskb_drops];1832		core_stats->multicast = stats[EF10_STAT_port_rx_multicast];1833		core_stats->rx_length_errors =1834				stats[EF10_STAT_port_rx_gtjumbo] +1835				stats[EF10_STAT_port_rx_length_error];1836		core_stats->rx_crc_errors = stats[EF10_STAT_port_rx_bad];1837		core_stats->rx_frame_errors =1838				stats[EF10_STAT_port_rx_align_error];1839		core_stats->rx_fifo_errors = stats[EF10_STAT_port_rx_overflow];1840		core_stats->rx_errors = (core_stats->rx_length_errors +1841					 core_stats->rx_crc_errors +1842					 core_stats->rx_frame_errors);1843	}1844 1845	return stats_count;1846}1847 1848static size_t efx_ef10_update_stats_pf(struct efx_nic *efx, u64 *full_stats,1849				       struct rtnl_link_stats64 *core_stats)1850{1851	struct efx_ef10_nic_data *nic_data = efx->nic_data;1852	DECLARE_BITMAP(mask, EF10_STAT_COUNT);1853	u64 *stats = nic_data->stats;1854 1855	efx_ef10_get_stat_mask(efx, mask);1856 1857	/* If NIC was fini'd (probably resetting), then we can't read1858	 * updated stats right now.1859	 */1860	if (nic_data->mc_stats) {1861		efx_nic_copy_stats(efx, nic_data->mc_stats);1862		efx_nic_update_stats(efx_ef10_stat_desc, EF10_STAT_COUNT,1863				     mask, stats, nic_data->mc_stats, false);1864	}1865 1866	/* Update derived statistics */1867	efx_nic_fix_nodesc_drop_stat(efx,1868				     &stats[EF10_STAT_port_rx_nodesc_drops]);1869	/* MC Firmware reads RX_BYTES and RX_GOOD_BYTES from the MAC.1870	 * It then calculates RX_BAD_BYTES and DMAs it to us with RX_BYTES.1871	 * We report these as port_rx_ stats. We are not given RX_GOOD_BYTES.1872	 * Here we calculate port_rx_good_bytes.1873	 */1874	stats[EF10_STAT_port_rx_good_bytes] =1875		stats[EF10_STAT_port_rx_bytes] -1876		stats[EF10_STAT_port_rx_bytes_minus_good_bytes];1877 1878	/* The asynchronous reads used to calculate RX_BAD_BYTES in1879	 * MC Firmware are done such that we should not see an increase in1880	 * RX_BAD_BYTES when a good packet has arrived. Unfortunately this1881	 * does mean that the stat can decrease at times. Here we do not1882	 * update the stat unless it has increased or has gone to zero1883	 * (In the case of the NIC rebooting).1884	 * Please see Bug 33781 for a discussion of why things work this way.1885	 */1886	efx_update_diff_stat(&stats[EF10_STAT_port_rx_bad_bytes],1887			     stats[EF10_STAT_port_rx_bytes_minus_good_bytes]);1888	efx_update_sw_stats(efx, stats);1889 1890	return efx_ef10_update_stats_common(efx, full_stats, core_stats);1891}1892 1893static int efx_ef10_try_update_nic_stats_vf(struct efx_nic *efx)1894	__must_hold(&efx->stats_lock)1895{1896	MCDI_DECLARE_BUF(inbuf, MC_CMD_MAC_STATS_IN_LEN);1897	struct efx_ef10_nic_data *nic_data = efx->nic_data;1898	DECLARE_BITMAP(mask, EF10_STAT_COUNT);1899	__le64 generation_start, generation_end;1900	u64 *stats = nic_data->stats;1901	u32 dma_len = efx->num_mac_stats * sizeof(u64);1902	struct efx_buffer stats_buf;1903	__le64 *dma_stats;1904	int rc;1905 1906	spin_unlock_bh(&efx->stats_lock);1907 1908	efx_ef10_get_stat_mask(efx, mask);1909 1910	rc = efx_nic_alloc_buffer(efx, &stats_buf, dma_len, GFP_KERNEL);1911	if (rc) {1912		spin_lock_bh(&efx->stats_lock);1913		return rc;1914	}1915 1916	dma_stats = stats_buf.addr;1917	dma_stats[efx->num_mac_stats - 1] = EFX_MC_STATS_GENERATION_INVALID;1918 1919	MCDI_SET_QWORD(inbuf, MAC_STATS_IN_DMA_ADDR, stats_buf.dma_addr);1920	MCDI_POPULATE_DWORD_1(inbuf, MAC_STATS_IN_CMD,1921			      MAC_STATS_IN_DMA, 1);1922	MCDI_SET_DWORD(inbuf, MAC_STATS_IN_DMA_LEN, dma_len);1923	MCDI_SET_DWORD(inbuf, MAC_STATS_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);1924 1925	rc = efx_mcdi_rpc_quiet(efx, MC_CMD_MAC_STATS, inbuf, sizeof(inbuf),1926				NULL, 0, NULL);1927	spin_lock_bh(&efx->stats_lock);1928	if (rc) {1929		/* Expect ENOENT if DMA queues have not been set up */1930		if (rc != -ENOENT || atomic_read(&efx->active_queues))1931			efx_mcdi_display_error(efx, MC_CMD_MAC_STATS,1932					       sizeof(inbuf), NULL, 0, rc);1933		goto out;1934	}1935 1936	generation_end = dma_stats[efx->num_mac_stats - 1];1937	if (generation_end == EFX_MC_STATS_GENERATION_INVALID) {1938		WARN_ON_ONCE(1);1939		goto out;1940	}1941	rmb();1942	efx_nic_update_stats(efx_ef10_stat_desc, EF10_STAT_COUNT, mask,1943			     stats, stats_buf.addr, false);1944	rmb();1945	generation_start = dma_stats[MC_CMD_MAC_GENERATION_START];1946	if (generation_end != generation_start) {1947		rc = -EAGAIN;1948		goto out;1949	}1950 1951	efx_update_sw_stats(efx, stats);1952out:1953	/* releasing a DMA coherent buffer with BH disabled can panic */1954	spin_unlock_bh(&efx->stats_lock);1955	efx_nic_free_buffer(efx, &stats_buf);1956	spin_lock_bh(&efx->stats_lock);1957	return rc;1958}1959 1960static size_t efx_ef10_update_stats_vf(struct efx_nic *efx, u64 *full_stats,1961				       struct rtnl_link_stats64 *core_stats)1962{1963	if (efx_ef10_try_update_nic_stats_vf(efx))1964		return 0;1965 1966	return efx_ef10_update_stats_common(efx, full_stats, core_stats);1967}1968 1969static size_t efx_ef10_update_stats_atomic_vf(struct efx_nic *efx, u64 *full_stats,1970					      struct rtnl_link_stats64 *core_stats)1971{1972	struct efx_ef10_nic_data *nic_data = efx->nic_data;1973 1974	/* In atomic context, cannot update HW stats.  Just update the1975	 * software stats and return so the caller can continue.1976	 */1977	efx_update_sw_stats(efx, nic_data->stats);1978	return efx_ef10_update_stats_common(efx, full_stats, core_stats);1979}1980 1981static void efx_ef10_push_irq_moderation(struct efx_channel *channel)1982{1983	struct efx_nic *efx = channel->efx;1984	unsigned int mode, usecs;1985	efx_dword_t timer_cmd;1986 1987	if (channel->irq_moderation_us) {1988		mode = 3;1989		usecs = channel->irq_moderation_us;1990	} else {1991		mode = 0;1992		usecs = 0;1993	}1994 1995	if (EFX_EF10_WORKAROUND_61265(efx)) {1996		MCDI_DECLARE_BUF(inbuf, MC_CMD_SET_EVQ_TMR_IN_LEN);1997		unsigned int ns = usecs * 1000;1998 1999		MCDI_SET_DWORD(inbuf, SET_EVQ_TMR_IN_INSTANCE,2000			       channel->channel);2001		MCDI_SET_DWORD(inbuf, SET_EVQ_TMR_IN_TMR_LOAD_REQ_NS, ns);2002		MCDI_SET_DWORD(inbuf, SET_EVQ_TMR_IN_TMR_RELOAD_REQ_NS, ns);2003		MCDI_SET_DWORD(inbuf, SET_EVQ_TMR_IN_TMR_MODE, mode);2004 2005		efx_mcdi_rpc_async(efx, MC_CMD_SET_EVQ_TMR,2006				   inbuf, sizeof(inbuf), 0, NULL, 0);2007	} else if (EFX_EF10_WORKAROUND_35388(efx)) {2008		unsigned int ticks = efx_usecs_to_ticks(efx, usecs);2009 2010		EFX_POPULATE_DWORD_3(timer_cmd, ERF_DD_EVQ_IND_TIMER_FLAGS,2011				     EFE_DD_EVQ_IND_TIMER_FLAGS,2012				     ERF_DD_EVQ_IND_TIMER_MODE, mode,2013				     ERF_DD_EVQ_IND_TIMER_VAL, ticks);2014		efx_writed_page(efx, &timer_cmd, ER_DD_EVQ_INDIRECT,2015				channel->channel);2016	} else {2017		unsigned int ticks = efx_usecs_to_ticks(efx, usecs);2018 2019		EFX_POPULATE_DWORD_3(timer_cmd, ERF_DZ_TC_TIMER_MODE, mode,2020				     ERF_DZ_TC_TIMER_VAL, ticks,2021				     ERF_FZ_TC_TMR_REL_VAL, ticks);2022		efx_writed_page(efx, &timer_cmd, ER_DZ_EVQ_TMR,2023				channel->channel);2024	}2025}2026 2027static void efx_ef10_get_wol_vf(struct efx_nic *efx,2028				struct ethtool_wolinfo *wol) {}2029 2030static int efx_ef10_set_wol_vf(struct efx_nic *efx, u32 type)2031{2032	return -EOPNOTSUPP;2033}2034 2035static void efx_ef10_get_wol(struct efx_nic *efx, struct ethtool_wolinfo *wol)2036{2037	wol->supported = 0;2038	wol->wolopts = 0;2039	memset(&wol->sopass, 0, sizeof(wol->sopass));2040}2041 2042static int efx_ef10_set_wol(struct efx_nic *efx, u32 type)2043{2044	if (type != 0)2045		return -EINVAL;2046	return 0;2047}2048 2049static void efx_ef10_mcdi_request(struct efx_nic *efx,2050				  const efx_dword_t *hdr, size_t hdr_len,2051				  const efx_dword_t *sdu, size_t sdu_len)2052{2053	struct efx_ef10_nic_data *nic_data = efx->nic_data;2054	u8 *pdu = nic_data->mcdi_buf.addr;2055 2056	memcpy(pdu, hdr, hdr_len);2057	memcpy(pdu + hdr_len, sdu, sdu_len);2058	wmb();2059 2060	/* The hardware provides 'low' and 'high' (doorbell) registers2061	 * for passing the 64-bit address of an MCDI request to2062	 * firmware.  However the dwords are swapped by firmware.  The2063	 * least significant bits of the doorbell are then 0 for all2064	 * MCDI requests due to alignment.2065	 */2066	_efx_writed(efx, cpu_to_le32((u64)nic_data->mcdi_buf.dma_addr >> 32),2067		    ER_DZ_MC_DB_LWRD);2068	_efx_writed(efx, cpu_to_le32((u32)nic_data->mcdi_buf.dma_addr),2069		    ER_DZ_MC_DB_HWRD);2070}2071 2072static bool efx_ef10_mcdi_poll_response(struct efx_nic *efx)2073{2074	struct efx_ef10_nic_data *nic_data = efx->nic_data;2075	const efx_dword_t hdr = *(const efx_dword_t *)nic_data->mcdi_buf.addr;2076 2077	rmb();2078	return EFX_DWORD_FIELD(hdr, MCDI_HEADER_RESPONSE);2079}2080 2081static void2082efx_ef10_mcdi_read_response(struct efx_nic *efx, efx_dword_t *outbuf,2083			    size_t offset, size_t outlen)2084{2085	struct efx_ef10_nic_data *nic_data = efx->nic_data;2086	const u8 *pdu = nic_data->mcdi_buf.addr;2087 2088	memcpy(outbuf, pdu + offset, outlen);2089}2090 2091static void efx_ef10_mcdi_reboot_detected(struct efx_nic *efx)2092{2093	struct efx_ef10_nic_data *nic_data = efx->nic_data;2094 2095	/* All our allocations have been reset */2096	efx_ef10_table_reset_mc_allocations(efx);2097 2098	/* The datapath firmware might have been changed */2099	nic_data->must_check_datapath_caps = true;2100 2101	/* MAC statistics have been cleared on the NIC; clear the local2102	 * statistic that we update with efx_update_diff_stat().2103	 */2104	nic_data->stats[EF10_STAT_port_rx_bad_bytes] = 0;2105}2106 2107static int efx_ef10_mcdi_poll_reboot(struct efx_nic *efx)2108{2109	struct efx_ef10_nic_data *nic_data = efx->nic_data;2110	int rc;2111 2112	rc = efx_ef10_get_warm_boot_count(efx);2113	if (rc < 0) {2114		/* The firmware is presumably in the process of2115		 * rebooting.  However, we are supposed to report each2116		 * reboot just once, so we must only do that once we2117		 * can read and store the updated warm boot count.2118		 */2119		return 0;2120	}2121 2122	if (rc == nic_data->warm_boot_count)2123		return 0;2124 2125	nic_data->warm_boot_count = rc;2126	efx_ef10_mcdi_reboot_detected(efx);2127 2128	return -EIO;2129}2130 2131/* Handle an MSI interrupt2132 *2133 * Handle an MSI hardware interrupt.  This routine schedules event2134 * queue processing.  No interrupt acknowledgement cycle is necessary.2135 * Also, we never need to check that the interrupt is for us, since2136 * MSI interrupts cannot be shared.2137 */2138static irqreturn_t efx_ef10_msi_interrupt(int irq, void *dev_id)2139{2140	struct efx_msi_context *context = dev_id;2141	struct efx_nic *efx = context->efx;2142 2143	netif_vdbg(efx, intr, efx->net_dev,2144		   "IRQ %d on CPU %d\n", irq, raw_smp_processor_id());2145 2146	if (likely(READ_ONCE(efx->irq_soft_enabled))) {2147		/* Note test interrupts */2148		if (context->index == efx->irq_level)2149			efx->last_irq_cpu = raw_smp_processor_id();2150 2151		/* Schedule processing of the channel */2152		efx_schedule_channel_irq(efx->channel[context->index]);2153	}2154 2155	return IRQ_HANDLED;2156}2157 2158static irqreturn_t efx_ef10_legacy_interrupt(int irq, void *dev_id)2159{2160	struct efx_nic *efx = dev_id;2161	bool soft_enabled = READ_ONCE(efx->irq_soft_enabled);2162	struct efx_channel *channel;2163	efx_dword_t reg;2164	u32 queues;2165 2166	/* Read the ISR which also ACKs the interrupts */2167	efx_readd(efx, &reg, ER_DZ_BIU_INT_ISR);2168	queues = EFX_DWORD_FIELD(reg, ERF_DZ_ISR_REG);2169 2170	if (queues == 0)2171		return IRQ_NONE;2172 2173	if (likely(soft_enabled)) {2174		/* Note test interrupts */2175		if (queues & (1U << efx->irq_level))2176			efx->last_irq_cpu = raw_smp_processor_id();2177 2178		efx_for_each_channel(channel, efx) {2179			if (queues & 1)2180				efx_schedule_channel_irq(channel);2181			queues >>= 1;2182		}2183	}2184 2185	netif_vdbg(efx, intr, efx->net_dev,2186		   "IRQ %d on CPU %d status " EFX_DWORD_FMT "\n",2187		   irq, raw_smp_processor_id(), EFX_DWORD_VAL(reg));2188 2189	return IRQ_HANDLED;2190}2191 2192static int efx_ef10_irq_test_generate(struct efx_nic *efx)2193{2194	MCDI_DECLARE_BUF(inbuf, MC_CMD_TRIGGER_INTERRUPT_IN_LEN);2195 2196	if (efx_mcdi_set_workaround(efx, MC_CMD_WORKAROUND_BUG41750, true,2197				    NULL) == 0)2198		return -ENOTSUPP;2199 2200	BUILD_BUG_ON(MC_CMD_TRIGGER_INTERRUPT_OUT_LEN != 0);2201 2202	MCDI_SET_DWORD(inbuf, TRIGGER_INTERRUPT_IN_INTR_LEVEL, efx->irq_level);2203	return efx_mcdi_rpc(efx, MC_CMD_TRIGGER_INTERRUPT,2204			    inbuf, sizeof(inbuf), NULL, 0, NULL);2205}2206 2207static int efx_ef10_tx_probe(struct efx_tx_queue *tx_queue)2208{2209	/* low two bits of label are what we want for type */2210	BUILD_BUG_ON((EFX_TXQ_TYPE_OUTER_CSUM | EFX_TXQ_TYPE_INNER_CSUM) != 3);2211	tx_queue->type = tx_queue->label & 3;2212	return efx_nic_alloc_buffer(tx_queue->efx, &tx_queue->txd,2213				    (tx_queue->ptr_mask + 1) *2214				    sizeof(efx_qword_t),2215				    GFP_KERNEL);2216}2217 2218/* This writes to the TX_DESC_WPTR and also pushes data */2219static inline void efx_ef10_push_tx_desc(struct efx_tx_queue *tx_queue,2220					 const efx_qword_t *txd)2221{2222	unsigned int write_ptr;2223	efx_oword_t reg;2224 2225	write_ptr = tx_queue->write_count & tx_queue->ptr_mask;2226	EFX_POPULATE_OWORD_1(reg, ERF_DZ_TX_DESC_WPTR, write_ptr);2227	reg.qword[0] = *txd;2228	efx_writeo_page(tx_queue->efx, &reg,2229			ER_DZ_TX_DESC_UPD, tx_queue->queue);2230}2231 2232/* Add Firmware-Assisted TSO v2 option descriptors to a queue.2233 */2234int efx_ef10_tx_tso_desc(struct efx_tx_queue *tx_queue, struct sk_buff *skb,2235			 bool *data_mapped)2236{2237	struct efx_tx_buffer *buffer;2238	u16 inner_ipv4_id = 0;2239	u16 outer_ipv4_id = 0;2240	struct tcphdr *tcp;2241	struct iphdr *ip;2242	u16 ip_tot_len;2243	u32 seqnum;2244	u32 mss;2245 2246	EFX_WARN_ON_ONCE_PARANOID(tx_queue->tso_version != 2);2247 2248	mss = skb_shinfo(skb)->gso_size;2249 2250	if (unlikely(mss < 4)) {2251		WARN_ONCE(1, "MSS of %u is too small for TSO v2\n", mss);2252		return -EINVAL;2253	}2254 2255	if (skb->encapsulation) {2256		if (!tx_queue->tso_encap)2257			return -EINVAL;2258		ip = ip_hdr(skb);2259		if (ip->version == 4)2260			outer_ipv4_id = ntohs(ip->id);2261 2262		ip = inner_ip_hdr(skb);2263		tcp = inner_tcp_hdr(skb);2264	} else {2265		ip = ip_hdr(skb);2266		tcp = tcp_hdr(skb);2267	}2268 2269	/* 8000-series EF10 hardware requires that IP Total Length be2270	 * greater than or equal to the value it will have in each segment2271	 * (which is at most mss + 208 + TCP header length), but also less2272	 * than (0x10000 - inner_network_header).  Otherwise the TCP2273	 * checksum calculation will be broken for encapsulated packets.2274	 * We fill in ip->tot_len with 0xff30, which should satisfy the2275	 * first requirement unless the MSS is ridiculously large (which2276	 * should be impossible as the driver max MTU is 9216); it is2277	 * guaranteed to satisfy the second as we only attempt TSO if2278	 * inner_network_header <= 208.2279	 */2280	ip_tot_len = 0x10000 - EFX_TSO2_MAX_HDRLEN;2281	EFX_WARN_ON_ONCE_PARANOID(mss + EFX_TSO2_MAX_HDRLEN +2282				  (tcp->doff << 2u) > ip_tot_len);2283 2284	if (ip->version == 4) {2285		ip->tot_len = htons(ip_tot_len);2286		ip->check = 0;2287		inner_ipv4_id = ntohs(ip->id);2288	} else {2289		((struct ipv6hdr *)ip)->payload_len = htons(ip_tot_len);2290	}2291 2292	seqnum = ntohl(tcp->seq);2293 2294	buffer = efx_tx_queue_get_insert_buffer(tx_queue);2295 2296	buffer->flags = EFX_TX_BUF_OPTION;2297	buffer->len = 0;2298	buffer->unmap_len = 0;2299	EFX_POPULATE_QWORD_5(buffer->option,2300			ESF_DZ_TX_DESC_IS_OPT, 1,2301			ESF_DZ_TX_OPTION_TYPE, ESE_DZ_TX_OPTION_DESC_TSO,2302			ESF_DZ_TX_TSO_OPTION_TYPE,2303			ESE_DZ_TX_TSO_OPTION_DESC_FATSO2A,2304			ESF_DZ_TX_TSO_IP_ID, inner_ipv4_id,2305			ESF_DZ_TX_TSO_TCP_SEQNO, seqnum2306			);2307	++tx_queue->insert_count;2308 2309	buffer = efx_tx_queue_get_insert_buffer(tx_queue);2310 2311	buffer->flags = EFX_TX_BUF_OPTION;2312	buffer->len = 0;2313	buffer->unmap_len = 0;2314	EFX_POPULATE_QWORD_5(buffer->option,2315			ESF_DZ_TX_DESC_IS_OPT, 1,2316			ESF_DZ_TX_OPTION_TYPE, ESE_DZ_TX_OPTION_DESC_TSO,2317			ESF_DZ_TX_TSO_OPTION_TYPE,2318			ESE_DZ_TX_TSO_OPTION_DESC_FATSO2B,2319			ESF_DZ_TX_TSO_OUTER_IPID, outer_ipv4_id,2320			ESF_DZ_TX_TSO_TCP_MSS, mss2321			);2322	++tx_queue->insert_count;2323 2324	return 0;2325}2326 2327static u32 efx_ef10_tso_versions(struct efx_nic *efx)2328{2329	struct efx_ef10_nic_data *nic_data = efx->nic_data;2330	u32 tso_versions = 0;2331 2332	if (nic_data->datapath_caps &2333	    (1 << MC_CMD_GET_CAPABILITIES_OUT_TX_TSO_LBN))2334		tso_versions |= BIT(1);2335	if (nic_data->datapath_caps2 &2336	    (1 << MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_V2_LBN))2337		tso_versions |= BIT(2);2338	return tso_versions;2339}2340 2341static void efx_ef10_tx_init(struct efx_tx_queue *tx_queue)2342{2343	bool csum_offload = tx_queue->type & EFX_TXQ_TYPE_OUTER_CSUM;2344	bool inner_csum = tx_queue->type & EFX_TXQ_TYPE_INNER_CSUM;2345	struct efx_channel *channel = tx_queue->channel;2346	struct efx_nic *efx = tx_queue->efx;2347	struct efx_ef10_nic_data *nic_data;2348	efx_qword_t *txd;2349	int rc;2350 2351	nic_data = efx->nic_data;2352 2353	/* Only attempt to enable TX timestamping if we have the license for it,2354	 * otherwise TXQ init will fail2355	 */2356	if (!(nic_data->licensed_features &2357	      (1 << LICENSED_V3_FEATURES_TX_TIMESTAMPS_LBN))) {2358		tx_queue->timestamping = false;2359		/* Disable sync events on this channel. */2360		if (efx->type->ptp_set_ts_sync_events)2361			efx->type->ptp_set_ts_sync_events(efx, false, false);2362	}2363 2364	/* TSOv2 is a limited resource that can only be configured on a limited2365	 * number of queues. TSO without checksum offload is not really a thing,2366	 * so we only enable it for those queues.2367	 * TSOv2 cannot be used with Hardware timestamping, and is never needed2368	 * for XDP tx.2369	 */2370	if (efx_has_cap(efx, TX_TSO_V2)) {2371		if ((csum_offload || inner_csum) &&2372		    !tx_queue->timestamping && !tx_queue->xdp_tx) {2373			tx_queue->tso_version = 2;2374			netif_dbg(efx, hw, efx->net_dev, "Using TSOv2 for channel %u\n",2375				  channel->channel);2376		}2377	} else if (efx_has_cap(efx, TX_TSO)) {2378		tx_queue->tso_version = 1;2379	}2380 2381	rc = efx_mcdi_tx_init(tx_queue);2382	if (rc)2383		goto fail;2384 2385	/* A previous user of this TX queue might have set us up the2386	 * bomb by writing a descriptor to the TX push collector but2387	 * not the doorbell.  (Each collector belongs to a port, not a2388	 * queue or function, so cannot easily be reset.)  We must2389	 * attempt to push a no-op descriptor in its place.2390	 */2391	tx_queue->buffer[0].flags = EFX_TX_BUF_OPTION;2392	tx_queue->insert_count = 1;2393	txd = efx_tx_desc(tx_queue, 0);2394	EFX_POPULATE_QWORD_7(*txd,2395			     ESF_DZ_TX_DESC_IS_OPT, true,2396			     ESF_DZ_TX_OPTION_TYPE,2397			     ESE_DZ_TX_OPTION_DESC_CRC_CSUM,2398			     ESF_DZ_TX_OPTION_UDP_TCP_CSUM, csum_offload,2399			     ESF_DZ_TX_OPTION_IP_CSUM, csum_offload && tx_queue->tso_version != 2,2400			     ESF_DZ_TX_OPTION_INNER_UDP_TCP_CSUM, inner_csum,2401			     ESF_DZ_TX_OPTION_INNER_IP_CSUM, inner_csum && tx_queue->tso_version != 2,2402			     ESF_DZ_TX_TIMESTAMP, tx_queue->timestamping);2403	tx_queue->write_count = 1;2404 2405	if (tx_queue->tso_version == 2 && efx_has_cap(efx, TX_TSO_V2_ENCAP))2406		tx_queue->tso_encap = true;2407 2408	wmb();2409	efx_ef10_push_tx_desc(tx_queue, txd);2410 2411	return;2412 2413fail:2414	netdev_WARN(efx->net_dev, "failed to initialise TXQ %d\n",2415		    tx_queue->queue);2416}2417 2418/* This writes to the TX_DESC_WPTR; write pointer for TX descriptor ring */2419static inline void efx_ef10_notify_tx_desc(struct efx_tx_queue *tx_queue)2420{2421	unsigned int write_ptr;2422	efx_dword_t reg;2423 2424	write_ptr = tx_queue->write_count & tx_queue->ptr_mask;2425	EFX_POPULATE_DWORD_1(reg, ERF_DZ_TX_DESC_WPTR_DWORD, write_ptr);2426	efx_writed_page(tx_queue->efx, &reg,2427			ER_DZ_TX_DESC_UPD_DWORD, tx_queue->queue);2428}2429 2430#define EFX_EF10_MAX_TX_DESCRIPTOR_LEN 0x3fff2431 2432static unsigned int efx_ef10_tx_limit_len(struct efx_tx_queue *tx_queue,2433					  dma_addr_t dma_addr, unsigned int len)2434{2435	if (len > EFX_EF10_MAX_TX_DESCRIPTOR_LEN) {2436		/* If we need to break across multiple descriptors we should2437		 * stop at a page boundary. This assumes the length limit is2438		 * greater than the page size.2439		 */2440		dma_addr_t end = dma_addr + EFX_EF10_MAX_TX_DESCRIPTOR_LEN;2441 2442		BUILD_BUG_ON(EFX_EF10_MAX_TX_DESCRIPTOR_LEN < EFX_PAGE_SIZE);2443		len = (end & (~(EFX_PAGE_SIZE - 1))) - dma_addr;2444	}2445 2446	return len;2447}2448 2449static void efx_ef10_tx_write(struct efx_tx_queue *tx_queue)2450{2451	unsigned int old_write_count = tx_queue->write_count;2452	struct efx_tx_buffer *buffer;2453	unsigned int write_ptr;2454	efx_qword_t *txd;2455 2456	tx_queue->xmit_pending = false;2457	if (unlikely(tx_queue->write_count == tx_queue->insert_count))2458		return;2459 2460	do {2461		write_ptr = tx_queue->write_count & tx_queue->ptr_mask;2462		buffer = &tx_queue->buffer[write_ptr];2463		txd = efx_tx_desc(tx_queue, write_ptr);2464		++tx_queue->write_count;2465 2466		/* Create TX descriptor ring entry */2467		if (buffer->flags & EFX_TX_BUF_OPTION) {2468			*txd = buffer->option;2469			if (EFX_QWORD_FIELD(*txd, ESF_DZ_TX_OPTION_TYPE) == 1)2470				/* PIO descriptor */2471				tx_queue->packet_write_count = tx_queue->write_count;2472		} else {2473			tx_queue->packet_write_count = tx_queue->write_count;2474			BUILD_BUG_ON(EFX_TX_BUF_CONT != 1);2475			EFX_POPULATE_QWORD_3(2476				*txd,2477				ESF_DZ_TX_KER_CONT,2478				buffer->flags & EFX_TX_BUF_CONT,2479				ESF_DZ_TX_KER_BYTE_CNT, buffer->len,2480				ESF_DZ_TX_KER_BUF_ADDR, buffer->dma_addr);2481		}2482	} while (tx_queue->write_count != tx_queue->insert_count);2483 2484	wmb(); /* Ensure descriptors are written before they are fetched */2485 2486	if (efx_nic_may_push_tx_desc(tx_queue, old_write_count)) {2487		txd = efx_tx_desc(tx_queue,2488				  old_write_count & tx_queue->ptr_mask);2489		efx_ef10_push_tx_desc(tx_queue, txd);2490		++tx_queue->pushes;2491	} else {2492		efx_ef10_notify_tx_desc(tx_queue);2493	}2494}2495 2496static int efx_ef10_probe_multicast_chaining(struct efx_nic *efx)2497{2498	struct efx_ef10_nic_data *nic_data = efx->nic_data;2499	unsigned int enabled, implemented;2500	bool want_workaround_26807;2501	int rc;2502 2503	rc = efx_mcdi_get_workarounds(efx, &implemented, &enabled);2504	if (rc == -ENOSYS) {2505		/* GET_WORKAROUNDS was implemented before this workaround,2506		 * thus it must be unavailable in this firmware.2507		 */2508		nic_data->workaround_26807 = false;2509		return 0;2510	}2511	if (rc)2512		return rc;2513	want_workaround_26807 =2514		implemented & MC_CMD_GET_WORKAROUNDS_OUT_BUG26807;2515	nic_data->workaround_26807 =2516		!!(enabled & MC_CMD_GET_WORKAROUNDS_OUT_BUG26807);2517 2518	if (want_workaround_26807 && !nic_data->workaround_26807) {2519		unsigned int flags;2520 2521		rc = efx_mcdi_set_workaround(efx,2522					     MC_CMD_WORKAROUND_BUG26807,2523					     true, &flags);2524		if (!rc) {2525			if (flags &2526			    1 << MC_CMD_WORKAROUND_EXT_OUT_FLR_DONE_LBN) {2527				netif_info(efx, drv, efx->net_dev,2528					   "other functions on NIC have been reset\n");2529 2530				/* With MCFW v4.6.x and earlier, the2531				 * boot count will have incremented,2532				 * so re-read the warm_boot_count2533				 * value now to ensure this function2534				 * doesn't think it has changed next2535				 * time it checks.2536				 */2537				rc = efx_ef10_get_warm_boot_count(efx);2538				if (rc >= 0) {2539					nic_data->warm_boot_count = rc;2540					rc = 0;2541				}2542			}2543			nic_data->workaround_26807 = true;2544		} else if (rc == -EPERM) {2545			rc = 0;2546		}2547	}2548	return rc;2549}2550 2551static int efx_ef10_filter_table_probe(struct efx_nic *efx)2552{2553	struct efx_ef10_nic_data *nic_data = efx->nic_data;2554	int rc = efx_ef10_probe_multicast_chaining(efx);2555	struct efx_mcdi_filter_vlan *vlan;2556 2557	if (rc)2558		return rc;2559	down_write(&efx->filter_sem);2560	rc = efx_mcdi_filter_table_probe(efx, nic_data->workaround_26807);2561 2562	if (rc)2563		goto out_unlock;2564 2565	list_for_each_entry(vlan, &nic_data->vlan_list, list) {2566		rc = efx_mcdi_filter_add_vlan(efx, vlan->vid);2567		if (rc)2568			goto fail_add_vlan;2569	}2570	goto out_unlock;2571 2572fail_add_vlan:2573	efx_mcdi_filter_table_remove(efx);2574out_unlock:2575	up_write(&efx->filter_sem);2576	return rc;2577}2578 2579static void efx_ef10_filter_table_remove(struct efx_nic *efx)2580{2581	down_write(&efx->filter_sem);2582	efx_mcdi_filter_table_remove(efx);2583	up_write(&efx->filter_sem);2584}2585 2586/* This creates an entry in the RX descriptor queue */2587static inline void2588efx_ef10_build_rx_desc(struct efx_rx_queue *rx_queue, unsigned int index)2589{2590	struct efx_rx_buffer *rx_buf;2591	efx_qword_t *rxd;2592 2593	rxd = efx_rx_desc(rx_queue, index);2594	rx_buf = efx_rx_buffer(rx_queue, index);2595	EFX_POPULATE_QWORD_2(*rxd,2596			     ESF_DZ_RX_KER_BYTE_CNT, rx_buf->len,2597			     ESF_DZ_RX_KER_BUF_ADDR, rx_buf->dma_addr);2598}2599 2600static void efx_ef10_rx_write(struct efx_rx_queue *rx_queue)2601{2602	struct efx_nic *efx = rx_queue->efx;2603	unsigned int write_count;2604	efx_dword_t reg;2605 2606	/* Firmware requires that RX_DESC_WPTR be a multiple of 8 */2607	write_count = rx_queue->added_count & ~7;2608	if (rx_queue->notified_count == write_count)2609		return;2610 2611	do2612		efx_ef10_build_rx_desc(2613			rx_queue,2614			rx_queue->notified_count & rx_queue->ptr_mask);2615	while (++rx_queue->notified_count != write_count);2616 2617	wmb();2618	EFX_POPULATE_DWORD_1(reg, ERF_DZ_RX_DESC_WPTR,2619			     write_count & rx_queue->ptr_mask);2620	efx_writed_page(efx, &reg, ER_DZ_RX_DESC_UPD,2621			efx_rx_queue_index(rx_queue));2622}2623 2624static efx_mcdi_async_completer efx_ef10_rx_defer_refill_complete;2625 2626static void efx_ef10_rx_defer_refill(struct efx_rx_queue *rx_queue)2627{2628	struct efx_channel *channel = efx_rx_queue_channel(rx_queue);2629	MCDI_DECLARE_BUF(inbuf, MC_CMD_DRIVER_EVENT_IN_LEN);2630	efx_qword_t event;2631 2632	EFX_POPULATE_QWORD_2(event,2633			     ESF_DZ_EV_CODE, EFX_EF10_DRVGEN_EV,2634			     ESF_DZ_EV_DATA, EFX_EF10_REFILL);2635 2636	MCDI_SET_DWORD(inbuf, DRIVER_EVENT_IN_EVQ, channel->channel);2637 2638	/* MCDI_SET_QWORD is not appropriate here since EFX_POPULATE_* has2639	 * already swapped the data to little-endian order.2640	 */2641	memcpy(MCDI_PTR(inbuf, DRIVER_EVENT_IN_DATA), &event.u64[0],2642	       sizeof(efx_qword_t));2643 2644	efx_mcdi_rpc_async(channel->efx, MC_CMD_DRIVER_EVENT,2645			   inbuf, sizeof(inbuf), 0,2646			   efx_ef10_rx_defer_refill_complete, 0);2647}2648 2649static void2650efx_ef10_rx_defer_refill_complete(struct efx_nic *efx, unsigned long cookie,2651				  int rc, efx_dword_t *outbuf,2652				  size_t outlen_actual)2653{2654	/* nothing to do */2655}2656 2657static int efx_ef10_ev_init(struct efx_channel *channel)2658{2659	struct efx_nic *efx = channel->efx;2660	struct efx_ef10_nic_data *nic_data;2661	bool use_v2, cut_thru;2662 2663	nic_data = efx->nic_data;2664	use_v2 = nic_data->datapath_caps2 &2665			    1 << MC_CMD_GET_CAPABILITIES_V2_OUT_INIT_EVQ_V2_LBN;2666	cut_thru = !(nic_data->datapath_caps &2667			      1 << MC_CMD_GET_CAPABILITIES_OUT_RX_BATCHING_LBN);2668	return efx_mcdi_ev_init(channel, cut_thru, use_v2);2669}2670 2671static void efx_ef10_handle_rx_wrong_queue(struct efx_rx_queue *rx_queue,2672					   unsigned int rx_queue_label)2673{2674	struct efx_nic *efx = rx_queue->efx;2675 2676	netif_info(efx, hw, efx->net_dev,2677		   "rx event arrived on queue %d labeled as queue %u\n",2678		   efx_rx_queue_index(rx_queue), rx_queue_label);2679 2680	efx_schedule_reset(efx, RESET_TYPE_DISABLE);2681}2682 2683static void2684efx_ef10_handle_rx_bad_lbits(struct efx_rx_queue *rx_queue,2685			     unsigned int actual, unsigned int expected)2686{2687	unsigned int dropped = (actual - expected) & rx_queue->ptr_mask;2688	struct efx_nic *efx = rx_queue->efx;2689 2690	netif_info(efx, hw, efx->net_dev,2691		   "dropped %d events (index=%d expected=%d)\n",2692		   dropped, actual, expected);2693 2694	efx_schedule_reset(efx, RESET_TYPE_DISABLE);2695}2696 2697/* partially received RX was aborted. clean up. */2698static void efx_ef10_handle_rx_abort(struct efx_rx_queue *rx_queue)2699{2700	unsigned int rx_desc_ptr;2701 2702	netif_dbg(rx_queue->efx, hw, rx_queue->efx->net_dev,2703		  "scattered RX aborted (dropping %u buffers)\n",2704		  rx_queue->scatter_n);2705 2706	rx_desc_ptr = rx_queue->removed_count & rx_queue->ptr_mask;2707 2708	efx_rx_packet(rx_queue, rx_desc_ptr, rx_queue->scatter_n,2709		      0, EFX_RX_PKT_DISCARD);2710 2711	rx_queue->removed_count += rx_queue->scatter_n;2712	rx_queue->scatter_n = 0;2713	rx_queue->scatter_len = 0;2714	++efx_rx_queue_channel(rx_queue)->n_rx_nodesc_trunc;2715}2716 2717static u16 efx_ef10_handle_rx_event_errors(struct efx_channel *channel,2718					   unsigned int n_packets,2719					   unsigned int rx_encap_hdr,2720					   unsigned int rx_l3_class,2721					   unsigned int rx_l4_class,2722					   const efx_qword_t *event)2723{2724	struct efx_nic *efx = channel->efx;2725	bool handled = false;2726 2727	if (EFX_QWORD_FIELD(*event, ESF_DZ_RX_ECRC_ERR)) {2728		if (!(efx->net_dev->features & NETIF_F_RXALL)) {2729			if (!efx->loopback_selftest)2730				channel->n_rx_eth_crc_err += n_packets;2731			return EFX_RX_PKT_DISCARD;2732		}2733		handled = true;2734	}2735	if (EFX_QWORD_FIELD(*event, ESF_DZ_RX_IPCKSUM_ERR)) {2736		if (unlikely(rx_encap_hdr != ESE_EZ_ENCAP_HDR_VXLAN &&2737			     rx_l3_class != ESE_DZ_L3_CLASS_IP4 &&2738			     rx_l3_class != ESE_DZ_L3_CLASS_IP4_FRAG &&2739			     rx_l3_class != ESE_DZ_L3_CLASS_IP6 &&2740			     rx_l3_class != ESE_DZ_L3_CLASS_IP6_FRAG))2741			netdev_WARN(efx->net_dev,2742				    "invalid class for RX_IPCKSUM_ERR: event="2743				    EFX_QWORD_FMT "\n",2744				    EFX_QWORD_VAL(*event));2745		if (!efx->loopback_selftest)2746			*(rx_encap_hdr ?2747			  &channel->n_rx_outer_ip_hdr_chksum_err :2748			  &channel->n_rx_ip_hdr_chksum_err) += n_packets;2749		return 0;2750	}2751	if (EFX_QWORD_FIELD(*event, ESF_DZ_RX_TCPUDP_CKSUM_ERR)) {2752		if (unlikely(rx_encap_hdr != ESE_EZ_ENCAP_HDR_VXLAN &&2753			     ((rx_l3_class != ESE_DZ_L3_CLASS_IP4 &&2754			       rx_l3_class != ESE_DZ_L3_CLASS_IP6) ||2755			      (rx_l4_class != ESE_FZ_L4_CLASS_TCP &&2756			       rx_l4_class != ESE_FZ_L4_CLASS_UDP))))2757			netdev_WARN(efx->net_dev,2758				    "invalid class for RX_TCPUDP_CKSUM_ERR: event="2759				    EFX_QWORD_FMT "\n",2760				    EFX_QWORD_VAL(*event));2761		if (!efx->loopback_selftest)2762			*(rx_encap_hdr ?2763			  &channel->n_rx_outer_tcp_udp_chksum_err :2764			  &channel->n_rx_tcp_udp_chksum_err) += n_packets;2765		return 0;2766	}2767	if (EFX_QWORD_FIELD(*event, ESF_EZ_RX_IP_INNER_CHKSUM_ERR)) {2768		if (unlikely(!rx_encap_hdr))2769			netdev_WARN(efx->net_dev,2770				    "invalid encapsulation type for RX_IP_INNER_CHKSUM_ERR: event="2771				    EFX_QWORD_FMT "\n",2772				    EFX_QWORD_VAL(*event));2773		else if (unlikely(rx_l3_class != ESE_DZ_L3_CLASS_IP4 &&2774				  rx_l3_class != ESE_DZ_L3_CLASS_IP4_FRAG &&2775				  rx_l3_class != ESE_DZ_L3_CLASS_IP6 &&2776				  rx_l3_class != ESE_DZ_L3_CLASS_IP6_FRAG))2777			netdev_WARN(efx->net_dev,2778				    "invalid class for RX_IP_INNER_CHKSUM_ERR: event="2779				    EFX_QWORD_FMT "\n",2780				    EFX_QWORD_VAL(*event));2781		if (!efx->loopback_selftest)2782			channel->n_rx_inner_ip_hdr_chksum_err += n_packets;2783		return 0;2784	}2785	if (EFX_QWORD_FIELD(*event, ESF_EZ_RX_TCP_UDP_INNER_CHKSUM_ERR)) {2786		if (unlikely(!rx_encap_hdr))2787			netdev_WARN(efx->net_dev,2788				    "invalid encapsulation type for RX_TCP_UDP_INNER_CHKSUM_ERR: event="2789				    EFX_QWORD_FMT "\n",2790				    EFX_QWORD_VAL(*event));2791		else if (unlikely((rx_l3_class != ESE_DZ_L3_CLASS_IP4 &&2792				   rx_l3_class != ESE_DZ_L3_CLASS_IP6) ||2793				  (rx_l4_class != ESE_FZ_L4_CLASS_TCP &&2794				   rx_l4_class != ESE_FZ_L4_CLASS_UDP)))2795			netdev_WARN(efx->net_dev,2796				    "invalid class for RX_TCP_UDP_INNER_CHKSUM_ERR: event="2797				    EFX_QWORD_FMT "\n",2798				    EFX_QWORD_VAL(*event));2799		if (!efx->loopback_selftest)2800			channel->n_rx_inner_tcp_udp_chksum_err += n_packets;2801		return 0;2802	}2803 2804	WARN_ON(!handled); /* No error bits were recognised */2805	return 0;2806}2807 2808static int efx_ef10_handle_rx_event(struct efx_channel *channel,2809				    const efx_qword_t *event)2810{2811	unsigned int rx_bytes, next_ptr_lbits, rx_queue_label;2812	unsigned int rx_l3_class, rx_l4_class, rx_encap_hdr;2813	unsigned int n_descs, n_packets, i;2814	struct efx_nic *efx = channel->efx;2815	struct efx_ef10_nic_data *nic_data = efx->nic_data;2816	struct efx_rx_queue *rx_queue;2817	efx_qword_t errors;2818	bool rx_cont;2819	u16 flags = 0;2820 2821	if (unlikely(READ_ONCE(efx->reset_pending)))2822		return 0;2823 2824	/* Basic packet information */2825	rx_bytes = EFX_QWORD_FIELD(*event, ESF_DZ_RX_BYTES);2826	next_ptr_lbits = EFX_QWORD_FIELD(*event, ESF_DZ_RX_DSC_PTR_LBITS);2827	rx_queue_label = EFX_QWORD_FIELD(*event, ESF_DZ_RX_QLABEL);2828	rx_l3_class = EFX_QWORD_FIELD(*event, ESF_DZ_RX_L3_CLASS);2829	rx_l4_class = EFX_QWORD_FIELD(*event, ESF_FZ_RX_L4_CLASS);2830	rx_cont = EFX_QWORD_FIELD(*event, ESF_DZ_RX_CONT);2831	rx_encap_hdr =2832		nic_data->datapath_caps &2833			(1 << MC_CMD_GET_CAPABILITIES_OUT_VXLAN_NVGRE_LBN) ?2834		EFX_QWORD_FIELD(*event, ESF_EZ_RX_ENCAP_HDR) :2835		ESE_EZ_ENCAP_HDR_NONE;2836 2837	if (EFX_QWORD_FIELD(*event, ESF_DZ_RX_DROP_EVENT))2838		netdev_WARN(efx->net_dev, "saw RX_DROP_EVENT: event="2839			    EFX_QWORD_FMT "\n",2840			    EFX_QWORD_VAL(*event));2841 2842	rx_queue = efx_channel_get_rx_queue(channel);2843 2844	if (unlikely(rx_queue_label != efx_rx_queue_index(rx_queue)))2845		efx_ef10_handle_rx_wrong_queue(rx_queue, rx_queue_label);2846 2847	n_descs = ((next_ptr_lbits - rx_queue->removed_count) &2848		   ((1 << ESF_DZ_RX_DSC_PTR_LBITS_WIDTH) - 1));2849 2850	if (n_descs != rx_queue->scatter_n + 1) {2851		struct efx_ef10_nic_data *nic_data = efx->nic_data;2852 2853		/* detect rx abort */2854		if (unlikely(n_descs == rx_queue->scatter_n)) {2855			if (rx_queue->scatter_n == 0 || rx_bytes != 0)2856				netdev_WARN(efx->net_dev,2857					    "invalid RX abort: scatter_n=%u event="2858					    EFX_QWORD_FMT "\n",2859					    rx_queue->scatter_n,2860					    EFX_QWORD_VAL(*event));2861			efx_ef10_handle_rx_abort(rx_queue);2862			return 0;2863		}2864 2865		/* Check that RX completion merging is valid, i.e.2866		 * the current firmware supports it and this is a2867		 * non-scattered packet.2868		 */2869		if (!(nic_data->datapath_caps &2870		      (1 << MC_CMD_GET_CAPABILITIES_OUT_RX_BATCHING_LBN)) ||2871		    rx_queue->scatter_n != 0 || rx_cont) {2872			efx_ef10_handle_rx_bad_lbits(2873				rx_queue, next_ptr_lbits,2874				(rx_queue->removed_count +2875				 rx_queue->scatter_n + 1) &2876				((1 << ESF_DZ_RX_DSC_PTR_LBITS_WIDTH) - 1));2877			return 0;2878		}2879 2880		/* Merged completion for multiple non-scattered packets */2881		rx_queue->scatter_n = 1;2882		rx_queue->scatter_len = 0;2883		n_packets = n_descs;2884		++channel->n_rx_merge_events;2885		channel->n_rx_merge_packets += n_packets;2886		flags |= EFX_RX_PKT_PREFIX_LEN;2887	} else {2888		++rx_queue->scatter_n;2889		rx_queue->scatter_len += rx_bytes;2890		if (rx_cont)2891			return 0;2892		n_packets = 1;2893	}2894 2895	EFX_POPULATE_QWORD_5(errors, ESF_DZ_RX_ECRC_ERR, 1,2896				     ESF_DZ_RX_IPCKSUM_ERR, 1,2897				     ESF_DZ_RX_TCPUDP_CKSUM_ERR, 1,2898				     ESF_EZ_RX_IP_INNER_CHKSUM_ERR, 1,2899				     ESF_EZ_RX_TCP_UDP_INNER_CHKSUM_ERR, 1);2900	EFX_AND_QWORD(errors, *event, errors);2901	if (unlikely(!EFX_QWORD_IS_ZERO(errors))) {2902		flags |= efx_ef10_handle_rx_event_errors(channel, n_packets,2903							 rx_encap_hdr,2904							 rx_l3_class, rx_l4_class,2905							 event);2906	} else {2907		bool tcpudp = rx_l4_class == ESE_FZ_L4_CLASS_TCP ||2908			      rx_l4_class == ESE_FZ_L4_CLASS_UDP;2909 2910		switch (rx_encap_hdr) {2911		case ESE_EZ_ENCAP_HDR_VXLAN: /* VxLAN or GENEVE */2912			flags |= EFX_RX_PKT_CSUMMED; /* outer UDP csum */2913			if (tcpudp)2914				flags |= EFX_RX_PKT_CSUM_LEVEL; /* inner L4 */2915			break;2916		case ESE_EZ_ENCAP_HDR_GRE:2917		case ESE_EZ_ENCAP_HDR_NONE:2918			if (tcpudp)2919				flags |= EFX_RX_PKT_CSUMMED;2920			break;2921		default:2922			netdev_WARN(efx->net_dev,2923				    "unknown encapsulation type: event="2924				    EFX_QWORD_FMT "\n",2925				    EFX_QWORD_VAL(*event));2926		}2927	}2928 2929	if (rx_l4_class == ESE_FZ_L4_CLASS_TCP)2930		flags |= EFX_RX_PKT_TCP;2931 2932	channel->irq_mod_score += 2 * n_packets;2933 2934	/* Handle received packet(s) */2935	for (i = 0; i < n_packets; i++) {2936		efx_rx_packet(rx_queue,2937			      rx_queue->removed_count & rx_queue->ptr_mask,2938			      rx_queue->scatter_n, rx_queue->scatter_len,2939			      flags);2940		rx_queue->removed_count += rx_queue->scatter_n;2941	}2942 2943	rx_queue->scatter_n = 0;2944	rx_queue->scatter_len = 0;2945 2946	return n_packets;2947}2948 2949static u32 efx_ef10_extract_event_ts(efx_qword_t *event)2950{2951	u32 tstamp;2952 2953	tstamp = EFX_QWORD_FIELD(*event, TX_TIMESTAMP_EVENT_TSTAMP_DATA_HI);2954	tstamp <<= 16;2955	tstamp |= EFX_QWORD_FIELD(*event, TX_TIMESTAMP_EVENT_TSTAMP_DATA_LO);2956 2957	return tstamp;2958}2959 2960static int2961efx_ef10_handle_tx_event(struct efx_channel *channel, efx_qword_t *event)2962{2963	struct efx_nic *efx = channel->efx;2964	struct efx_tx_queue *tx_queue;2965	unsigned int tx_ev_desc_ptr;2966	unsigned int tx_ev_q_label;2967	unsigned int tx_ev_type;2968	int work_done;2969	u64 ts_part;2970 2971	if (unlikely(READ_ONCE(efx->reset_pending)))2972		return 0;2973 2974	if (unlikely(EFX_QWORD_FIELD(*event, ESF_DZ_TX_DROP_EVENT)))2975		return 0;2976 2977	/* Get the transmit queue */2978	tx_ev_q_label = EFX_QWORD_FIELD(*event, ESF_DZ_TX_QLABEL);2979	tx_queue = channel->tx_queue + (tx_ev_q_label % EFX_MAX_TXQ_PER_CHANNEL);2980 2981	if (!tx_queue->timestamping) {2982		/* Transmit completion */2983		tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, ESF_DZ_TX_DESCR_INDX);2984		return efx_xmit_done(tx_queue, tx_ev_desc_ptr & tx_queue->ptr_mask);2985	}2986 2987	/* Transmit timestamps are only available for 8XXX series. They result2988	 * in up to three events per packet. These occur in order, and are:2989	 *  - the normal completion event (may be omitted)2990	 *  - the low part of the timestamp2991	 *  - the high part of the timestamp2992	 *2993	 * It's possible for multiple completion events to appear before the2994	 * corresponding timestamps. So we can for example get:2995	 *  COMP N2996	 *  COMP N+12997	 *  TS_LO N2998	 *  TS_HI N2999	 *  TS_LO N+13000	 *  TS_HI N+13001	 *3002	 * In addition it's also possible for the adjacent completions to be3003	 * merged, so we may not see COMP N above. As such, the completion3004	 * events are not very useful here.3005	 *3006	 * Each part of the timestamp is itself split across two 16 bit3007	 * fields in the event.3008	 */3009	tx_ev_type = EFX_QWORD_FIELD(*event, ESF_EZ_TX_SOFT1);3010	work_done = 0;3011 3012	switch (tx_ev_type) {3013	case TX_TIMESTAMP_EVENT_TX_EV_COMPLETION:3014		/* Ignore this event - see above. */3015		break;3016 3017	case TX_TIMESTAMP_EVENT_TX_EV_TSTAMP_LO:3018		ts_part = efx_ef10_extract_event_ts(event);3019		tx_queue->completed_timestamp_minor = ts_part;3020		break;3021 3022	case TX_TIMESTAMP_EVENT_TX_EV_TSTAMP_HI:3023		ts_part = efx_ef10_extract_event_ts(event);3024		tx_queue->completed_timestamp_major = ts_part;3025 3026		efx_xmit_done_single(tx_queue);3027		work_done = 1;3028		break;3029 3030	default:3031		netif_err(efx, hw, efx->net_dev,3032			  "channel %d unknown tx event type %d (data "3033			  EFX_QWORD_FMT ")\n",3034			  channel->channel, tx_ev_type,3035			  EFX_QWORD_VAL(*event));3036		break;3037	}3038 3039	return work_done;3040}3041 3042static void3043efx_ef10_handle_driver_event(struct efx_channel *channel, efx_qword_t *event)3044{3045	struct efx_nic *efx = channel->efx;3046	int subcode;3047 3048	subcode = EFX_QWORD_FIELD(*event, ESF_DZ_DRV_SUB_CODE);3049 3050	switch (subcode) {3051	case ESE_DZ_DRV_TIMER_EV:3052	case ESE_DZ_DRV_WAKE_UP_EV:3053		break;3054	case ESE_DZ_DRV_START_UP_EV:3055		/* event queue init complete. ok. */3056		break;3057	default:3058		netif_err(efx, hw, efx->net_dev,3059			  "channel %d unknown driver event type %d"3060			  " (data " EFX_QWORD_FMT ")\n",3061			  channel->channel, subcode,3062			  EFX_QWORD_VAL(*event));3063 3064	}3065}3066 3067static void efx_ef10_handle_driver_generated_event(struct efx_channel *channel,3068						   efx_qword_t *event)3069{3070	struct efx_nic *efx = channel->efx;3071	u32 subcode;3072 3073	subcode = EFX_QWORD_FIELD(*event, EFX_DWORD_0);3074 3075	switch (subcode) {3076	case EFX_EF10_TEST:3077		channel->event_test_cpu = raw_smp_processor_id();3078		break;3079	case EFX_EF10_REFILL:3080		/* The queue must be empty, so we won't receive any rx3081		 * events, so efx_process_channel() won't refill the3082		 * queue. Refill it here3083		 */3084		efx_fast_push_rx_descriptors(&channel->rx_queue, true);3085		break;3086	default:3087		netif_err(efx, hw, efx->net_dev,3088			  "channel %d unknown driver event type %u"3089			  " (data " EFX_QWORD_FMT ")\n",3090			  channel->channel, (unsigned) subcode,3091			  EFX_QWORD_VAL(*event));3092	}3093}3094 3095#define EFX_NAPI_MAX_TX 5123096 3097static int efx_ef10_ev_process(struct efx_channel *channel, int quota)3098{3099	struct efx_nic *efx = channel->efx;3100	efx_qword_t event, *p_event;3101	unsigned int read_ptr;3102	int spent_tx = 0;3103	int spent = 0;3104	int ev_code;3105 3106	if (quota <= 0)3107		return spent;3108 3109	read_ptr = channel->eventq_read_ptr;3110 3111	for (;;) {3112		p_event = efx_event(channel, read_ptr);3113		event = *p_event;3114 3115		if (!efx_event_present(&event))3116			break;3117 3118		EFX_SET_QWORD(*p_event);3119 3120		++read_ptr;3121 3122		ev_code = EFX_QWORD_FIELD(event, ESF_DZ_EV_CODE);3123 3124		netif_vdbg(efx, drv, efx->net_dev,3125			   "processing event on %d " EFX_QWORD_FMT "\n",3126			   channel->channel, EFX_QWORD_VAL(event));3127 3128		switch (ev_code) {3129		case ESE_DZ_EV_CODE_MCDI_EV:3130			efx_mcdi_process_event(channel, &event);3131			break;3132		case ESE_DZ_EV_CODE_RX_EV:3133			spent += efx_ef10_handle_rx_event(channel, &event);3134			if (spent >= quota) {3135				/* XXX can we split a merged event to3136				 * avoid going over-quota?3137				 */3138				spent = quota;3139				goto out;3140			}3141			break;3142		case ESE_DZ_EV_CODE_TX_EV:3143			spent_tx += efx_ef10_handle_tx_event(channel, &event);3144			if (spent_tx >= EFX_NAPI_MAX_TX) {3145				spent = quota;3146				goto out;3147			}3148			break;3149		case ESE_DZ_EV_CODE_DRIVER_EV:3150			efx_ef10_handle_driver_event(channel, &event);3151			if (++spent == quota)3152				goto out;3153			break;3154		case EFX_EF10_DRVGEN_EV:3155			efx_ef10_handle_driver_generated_event(channel, &event);3156			break;3157		default:3158			netif_err(efx, hw, efx->net_dev,3159				  "channel %d unknown event type %d"3160				  " (data " EFX_QWORD_FMT ")\n",3161				  channel->channel, ev_code,3162				  EFX_QWORD_VAL(event));3163		}3164	}3165 3166out:3167	channel->eventq_read_ptr = read_ptr;3168	return spent;3169}3170 3171static void efx_ef10_ev_read_ack(struct efx_channel *channel)3172{3173	struct efx_nic *efx = channel->efx;3174	efx_dword_t rptr;3175 3176	if (EFX_EF10_WORKAROUND_35388(efx)) {3177		BUILD_BUG_ON(EFX_MIN_EVQ_SIZE <3178			     (1 << ERF_DD_EVQ_IND_RPTR_WIDTH));3179		BUILD_BUG_ON(EFX_MAX_EVQ_SIZE >3180			     (1 << 2 * ERF_DD_EVQ_IND_RPTR_WIDTH));3181 3182		EFX_POPULATE_DWORD_2(rptr, ERF_DD_EVQ_IND_RPTR_FLAGS,3183				     EFE_DD_EVQ_IND_RPTR_FLAGS_HIGH,3184				     ERF_DD_EVQ_IND_RPTR,3185				     (channel->eventq_read_ptr &3186				      channel->eventq_mask) >>3187				     ERF_DD_EVQ_IND_RPTR_WIDTH);3188		efx_writed_page(efx, &rptr, ER_DD_EVQ_INDIRECT,3189				channel->channel);3190		EFX_POPULATE_DWORD_2(rptr, ERF_DD_EVQ_IND_RPTR_FLAGS,3191				     EFE_DD_EVQ_IND_RPTR_FLAGS_LOW,3192				     ERF_DD_EVQ_IND_RPTR,3193				     channel->eventq_read_ptr &3194				     ((1 << ERF_DD_EVQ_IND_RPTR_WIDTH) - 1));3195		efx_writed_page(efx, &rptr, ER_DD_EVQ_INDIRECT,3196				channel->channel);3197	} else {3198		EFX_POPULATE_DWORD_1(rptr, ERF_DZ_EVQ_RPTR,3199				     channel->eventq_read_ptr &3200				     channel->eventq_mask);3201		efx_writed_page(efx, &rptr, ER_DZ_EVQ_RPTR, channel->channel);3202	}3203}3204 3205static void efx_ef10_ev_test_generate(struct efx_channel *channel)3206{3207	MCDI_DECLARE_BUF(inbuf, MC_CMD_DRIVER_EVENT_IN_LEN);3208	struct efx_nic *efx = channel->efx;3209	efx_qword_t event;3210	int rc;3211 3212	EFX_POPULATE_QWORD_2(event,3213			     ESF_DZ_EV_CODE, EFX_EF10_DRVGEN_EV,3214			     ESF_DZ_EV_DATA, EFX_EF10_TEST);3215 3216	MCDI_SET_DWORD(inbuf, DRIVER_EVENT_IN_EVQ, channel->channel);3217 3218	/* MCDI_SET_QWORD is not appropriate here since EFX_POPULATE_* has3219	 * already swapped the data to little-endian order.3220	 */3221	memcpy(MCDI_PTR(inbuf, DRIVER_EVENT_IN_DATA), &event.u64[0],3222	       sizeof(efx_qword_t));3223 3224	rc = efx_mcdi_rpc(efx, MC_CMD_DRIVER_EVENT, inbuf, sizeof(inbuf),3225			  NULL, 0, NULL);3226	if (rc != 0)3227		goto fail;3228 3229	return;3230 3231fail:3232	WARN_ON(true);3233	netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);3234}3235 3236static void efx_ef10_prepare_flr(struct efx_nic *efx)3237{3238	atomic_set(&efx->active_queues, 0);3239}3240 3241static int efx_ef10_vport_set_mac_address(struct efx_nic *efx)3242{3243	struct efx_ef10_nic_data *nic_data = efx->nic_data;3244	u8 mac_old[ETH_ALEN];3245	int rc, rc2;3246 3247	/* Only reconfigure a PF-created vport */3248	if (is_zero_ether_addr(nic_data->vport_mac))3249		return 0;3250 3251	efx_device_detach_sync(efx);3252	efx_net_stop(efx->net_dev);3253	efx_ef10_filter_table_remove(efx);3254 3255	rc = efx_ef10_vadaptor_free(efx, efx->vport_id);3256	if (rc)3257		goto restore_filters;3258 3259	ether_addr_copy(mac_old, nic_data->vport_mac);3260	rc = efx_ef10_vport_del_mac(efx, efx->vport_id,3261				    nic_data->vport_mac);3262	if (rc)3263		goto restore_vadaptor;3264 3265	rc = efx_ef10_vport_add_mac(efx, efx->vport_id,3266				    efx->net_dev->dev_addr);3267	if (!rc) {3268		ether_addr_copy(nic_data->vport_mac, efx->net_dev->dev_addr);3269	} else {3270		rc2 = efx_ef10_vport_add_mac(efx, efx->vport_id, mac_old);3271		if (rc2) {3272			/* Failed to add original MAC, so clear vport_mac */3273			eth_zero_addr(nic_data->vport_mac);3274			goto reset_nic;3275		}3276	}3277 3278restore_vadaptor:3279	rc2 = efx_ef10_vadaptor_alloc(efx, efx->vport_id);3280	if (rc2)3281		goto reset_nic;3282restore_filters:3283	rc2 = efx_ef10_filter_table_probe(efx);3284	if (rc2)3285		goto reset_nic;3286 3287	rc2 = efx_net_open(efx->net_dev);3288	if (rc2)3289		goto reset_nic;3290 3291	efx_device_attach_if_not_resetting(efx);3292 3293	return rc;3294 3295reset_nic:3296	netif_err(efx, drv, efx->net_dev,3297		  "Failed to restore when changing MAC address - scheduling reset\n");3298	efx_schedule_reset(efx, RESET_TYPE_DATAPATH);3299 3300	return rc ? rc : rc2;3301}3302 3303static int efx_ef10_set_mac_address(struct efx_nic *efx)3304{3305	MCDI_DECLARE_BUF(inbuf, MC_CMD_VADAPTOR_SET_MAC_IN_LEN);3306	bool was_enabled = efx->port_enabled;3307	int rc;3308 3309#ifdef CONFIG_SFC_SRIOV3310	/* If this function is a VF and we have access to the parent PF,3311	 * then use the PF control path to attempt to change the VF MAC address.3312	 */3313	if (efx->pci_dev->is_virtfn && efx->pci_dev->physfn) {3314		struct efx_nic *efx_pf = pci_get_drvdata(efx->pci_dev->physfn);3315		struct efx_ef10_nic_data *nic_data = efx->nic_data;3316		u8 mac[ETH_ALEN];3317 3318		/* net_dev->dev_addr can be zeroed by efx_net_stop in3319		 * efx_ef10_sriov_set_vf_mac, so pass in a copy.3320		 */3321		ether_addr_copy(mac, efx->net_dev->dev_addr);3322 3323		rc = efx_ef10_sriov_set_vf_mac(efx_pf, nic_data->vf_index, mac);3324		if (!rc)3325			return 0;3326 3327		netif_dbg(efx, drv, efx->net_dev,3328			  "Updating VF mac via PF failed (%d), setting directly\n",3329			  rc);3330	}3331#endif3332 3333	efx_device_detach_sync(efx);3334	efx_net_stop(efx->net_dev);3335 3336	mutex_lock(&efx->mac_lock);3337	efx_ef10_filter_table_remove(efx);3338 3339	ether_addr_copy(MCDI_PTR(inbuf, VADAPTOR_SET_MAC_IN_MACADDR),3340			efx->net_dev->dev_addr);3341	MCDI_SET_DWORD(inbuf, VADAPTOR_SET_MAC_IN_UPSTREAM_PORT_ID,3342		       efx->vport_id);3343	rc = efx_mcdi_rpc_quiet(efx, MC_CMD_VADAPTOR_SET_MAC, inbuf,3344				sizeof(inbuf), NULL, 0, NULL);3345 3346	efx_ef10_filter_table_probe(efx);3347	mutex_unlock(&efx->mac_lock);3348 3349	if (was_enabled)3350		efx_net_open(efx->net_dev);3351	efx_device_attach_if_not_resetting(efx);3352 3353	if (rc == -EPERM) {3354		netif_err(efx, drv, efx->net_dev,3355			  "Cannot change MAC address; use sfboot to enable"3356			  " mac-spoofing on this interface\n");3357	} else if (rc == -ENOSYS && !efx_ef10_is_vf(efx)) {3358		/* If the active MCFW does not support MC_CMD_VADAPTOR_SET_MAC3359		 * fall-back to the method of changing the MAC address on the3360		 * vport.  This only applies to PFs because such versions of3361		 * MCFW do not support VFs.3362		 */3363		rc = efx_ef10_vport_set_mac_address(efx);3364	} else if (rc) {3365		efx_mcdi_display_error(efx, MC_CMD_VADAPTOR_SET_MAC,3366				       sizeof(inbuf), NULL, 0, rc);3367	}3368 3369	return rc;3370}3371 3372static int efx_ef10_mac_reconfigure(struct efx_nic *efx, bool mtu_only)3373{3374	WARN_ON(!mutex_is_locked(&efx->mac_lock));3375 3376	efx_mcdi_filter_sync_rx_mode(efx);3377 3378	if (mtu_only && efx_has_cap(efx, SET_MAC_ENHANCED))3379		return efx_mcdi_set_mtu(efx);3380	return efx_mcdi_set_mac(efx);3381}3382 3383static int efx_ef10_start_bist(struct efx_nic *efx, u32 bist_type)3384{3385	MCDI_DECLARE_BUF(inbuf, MC_CMD_START_BIST_IN_LEN);3386 3387	MCDI_SET_DWORD(inbuf, START_BIST_IN_TYPE, bist_type);3388	return efx_mcdi_rpc(efx, MC_CMD_START_BIST, inbuf, sizeof(inbuf),3389			    NULL, 0, NULL);3390}3391 3392/* MC BISTs follow a different poll mechanism to phy BISTs.3393 * The BIST is done in the poll handler on the MC, and the MCDI command3394 * will block until the BIST is done.3395 */3396static int efx_ef10_poll_bist(struct efx_nic *efx)3397{3398	int rc;3399	MCDI_DECLARE_BUF(outbuf, MC_CMD_POLL_BIST_OUT_LEN);3400	size_t outlen;3401	u32 result;3402 3403	rc = efx_mcdi_rpc(efx, MC_CMD_POLL_BIST, NULL, 0,3404			   outbuf, sizeof(outbuf), &outlen);3405	if (rc != 0)3406		return rc;3407 3408	if (outlen < MC_CMD_POLL_BIST_OUT_LEN)3409		return -EIO;3410 3411	result = MCDI_DWORD(outbuf, POLL_BIST_OUT_RESULT);3412	switch (result) {3413	case MC_CMD_POLL_BIST_PASSED:3414		netif_dbg(efx, hw, efx->net_dev, "BIST passed.\n");3415		return 0;3416	case MC_CMD_POLL_BIST_TIMEOUT:3417		netif_err(efx, hw, efx->net_dev, "BIST timed out\n");3418		return -EIO;3419	case MC_CMD_POLL_BIST_FAILED:3420		netif_err(efx, hw, efx->net_dev, "BIST failed.\n");3421		return -EIO;3422	default:3423		netif_err(efx, hw, efx->net_dev,3424			  "BIST returned unknown result %u", result);3425		return -EIO;3426	}3427}3428 3429static int efx_ef10_run_bist(struct efx_nic *efx, u32 bist_type)3430{3431	int rc;3432 3433	netif_dbg(efx, drv, efx->net_dev, "starting BIST type %u\n", bist_type);3434 3435	rc = efx_ef10_start_bist(efx, bist_type);3436	if (rc != 0)3437		return rc;3438 3439	return efx_ef10_poll_bist(efx);3440}3441 3442static int3443efx_ef10_test_chip(struct efx_nic *efx, struct efx_self_tests *tests)3444{3445	int rc, rc2;3446 3447	efx_reset_down(efx, RESET_TYPE_WORLD);3448 3449	rc = efx_mcdi_rpc(efx, MC_CMD_ENABLE_OFFLINE_BIST,3450			  NULL, 0, NULL, 0, NULL);3451	if (rc != 0)3452		goto out;3453 3454	tests->memory = efx_ef10_run_bist(efx, MC_CMD_MC_MEM_BIST) ? -1 : 1;3455	tests->registers = efx_ef10_run_bist(efx, MC_CMD_REG_BIST) ? -1 : 1;3456 3457	rc = efx_mcdi_reset(efx, RESET_TYPE_WORLD);3458 3459out:3460	if (rc == -EPERM)3461		rc = 0;3462	rc2 = efx_reset_up(efx, RESET_TYPE_WORLD, rc == 0);3463	return rc ? rc : rc2;3464}3465 3466#ifdef CONFIG_SFC_MTD3467 3468struct efx_ef10_nvram_type_info {3469	u16 type, type_mask;3470	u8 port;3471	const char *name;3472};3473 3474static const struct efx_ef10_nvram_type_info efx_ef10_nvram_types[] = {3475	{ NVRAM_PARTITION_TYPE_MC_FIRMWARE,	   0,    0, "sfc_mcfw" },3476	{ NVRAM_PARTITION_TYPE_MC_FIRMWARE_BACKUP, 0,    0, "sfc_mcfw_backup" },3477	{ NVRAM_PARTITION_TYPE_EXPANSION_ROM,	   0,    0, "sfc_exp_rom" },3478	{ NVRAM_PARTITION_TYPE_STATIC_CONFIG,	   0,    0, "sfc_static_cfg" },3479	{ NVRAM_PARTITION_TYPE_DYNAMIC_CONFIG,	   0,    0, "sfc_dynamic_cfg" },3480	{ NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT0, 0,   0, "sfc_exp_rom_cfg" },3481	{ NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT1, 0,   1, "sfc_exp_rom_cfg" },3482	{ NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT2, 0,   2, "sfc_exp_rom_cfg" },3483	{ NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT3, 0,   3, "sfc_exp_rom_cfg" },3484	{ NVRAM_PARTITION_TYPE_LICENSE,		   0,    0, "sfc_license" },3485	{ NVRAM_PARTITION_TYPE_PHY_MIN,		   0xff, 0, "sfc_phy_fw" },3486	{ NVRAM_PARTITION_TYPE_MUM_FIRMWARE,	   0,    0, "sfc_mumfw" },3487	{ NVRAM_PARTITION_TYPE_EXPANSION_UEFI,	   0,    0, "sfc_uefi" },3488	{ NVRAM_PARTITION_TYPE_DYNCONFIG_DEFAULTS, 0,    0, "sfc_dynamic_cfg_dflt" },3489	{ NVRAM_PARTITION_TYPE_ROMCONFIG_DEFAULTS, 0,    0, "sfc_exp_rom_cfg_dflt" },3490	{ NVRAM_PARTITION_TYPE_STATUS,		   0,    0, "sfc_status" },3491	{ NVRAM_PARTITION_TYPE_BUNDLE,		   0,    0, "sfc_bundle" },3492	{ NVRAM_PARTITION_TYPE_BUNDLE_METADATA,	   0,    0, "sfc_bundle_metadata" },3493};3494#define EF10_NVRAM_PARTITION_COUNT	ARRAY_SIZE(efx_ef10_nvram_types)3495 3496static int efx_ef10_mtd_probe_partition(struct efx_nic *efx,3497					struct efx_mcdi_mtd_partition *part,3498					unsigned int type,3499					unsigned long *found)3500{3501	MCDI_DECLARE_BUF(inbuf, MC_CMD_NVRAM_METADATA_IN_LEN);3502	MCDI_DECLARE_BUF(outbuf, MC_CMD_NVRAM_METADATA_OUT_LENMAX);3503	const struct efx_ef10_nvram_type_info *info;3504	size_t size, erase_size, outlen;3505	int type_idx = 0;3506	bool protected;3507	int rc;3508 3509	for (type_idx = 0; ; type_idx++) {3510		if (type_idx == EF10_NVRAM_PARTITION_COUNT)3511			return -ENODEV;3512		info = efx_ef10_nvram_types + type_idx;3513		if ((type & ~info->type_mask) == info->type)3514			break;3515	}3516	if (info->port != efx_port_num(efx))3517		return -ENODEV;3518 3519	rc = efx_mcdi_nvram_info(efx, type, &size, &erase_size, &protected);3520	if (rc)3521		return rc;3522	if (protected &&3523	    (type != NVRAM_PARTITION_TYPE_DYNCONFIG_DEFAULTS &&3524	     type != NVRAM_PARTITION_TYPE_ROMCONFIG_DEFAULTS))3525		/* Hide protected partitions that don't provide defaults. */3526		return -ENODEV;3527 3528	if (protected)3529		/* Protected partitions are read only. */3530		erase_size = 0;3531 3532	/* If we've already exposed a partition of this type, hide this3533	 * duplicate.  All operations on MTDs are keyed by the type anyway,3534	 * so we can't act on the duplicate.3535	 */3536	if (__test_and_set_bit(type_idx, found))3537		return -EEXIST;3538 3539	part->nvram_type = type;3540 3541	MCDI_SET_DWORD(inbuf, NVRAM_METADATA_IN_TYPE, type);3542	rc = efx_mcdi_rpc(efx, MC_CMD_NVRAM_METADATA, inbuf, sizeof(inbuf),3543			  outbuf, sizeof(outbuf), &outlen);3544	if (rc)3545		return rc;3546	if (outlen < MC_CMD_NVRAM_METADATA_OUT_LENMIN)3547		return -EIO;3548	if (MCDI_DWORD(outbuf, NVRAM_METADATA_OUT_FLAGS) &3549	    (1 << MC_CMD_NVRAM_METADATA_OUT_SUBTYPE_VALID_LBN))3550		part->fw_subtype = MCDI_DWORD(outbuf,3551					      NVRAM_METADATA_OUT_SUBTYPE);3552 3553	part->common.dev_type_name = "EF10 NVRAM manager";3554	part->common.type_name = info->name;3555 3556	part->common.mtd.type = MTD_NORFLASH;3557	part->common.mtd.flags = MTD_CAP_NORFLASH;3558	part->common.mtd.size = size;3559	part->common.mtd.erasesize = erase_size;3560	/* sfc_status is read-only */3561	if (!erase_size)3562		part->common.mtd.flags |= MTD_NO_ERASE;3563 3564	return 0;3565}3566 3567static int efx_ef10_mtd_probe(struct efx_nic *efx)3568{3569	MCDI_DECLARE_BUF(outbuf, MC_CMD_NVRAM_PARTITIONS_OUT_LENMAX);3570	DECLARE_BITMAP(found, EF10_NVRAM_PARTITION_COUNT) = { 0 };3571	struct efx_mcdi_mtd_partition *parts;3572	size_t outlen, n_parts_total, i, n_parts;3573	unsigned int type;3574	int rc;3575 3576	ASSERT_RTNL();3577 3578	BUILD_BUG_ON(MC_CMD_NVRAM_PARTITIONS_IN_LEN != 0);3579	rc = efx_mcdi_rpc(efx, MC_CMD_NVRAM_PARTITIONS, NULL, 0,3580			  outbuf, sizeof(outbuf), &outlen);3581	if (rc)3582		return rc;3583	if (outlen < MC_CMD_NVRAM_PARTITIONS_OUT_LENMIN)3584		return -EIO;3585 3586	n_parts_total = MCDI_DWORD(outbuf, NVRAM_PARTITIONS_OUT_NUM_PARTITIONS);3587	if (n_parts_total >3588	    MCDI_VAR_ARRAY_LEN(outlen, NVRAM_PARTITIONS_OUT_TYPE_ID))3589		return -EIO;3590 3591	parts = kcalloc(n_parts_total, sizeof(*parts), GFP_KERNEL);3592	if (!parts)3593		return -ENOMEM;3594 3595	n_parts = 0;3596	for (i = 0; i < n_parts_total; i++) {3597		type = MCDI_ARRAY_DWORD(outbuf, NVRAM_PARTITIONS_OUT_TYPE_ID,3598					i);3599		rc = efx_ef10_mtd_probe_partition(efx, &parts[n_parts], type,3600						  found);3601		if (rc == -EEXIST || rc == -ENODEV)3602			continue;3603		if (rc)3604			goto fail;3605		n_parts++;3606	}3607 3608	if (!n_parts) {3609		kfree(parts);3610		return 0;3611	}3612 3613	rc = efx_mtd_add(efx, &parts[0].common, n_parts, sizeof(*parts));3614fail:3615	if (rc)3616		kfree(parts);3617	return rc;3618}3619 3620#endif /* CONFIG_SFC_MTD */3621 3622static void efx_ef10_ptp_write_host_time(struct efx_nic *efx, u32 host_time)3623{3624	_efx_writed(efx, cpu_to_le32(host_time), ER_DZ_MC_DB_LWRD);3625}3626 3627static void efx_ef10_ptp_write_host_time_vf(struct efx_nic *efx,3628					    u32 host_time) {}3629 3630static int efx_ef10_rx_enable_timestamping(struct efx_channel *channel,3631					   bool temp)3632{3633	MCDI_DECLARE_BUF(inbuf, MC_CMD_PTP_IN_TIME_EVENT_SUBSCRIBE_LEN);3634	int rc;3635 3636	if (channel->sync_events_state == SYNC_EVENTS_REQUESTED ||3637	    channel->sync_events_state == SYNC_EVENTS_VALID ||3638	    (temp && channel->sync_events_state == SYNC_EVENTS_DISABLED))3639		return 0;3640	channel->sync_events_state = SYNC_EVENTS_REQUESTED;3641 3642	MCDI_SET_DWORD(inbuf, PTP_IN_OP, MC_CMD_PTP_OP_TIME_EVENT_SUBSCRIBE);3643	MCDI_SET_DWORD(inbuf, PTP_IN_PERIPH_ID, 0);3644	MCDI_SET_DWORD(inbuf, PTP_IN_TIME_EVENT_SUBSCRIBE_QUEUE,3645		       channel->channel);3646 3647	rc = efx_mcdi_rpc(channel->efx, MC_CMD_PTP,3648			  inbuf, sizeof(inbuf), NULL, 0, NULL);3649 3650	if (rc != 0)3651		channel->sync_events_state = temp ? SYNC_EVENTS_QUIESCENT :3652						    SYNC_EVENTS_DISABLED;3653 3654	return rc;3655}3656 3657static int efx_ef10_rx_disable_timestamping(struct efx_channel *channel,3658					    bool temp)3659{3660	MCDI_DECLARE_BUF(inbuf, MC_CMD_PTP_IN_TIME_EVENT_UNSUBSCRIBE_LEN);3661	int rc;3662 3663	if (channel->sync_events_state == SYNC_EVENTS_DISABLED ||3664	    (temp && channel->sync_events_state == SYNC_EVENTS_QUIESCENT))3665		return 0;3666	if (channel->sync_events_state == SYNC_EVENTS_QUIESCENT) {3667		channel->sync_events_state = SYNC_EVENTS_DISABLED;3668		return 0;3669	}3670	channel->sync_events_state = temp ? SYNC_EVENTS_QUIESCENT :3671					    SYNC_EVENTS_DISABLED;3672 3673	MCDI_SET_DWORD(inbuf, PTP_IN_OP, MC_CMD_PTP_OP_TIME_EVENT_UNSUBSCRIBE);3674	MCDI_SET_DWORD(inbuf, PTP_IN_PERIPH_ID, 0);3675	MCDI_SET_DWORD(inbuf, PTP_IN_TIME_EVENT_UNSUBSCRIBE_CONTROL,3676		       MC_CMD_PTP_IN_TIME_EVENT_UNSUBSCRIBE_SINGLE);3677	MCDI_SET_DWORD(inbuf, PTP_IN_TIME_EVENT_UNSUBSCRIBE_QUEUE,3678		       channel->channel);3679 3680	rc = efx_mcdi_rpc(channel->efx, MC_CMD_PTP,3681			  inbuf, sizeof(inbuf), NULL, 0, NULL);3682 3683	return rc;3684}3685 3686static int efx_ef10_ptp_set_ts_sync_events(struct efx_nic *efx, bool en,3687					   bool temp)3688{3689	int (*set)(struct efx_channel *channel, bool temp);3690	struct efx_channel *channel;3691 3692	set = en ?3693	      efx_ef10_rx_enable_timestamping :3694	      efx_ef10_rx_disable_timestamping;3695 3696	channel = efx_ptp_channel(efx);3697	if (channel) {3698		int rc = set(channel, temp);3699		if (en && rc != 0) {3700			efx_ef10_ptp_set_ts_sync_events(efx, false, temp);3701			return rc;3702		}3703	}3704 3705	return 0;3706}3707 3708static int efx_ef10_ptp_set_ts_config_vf(struct efx_nic *efx,3709					 struct kernel_hwtstamp_config *init)3710{3711	return -EOPNOTSUPP;3712}3713 3714static int efx_ef10_ptp_set_ts_config(struct efx_nic *efx,3715				      struct kernel_hwtstamp_config *init)3716{3717	int rc;3718 3719	switch (init->rx_filter) {3720	case HWTSTAMP_FILTER_NONE:3721		efx_ef10_ptp_set_ts_sync_events(efx, false, false);3722		/* if TX timestamping is still requested then leave PTP on */3723		return efx_ptp_change_mode(efx,3724					   init->tx_type != HWTSTAMP_TX_OFF, 0);3725	case HWTSTAMP_FILTER_ALL:3726	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:3727	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:3728	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:3729	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:3730	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:3731	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:3732	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:3733	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:3734	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:3735	case HWTSTAMP_FILTER_PTP_V2_EVENT:3736	case HWTSTAMP_FILTER_PTP_V2_SYNC:3737	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:3738	case HWTSTAMP_FILTER_NTP_ALL:3739		init->rx_filter = HWTSTAMP_FILTER_ALL;3740		rc = efx_ptp_change_mode(efx, true, 0);3741		if (!rc)3742			rc = efx_ef10_ptp_set_ts_sync_events(efx, true, false);3743		if (rc)3744			efx_ptp_change_mode(efx, false, 0);3745		return rc;3746	default:3747		return -ERANGE;3748	}3749}3750 3751static int efx_ef10_get_phys_port_id(struct efx_nic *efx,3752				     struct netdev_phys_item_id *ppid)3753{3754	struct efx_ef10_nic_data *nic_data = efx->nic_data;3755 3756	if (!is_valid_ether_addr(nic_data->port_id))3757		return -EOPNOTSUPP;3758 3759	ppid->id_len = ETH_ALEN;3760	memcpy(ppid->id, nic_data->port_id, ppid->id_len);3761 3762	return 0;3763}3764 3765static int efx_ef10_vlan_rx_add_vid(struct efx_nic *efx, __be16 proto, u16 vid)3766{3767	if (proto != htons(ETH_P_8021Q))3768		return -EINVAL;3769 3770	return efx_ef10_add_vlan(efx, vid);3771}3772 3773static int efx_ef10_vlan_rx_kill_vid(struct efx_nic *efx, __be16 proto, u16 vid)3774{3775	if (proto != htons(ETH_P_8021Q))3776		return -EINVAL;3777 3778	return efx_ef10_del_vlan(efx, vid);3779}3780 3781/* We rely on the MCDI wiping out our TX rings if it made any changes to the3782 * ports table, ensuring that any TSO descriptors that were made on a now-3783 * removed tunnel port will be blown away and won't break things when we try3784 * to transmit them using the new ports table.3785 */3786static int efx_ef10_set_udp_tnl_ports(struct efx_nic *efx, bool unloading)3787{3788	struct efx_ef10_nic_data *nic_data = efx->nic_data;3789	MCDI_DECLARE_BUF(inbuf, MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_LENMAX);3790	MCDI_DECLARE_BUF(outbuf, MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_OUT_LEN);3791	bool will_reset = false;3792	size_t num_entries = 0;3793	size_t inlen, outlen;3794	size_t i;3795	int rc;3796	efx_dword_t flags_and_num_entries;3797 3798	WARN_ON(!mutex_is_locked(&nic_data->udp_tunnels_lock));3799 3800	nic_data->udp_tunnels_dirty = false;3801 3802	if (!(nic_data->datapath_caps &3803	    (1 << MC_CMD_GET_CAPABILITIES_OUT_VXLAN_NVGRE_LBN))) {3804		efx_device_attach_if_not_resetting(efx);3805		return 0;3806	}3807 3808	BUILD_BUG_ON(ARRAY_SIZE(nic_data->udp_tunnels) >3809		     MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_ENTRIES_MAXNUM);3810 3811	for (i = 0; i < ARRAY_SIZE(nic_data->udp_tunnels); ++i) {3812		if (nic_data->udp_tunnels[i].type !=3813		    TUNNEL_ENCAP_UDP_PORT_ENTRY_INVALID) {3814			efx_dword_t entry;3815 3816			EFX_POPULATE_DWORD_2(entry,3817				TUNNEL_ENCAP_UDP_PORT_ENTRY_UDP_PORT,3818					ntohs(nic_data->udp_tunnels[i].port),3819				TUNNEL_ENCAP_UDP_PORT_ENTRY_PROTOCOL,3820					nic_data->udp_tunnels[i].type);3821			*_MCDI_ARRAY_DWORD(inbuf,3822				SET_TUNNEL_ENCAP_UDP_PORTS_IN_ENTRIES,3823				num_entries++) = entry;3824		}3825	}3826 3827	BUILD_BUG_ON((MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_NUM_ENTRIES_OFST -3828		      MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_FLAGS_OFST) * 8 !=3829		     EFX_WORD_1_LBN);3830	BUILD_BUG_ON(MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_NUM_ENTRIES_LEN * 8 !=3831		     EFX_WORD_1_WIDTH);3832	EFX_POPULATE_DWORD_2(flags_and_num_entries,3833			     MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_UNLOADING,3834				!!unloading,3835			     EFX_WORD_1, num_entries);3836	*_MCDI_DWORD(inbuf, SET_TUNNEL_ENCAP_UDP_PORTS_IN_FLAGS) =3837		flags_and_num_entries;3838 3839	inlen = MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_LEN(num_entries);3840 3841	rc = efx_mcdi_rpc_quiet(efx, MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS,3842				inbuf, inlen, outbuf, sizeof(outbuf), &outlen);3843	if (rc == -EIO) {3844		/* Most likely the MC rebooted due to another function also3845		 * setting its tunnel port list. Mark the tunnel port list as3846		 * dirty, so it will be pushed upon coming up from the reboot.3847		 */3848		nic_data->udp_tunnels_dirty = true;3849		return 0;3850	}3851 3852	if (rc) {3853		/* expected not available on unprivileged functions */3854		if (rc != -EPERM)3855			netif_warn(efx, drv, efx->net_dev,3856				   "Unable to set UDP tunnel ports; rc=%d.\n", rc);3857	} else if (MCDI_DWORD(outbuf, SET_TUNNEL_ENCAP_UDP_PORTS_OUT_FLAGS) &3858		   (1 << MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_OUT_RESETTING_LBN)) {3859		netif_info(efx, drv, efx->net_dev,3860			   "Rebooting MC due to UDP tunnel port list change\n");3861		will_reset = true;3862		if (unloading)3863			/* Delay for the MC reset to complete. This will make3864			 * unloading other functions a bit smoother. This is a3865			 * race, but the other unload will work whichever way3866			 * it goes, this just avoids an unnecessary error3867			 * message.3868			 */3869			msleep(100);3870	}3871	if (!will_reset && !unloading) {3872		/* The caller will have detached, relying on the MC reset to3873		 * trigger a re-attach.  Since there won't be an MC reset, we3874		 * have to do the attach ourselves.3875		 */3876		efx_device_attach_if_not_resetting(efx);3877	}3878 3879	return rc;3880}3881 3882static int efx_ef10_udp_tnl_push_ports(struct efx_nic *efx)3883{3884	struct efx_ef10_nic_data *nic_data = efx->nic_data;3885	int rc = 0;3886 3887	mutex_lock(&nic_data->udp_tunnels_lock);3888	if (nic_data->udp_tunnels_dirty) {3889		/* Make sure all TX are stopped while we modify the table, else3890		 * we might race against an efx_features_check().3891		 */3892		efx_device_detach_sync(efx);3893		rc = efx_ef10_set_udp_tnl_ports(efx, false);3894	}3895	mutex_unlock(&nic_data->udp_tunnels_lock);3896	return rc;3897}3898 3899static int efx_ef10_udp_tnl_set_port(struct net_device *dev,3900				     unsigned int table, unsigned int entry,3901				     struct udp_tunnel_info *ti)3902{3903	struct efx_nic *efx = efx_netdev_priv(dev);3904	struct efx_ef10_nic_data *nic_data;3905	int efx_tunnel_type, rc;3906 3907	if (ti->type == UDP_TUNNEL_TYPE_VXLAN)3908		efx_tunnel_type = TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN;3909	else3910		efx_tunnel_type = TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE;3911 3912	nic_data = efx->nic_data;3913	if (!(nic_data->datapath_caps &3914	      (1 << MC_CMD_GET_CAPABILITIES_OUT_VXLAN_NVGRE_LBN)))3915		return -EOPNOTSUPP;3916 3917	mutex_lock(&nic_data->udp_tunnels_lock);3918	/* Make sure all TX are stopped while we add to the table, else we3919	 * might race against an efx_features_check().3920	 */3921	efx_device_detach_sync(efx);3922	nic_data->udp_tunnels[entry].type = efx_tunnel_type;3923	nic_data->udp_tunnels[entry].port = ti->port;3924	rc = efx_ef10_set_udp_tnl_ports(efx, false);3925	mutex_unlock(&nic_data->udp_tunnels_lock);3926 3927	return rc;3928}3929 3930/* Called under the TX lock with the TX queue running, hence no-one can be3931 * in the middle of updating the UDP tunnels table.  However, they could3932 * have tried and failed the MCDI, in which case they'll have set the dirty3933 * flag before dropping their locks.3934 */3935static bool efx_ef10_udp_tnl_has_port(struct efx_nic *efx, __be16 port)3936{3937	struct efx_ef10_nic_data *nic_data = efx->nic_data;3938	size_t i;3939 3940	if (!(nic_data->datapath_caps &3941	      (1 << MC_CMD_GET_CAPABILITIES_OUT_VXLAN_NVGRE_LBN)))3942		return false;3943 3944	if (nic_data->udp_tunnels_dirty)3945		/* SW table may not match HW state, so just assume we can't3946		 * use any UDP tunnel offloads.3947		 */3948		return false;3949 3950	for (i = 0; i < ARRAY_SIZE(nic_data->udp_tunnels); ++i)3951		if (nic_data->udp_tunnels[i].type !=3952		    TUNNEL_ENCAP_UDP_PORT_ENTRY_INVALID &&3953		    nic_data->udp_tunnels[i].port == port)3954			return true;3955 3956	return false;3957}3958 3959static int efx_ef10_udp_tnl_unset_port(struct net_device *dev,3960				       unsigned int table, unsigned int entry,3961				       struct udp_tunnel_info *ti)3962{3963	struct efx_nic *efx = efx_netdev_priv(dev);3964	struct efx_ef10_nic_data *nic_data;3965	int rc;3966 3967	nic_data = efx->nic_data;3968 3969	mutex_lock(&nic_data->udp_tunnels_lock);3970	/* Make sure all TX are stopped while we remove from the table, else we3971	 * might race against an efx_features_check().3972	 */3973	efx_device_detach_sync(efx);3974	nic_data->udp_tunnels[entry].type = TUNNEL_ENCAP_UDP_PORT_ENTRY_INVALID;3975	nic_data->udp_tunnels[entry].port = 0;3976	rc = efx_ef10_set_udp_tnl_ports(efx, false);3977	mutex_unlock(&nic_data->udp_tunnels_lock);3978 3979	return rc;3980}3981 3982static const struct udp_tunnel_nic_info efx_ef10_udp_tunnels = {3983	.set_port	= efx_ef10_udp_tnl_set_port,3984	.unset_port	= efx_ef10_udp_tnl_unset_port,3985	.flags          = UDP_TUNNEL_NIC_INFO_MAY_SLEEP,3986	.tables         = {3987		{3988			.n_entries = 16,3989			.tunnel_types = UDP_TUNNEL_TYPE_VXLAN |3990					UDP_TUNNEL_TYPE_GENEVE,3991		},3992	},3993};3994 3995/* EF10 may have multiple datapath firmware variants within a3996 * single version.  Report which variants are running.3997 */3998static size_t efx_ef10_print_additional_fwver(struct efx_nic *efx, char *buf,3999					      size_t len)4000{4001	struct efx_ef10_nic_data *nic_data = efx->nic_data;4002 4003	return scnprintf(buf, len, " rx%x tx%x",4004			 nic_data->rx_dpcpu_fw_id,4005			 nic_data->tx_dpcpu_fw_id);4006}4007 4008static unsigned int ef10_check_caps(const struct efx_nic *efx,4009				    u8 flag,4010				    u32 offset)4011{4012	const struct efx_ef10_nic_data *nic_data = efx->nic_data;4013 4014	switch (offset) {4015	case(MC_CMD_GET_CAPABILITIES_V4_OUT_FLAGS1_OFST):4016		return nic_data->datapath_caps & BIT_ULL(flag);4017	case(MC_CMD_GET_CAPABILITIES_V4_OUT_FLAGS2_OFST):4018		return nic_data->datapath_caps2 & BIT_ULL(flag);4019	default:4020		return 0;4021	}4022}4023 4024static unsigned int efx_ef10_recycle_ring_size(const struct efx_nic *efx)4025{4026	unsigned int ret = EFX_RECYCLE_RING_SIZE_10G;4027 4028	/* There is no difference between PFs and VFs. The side is based on4029	 * the maximum link speed of a given NIC.4030	 */4031	switch (efx->pci_dev->device & 0xfff) {4032	case 0x0903:	/* Farmingdale can do up to 10G */4033		break;4034	case 0x0923:	/* Greenport can do up to 40G */4035	case 0x0a03:	/* Medford can do up to 40G */4036		ret *= 4;4037		break;4038	default:	/* Medford2 can do up to 100G */4039		ret *= 10;4040	}4041 4042	if (IS_ENABLED(CONFIG_PPC64))4043		ret *= 4;4044 4045	return ret;4046}4047 4048#define EF10_OFFLOAD_FEATURES		\4049	(NETIF_F_IP_CSUM |		\4050	 NETIF_F_HW_VLAN_CTAG_FILTER |	\4051	 NETIF_F_IPV6_CSUM |		\4052	 NETIF_F_RXHASH |		\4053	 NETIF_F_NTUPLE |		\4054	 NETIF_F_SG |			\4055	 NETIF_F_RXCSUM |		\4056	 NETIF_F_RXALL)4057 4058const struct efx_nic_type efx_hunt_a0_vf_nic_type = {4059	.is_vf = true,4060	.mem_bar = efx_ef10_vf_mem_bar,4061	.mem_map_size = efx_ef10_mem_map_size,4062	.probe = efx_ef10_probe_vf,4063	.remove = efx_ef10_remove,4064	.dimension_resources = efx_ef10_dimension_resources,4065	.init = efx_ef10_init_nic,4066	.fini = efx_ef10_fini_nic,4067	.map_reset_reason = efx_ef10_map_reset_reason,4068	.map_reset_flags = efx_ef10_map_reset_flags,4069	.reset = efx_ef10_reset,4070	.probe_port = efx_mcdi_port_probe,4071	.remove_port = efx_mcdi_port_remove,4072	.fini_dmaq = efx_fini_dmaq,4073	.prepare_flr = efx_ef10_prepare_flr,4074	.finish_flr = efx_port_dummy_op_void,4075	.describe_stats = efx_ef10_describe_stats,4076	.update_stats = efx_ef10_update_stats_vf,4077	.update_stats_atomic = efx_ef10_update_stats_atomic_vf,4078	.start_stats = efx_port_dummy_op_void,4079	.pull_stats = efx_port_dummy_op_void,4080	.stop_stats = efx_port_dummy_op_void,4081	.push_irq_moderation = efx_ef10_push_irq_moderation,4082	.reconfigure_mac = efx_ef10_mac_reconfigure,4083	.check_mac_fault = efx_mcdi_mac_check_fault,4084	.reconfigure_port = efx_mcdi_port_reconfigure,4085	.get_wol = efx_ef10_get_wol_vf,4086	.set_wol = efx_ef10_set_wol_vf,4087	.resume_wol = efx_port_dummy_op_void,4088	.mcdi_request = efx_ef10_mcdi_request,4089	.mcdi_poll_response = efx_ef10_mcdi_poll_response,4090	.mcdi_read_response = efx_ef10_mcdi_read_response,4091	.mcdi_poll_reboot = efx_ef10_mcdi_poll_reboot,4092	.mcdi_reboot_detected = efx_ef10_mcdi_reboot_detected,4093	.irq_enable_master = efx_port_dummy_op_void,4094	.irq_test_generate = efx_ef10_irq_test_generate,4095	.irq_disable_non_ev = efx_port_dummy_op_void,4096	.irq_handle_msi = efx_ef10_msi_interrupt,4097	.irq_handle_legacy = efx_ef10_legacy_interrupt,4098	.tx_probe = efx_ef10_tx_probe,4099	.tx_init = efx_ef10_tx_init,4100	.tx_remove = efx_mcdi_tx_remove,4101	.tx_write = efx_ef10_tx_write,4102	.tx_limit_len = efx_ef10_tx_limit_len,4103	.tx_enqueue = __efx_enqueue_skb,4104	.rx_push_rss_config = efx_mcdi_vf_rx_push_rss_config,4105	.rx_pull_rss_config = efx_mcdi_rx_pull_rss_config,4106	.rx_probe = efx_mcdi_rx_probe,4107	.rx_init = efx_mcdi_rx_init,4108	.rx_remove = efx_mcdi_rx_remove,4109	.rx_write = efx_ef10_rx_write,4110	.rx_defer_refill = efx_ef10_rx_defer_refill,4111	.rx_packet = __efx_rx_packet,4112	.ev_probe = efx_mcdi_ev_probe,4113	.ev_init = efx_ef10_ev_init,4114	.ev_fini = efx_mcdi_ev_fini,4115	.ev_remove = efx_mcdi_ev_remove,4116	.ev_process = efx_ef10_ev_process,4117	.ev_read_ack = efx_ef10_ev_read_ack,4118	.ev_test_generate = efx_ef10_ev_test_generate,4119	.filter_table_probe = efx_ef10_filter_table_probe,4120	.filter_table_restore = efx_mcdi_filter_table_restore,4121	.filter_table_remove = efx_ef10_filter_table_remove,4122	.filter_update_rx_scatter = efx_mcdi_update_rx_scatter,4123	.filter_insert = efx_mcdi_filter_insert,4124	.filter_remove_safe = efx_mcdi_filter_remove_safe,4125	.filter_get_safe = efx_mcdi_filter_get_safe,4126	.filter_clear_rx = efx_mcdi_filter_clear_rx,4127	.filter_count_rx_used = efx_mcdi_filter_count_rx_used,4128	.filter_get_rx_id_limit = efx_mcdi_filter_get_rx_id_limit,4129	.filter_get_rx_ids = efx_mcdi_filter_get_rx_ids,4130#ifdef CONFIG_RFS_ACCEL4131	.filter_rfs_expire_one = efx_mcdi_filter_rfs_expire_one,4132#endif4133#ifdef CONFIG_SFC_MTD4134	.mtd_probe = efx_port_dummy_op_int,4135#endif4136	.ptp_write_host_time = efx_ef10_ptp_write_host_time_vf,4137	.ptp_set_ts_config = efx_ef10_ptp_set_ts_config_vf,4138	.vlan_rx_add_vid = efx_ef10_vlan_rx_add_vid,4139	.vlan_rx_kill_vid = efx_ef10_vlan_rx_kill_vid,4140#ifdef CONFIG_SFC_SRIOV4141	.vswitching_probe = efx_ef10_vswitching_probe_vf,4142	.vswitching_restore = efx_ef10_vswitching_restore_vf,4143	.vswitching_remove = efx_ef10_vswitching_remove_vf,4144#endif4145	.get_mac_address = efx_ef10_get_mac_address_vf,4146	.set_mac_address = efx_ef10_set_mac_address,4147 4148	.get_phys_port_id = efx_ef10_get_phys_port_id,4149	.revision = EFX_REV_HUNT_A0,4150	.max_dma_mask = DMA_BIT_MASK(ESF_DZ_TX_KER_BUF_ADDR_WIDTH),4151	.rx_prefix_size = ES_DZ_RX_PREFIX_SIZE,4152	.rx_hash_offset = ES_DZ_RX_PREFIX_HASH_OFST,4153	.rx_ts_offset = ES_DZ_RX_PREFIX_TSTAMP_OFST,4154	.can_rx_scatter = true,4155	.always_rx_scatter = true,4156	.min_interrupt_mode = EFX_INT_MODE_MSIX,4157	.timer_period_max = 1 << ERF_DD_EVQ_IND_TIMER_VAL_WIDTH,4158	.offload_features = EF10_OFFLOAD_FEATURES,4159	.mcdi_max_ver = 2,4160	.max_rx_ip_filters = EFX_MCDI_FILTER_TBL_ROWS,4161	.hwtstamp_filters = 1 << HWTSTAMP_FILTER_NONE |4162			    1 << HWTSTAMP_FILTER_ALL,4163	.rx_hash_key_size = 40,4164	.check_caps = ef10_check_caps,4165	.print_additional_fwver = efx_ef10_print_additional_fwver,4166	.sensor_event = efx_mcdi_sensor_event,4167	.rx_recycle_ring_size = efx_ef10_recycle_ring_size,4168};4169 4170const struct efx_nic_type efx_hunt_a0_nic_type = {4171	.is_vf = false,4172	.mem_bar = efx_ef10_pf_mem_bar,4173	.mem_map_size = efx_ef10_mem_map_size,4174	.probe = efx_ef10_probe_pf,4175	.remove = efx_ef10_remove,4176	.dimension_resources = efx_ef10_dimension_resources,4177	.init = efx_ef10_init_nic,4178	.fini = efx_ef10_fini_nic,4179	.map_reset_reason = efx_ef10_map_reset_reason,4180	.map_reset_flags = efx_ef10_map_reset_flags,4181	.reset = efx_ef10_reset,4182	.probe_port = efx_mcdi_port_probe,4183	.remove_port = efx_mcdi_port_remove,4184	.fini_dmaq = efx_fini_dmaq,4185	.prepare_flr = efx_ef10_prepare_flr,4186	.finish_flr = efx_port_dummy_op_void,4187	.describe_stats = efx_ef10_describe_stats,4188	.update_stats = efx_ef10_update_stats_pf,4189	.start_stats = efx_mcdi_mac_start_stats,4190	.pull_stats = efx_mcdi_mac_pull_stats,4191	.stop_stats = efx_mcdi_mac_stop_stats,4192	.push_irq_moderation = efx_ef10_push_irq_moderation,4193	.reconfigure_mac = efx_ef10_mac_reconfigure,4194	.check_mac_fault = efx_mcdi_mac_check_fault,4195	.reconfigure_port = efx_mcdi_port_reconfigure,4196	.get_wol = efx_ef10_get_wol,4197	.set_wol = efx_ef10_set_wol,4198	.resume_wol = efx_port_dummy_op_void,4199	.get_fec_stats = efx_ef10_get_fec_stats,4200	.test_chip = efx_ef10_test_chip,4201	.test_nvram = efx_mcdi_nvram_test_all,4202	.mcdi_request = efx_ef10_mcdi_request,4203	.mcdi_poll_response = efx_ef10_mcdi_poll_response,4204	.mcdi_read_response = efx_ef10_mcdi_read_response,4205	.mcdi_poll_reboot = efx_ef10_mcdi_poll_reboot,4206	.mcdi_reboot_detected = efx_ef10_mcdi_reboot_detected,4207	.irq_enable_master = efx_port_dummy_op_void,4208	.irq_test_generate = efx_ef10_irq_test_generate,4209	.irq_disable_non_ev = efx_port_dummy_op_void,4210	.irq_handle_msi = efx_ef10_msi_interrupt,4211	.irq_handle_legacy = efx_ef10_legacy_interrupt,4212	.tx_probe = efx_ef10_tx_probe,4213	.tx_init = efx_ef10_tx_init,4214	.tx_remove = efx_mcdi_tx_remove,4215	.tx_write = efx_ef10_tx_write,4216	.tx_limit_len = efx_ef10_tx_limit_len,4217	.tx_enqueue = __efx_enqueue_skb,4218	.rx_push_rss_config = efx_mcdi_pf_rx_push_rss_config,4219	.rx_pull_rss_config = efx_mcdi_rx_pull_rss_config,4220	.rx_push_rss_context_config = efx_mcdi_rx_push_rss_context_config,4221	.rx_pull_rss_context_config = efx_mcdi_rx_pull_rss_context_config,4222	.rx_restore_rss_contexts = efx_mcdi_rx_restore_rss_contexts,4223	.rx_probe = efx_mcdi_rx_probe,4224	.rx_init = efx_mcdi_rx_init,4225	.rx_remove = efx_mcdi_rx_remove,4226	.rx_write = efx_ef10_rx_write,4227	.rx_defer_refill = efx_ef10_rx_defer_refill,4228	.rx_packet = __efx_rx_packet,4229	.ev_probe = efx_mcdi_ev_probe,4230	.ev_init = efx_ef10_ev_init,4231	.ev_fini = efx_mcdi_ev_fini,4232	.ev_remove = efx_mcdi_ev_remove,4233	.ev_process = efx_ef10_ev_process,4234	.ev_read_ack = efx_ef10_ev_read_ack,4235	.ev_test_generate = efx_ef10_ev_test_generate,4236	.filter_table_probe = efx_ef10_filter_table_probe,4237	.filter_table_restore = efx_mcdi_filter_table_restore,4238	.filter_table_remove = efx_ef10_filter_table_remove,4239	.filter_update_rx_scatter = efx_mcdi_update_rx_scatter,4240	.filter_insert = efx_mcdi_filter_insert,4241	.filter_remove_safe = efx_mcdi_filter_remove_safe,4242	.filter_get_safe = efx_mcdi_filter_get_safe,4243	.filter_clear_rx = efx_mcdi_filter_clear_rx,4244	.filter_count_rx_used = efx_mcdi_filter_count_rx_used,4245	.filter_get_rx_id_limit = efx_mcdi_filter_get_rx_id_limit,4246	.filter_get_rx_ids = efx_mcdi_filter_get_rx_ids,4247#ifdef CONFIG_RFS_ACCEL4248	.filter_rfs_expire_one = efx_mcdi_filter_rfs_expire_one,4249#endif4250#ifdef CONFIG_SFC_MTD4251	.mtd_probe = efx_ef10_mtd_probe,4252	.mtd_rename = efx_mcdi_mtd_rename,4253	.mtd_read = efx_mcdi_mtd_read,4254	.mtd_erase = efx_mcdi_mtd_erase,4255	.mtd_write = efx_mcdi_mtd_write,4256	.mtd_sync = efx_mcdi_mtd_sync,4257#endif4258	.ptp_write_host_time = efx_ef10_ptp_write_host_time,4259	.ptp_set_ts_sync_events = efx_ef10_ptp_set_ts_sync_events,4260	.ptp_set_ts_config = efx_ef10_ptp_set_ts_config,4261	.vlan_rx_add_vid = efx_ef10_vlan_rx_add_vid,4262	.vlan_rx_kill_vid = efx_ef10_vlan_rx_kill_vid,4263	.udp_tnl_push_ports = efx_ef10_udp_tnl_push_ports,4264	.udp_tnl_has_port = efx_ef10_udp_tnl_has_port,4265#ifdef CONFIG_SFC_SRIOV4266	.sriov_configure = efx_ef10_sriov_configure,4267	.sriov_init = efx_ef10_sriov_init,4268	.sriov_fini = efx_ef10_sriov_fini,4269	.sriov_wanted = efx_ef10_sriov_wanted,4270	.sriov_set_vf_mac = efx_ef10_sriov_set_vf_mac,4271	.sriov_set_vf_vlan = efx_ef10_sriov_set_vf_vlan,4272	.sriov_set_vf_spoofchk = efx_ef10_sriov_set_vf_spoofchk,4273	.sriov_get_vf_config = efx_ef10_sriov_get_vf_config,4274	.sriov_set_vf_link_state = efx_ef10_sriov_set_vf_link_state,4275	.vswitching_probe = efx_ef10_vswitching_probe_pf,4276	.vswitching_restore = efx_ef10_vswitching_restore_pf,4277	.vswitching_remove = efx_ef10_vswitching_remove_pf,4278#endif4279	.get_mac_address = efx_ef10_get_mac_address_pf,4280	.set_mac_address = efx_ef10_set_mac_address,4281	.tso_versions = efx_ef10_tso_versions,4282 4283	.get_phys_port_id = efx_ef10_get_phys_port_id,4284	.revision = EFX_REV_HUNT_A0,4285	.max_dma_mask = DMA_BIT_MASK(ESF_DZ_TX_KER_BUF_ADDR_WIDTH),4286	.rx_prefix_size = ES_DZ_RX_PREFIX_SIZE,4287	.rx_hash_offset = ES_DZ_RX_PREFIX_HASH_OFST,4288	.rx_ts_offset = ES_DZ_RX_PREFIX_TSTAMP_OFST,4289	.can_rx_scatter = true,4290	.always_rx_scatter = true,4291	.option_descriptors = true,4292	.min_interrupt_mode = EFX_INT_MODE_LEGACY,4293	.timer_period_max = 1 << ERF_DD_EVQ_IND_TIMER_VAL_WIDTH,4294	.offload_features = EF10_OFFLOAD_FEATURES,4295	.mcdi_max_ver = 2,4296	.max_rx_ip_filters = EFX_MCDI_FILTER_TBL_ROWS,4297	.hwtstamp_filters = 1 << HWTSTAMP_FILTER_NONE |4298			    1 << HWTSTAMP_FILTER_ALL,4299	.rx_hash_key_size = 40,4300	.check_caps = ef10_check_caps,4301	.print_additional_fwver = efx_ef10_print_additional_fwver,4302	.sensor_event = efx_mcdi_sensor_event,4303	.rx_recycle_ring_size = efx_ef10_recycle_ring_size,4304};4305 4306const struct efx_nic_type efx_x4_nic_type = {4307	.is_vf = false,4308	.mem_bar = efx_ef10_pf_mem_bar,4309	.mem_map_size = efx_ef10_mem_map_size,4310	.probe = efx_ef10_probe_pf,4311	.remove = efx_ef10_remove,4312	.dimension_resources = efx_ef10_dimension_resources,4313	.init = efx_ef10_init_nic,4314	.fini = efx_ef10_fini_nic,4315	.map_reset_reason = efx_ef10_map_reset_reason,4316	.map_reset_flags = efx_ef10_map_reset_flags,4317	.reset = efx_ef10_reset,4318	.probe_port = efx_mcdi_port_probe,4319	.remove_port = efx_mcdi_port_remove,4320	.fini_dmaq = efx_fini_dmaq,4321	.prepare_flr = efx_ef10_prepare_flr,4322	.finish_flr = efx_port_dummy_op_void,4323	.describe_stats = efx_ef10_describe_stats,4324	.update_stats = efx_ef10_update_stats_pf,4325	.start_stats = efx_mcdi_mac_start_stats,4326	.pull_stats = efx_mcdi_mac_pull_stats,4327	.stop_stats = efx_mcdi_mac_stop_stats,4328	.push_irq_moderation = efx_ef10_push_irq_moderation,4329	.reconfigure_mac = efx_ef10_mac_reconfigure,4330	.check_mac_fault = efx_mcdi_mac_check_fault,4331	.reconfigure_port = efx_mcdi_port_reconfigure,4332	.get_wol = efx_ef10_get_wol,4333	.set_wol = efx_ef10_set_wol,4334	.resume_wol = efx_port_dummy_op_void,4335	.get_fec_stats = efx_ef10_get_fec_stats,4336	.test_chip = efx_ef10_test_chip,4337	.test_nvram = efx_mcdi_nvram_test_all,4338	.mcdi_request = efx_ef10_mcdi_request,4339	.mcdi_poll_response = efx_ef10_mcdi_poll_response,4340	.mcdi_read_response = efx_ef10_mcdi_read_response,4341	.mcdi_poll_reboot = efx_ef10_mcdi_poll_reboot,4342	.mcdi_reboot_detected = efx_ef10_mcdi_reboot_detected,4343	.irq_enable_master = efx_port_dummy_op_void,4344	.irq_test_generate = efx_ef10_irq_test_generate,4345	.irq_disable_non_ev = efx_port_dummy_op_void,4346	.irq_handle_msi = efx_ef10_msi_interrupt,4347	.tx_probe = efx_ef10_tx_probe,4348	.tx_init = efx_ef10_tx_init,4349	.tx_write = efx_ef10_tx_write,4350	.tx_limit_len = efx_ef10_tx_limit_len,4351	.tx_enqueue = __efx_enqueue_skb,4352	.rx_push_rss_config = efx_mcdi_pf_rx_push_rss_config,4353	.rx_pull_rss_config = efx_mcdi_rx_pull_rss_config,4354	.rx_push_rss_context_config = efx_mcdi_rx_push_rss_context_config,4355	.rx_pull_rss_context_config = efx_mcdi_rx_pull_rss_context_config,4356	.rx_restore_rss_contexts = efx_mcdi_rx_restore_rss_contexts,4357	.rx_probe = efx_mcdi_rx_probe,4358	.rx_init = efx_mcdi_rx_init,4359	.rx_remove = efx_mcdi_rx_remove,4360	.rx_write = efx_ef10_rx_write,4361	.rx_defer_refill = efx_ef10_rx_defer_refill,4362	.rx_packet = __efx_rx_packet,4363	.ev_probe = efx_mcdi_ev_probe,4364	.ev_init = efx_ef10_ev_init,4365	.ev_fini = efx_mcdi_ev_fini,4366	.ev_remove = efx_mcdi_ev_remove,4367	.ev_process = efx_ef10_ev_process,4368	.ev_read_ack = efx_ef10_ev_read_ack,4369	.ev_test_generate = efx_ef10_ev_test_generate,4370	.filter_table_probe = efx_ef10_filter_table_probe,4371	.filter_table_restore = efx_mcdi_filter_table_restore,4372	.filter_table_remove = efx_ef10_filter_table_remove,4373	.filter_insert = efx_mcdi_filter_insert,4374	.filter_remove_safe = efx_mcdi_filter_remove_safe,4375	.filter_get_safe = efx_mcdi_filter_get_safe,4376	.filter_clear_rx = efx_mcdi_filter_clear_rx,4377	.filter_count_rx_used = efx_mcdi_filter_count_rx_used,4378	.filter_get_rx_id_limit = efx_mcdi_filter_get_rx_id_limit,4379	.filter_get_rx_ids = efx_mcdi_filter_get_rx_ids,4380#ifdef CONFIG_RFS_ACCEL4381	.filter_rfs_expire_one = efx_mcdi_filter_rfs_expire_one,4382#endif4383#ifdef CONFIG_SFC_MTD4384	.mtd_probe = efx_ef10_mtd_probe,4385	.mtd_rename = efx_mcdi_mtd_rename,4386	.mtd_read = efx_mcdi_mtd_read,4387	.mtd_erase = efx_mcdi_mtd_erase,4388	.mtd_write = efx_mcdi_mtd_write,4389	.mtd_sync = efx_mcdi_mtd_sync,4390#endif4391	.ptp_write_host_time = efx_ef10_ptp_write_host_time,4392	.ptp_set_ts_sync_events = efx_ef10_ptp_set_ts_sync_events,4393	.ptp_set_ts_config = efx_ef10_ptp_set_ts_config,4394	.vlan_rx_add_vid = efx_ef10_vlan_rx_add_vid,4395	.vlan_rx_kill_vid = efx_ef10_vlan_rx_kill_vid,4396	.udp_tnl_push_ports = efx_ef10_udp_tnl_push_ports,4397	.udp_tnl_has_port = efx_ef10_udp_tnl_has_port,4398#ifdef CONFIG_SFC_SRIOV4399	/* currently set to the VF versions of these functions4400	 * because SRIOV will be reimplemented later.4401	 */4402	.vswitching_probe = efx_ef10_vswitching_probe_vf,4403	.vswitching_restore = efx_ef10_vswitching_restore_vf,4404	.vswitching_remove = efx_ef10_vswitching_remove_vf,4405#endif4406	.get_mac_address = efx_ef10_get_mac_address_pf,4407	.set_mac_address = efx_ef10_set_mac_address,4408	.tso_versions = efx_ef10_tso_versions,4409 4410	.get_phys_port_id = efx_ef10_get_phys_port_id,4411	.revision = EFX_REV_X4,4412	.max_dma_mask = DMA_BIT_MASK(ESF_DZ_TX_KER_BUF_ADDR_WIDTH),4413	.rx_prefix_size = ES_DZ_RX_PREFIX_SIZE,4414	.rx_hash_offset = ES_DZ_RX_PREFIX_HASH_OFST,4415	.rx_ts_offset = ES_DZ_RX_PREFIX_TSTAMP_OFST,4416	.can_rx_scatter = true,4417	.always_rx_scatter = true,4418	.option_descriptors = true,4419	.min_interrupt_mode = EFX_INT_MODE_MSIX,4420	.timer_period_max = 1 << ERF_DD_EVQ_IND_TIMER_VAL_WIDTH,4421	.offload_features = EF10_OFFLOAD_FEATURES,4422	.mcdi_max_ver = 2,4423	.max_rx_ip_filters = EFX_MCDI_FILTER_TBL_ROWS,4424	.hwtstamp_filters = 1 << HWTSTAMP_FILTER_NONE |4425			    1 << HWTSTAMP_FILTER_ALL,4426	.check_caps = ef10_check_caps,4427	.print_additional_fwver = efx_ef10_print_additional_fwver,4428	.sensor_event = efx_mcdi_sensor_event,4429	.rx_recycle_ring_size = efx_ef10_recycle_ring_size,4430};4431 4432