brintos / linux-shallow public Read only

1/**********************************************************************2 * Author: Cavium, Inc.3 *4 * Contact: support@cavium.com5 *          Please include "LiquidIO" in the subject.6 *7 * Copyright (c) 2003-2016 Cavium, Inc.8 *9 * This file is free software; you can redistribute it and/or modify10 * it under the terms of the GNU General Public License, Version 2, as11 * published by the Free Software Foundation.12 *13 * This file is distributed in the hope that it will be useful, but14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or16 * NONINFRINGEMENT.  See the GNU General Public License for more details.17 ***********************************************************************/18#include <linux/module.h>19#include <linux/interrupt.h>20#include <linux/pci.h>21#include <linux/firmware.h>22#include <net/vxlan.h>23#include <linux/kthread.h>24#include "liquidio_common.h"25#include "octeon_droq.h"26#include "octeon_iq.h"27#include "response_manager.h"28#include "octeon_device.h"29#include "octeon_nic.h"30#include "octeon_main.h"31#include "octeon_network.h"32#include "cn66xx_regs.h"33#include "cn66xx_device.h"34#include "cn68xx_device.h"35#include "cn23xx_pf_device.h"36#include "liquidio_image.h"37#include "lio_vf_rep.h"38 39MODULE_AUTHOR("Cavium Networks, <support@cavium.com>");40MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Driver");41MODULE_LICENSE("GPL");42MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210SV_NAME43		"_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);44MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210NV_NAME45		"_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);46MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_410NV_NAME47		"_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);48MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_23XX_NAME49		"_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);50 51static int ddr_timeout = 10000;52module_param(ddr_timeout, int, 0644);53MODULE_PARM_DESC(ddr_timeout,54		 "Number of milliseconds to wait for DDR initialization. 0 waits for ddr_timeout to be set to non-zero value before starting to check");55 56#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)57 58static int debug = -1;59module_param(debug, int, 0644);60MODULE_PARM_DESC(debug, "NETIF_MSG debug bits");61 62static char fw_type[LIO_MAX_FW_TYPE_LEN] = LIO_FW_NAME_TYPE_AUTO;63module_param_string(fw_type, fw_type, sizeof(fw_type), 0444);64MODULE_PARM_DESC(fw_type, "Type of firmware to be loaded (default is \"auto\"), which uses firmware in flash, if present, else loads \"nic\".");65 66static u32 console_bitmask;67module_param(console_bitmask, int, 0644);68MODULE_PARM_DESC(console_bitmask,69		 "Bitmask indicating which consoles have debug output redirected to syslog.");70 71/**72 * octeon_console_debug_enabled - determines if a given console has debug enabled.73 * @console: console to check74 * Return:  1 = enabled. 0 otherwise75 */76static int octeon_console_debug_enabled(u32 console)77{78	return (console_bitmask >> (console)) & 0x1;79}80 81/* Polling interval for determining when NIC application is alive */82#define LIQUIDIO_STARTER_POLL_INTERVAL_MS 10083 84/* runtime link query interval */85#define LIQUIDIO_LINK_QUERY_INTERVAL_MS         100086/* update localtime to octeon firmware every 60 seconds.87 * make firmware to use same time reference, so that it will be easy to88 * correlate firmware logged events/errors with host events, for debugging.89 */90#define LIO_SYNC_OCTEON_TIME_INTERVAL_MS 6000091 92/* time to wait for possible in-flight requests in milliseconds */93#define WAIT_INFLIGHT_REQUEST	msecs_to_jiffies(1000)94 95struct oct_timestamp_resp {96	u64 rh;97	u64 timestamp;98	u64 status;99};100 101#define OCT_TIMESTAMP_RESP_SIZE (sizeof(struct oct_timestamp_resp))102 103union tx_info {104	u64 u64;105	struct {106#ifdef __BIG_ENDIAN_BITFIELD107		u16 gso_size;108		u16 gso_segs;109		u32 reserved;110#else111		u32 reserved;112		u16 gso_segs;113		u16 gso_size;114#endif115	} s;116};117 118/* Octeon device properties to be used by the NIC module.119 * Each octeon device in the system will be represented120 * by this structure in the NIC module.121 */122 123#define OCTNIC_GSO_MAX_HEADER_SIZE 128124#define OCTNIC_GSO_MAX_SIZE                                                    \125	(CN23XX_DEFAULT_INPUT_JABBER - OCTNIC_GSO_MAX_HEADER_SIZE)126 127struct handshake {128	struct completion init;129	struct completion started;130	struct pci_dev *pci_dev;131	int init_ok;132	int started_ok;133};134 135#ifdef CONFIG_PCI_IOV136static int liquidio_enable_sriov(struct pci_dev *dev, int num_vfs);137#endif138 139static int octeon_dbg_console_print(struct octeon_device *oct, u32 console_num,140				    char *prefix, char *suffix);141 142static int octeon_device_init(struct octeon_device *);143static int liquidio_stop(struct net_device *netdev);144static void liquidio_remove(struct pci_dev *pdev);145static int liquidio_probe(struct pci_dev *pdev,146			  const struct pci_device_id *ent);147static int liquidio_set_vf_link_state(struct net_device *netdev, int vfidx,148				      int linkstate);149 150static struct handshake handshake[MAX_OCTEON_DEVICES];151static struct completion first_stage;152 153static void octeon_droq_bh(struct tasklet_struct *t)154{155	int q_no;156	int reschedule = 0;157	struct octeon_device_priv *oct_priv = from_tasklet(oct_priv, t,158							  droq_tasklet);159	struct octeon_device *oct = oct_priv->dev;160 161	for (q_no = 0; q_no < MAX_OCTEON_OUTPUT_QUEUES(oct); q_no++) {162		if (!(oct->io_qmask.oq & BIT_ULL(q_no)))163			continue;164		reschedule |= octeon_droq_process_packets(oct, oct->droq[q_no],165							  MAX_PACKET_BUDGET);166		lio_enable_irq(oct->droq[q_no], NULL);167 168		if (OCTEON_CN23XX_PF(oct) && oct->msix_on) {169			/* set time and cnt interrupt thresholds for this DROQ170			 * for NAPI171			 */172			int adjusted_q_no = q_no + oct->sriov_info.pf_srn;173 174			octeon_write_csr64(175			    oct, CN23XX_SLI_OQ_PKT_INT_LEVELS(adjusted_q_no),176			    0x5700000040ULL);177			octeon_write_csr64(178			    oct, CN23XX_SLI_OQ_PKTS_SENT(adjusted_q_no), 0);179		}180	}181 182	if (reschedule)183		tasklet_schedule(&oct_priv->droq_tasklet);184}185 186static int lio_wait_for_oq_pkts(struct octeon_device *oct)187{188	struct octeon_device_priv *oct_priv = oct->priv;189	int retry = 100, pkt_cnt = 0, pending_pkts = 0;190	int i;191 192	do {193		pending_pkts = 0;194 195		for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {196			if (!(oct->io_qmask.oq & BIT_ULL(i)))197				continue;198			pkt_cnt += octeon_droq_check_hw_for_pkts(oct->droq[i]);199		}200		if (pkt_cnt > 0) {201			pending_pkts += pkt_cnt;202			tasklet_schedule(&oct_priv->droq_tasklet);203		}204		pkt_cnt = 0;205		schedule_timeout_uninterruptible(1);206 207	} while (retry-- && pending_pkts);208 209	return pkt_cnt;210}211 212/**213 * force_io_queues_off - Forces all IO queues off on a given device214 * @oct: Pointer to Octeon device215 */216static void force_io_queues_off(struct octeon_device *oct)217{218	if ((oct->chip_id == OCTEON_CN66XX) ||219	    (oct->chip_id == OCTEON_CN68XX)) {220		/* Reset the Enable bits for Input Queues. */221		octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, 0);222 223		/* Reset the Enable bits for Output Queues. */224		octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, 0);225	}226}227 228/**229 * pcierror_quiesce_device - Cause device to go quiet so it can be safely removed/reset/etc230 * @oct: Pointer to Octeon device231 */232static inline void pcierror_quiesce_device(struct octeon_device *oct)233{234	int i;235 236	/* Disable the input and output queues now. No more packets will237	 * arrive from Octeon, but we should wait for all packet processing238	 * to finish.239	 */240	force_io_queues_off(oct);241 242	/* To allow for in-flight requests */243	schedule_timeout_uninterruptible(WAIT_INFLIGHT_REQUEST);244 245	if (wait_for_pending_requests(oct))246		dev_err(&oct->pci_dev->dev, "There were pending requests\n");247 248	/* Force all requests waiting to be fetched by OCTEON to complete. */249	for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {250		struct octeon_instr_queue *iq;251 252		if (!(oct->io_qmask.iq & BIT_ULL(i)))253			continue;254		iq = oct->instr_queue[i];255 256		if (atomic_read(&iq->instr_pending)) {257			spin_lock_bh(&iq->lock);258			iq->fill_cnt = 0;259			iq->octeon_read_index = iq->host_write_index;260			iq->stats.instr_processed +=261				atomic_read(&iq->instr_pending);262			lio_process_iq_request_list(oct, iq, 0);263			spin_unlock_bh(&iq->lock);264		}265	}266 267	/* Force all pending ordered list requests to time out. */268	lio_process_ordered_list(oct, 1);269 270	/* We do not need to wait for output queue packets to be processed. */271}272 273/**274 * cleanup_aer_uncorrect_error_status - Cleanup PCI AER uncorrectable error status275 * @dev: Pointer to PCI device276 */277static void cleanup_aer_uncorrect_error_status(struct pci_dev *dev)278{279	int pos = 0x100;280	u32 status, mask;281 282	pr_info("%s :\n", __func__);283 284	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);285	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask);286	if (dev->error_state == pci_channel_io_normal)287		status &= ~mask;        /* Clear corresponding nonfatal bits */288	else289		status &= mask;         /* Clear corresponding fatal bits */290	pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);291}292 293/**294 * stop_pci_io - Stop all PCI IO to a given device295 * @oct: Pointer to Octeon device296 */297static void stop_pci_io(struct octeon_device *oct)298{299	/* No more instructions will be forwarded. */300	atomic_set(&oct->status, OCT_DEV_IN_RESET);301 302	pci_disable_device(oct->pci_dev);303 304	/* Disable interrupts  */305	oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);306 307	pcierror_quiesce_device(oct);308 309	/* Release the interrupt line */310	free_irq(oct->pci_dev->irq, oct);311 312	if (oct->flags & LIO_FLAG_MSI_ENABLED)313		pci_disable_msi(oct->pci_dev);314 315	dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",316		lio_get_state_string(&oct->status));317 318	/* making it a common function for all OCTEON models */319	cleanup_aer_uncorrect_error_status(oct->pci_dev);320}321 322/**323 * liquidio_pcie_error_detected - called when PCI error is detected324 * @pdev: Pointer to PCI device325 * @state: The current pci connection state326 *327 * This function is called after a PCI bus error affecting328 * this device has been detected.329 */330static pci_ers_result_t liquidio_pcie_error_detected(struct pci_dev *pdev,331						     pci_channel_state_t state)332{333	struct octeon_device *oct = pci_get_drvdata(pdev);334 335	/* Non-correctable Non-fatal errors */336	if (state == pci_channel_io_normal) {337		dev_err(&oct->pci_dev->dev, "Non-correctable non-fatal error reported:\n");338		cleanup_aer_uncorrect_error_status(oct->pci_dev);339		return PCI_ERS_RESULT_CAN_RECOVER;340	}341 342	/* Non-correctable Fatal errors */343	dev_err(&oct->pci_dev->dev, "Non-correctable FATAL reported by PCI AER driver\n");344	stop_pci_io(oct);345 346	/* Always return a DISCONNECT. There is no support for recovery but only347	 * for a clean shutdown.348	 */349	return PCI_ERS_RESULT_DISCONNECT;350}351 352/**353 * liquidio_pcie_mmio_enabled - mmio handler354 * @pdev: Pointer to PCI device355 */356static pci_ers_result_t liquidio_pcie_mmio_enabled(struct pci_dev __maybe_unused *pdev)357{358	/* We should never hit this since we never ask for a reset for a Fatal359	 * Error. We always return DISCONNECT in io_error above.360	 * But play safe and return RECOVERED for now.361	 */362	return PCI_ERS_RESULT_RECOVERED;363}364 365/**366 * liquidio_pcie_slot_reset - called after the pci bus has been reset.367 * @pdev: Pointer to PCI device368 *369 * Restart the card from scratch, as if from a cold-boot. Implementation370 * resembles the first-half of the octeon_resume routine.371 */372static pci_ers_result_t liquidio_pcie_slot_reset(struct pci_dev __maybe_unused *pdev)373{374	/* We should never hit this since we never ask for a reset for a Fatal375	 * Error. We always return DISCONNECT in io_error above.376	 * But play safe and return RECOVERED for now.377	 */378	return PCI_ERS_RESULT_RECOVERED;379}380 381/**382 * liquidio_pcie_resume - called when traffic can start flowing again.383 * @pdev: Pointer to PCI device384 *385 * This callback is called when the error recovery driver tells us that386 * its OK to resume normal operation. Implementation resembles the387 * second-half of the octeon_resume routine.388 */389static void liquidio_pcie_resume(struct pci_dev __maybe_unused *pdev)390{391	/* Nothing to be done here. */392}393 394#define liquidio_suspend NULL395#define liquidio_resume NULL396 397/* For PCI-E Advanced Error Recovery (AER) Interface */398static const struct pci_error_handlers liquidio_err_handler = {399	.error_detected = liquidio_pcie_error_detected,400	.mmio_enabled	= liquidio_pcie_mmio_enabled,401	.slot_reset	= liquidio_pcie_slot_reset,402	.resume		= liquidio_pcie_resume,403};404 405static const struct pci_device_id liquidio_pci_tbl[] = {406	{       /* 68xx */407		PCI_VENDOR_ID_CAVIUM, 0x91, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0408	},409	{       /* 66xx */410		PCI_VENDOR_ID_CAVIUM, 0x92, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0411	},412	{       /* 23xx pf */413		PCI_VENDOR_ID_CAVIUM, 0x9702, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0414	},415	{416		0, 0, 0, 0, 0, 0, 0417	}418};419MODULE_DEVICE_TABLE(pci, liquidio_pci_tbl);420 421static SIMPLE_DEV_PM_OPS(liquidio_pm_ops, liquidio_suspend, liquidio_resume);422 423static struct pci_driver liquidio_pci_driver = {424	.name		= "LiquidIO",425	.id_table	= liquidio_pci_tbl,426	.probe		= liquidio_probe,427	.remove		= liquidio_remove,428	.err_handler	= &liquidio_err_handler,    /* For AER */429	.driver.pm	= &liquidio_pm_ops,430#ifdef CONFIG_PCI_IOV431	.sriov_configure = liquidio_enable_sriov,432#endif433};434 435/**436 * liquidio_init_pci - register PCI driver437 */438static int liquidio_init_pci(void)439{440	return pci_register_driver(&liquidio_pci_driver);441}442 443/**444 * liquidio_deinit_pci - unregister PCI driver445 */446static void liquidio_deinit_pci(void)447{448	pci_unregister_driver(&liquidio_pci_driver);449}450 451/**452 * check_txq_status - Check Tx queue status, and take appropriate action453 * @lio: per-network private data454 * Return: 0 if full, number of queues woken up otherwise455 */456static inline int check_txq_status(struct lio *lio)457{458	int numqs = lio->netdev->real_num_tx_queues;459	int ret_val = 0;460	int q, iq;461 462	/* check each sub-queue state */463	for (q = 0; q < numqs; q++) {464		iq = lio->linfo.txpciq[q %465			lio->oct_dev->num_iqs].s.q_no;466		if (octnet_iq_is_full(lio->oct_dev, iq))467			continue;468		if (__netif_subqueue_stopped(lio->netdev, q)) {469			netif_wake_subqueue(lio->netdev, q);470			INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq,471						  tx_restart, 1);472			ret_val++;473		}474	}475 476	return ret_val;477}478 479/**480 * print_link_info -  Print link information481 * @netdev: network device482 */483static void print_link_info(struct net_device *netdev)484{485	struct lio *lio = GET_LIO(netdev);486 487	if (!ifstate_check(lio, LIO_IFSTATE_RESETTING) &&488	    ifstate_check(lio, LIO_IFSTATE_REGISTERED)) {489		struct oct_link_info *linfo = &lio->linfo;490 491		if (linfo->link.s.link_up) {492			netif_info(lio, link, lio->netdev, "%d Mbps %s Duplex UP\n",493				   linfo->link.s.speed,494				   (linfo->link.s.duplex) ? "Full" : "Half");495		} else {496			netif_info(lio, link, lio->netdev, "Link Down\n");497		}498	}499}500 501/**502 * octnet_link_status_change - Routine to notify MTU change503 * @work: work_struct data structure504 */505static void octnet_link_status_change(struct work_struct *work)506{507	struct cavium_wk *wk = (struct cavium_wk *)work;508	struct lio *lio = (struct lio *)wk->ctxptr;509 510	/* lio->linfo.link.s.mtu always contains max MTU of the lio interface.511	 * this API is invoked only when new max-MTU of the interface is512	 * less than current MTU.513	 */514	rtnl_lock();515	dev_set_mtu(lio->netdev, lio->linfo.link.s.mtu);516	rtnl_unlock();517}518 519/**520 * setup_link_status_change_wq - Sets up the mtu status change work521 * @netdev: network device522 */523static inline int setup_link_status_change_wq(struct net_device *netdev)524{525	struct lio *lio = GET_LIO(netdev);526	struct octeon_device *oct = lio->oct_dev;527 528	lio->link_status_wq.wq = alloc_workqueue("link-status",529						 WQ_MEM_RECLAIM, 0);530	if (!lio->link_status_wq.wq) {531		dev_err(&oct->pci_dev->dev, "unable to create cavium link status wq\n");532		return -1;533	}534	INIT_DELAYED_WORK(&lio->link_status_wq.wk.work,535			  octnet_link_status_change);536	lio->link_status_wq.wk.ctxptr = lio;537 538	return 0;539}540 541static inline void cleanup_link_status_change_wq(struct net_device *netdev)542{543	struct lio *lio = GET_LIO(netdev);544 545	if (lio->link_status_wq.wq) {546		cancel_delayed_work_sync(&lio->link_status_wq.wk.work);547		destroy_workqueue(lio->link_status_wq.wq);548	}549}550 551/**552 * update_link_status - Update link status553 * @netdev: network device554 * @ls: link status structure555 *556 * Called on receipt of a link status response from the core application to557 * update each interface's link status.558 */559static inline void update_link_status(struct net_device *netdev,560				      union oct_link_status *ls)561{562	struct lio *lio = GET_LIO(netdev);563	int changed = (lio->linfo.link.u64 != ls->u64);564	int current_max_mtu = lio->linfo.link.s.mtu;565	struct octeon_device *oct = lio->oct_dev;566 567	dev_dbg(&oct->pci_dev->dev, "%s: lio->linfo.link.u64=%llx, ls->u64=%llx\n",568		__func__, lio->linfo.link.u64, ls->u64);569	lio->linfo.link.u64 = ls->u64;570 571	if ((lio->intf_open) && (changed)) {572		print_link_info(netdev);573		lio->link_changes++;574 575		if (lio->linfo.link.s.link_up) {576			dev_dbg(&oct->pci_dev->dev, "%s: link_up", __func__);577			netif_carrier_on(netdev);578			wake_txqs(netdev);579		} else {580			dev_dbg(&oct->pci_dev->dev, "%s: link_off", __func__);581			netif_carrier_off(netdev);582			stop_txqs(netdev);583		}584		if (lio->linfo.link.s.mtu != current_max_mtu) {585			netif_info(lio, probe, lio->netdev, "Max MTU changed from %d to %d\n",586				   current_max_mtu, lio->linfo.link.s.mtu);587			netdev->max_mtu = lio->linfo.link.s.mtu;588		}589		if (lio->linfo.link.s.mtu < netdev->mtu) {590			dev_warn(&oct->pci_dev->dev,591				 "Current MTU is higher than new max MTU; Reducing the current mtu from %d to %d\n",592				     netdev->mtu, lio->linfo.link.s.mtu);593			queue_delayed_work(lio->link_status_wq.wq,594					   &lio->link_status_wq.wk.work, 0);595		}596	}597}598 599/**600 * lio_sync_octeon_time - send latest localtime to octeon firmware so that601 * firmware will correct it's time, in case there is a time skew602 *603 * @work: work scheduled to send time update to octeon firmware604 **/605static void lio_sync_octeon_time(struct work_struct *work)606{607	struct cavium_wk *wk = (struct cavium_wk *)work;608	struct lio *lio = (struct lio *)wk->ctxptr;609	struct octeon_device *oct = lio->oct_dev;610	struct octeon_soft_command *sc;611	struct timespec64 ts;612	struct lio_time *lt;613	int ret;614 615	sc = octeon_alloc_soft_command(oct, sizeof(struct lio_time), 16, 0);616	if (!sc) {617		dev_err(&oct->pci_dev->dev,618			"Failed to sync time to octeon: soft command allocation failed\n");619		return;620	}621 622	lt = (struct lio_time *)sc->virtdptr;623 624	/* Get time of the day */625	ktime_get_real_ts64(&ts);626	lt->sec = ts.tv_sec;627	lt->nsec = ts.tv_nsec;628	octeon_swap_8B_data((u64 *)lt, (sizeof(struct lio_time)) / 8);629 630	sc->iq_no = lio->linfo.txpciq[0].s.q_no;631	octeon_prepare_soft_command(oct, sc, OPCODE_NIC,632				    OPCODE_NIC_SYNC_OCTEON_TIME, 0, 0, 0);633 634	init_completion(&sc->complete);635	sc->sc_status = OCTEON_REQUEST_PENDING;636 637	ret = octeon_send_soft_command(oct, sc);638	if (ret == IQ_SEND_FAILED) {639		dev_err(&oct->pci_dev->dev,640			"Failed to sync time to octeon: failed to send soft command\n");641		octeon_free_soft_command(oct, sc);642	} else {643		WRITE_ONCE(sc->caller_is_done, true);644	}645 646	queue_delayed_work(lio->sync_octeon_time_wq.wq,647			   &lio->sync_octeon_time_wq.wk.work,648			   msecs_to_jiffies(LIO_SYNC_OCTEON_TIME_INTERVAL_MS));649}650 651/**652 * setup_sync_octeon_time_wq - prepare work to periodically update local time to octeon firmware653 *654 * @netdev: network device which should send time update to firmware655 **/656static inline int setup_sync_octeon_time_wq(struct net_device *netdev)657{658	struct lio *lio = GET_LIO(netdev);659	struct octeon_device *oct = lio->oct_dev;660 661	lio->sync_octeon_time_wq.wq =662		alloc_workqueue("update-octeon-time", WQ_MEM_RECLAIM, 0);663	if (!lio->sync_octeon_time_wq.wq) {664		dev_err(&oct->pci_dev->dev, "Unable to create wq to update octeon time\n");665		return -1;666	}667	INIT_DELAYED_WORK(&lio->sync_octeon_time_wq.wk.work,668			  lio_sync_octeon_time);669	lio->sync_octeon_time_wq.wk.ctxptr = lio;670	queue_delayed_work(lio->sync_octeon_time_wq.wq,671			   &lio->sync_octeon_time_wq.wk.work,672			   msecs_to_jiffies(LIO_SYNC_OCTEON_TIME_INTERVAL_MS));673 674	return 0;675}676 677/**678 * cleanup_sync_octeon_time_wq - destroy wq679 *680 * @netdev: network device which should send time update to firmware681 *682 * Stop scheduling and destroy the work created to periodically update local683 * time to octeon firmware.684 **/685static inline void cleanup_sync_octeon_time_wq(struct net_device *netdev)686{687	struct lio *lio = GET_LIO(netdev);688	struct cavium_wq *time_wq = &lio->sync_octeon_time_wq;689 690	if (time_wq->wq) {691		cancel_delayed_work_sync(&time_wq->wk.work);692		destroy_workqueue(time_wq->wq);693	}694}695 696static struct octeon_device *get_other_octeon_device(struct octeon_device *oct)697{698	struct octeon_device *other_oct;699 700	other_oct = lio_get_device(oct->octeon_id + 1);701 702	if (other_oct && other_oct->pci_dev) {703		int oct_busnum, other_oct_busnum;704 705		oct_busnum = oct->pci_dev->bus->number;706		other_oct_busnum = other_oct->pci_dev->bus->number;707 708		if (oct_busnum == other_oct_busnum) {709			int oct_slot, other_oct_slot;710 711			oct_slot = PCI_SLOT(oct->pci_dev->devfn);712			other_oct_slot = PCI_SLOT(other_oct->pci_dev->devfn);713 714			if (oct_slot == other_oct_slot)715				return other_oct;716		}717	}718 719	return NULL;720}721 722static void disable_all_vf_links(struct octeon_device *oct)723{724	struct net_device *netdev;725	int max_vfs, vf, i;726 727	if (!oct)728		return;729 730	max_vfs = oct->sriov_info.max_vfs;731 732	for (i = 0; i < oct->ifcount; i++) {733		netdev = oct->props[i].netdev;734		if (!netdev)735			continue;736 737		for (vf = 0; vf < max_vfs; vf++)738			liquidio_set_vf_link_state(netdev, vf,739						   IFLA_VF_LINK_STATE_DISABLE);740	}741}742 743static int liquidio_watchdog(void *param)744{745	bool err_msg_was_printed[LIO_MAX_CORES];746	u16 mask_of_crashed_or_stuck_cores = 0;747	bool all_vf_links_are_disabled = false;748	struct octeon_device *oct = param;749	struct octeon_device *other_oct;750#ifdef CONFIG_MODULE_UNLOAD751	long refcount, vfs_referencing_pf;752	u64 vfs_mask1, vfs_mask2;753#endif754	int core;755 756	memset(err_msg_was_printed, 0, sizeof(err_msg_was_printed));757 758	while (!kthread_should_stop()) {759		/* sleep for a couple of seconds so that we don't hog the CPU */760		set_current_state(TASK_INTERRUPTIBLE);761		schedule_timeout(msecs_to_jiffies(2000));762 763		mask_of_crashed_or_stuck_cores =764		    (u16)octeon_read_csr64(oct, CN23XX_SLI_SCRATCH2);765 766		if (!mask_of_crashed_or_stuck_cores)767			continue;768 769		WRITE_ONCE(oct->cores_crashed, true);770		other_oct = get_other_octeon_device(oct);771		if (other_oct)772			WRITE_ONCE(other_oct->cores_crashed, true);773 774		for (core = 0; core < LIO_MAX_CORES; core++) {775			bool core_crashed_or_got_stuck;776 777			core_crashed_or_got_stuck =778						(mask_of_crashed_or_stuck_cores779						 >> core) & 1;780 781			if (core_crashed_or_got_stuck &&782			    !err_msg_was_printed[core]) {783				dev_err(&oct->pci_dev->dev,784					"ERROR: Octeon core %d crashed or got stuck!  See oct-fwdump for details.\n",785					core);786				err_msg_was_printed[core] = true;787			}788		}789 790		if (all_vf_links_are_disabled)791			continue;792 793		disable_all_vf_links(oct);794		disable_all_vf_links(other_oct);795		all_vf_links_are_disabled = true;796 797#ifdef CONFIG_MODULE_UNLOAD798		vfs_mask1 = READ_ONCE(oct->sriov_info.vf_drv_loaded_mask);799		vfs_mask2 = READ_ONCE(other_oct->sriov_info.vf_drv_loaded_mask);800 801		vfs_referencing_pf  = hweight64(vfs_mask1);802		vfs_referencing_pf += hweight64(vfs_mask2);803 804		refcount = module_refcount(THIS_MODULE);805		if (refcount >= vfs_referencing_pf) {806			while (vfs_referencing_pf) {807				module_put(THIS_MODULE);808				vfs_referencing_pf--;809			}810		}811#endif812	}813 814	return 0;815}816 817/**818 * liquidio_probe - PCI probe handler819 * @pdev: PCI device structure820 * @ent: unused821 */822static int823liquidio_probe(struct pci_dev *pdev, const struct pci_device_id __maybe_unused *ent)824{825	struct octeon_device *oct_dev = NULL;826	struct handshake *hs;827 828	oct_dev = octeon_allocate_device(pdev->device,829					 sizeof(struct octeon_device_priv));830	if (!oct_dev) {831		dev_err(&pdev->dev, "Unable to allocate device\n");832		return -ENOMEM;833	}834 835	if (pdev->device == OCTEON_CN23XX_PF_VID)836		oct_dev->msix_on = LIO_FLAG_MSIX_ENABLED;837 838	/* Enable PTP for 6XXX Device */839	if (((pdev->device == OCTEON_CN66XX) ||840	     (pdev->device == OCTEON_CN68XX)))841		oct_dev->ptp_enable = true;842	else843		oct_dev->ptp_enable = false;844 845	dev_info(&pdev->dev, "Initializing device %x:%x.\n",846		 (u32)pdev->vendor, (u32)pdev->device);847 848	/* Assign octeon_device for this device to the private data area. */849	pci_set_drvdata(pdev, oct_dev);850 851	/* set linux specific device pointer */852	oct_dev->pci_dev = (void *)pdev;853 854	oct_dev->subsystem_id = pdev->subsystem_vendor |855		(pdev->subsystem_device << 16);856 857	hs = &handshake[oct_dev->octeon_id];858	init_completion(&hs->init);859	init_completion(&hs->started);860	hs->pci_dev = pdev;861 862	if (oct_dev->octeon_id == 0)863		/* first LiquidIO NIC is detected */864		complete(&first_stage);865 866	if (octeon_device_init(oct_dev)) {867		complete(&hs->init);868		liquidio_remove(pdev);869		return -ENOMEM;870	}871 872	if (OCTEON_CN23XX_PF(oct_dev)) {873		u8 bus, device, function;874 875		if (atomic_read(oct_dev->adapter_refcount) == 1) {876			/* Each NIC gets one watchdog kernel thread.  The first877			 * PF (of each NIC) that gets pci_driver->probe()'d878			 * creates that thread.879			 */880			bus = pdev->bus->number;881			device = PCI_SLOT(pdev->devfn);882			function = PCI_FUNC(pdev->devfn);883			oct_dev->watchdog_task = kthread_run(liquidio_watchdog,884							     oct_dev,885							     "liowd/%02hhx:%02hhx.%hhx",886							     bus, device, function);887			if (IS_ERR(oct_dev->watchdog_task)) {888				oct_dev->watchdog_task = NULL;889				dev_err(&oct_dev->pci_dev->dev,890					"failed to create kernel_thread\n");891				liquidio_remove(pdev);892				return -1;893			}894		}895	}896 897	oct_dev->rx_pause = 1;898	oct_dev->tx_pause = 1;899 900	dev_dbg(&oct_dev->pci_dev->dev, "Device is ready\n");901 902	return 0;903}904 905static bool fw_type_is_auto(void)906{907	return strncmp(fw_type, LIO_FW_NAME_TYPE_AUTO,908		       sizeof(LIO_FW_NAME_TYPE_AUTO)) == 0;909}910 911/**912 * octeon_pci_flr - PCI FLR for each Octeon device.913 * @oct: octeon device914 */915static void octeon_pci_flr(struct octeon_device *oct)916{917	int rc;918 919	pci_save_state(oct->pci_dev);920 921	pci_cfg_access_lock(oct->pci_dev);922 923	/* Quiesce the device completely */924	pci_write_config_word(oct->pci_dev, PCI_COMMAND,925			      PCI_COMMAND_INTX_DISABLE);926 927	rc = __pci_reset_function_locked(oct->pci_dev);928 929	if (rc != 0)930		dev_err(&oct->pci_dev->dev, "Error %d resetting PCI function %d\n",931			rc, oct->pf_num);932 933	pci_cfg_access_unlock(oct->pci_dev);934 935	pci_restore_state(oct->pci_dev);936}937 938/**939 * octeon_destroy_resources - Destroy resources associated with octeon device940 * @oct: octeon device941 */942static void octeon_destroy_resources(struct octeon_device *oct)943{944	int i, refcount;945	struct msix_entry *msix_entries;946	struct octeon_device_priv *oct_priv = oct->priv;947 948	struct handshake *hs;949 950	switch (atomic_read(&oct->status)) {951	case OCT_DEV_RUNNING:952	case OCT_DEV_CORE_OK:953 954		/* No more instructions will be forwarded. */955		atomic_set(&oct->status, OCT_DEV_IN_RESET);956 957		oct->app_mode = CVM_DRV_INVALID_APP;958		dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",959			lio_get_state_string(&oct->status));960 961		schedule_timeout_uninterruptible(HZ / 10);962 963		fallthrough;964	case OCT_DEV_HOST_OK:965 966	case OCT_DEV_CONSOLE_INIT_DONE:967		/* Remove any consoles */968		octeon_remove_consoles(oct);969 970		fallthrough;971	case OCT_DEV_IO_QUEUES_DONE:972		if (lio_wait_for_instr_fetch(oct))973			dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n");974 975		if (wait_for_pending_requests(oct))976			dev_err(&oct->pci_dev->dev, "There were pending requests\n");977 978		/* Disable the input and output queues now. No more packets will979		 * arrive from Octeon, but we should wait for all packet980		 * processing to finish.981		 */982		oct->fn_list.disable_io_queues(oct);983 984		if (lio_wait_for_oq_pkts(oct))985			dev_err(&oct->pci_dev->dev, "OQ had pending packets\n");986 987		/* Force all requests waiting to be fetched by OCTEON to988		 * complete.989		 */990		for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {991			struct octeon_instr_queue *iq;992 993			if (!(oct->io_qmask.iq & BIT_ULL(i)))994				continue;995			iq = oct->instr_queue[i];996 997			if (atomic_read(&iq->instr_pending)) {998				spin_lock_bh(&iq->lock);999				iq->fill_cnt = 0;1000				iq->octeon_read_index = iq->host_write_index;1001				iq->stats.instr_processed +=1002					atomic_read(&iq->instr_pending);1003				lio_process_iq_request_list(oct, iq, 0);1004				spin_unlock_bh(&iq->lock);1005			}1006		}1007 1008		lio_process_ordered_list(oct, 1);1009		octeon_free_sc_done_list(oct);1010		octeon_free_sc_zombie_list(oct);1011 1012		fallthrough;1013	case OCT_DEV_INTR_SET_DONE:1014		/* Disable interrupts  */1015		oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);1016 1017		if (oct->msix_on) {1018			msix_entries = (struct msix_entry *)oct->msix_entries;1019			for (i = 0; i < oct->num_msix_irqs - 1; i++) {1020				if (oct->ioq_vector[i].vector) {1021					/* clear the affinity_cpumask */1022					irq_set_affinity_hint(1023							msix_entries[i].vector,1024							NULL);1025					free_irq(msix_entries[i].vector,1026						 &oct->ioq_vector[i]);1027					oct->ioq_vector[i].vector = 0;1028				}1029			}1030			/* non-iov vector's argument is oct struct */1031			free_irq(msix_entries[i].vector, oct);1032 1033			pci_disable_msix(oct->pci_dev);1034			kfree(oct->msix_entries);1035			oct->msix_entries = NULL;1036		} else {1037			/* Release the interrupt line */1038			free_irq(oct->pci_dev->irq, oct);1039 1040			if (oct->flags & LIO_FLAG_MSI_ENABLED)1041				pci_disable_msi(oct->pci_dev);1042		}1043 1044		kfree(oct->irq_name_storage);1045		oct->irq_name_storage = NULL;1046 1047		fallthrough;1048	case OCT_DEV_MSIX_ALLOC_VECTOR_DONE:1049		if (OCTEON_CN23XX_PF(oct))1050			octeon_free_ioq_vector(oct);1051 1052		fallthrough;1053	case OCT_DEV_MBOX_SETUP_DONE:1054		if (OCTEON_CN23XX_PF(oct))1055			oct->fn_list.free_mbox(oct);1056 1057		fallthrough;1058	case OCT_DEV_IN_RESET:1059	case OCT_DEV_DROQ_INIT_DONE:1060		/* Wait for any pending operations */1061		mdelay(100);1062		for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {1063			if (!(oct->io_qmask.oq & BIT_ULL(i)))1064				continue;1065			octeon_delete_droq(oct, i);1066		}1067 1068		/* Force any pending handshakes to complete */1069		for (i = 0; i < MAX_OCTEON_DEVICES; i++) {1070			hs = &handshake[i];1071 1072			if (hs->pci_dev) {1073				handshake[oct->octeon_id].init_ok = 0;1074				complete(&handshake[oct->octeon_id].init);1075				handshake[oct->octeon_id].started_ok = 0;1076				complete(&handshake[oct->octeon_id].started);1077			}1078		}1079 1080		fallthrough;1081	case OCT_DEV_RESP_LIST_INIT_DONE:1082		octeon_delete_response_list(oct);1083 1084		fallthrough;1085	case OCT_DEV_INSTR_QUEUE_INIT_DONE:1086		for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {1087			if (!(oct->io_qmask.iq & BIT_ULL(i)))1088				continue;1089			octeon_delete_instr_queue(oct, i);1090		}1091#ifdef CONFIG_PCI_IOV1092		if (oct->sriov_info.sriov_enabled)1093			pci_disable_sriov(oct->pci_dev);1094#endif1095		fallthrough;1096	case OCT_DEV_SC_BUFF_POOL_INIT_DONE:1097		octeon_free_sc_buffer_pool(oct);1098 1099		fallthrough;1100	case OCT_DEV_DISPATCH_INIT_DONE:1101		octeon_delete_dispatch_list(oct);1102		cancel_delayed_work_sync(&oct->nic_poll_work.work);1103 1104		fallthrough;1105	case OCT_DEV_PCI_MAP_DONE:1106		refcount = octeon_deregister_device(oct);1107 1108		/* Soft reset the octeon device before exiting.1109		 * However, if fw was loaded from card (i.e. autoboot),1110		 * perform an FLR instead.1111		 * Implementation note: only soft-reset the device1112		 * if it is a CN6XXX OR the LAST CN23XX device.1113		 */1114		if (atomic_read(oct->adapter_fw_state) == FW_IS_PRELOADED)1115			octeon_pci_flr(oct);1116		else if (OCTEON_CN6XXX(oct) || !refcount)1117			oct->fn_list.soft_reset(oct);1118 1119		octeon_unmap_pci_barx(oct, 0);1120		octeon_unmap_pci_barx(oct, 1);1121 1122		fallthrough;1123	case OCT_DEV_PCI_ENABLE_DONE:1124		/* Disable the device, releasing the PCI INT */1125		pci_disable_device(oct->pci_dev);1126 1127		fallthrough;1128	case OCT_DEV_BEGIN_STATE:1129		/* Nothing to be done here either */1130		break;1131	}                       /* end switch (oct->status) */1132 1133	tasklet_kill(&oct_priv->droq_tasklet);1134}1135 1136/**1137 * send_rx_ctrl_cmd - Send Rx control command1138 * @lio: per-network private data1139 * @start_stop: whether to start or stop1140 */1141static int send_rx_ctrl_cmd(struct lio *lio, int start_stop)1142{1143	struct octeon_soft_command *sc;1144	union octnet_cmd *ncmd;1145	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;1146	int retval;1147 1148	if (oct->props[lio->ifidx].rx_on == start_stop)1149		return 0;1150 1151	sc = (struct octeon_soft_command *)1152		octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE,1153					  16, 0);1154	if (!sc) {1155		netif_info(lio, rx_err, lio->netdev,1156			   "Failed to allocate octeon_soft_command struct\n");1157		return -ENOMEM;1158	}1159 1160	ncmd = (union octnet_cmd *)sc->virtdptr;1161 1162	ncmd->u64 = 0;1163	ncmd->s.cmd = OCTNET_CMD_RX_CTL;1164	ncmd->s.param1 = start_stop;1165 1166	octeon_swap_8B_data((u64 *)ncmd, (OCTNET_CMD_SIZE >> 3));1167 1168	sc->iq_no = lio->linfo.txpciq[0].s.q_no;1169 1170	octeon_prepare_soft_command(oct, sc, OPCODE_NIC,1171				    OPCODE_NIC_CMD, 0, 0, 0);1172 1173	init_completion(&sc->complete);1174	sc->sc_status = OCTEON_REQUEST_PENDING;1175 1176	retval = octeon_send_soft_command(oct, sc);1177	if (retval == IQ_SEND_FAILED) {1178		netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n");1179		octeon_free_soft_command(oct, sc);1180	} else {1181		/* Sleep on a wait queue till the cond flag indicates that the1182		 * response arrived or timed-out.1183		 */1184		retval = wait_for_sc_completion_timeout(oct, sc, 0);1185		if (retval)1186			return retval;1187 1188		oct->props[lio->ifidx].rx_on = start_stop;1189		WRITE_ONCE(sc->caller_is_done, true);1190	}1191 1192	return retval;1193}1194 1195/**1196 * liquidio_destroy_nic_device - Destroy NIC device interface1197 * @oct: octeon device1198 * @ifidx: which interface to destroy1199 *1200 * Cleanup associated with each interface for an Octeon device  when NIC1201 * module is being unloaded or if initialization fails during load.1202 */1203static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)1204{1205	struct net_device *netdev = oct->props[ifidx].netdev;1206	struct octeon_device_priv *oct_priv = oct->priv;1207	struct napi_struct *napi, *n;1208	struct lio *lio;1209 1210	if (!netdev) {1211		dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n",1212			__func__, ifidx);1213		return;1214	}1215 1216	lio = GET_LIO(netdev);1217 1218	dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n");1219 1220	if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING)1221		liquidio_stop(netdev);1222 1223	if (oct->props[lio->ifidx].napi_enabled == 1) {1224		list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)1225			napi_disable(napi);1226 1227		oct->props[lio->ifidx].napi_enabled = 0;1228 1229		if (OCTEON_CN23XX_PF(oct))1230			oct->droq[0]->ops.poll_mode = 0;1231	}1232 1233	/* Delete NAPI */1234	list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)1235		netif_napi_del(napi);1236 1237	tasklet_enable(&oct_priv->droq_tasklet);1238 1239	if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED)1240		unregister_netdev(netdev);1241 1242	cleanup_sync_octeon_time_wq(netdev);1243	cleanup_link_status_change_wq(netdev);1244 1245	cleanup_rx_oom_poll_fn(netdev);1246 1247	lio_delete_glists(lio);1248 1249	free_netdev(netdev);1250 1251	oct->props[ifidx].gmxport = -1;1252 1253	oct->props[ifidx].netdev = NULL;1254}1255 1256/**1257 * liquidio_stop_nic_module - Stop complete NIC functionality1258 * @oct: octeon device1259 */1260static int liquidio_stop_nic_module(struct octeon_device *oct)1261{1262	int i, j;1263	struct lio *lio;1264 1265	dev_dbg(&oct->pci_dev->dev, "Stopping network interfaces\n");1266	device_lock(&oct->pci_dev->dev);1267	if (oct->devlink) {1268		devlink_unregister(oct->devlink);1269		devlink_free(oct->devlink);1270		oct->devlink = NULL;1271	}1272	device_unlock(&oct->pci_dev->dev);1273 1274	if (!oct->ifcount) {1275		dev_err(&oct->pci_dev->dev, "Init for Octeon was not completed\n");1276		return 1;1277	}1278 1279	spin_lock_bh(&oct->cmd_resp_wqlock);1280	oct->cmd_resp_state = OCT_DRV_OFFLINE;1281	spin_unlock_bh(&oct->cmd_resp_wqlock);1282 1283	lio_vf_rep_destroy(oct);1284 1285	for (i = 0; i < oct->ifcount; i++) {1286		lio = GET_LIO(oct->props[i].netdev);1287		for (j = 0; j < oct->num_oqs; j++)1288			octeon_unregister_droq_ops(oct,1289						   lio->linfo.rxpciq[j].s.q_no);1290	}1291 1292	for (i = 0; i < oct->ifcount; i++)1293		liquidio_destroy_nic_device(oct, i);1294 1295	dev_dbg(&oct->pci_dev->dev, "Network interfaces stopped\n");1296	return 0;1297}1298 1299/**1300 * liquidio_remove - Cleans up resources at unload time1301 * @pdev: PCI device structure1302 */1303static void liquidio_remove(struct pci_dev *pdev)1304{1305	struct octeon_device *oct_dev = pci_get_drvdata(pdev);1306 1307	dev_dbg(&oct_dev->pci_dev->dev, "Stopping device\n");1308 1309	if (oct_dev->watchdog_task)1310		kthread_stop(oct_dev->watchdog_task);1311 1312	if (!oct_dev->octeon_id &&1313	    oct_dev->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP)1314		lio_vf_rep_modexit();1315 1316	if (oct_dev->app_mode && (oct_dev->app_mode == CVM_DRV_NIC_APP))1317		liquidio_stop_nic_module(oct_dev);1318 1319	/* Reset the octeon device and cleanup all memory allocated for1320	 * the octeon device by driver.1321	 */1322	octeon_destroy_resources(oct_dev);1323 1324	dev_info(&oct_dev->pci_dev->dev, "Device removed\n");1325 1326	/* This octeon device has been removed. Update the global1327	 * data structure to reflect this. Free the device structure.1328	 */1329	octeon_free_device_mem(oct_dev);1330}1331 1332/**1333 * octeon_chip_specific_setup - Identify the Octeon device and to map the BAR address space1334 * @oct: octeon device1335 */1336static int octeon_chip_specific_setup(struct octeon_device *oct)1337{1338	u32 dev_id, rev_id;1339	int ret = 1;1340 1341	pci_read_config_dword(oct->pci_dev, 0, &dev_id);1342	pci_read_config_dword(oct->pci_dev, 8, &rev_id);1343	oct->rev_id = rev_id & 0xff;1344 1345	switch (dev_id) {1346	case OCTEON_CN68XX_PCIID:1347		oct->chip_id = OCTEON_CN68XX;1348		ret = lio_setup_cn68xx_octeon_device(oct);1349		break;1350 1351	case OCTEON_CN66XX_PCIID:1352		oct->chip_id = OCTEON_CN66XX;1353		ret = lio_setup_cn66xx_octeon_device(oct);1354		break;1355 1356	case OCTEON_CN23XX_PCIID_PF:1357		oct->chip_id = OCTEON_CN23XX_PF_VID;1358		ret = setup_cn23xx_octeon_pf_device(oct);1359		if (ret)1360			break;1361#ifdef CONFIG_PCI_IOV1362		if (!ret)1363			pci_sriov_set_totalvfs(oct->pci_dev,1364					       oct->sriov_info.max_vfs);1365#endif1366		break;1367 1368	default:1369		dev_err(&oct->pci_dev->dev, "Unknown device found (dev_id: %x)\n",1370			dev_id);1371	}1372 1373	return ret;1374}1375 1376/**1377 * octeon_pci_os_setup - PCI initialization for each Octeon device.1378 * @oct: octeon device1379 */1380static int octeon_pci_os_setup(struct octeon_device *oct)1381{1382	/* setup PCI stuff first */1383	if (pci_enable_device(oct->pci_dev)) {1384		dev_err(&oct->pci_dev->dev, "pci_enable_device failed\n");1385		return 1;1386	}1387 1388	if (dma_set_mask_and_coherent(&oct->pci_dev->dev, DMA_BIT_MASK(64))) {1389		dev_err(&oct->pci_dev->dev, "Unexpected DMA device capability\n");1390		pci_disable_device(oct->pci_dev);1391		return 1;1392	}1393 1394	/* Enable PCI DMA Master. */1395	pci_set_master(oct->pci_dev);1396 1397	return 0;1398}1399 1400/**1401 * free_netbuf - Unmap and free network buffer1402 * @buf: buffer1403 */1404static void free_netbuf(void *buf)1405{1406	struct sk_buff *skb;1407	struct octnet_buf_free_info *finfo;1408	struct lio *lio;1409 1410	finfo = (struct octnet_buf_free_info *)buf;1411	skb = finfo->skb;1412	lio = finfo->lio;1413 1414	dma_unmap_single(&lio->oct_dev->pci_dev->dev, finfo->dptr, skb->len,1415			 DMA_TO_DEVICE);1416 1417	tx_buffer_free(skb);1418}1419 1420/**1421 * free_netsgbuf - Unmap and free gather buffer1422 * @buf: buffer1423 */1424static void free_netsgbuf(void *buf)1425{1426	struct octnet_buf_free_info *finfo;1427	struct sk_buff *skb;1428	struct lio *lio;1429	struct octnic_gather *g;1430	int i, frags, iq;1431 1432	finfo = (struct octnet_buf_free_info *)buf;1433	skb = finfo->skb;1434	lio = finfo->lio;1435	g = finfo->g;1436	frags = skb_shinfo(skb)->nr_frags;1437 1438	dma_unmap_single(&lio->oct_dev->pci_dev->dev,1439			 g->sg[0].ptr[0], (skb->len - skb->data_len),1440			 DMA_TO_DEVICE);1441 1442	i = 1;1443	while (frags--) {1444		skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];1445 1446		dma_unmap_page(&lio->oct_dev->pci_dev->dev,1447			       g->sg[(i >> 2)].ptr[(i & 3)],1448			       skb_frag_size(frag), DMA_TO_DEVICE);1449		i++;1450	}1451 1452	iq = skb_iq(lio->oct_dev, skb);1453	spin_lock(&lio->glist_lock[iq]);1454	list_add_tail(&g->list, &lio->glist[iq]);1455	spin_unlock(&lio->glist_lock[iq]);1456 1457	tx_buffer_free(skb);1458}1459 1460/**1461 * free_netsgbuf_with_resp - Unmap and free gather buffer with response1462 * @buf: buffer1463 */1464static void free_netsgbuf_with_resp(void *buf)1465{1466	struct octeon_soft_command *sc;1467	struct octnet_buf_free_info *finfo;1468	struct sk_buff *skb;1469	struct lio *lio;1470	struct octnic_gather *g;1471	int i, frags, iq;1472 1473	sc = (struct octeon_soft_command *)buf;1474	skb = (struct sk_buff *)sc->callback_arg;1475	finfo = (struct octnet_buf_free_info *)&skb->cb;1476 1477	lio = finfo->lio;1478	g = finfo->g;1479	frags = skb_shinfo(skb)->nr_frags;1480 1481	dma_unmap_single(&lio->oct_dev->pci_dev->dev,1482			 g->sg[0].ptr[0], (skb->len - skb->data_len),1483			 DMA_TO_DEVICE);1484 1485	i = 1;1486	while (frags--) {1487		skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];1488 1489		dma_unmap_page(&lio->oct_dev->pci_dev->dev,1490			       g->sg[(i >> 2)].ptr[(i & 3)],1491			       skb_frag_size(frag), DMA_TO_DEVICE);1492		i++;1493	}1494 1495	iq = skb_iq(lio->oct_dev, skb);1496 1497	spin_lock(&lio->glist_lock[iq]);1498	list_add_tail(&g->list, &lio->glist[iq]);1499	spin_unlock(&lio->glist_lock[iq]);1500 1501	/* Don't free the skb yet */1502}1503 1504/**1505 * liquidio_ptp_adjfine - Adjust ptp frequency1506 * @ptp: PTP clock info1507 * @scaled_ppm: how much to adjust by, in scaled parts-per-million1508 *1509 * Scaled parts per million is ppm with a 16-bit binary fractional field.1510 */1511static int liquidio_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm)1512{1513	struct lio *lio = container_of(ptp, struct lio, ptp_info);1514	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;1515	s32 ppb = scaled_ppm_to_ppb(scaled_ppm);1516	u64 comp, delta;1517	unsigned long flags;1518	bool neg_adj = false;1519 1520	if (ppb < 0) {1521		neg_adj = true;1522		ppb = -ppb;1523	}1524 1525	/* The hardware adds the clock compensation value to the1526	 * PTP clock on every coprocessor clock cycle, so we1527	 * compute the delta in terms of coprocessor clocks.1528	 */1529	delta = (u64)ppb << 32;1530	do_div(delta, oct->coproc_clock_rate);1531 1532	spin_lock_irqsave(&lio->ptp_lock, flags);1533	comp = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_COMP);1534	if (neg_adj)1535		comp -= delta;1536	else1537		comp += delta;1538	lio_pci_writeq(oct, comp, CN6XXX_MIO_PTP_CLOCK_COMP);1539	spin_unlock_irqrestore(&lio->ptp_lock, flags);1540 1541	return 0;1542}1543 1544/**1545 * liquidio_ptp_adjtime - Adjust ptp time1546 * @ptp: PTP clock info1547 * @delta: how much to adjust by, in nanosecs1548 */1549static int liquidio_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)1550{1551	unsigned long flags;1552	struct lio *lio = container_of(ptp, struct lio, ptp_info);1553 1554	spin_lock_irqsave(&lio->ptp_lock, flags);1555	lio->ptp_adjust += delta;1556	spin_unlock_irqrestore(&lio->ptp_lock, flags);1557 1558	return 0;1559}1560 1561/**1562 * liquidio_ptp_gettime - Get hardware clock time, including any adjustment1563 * @ptp: PTP clock info1564 * @ts: timespec1565 */1566static int liquidio_ptp_gettime(struct ptp_clock_info *ptp,1567				struct timespec64 *ts)1568{1569	u64 ns;1570	unsigned long flags;1571	struct lio *lio = container_of(ptp, struct lio, ptp_info);1572	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;1573 1574	spin_lock_irqsave(&lio->ptp_lock, flags);1575	ns = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_HI);1576	ns += lio->ptp_adjust;1577	spin_unlock_irqrestore(&lio->ptp_lock, flags);1578 1579	*ts = ns_to_timespec64(ns);1580 1581	return 0;1582}1583 1584/**1585 * liquidio_ptp_settime - Set hardware clock time. Reset adjustment1586 * @ptp: PTP clock info1587 * @ts: timespec1588 */1589static int liquidio_ptp_settime(struct ptp_clock_info *ptp,1590				const struct timespec64 *ts)1591{1592	u64 ns;1593	unsigned long flags;1594	struct lio *lio = container_of(ptp, struct lio, ptp_info);1595	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;1596 1597	ns = timespec64_to_ns(ts);1598 1599	spin_lock_irqsave(&lio->ptp_lock, flags);1600	lio_pci_writeq(oct, ns, CN6XXX_MIO_PTP_CLOCK_HI);1601	lio->ptp_adjust = 0;1602	spin_unlock_irqrestore(&lio->ptp_lock, flags);1603 1604	return 0;1605}1606 1607/**1608 * liquidio_ptp_enable - Check if PTP is enabled1609 * @ptp: PTP clock info1610 * @rq: request1611 * @on: is it on1612 */1613static int1614liquidio_ptp_enable(struct ptp_clock_info __maybe_unused *ptp,1615		    struct ptp_clock_request __maybe_unused *rq,1616		    int __maybe_unused on)1617{1618	return -EOPNOTSUPP;1619}1620 1621/**1622 * oct_ptp_open - Open PTP clock source1623 * @netdev: network device1624 */1625static void oct_ptp_open(struct net_device *netdev)1626{1627	struct lio *lio = GET_LIO(netdev);1628	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;1629 1630	spin_lock_init(&lio->ptp_lock);1631 1632	snprintf(lio->ptp_info.name, 16, "%s", netdev->name);1633	lio->ptp_info.owner = THIS_MODULE;1634	lio->ptp_info.max_adj = 250000000;1635	lio->ptp_info.n_alarm = 0;1636	lio->ptp_info.n_ext_ts = 0;1637	lio->ptp_info.n_per_out = 0;1638	lio->ptp_info.pps = 0;1639	lio->ptp_info.adjfine = liquidio_ptp_adjfine;1640	lio->ptp_info.adjtime = liquidio_ptp_adjtime;1641	lio->ptp_info.gettime64 = liquidio_ptp_gettime;1642	lio->ptp_info.settime64 = liquidio_ptp_settime;1643	lio->ptp_info.enable = liquidio_ptp_enable;1644 1645	lio->ptp_adjust = 0;1646 1647	lio->ptp_clock = ptp_clock_register(&lio->ptp_info,1648					     &oct->pci_dev->dev);1649 1650	if (IS_ERR(lio->ptp_clock))1651		lio->ptp_clock = NULL;1652}1653 1654/**1655 * liquidio_ptp_init - Init PTP clock1656 * @oct: octeon device1657 */1658static void liquidio_ptp_init(struct octeon_device *oct)1659{1660	u64 clock_comp, cfg;1661 1662	clock_comp = (u64)NSEC_PER_SEC << 32;1663	do_div(clock_comp, oct->coproc_clock_rate);1664	lio_pci_writeq(oct, clock_comp, CN6XXX_MIO_PTP_CLOCK_COMP);1665 1666	/* Enable */1667	cfg = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_CFG);1668	lio_pci_writeq(oct, cfg | 0x01, CN6XXX_MIO_PTP_CLOCK_CFG);1669}1670 1671/**1672 * load_firmware - Load firmware to device1673 * @oct: octeon device1674 *1675 * Maps device to firmware filename, requests firmware, and downloads it1676 */1677static int load_firmware(struct octeon_device *oct)1678{1679	int ret = 0;1680	const struct firmware *fw;1681	char fw_name[LIO_MAX_FW_FILENAME_LEN];1682	char *tmp_fw_type;1683 1684	if (fw_type_is_auto()) {1685		tmp_fw_type = LIO_FW_NAME_TYPE_NIC;1686		strscpy_pad(fw_type, tmp_fw_type, sizeof(fw_type));1687	} else {1688		tmp_fw_type = fw_type;1689	}1690 1691	sprintf(fw_name, "%s%s%s_%s%s", LIO_FW_DIR, LIO_FW_BASE_NAME,1692		octeon_get_conf(oct)->card_name, tmp_fw_type,1693		LIO_FW_NAME_SUFFIX);1694 1695	ret = request_firmware(&fw, fw_name, &oct->pci_dev->dev);1696	if (ret) {1697		dev_err(&oct->pci_dev->dev, "Request firmware failed. Could not find file %s.\n",1698			fw_name);1699		release_firmware(fw);1700		return ret;1701	}1702 1703	ret = octeon_download_firmware(oct, fw->data, fw->size);1704 1705	release_firmware(fw);1706 1707	return ret;1708}1709 1710/**1711 * octnet_poll_check_txq_status - Poll routine for checking transmit queue status1712 * @work: work_struct data structure1713 */1714static void octnet_poll_check_txq_status(struct work_struct *work)1715{1716	struct cavium_wk *wk = (struct cavium_wk *)work;1717	struct lio *lio = (struct lio *)wk->ctxptr;1718 1719	if (!ifstate_check(lio, LIO_IFSTATE_RUNNING))1720		return;1721 1722	check_txq_status(lio);1723	queue_delayed_work(lio->txq_status_wq.wq,1724			   &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));1725}1726 1727/**1728 * setup_tx_poll_fn - Sets up the txq poll check1729 * @netdev: network device1730 */1731static inline int setup_tx_poll_fn(struct net_device *netdev)1732{1733	struct lio *lio = GET_LIO(netdev);1734	struct octeon_device *oct = lio->oct_dev;1735 1736	lio->txq_status_wq.wq = alloc_workqueue("txq-status",1737						WQ_MEM_RECLAIM, 0);1738	if (!lio->txq_status_wq.wq) {1739		dev_err(&oct->pci_dev->dev, "unable to create cavium txq status wq\n");1740		return -1;1741	}1742	INIT_DELAYED_WORK(&lio->txq_status_wq.wk.work,1743			  octnet_poll_check_txq_status);1744	lio->txq_status_wq.wk.ctxptr = lio;1745	queue_delayed_work(lio->txq_status_wq.wq,1746			   &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));1747	return 0;1748}1749 1750static inline void cleanup_tx_poll_fn(struct net_device *netdev)1751{1752	struct lio *lio = GET_LIO(netdev);1753 1754	if (lio->txq_status_wq.wq) {1755		cancel_delayed_work_sync(&lio->txq_status_wq.wk.work);1756		destroy_workqueue(lio->txq_status_wq.wq);1757	}1758}1759 1760/**1761 * liquidio_open - Net device open for LiquidIO1762 * @netdev: network device1763 */1764static int liquidio_open(struct net_device *netdev)1765{1766	struct lio *lio = GET_LIO(netdev);1767	struct octeon_device *oct = lio->oct_dev;1768	struct octeon_device_priv *oct_priv = oct->priv;1769	struct napi_struct *napi, *n;1770	int ret = 0;1771 1772	if (oct->props[lio->ifidx].napi_enabled == 0) {1773		tasklet_disable(&oct_priv->droq_tasklet);1774 1775		list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)1776			napi_enable(napi);1777 1778		oct->props[lio->ifidx].napi_enabled = 1;1779 1780		if (OCTEON_CN23XX_PF(oct))1781			oct->droq[0]->ops.poll_mode = 1;1782	}1783 1784	if (oct->ptp_enable)1785		oct_ptp_open(netdev);1786 1787	ifstate_set(lio, LIO_IFSTATE_RUNNING);1788 1789	if (!OCTEON_CN23XX_PF(oct) || !oct->msix_on) {1790		ret = setup_tx_poll_fn(netdev);1791		if (ret)1792			goto err_poll;1793	}1794 1795	netif_tx_start_all_queues(netdev);1796 1797	/* Ready for link status updates */1798	lio->intf_open = 1;1799 1800	netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n");1801 1802	/* tell Octeon to start forwarding packets to host */1803	ret = send_rx_ctrl_cmd(lio, 1);1804	if (ret)1805		goto err_rx_ctrl;1806 1807	/* start periodical statistics fetch */1808	INIT_DELAYED_WORK(&lio->stats_wk.work, lio_fetch_stats);1809	lio->stats_wk.ctxptr = lio;1810	schedule_delayed_work(&lio->stats_wk.work, msecs_to_jiffies1811					(LIQUIDIO_NDEV_STATS_POLL_TIME_MS));1812 1813	dev_info(&oct->pci_dev->dev, "%s interface is opened\n",1814		 netdev->name);1815 1816	return 0;1817 1818err_rx_ctrl:1819	if (!OCTEON_CN23XX_PF(oct) || !oct->msix_on)1820		cleanup_tx_poll_fn(netdev);1821err_poll:1822	if (lio->ptp_clock) {1823		ptp_clock_unregister(lio->ptp_clock);1824		lio->ptp_clock = NULL;1825	}1826 1827	if (oct->props[lio->ifidx].napi_enabled == 1) {1828		list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)1829			napi_disable(napi);1830 1831		oct->props[lio->ifidx].napi_enabled = 0;1832 1833		if (OCTEON_CN23XX_PF(oct))1834			oct->droq[0]->ops.poll_mode = 0;1835	}1836 1837	return ret;1838}1839 1840/**1841 * liquidio_stop - Net device stop for LiquidIO1842 * @netdev: network device1843 */1844static int liquidio_stop(struct net_device *netdev)1845{1846	struct lio *lio = GET_LIO(netdev);1847	struct octeon_device *oct = lio->oct_dev;1848	struct octeon_device_priv *oct_priv = oct->priv;1849	struct napi_struct *napi, *n;1850	int ret = 0;1851 1852	ifstate_reset(lio, LIO_IFSTATE_RUNNING);1853 1854	/* Stop any link updates */1855	lio->intf_open = 0;1856 1857	stop_txqs(netdev);1858 1859	/* Inform that netif carrier is down */1860	netif_carrier_off(netdev);1861	netif_tx_disable(netdev);1862 1863	lio->linfo.link.s.link_up = 0;1864	lio->link_changes++;1865 1866	/* Tell Octeon that nic interface is down. */1867	ret = send_rx_ctrl_cmd(lio, 0);1868	if (ret)1869		return ret;1870 1871	if (OCTEON_CN23XX_PF(oct)) {1872		if (!oct->msix_on)1873			cleanup_tx_poll_fn(netdev);1874	} else {1875		cleanup_tx_poll_fn(netdev);1876	}1877 1878	cancel_delayed_work_sync(&lio->stats_wk.work);1879 1880	if (lio->ptp_clock) {1881		ptp_clock_unregister(lio->ptp_clock);1882		lio->ptp_clock = NULL;1883	}1884 1885	/* Wait for any pending Rx descriptors */1886	if (lio_wait_for_clean_oq(oct))1887		netif_info(lio, rx_err, lio->netdev,1888			   "Proceeding with stop interface after partial RX desc processing\n");1889 1890	if (oct->props[lio->ifidx].napi_enabled == 1) {1891		list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)1892			napi_disable(napi);1893 1894		oct->props[lio->ifidx].napi_enabled = 0;1895 1896		if (OCTEON_CN23XX_PF(oct))1897			oct->droq[0]->ops.poll_mode = 0;1898 1899		tasklet_enable(&oct_priv->droq_tasklet);1900	}1901 1902	dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name);1903 1904	return ret;1905}1906 1907/**1908 * get_new_flags - Converts a mask based on net device flags1909 * @netdev: network device1910 *1911 * This routine generates a octnet_ifflags mask from the net device flags1912 * received from the OS.1913 */1914static inline enum octnet_ifflags get_new_flags(struct net_device *netdev)1915{1916	enum octnet_ifflags f = OCTNET_IFFLAG_UNICAST;1917 1918	if (netdev->flags & IFF_PROMISC)1919		f |= OCTNET_IFFLAG_PROMISC;1920 1921	if (netdev->flags & IFF_ALLMULTI)1922		f |= OCTNET_IFFLAG_ALLMULTI;1923 1924	if (netdev->flags & IFF_MULTICAST) {1925		f |= OCTNET_IFFLAG_MULTICAST;1926 1927		/* Accept all multicast addresses if there are more than we1928		 * can handle1929		 */1930		if (netdev_mc_count(netdev) > MAX_OCTEON_MULTICAST_ADDR)1931			f |= OCTNET_IFFLAG_ALLMULTI;1932	}1933 1934	if (netdev->flags & IFF_BROADCAST)1935		f |= OCTNET_IFFLAG_BROADCAST;1936 1937	return f;1938}1939 1940/**1941 * liquidio_set_mcast_list - Net device set_multicast_list1942 * @netdev: network device1943 */1944static void liquidio_set_mcast_list(struct net_device *netdev)1945{1946	struct lio *lio = GET_LIO(netdev);1947	struct octeon_device *oct = lio->oct_dev;1948	struct octnic_ctrl_pkt nctrl;1949	struct netdev_hw_addr *ha;1950	u64 *mc;1951	int ret;1952	int mc_count = min(netdev_mc_count(netdev), MAX_OCTEON_MULTICAST_ADDR);1953 1954	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));1955 1956	/* Create a ctrl pkt command to be sent to core app. */1957	nctrl.ncmd.u64 = 0;1958	nctrl.ncmd.s.cmd = OCTNET_CMD_SET_MULTI_LIST;1959	nctrl.ncmd.s.param1 = get_new_flags(netdev);1960	nctrl.ncmd.s.param2 = mc_count;1961	nctrl.ncmd.s.more = mc_count;1962	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;1963	nctrl.netpndev = (u64)netdev;1964	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;1965 1966	/* copy all the addresses into the udd */1967	mc = &nctrl.udd[0];1968	netdev_for_each_mc_addr(ha, netdev) {1969		*mc = 0;1970		memcpy(((u8 *)mc) + 2, ha->addr, ETH_ALEN);1971		/* no need to swap bytes */1972 1973		if (++mc > &nctrl.udd[mc_count])1974			break;1975	}1976 1977	/* Apparently, any activity in this call from the kernel has to1978	 * be atomic. So we won't wait for response.1979	 */1980 1981	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);1982	if (ret) {1983		dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n",1984			ret);1985	}1986}1987 1988/**1989 * liquidio_set_mac - Net device set_mac_address1990 * @netdev: network device1991 * @p: pointer to sockaddr1992 */1993static int liquidio_set_mac(struct net_device *netdev, void *p)1994{1995	int ret = 0;1996	struct lio *lio = GET_LIO(netdev);1997	struct octeon_device *oct = lio->oct_dev;1998	struct sockaddr *addr = (struct sockaddr *)p;1999	struct octnic_ctrl_pkt nctrl;2000 2001	if (!is_valid_ether_addr(addr->sa_data))2002		return -EADDRNOTAVAIL;2003 2004	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));2005 2006	nctrl.ncmd.u64 = 0;2007	nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;2008	nctrl.ncmd.s.param1 = 0;2009	nctrl.ncmd.s.more = 1;2010	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;2011	nctrl.netpndev = (u64)netdev;2012 2013	nctrl.udd[0] = 0;2014	/* The MAC Address is presented in network byte order. */2015	memcpy((u8 *)&nctrl.udd[0] + 2, addr->sa_data, ETH_ALEN);2016 2017	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);2018	if (ret < 0) {2019		dev_err(&oct->pci_dev->dev, "MAC Address change failed\n");2020		return -ENOMEM;2021	}2022 2023	if (nctrl.sc_status) {2024		dev_err(&oct->pci_dev->dev,2025			"%s: MAC Address change failed. sc return=%x\n",2026			 __func__, nctrl.sc_status);2027		return -EIO;2028	}2029 2030	eth_hw_addr_set(netdev, addr->sa_data);2031	memcpy(((u8 *)&lio->linfo.hw_addr) + 2, addr->sa_data, ETH_ALEN);2032 2033	return 0;2034}2035 2036static void2037liquidio_get_stats64(struct net_device *netdev,2038		     struct rtnl_link_stats64 *lstats)2039{2040	struct lio *lio = GET_LIO(netdev);2041	struct octeon_device *oct;2042	u64 pkts = 0, drop = 0, bytes = 0;2043	struct oct_droq_stats *oq_stats;2044	struct oct_iq_stats *iq_stats;2045	int i, iq_no, oq_no;2046 2047	oct = lio->oct_dev;2048 2049	if (ifstate_check(lio, LIO_IFSTATE_RESETTING))2050		return;2051 2052	for (i = 0; i < oct->num_iqs; i++) {2053		iq_no = lio->linfo.txpciq[i].s.q_no;2054		iq_stats = &oct->instr_queue[iq_no]->stats;2055		pkts += iq_stats->tx_done;2056		drop += iq_stats->tx_dropped;2057		bytes += iq_stats->tx_tot_bytes;2058	}2059 2060	lstats->tx_packets = pkts;2061	lstats->tx_bytes = bytes;2062	lstats->tx_dropped = drop;2063 2064	pkts = 0;2065	drop = 0;2066	bytes = 0;2067 2068	for (i = 0; i < oct->num_oqs; i++) {2069		oq_no = lio->linfo.rxpciq[i].s.q_no;2070		oq_stats = &oct->droq[oq_no]->stats;2071		pkts += oq_stats->rx_pkts_received;2072		drop += (oq_stats->rx_dropped +2073			 oq_stats->dropped_nodispatch +2074			 oq_stats->dropped_toomany +2075			 oq_stats->dropped_nomem);2076		bytes += oq_stats->rx_bytes_received;2077	}2078 2079	lstats->rx_bytes = bytes;2080	lstats->rx_packets = pkts;2081	lstats->rx_dropped = drop;2082 2083	lstats->multicast = oct->link_stats.fromwire.fw_total_mcast;2084	lstats->collisions = oct->link_stats.fromhost.total_collisions;2085 2086	/* detailed rx_errors: */2087	lstats->rx_length_errors = oct->link_stats.fromwire.l2_err;2088	/* recved pkt with crc error    */2089	lstats->rx_crc_errors = oct->link_stats.fromwire.fcs_err;2090	/* recv'd frame alignment error */2091	lstats->rx_frame_errors = oct->link_stats.fromwire.frame_err;2092	/* recv'r fifo overrun */2093	lstats->rx_fifo_errors = oct->link_stats.fromwire.fifo_err;2094 2095	lstats->rx_errors = lstats->rx_length_errors + lstats->rx_crc_errors +2096		lstats->rx_frame_errors + lstats->rx_fifo_errors;2097 2098	/* detailed tx_errors */2099	lstats->tx_aborted_errors = oct->link_stats.fromhost.fw_err_pko;2100	lstats->tx_carrier_errors = oct->link_stats.fromhost.fw_err_link;2101	lstats->tx_fifo_errors = oct->link_stats.fromhost.fifo_err;2102 2103	lstats->tx_errors = lstats->tx_aborted_errors +2104		lstats->tx_carrier_errors +2105		lstats->tx_fifo_errors;2106}2107 2108/**2109 * hwtstamp_ioctl - Handler for SIOCSHWTSTAMP ioctl2110 * @netdev: network device2111 * @ifr: interface request2112 */2113static int hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr)2114{2115	struct hwtstamp_config conf;2116	struct lio *lio = GET_LIO(netdev);2117 2118	if (copy_from_user(&conf, ifr->ifr_data, sizeof(conf)))2119		return -EFAULT;2120 2121	switch (conf.tx_type) {2122	case HWTSTAMP_TX_ON:2123	case HWTSTAMP_TX_OFF:2124		break;2125	default:2126		return -ERANGE;2127	}2128 2129	switch (conf.rx_filter) {2130	case HWTSTAMP_FILTER_NONE:2131		break;2132	case HWTSTAMP_FILTER_ALL:2133	case HWTSTAMP_FILTER_SOME:2134	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:2135	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:2136	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:2137	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:2138	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:2139	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:2140	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:2141	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:2142	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:2143	case HWTSTAMP_FILTER_PTP_V2_EVENT:2144	case HWTSTAMP_FILTER_PTP_V2_SYNC:2145	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:2146	case HWTSTAMP_FILTER_NTP_ALL:2147		conf.rx_filter = HWTSTAMP_FILTER_ALL;2148		break;2149	default:2150		return -ERANGE;2151	}2152 2153	if (conf.rx_filter == HWTSTAMP_FILTER_ALL)2154		ifstate_set(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);2155 2156	else2157		ifstate_reset(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);2158 2159	return copy_to_user(ifr->ifr_data, &conf, sizeof(conf)) ? -EFAULT : 0;2160}2161 2162/**2163 * liquidio_ioctl - ioctl handler2164 * @netdev: network device2165 * @ifr: interface request2166 * @cmd: command2167 */2168static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)2169{2170	struct lio *lio = GET_LIO(netdev);2171 2172	switch (cmd) {2173	case SIOCSHWTSTAMP:2174		if (lio->oct_dev->ptp_enable)2175			return hwtstamp_ioctl(netdev, ifr);2176		fallthrough;2177	default:2178		return -EOPNOTSUPP;2179	}2180}2181 2182/**2183 * handle_timestamp - handle a Tx timestamp response2184 * @oct: octeon device2185 * @status: response status2186 * @buf: pointer to skb2187 */2188static void handle_timestamp(struct octeon_device *oct,2189			     u32 status,2190			     void *buf)2191{2192	struct octnet_buf_free_info *finfo;2193	struct octeon_soft_command *sc;2194	struct oct_timestamp_resp *resp;2195	struct lio *lio;2196	struct sk_buff *skb = (struct sk_buff *)buf;2197 2198	finfo = (struct octnet_buf_free_info *)skb->cb;2199	lio = finfo->lio;2200	sc = finfo->sc;2201	oct = lio->oct_dev;2202	resp = (struct oct_timestamp_resp *)sc->virtrptr;2203 2204	if (status != OCTEON_REQUEST_DONE) {2205		dev_err(&oct->pci_dev->dev, "Tx timestamp instruction failed. Status: %llx\n",2206			CVM_CAST64(status));2207		resp->timestamp = 0;2208	}2209 2210	octeon_swap_8B_data(&resp->timestamp, 1);2211 2212	if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) != 0)) {2213		struct skb_shared_hwtstamps ts;2214		u64 ns = resp->timestamp;2215 2216		netif_info(lio, tx_done, lio->netdev,2217			   "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n",2218			   skb, (unsigned long long)ns);2219		ts.hwtstamp = ns_to_ktime(ns + lio->ptp_adjust);2220		skb_tstamp_tx(skb, &ts);2221	}2222 2223	octeon_free_soft_command(oct, sc);2224	tx_buffer_free(skb);2225}2226 2227/**2228 * send_nic_timestamp_pkt - Send a data packet that will be timestamped2229 * @oct: octeon device2230 * @ndata: pointer to network data2231 * @finfo: pointer to private network data2232 * @xmit_more: more is coming2233 */2234static inline int send_nic_timestamp_pkt(struct octeon_device *oct,2235					 struct octnic_data_pkt *ndata,2236					 struct octnet_buf_free_info *finfo,2237					 int xmit_more)2238{2239	int retval;2240	struct octeon_soft_command *sc;2241	struct lio *lio;2242	int ring_doorbell;2243	u32 len;2244 2245	lio = finfo->lio;2246 2247	sc = octeon_alloc_soft_command_resp(oct, &ndata->cmd,2248					    sizeof(struct oct_timestamp_resp));2249	finfo->sc = sc;2250 2251	if (!sc) {2252		dev_err(&oct->pci_dev->dev, "No memory for timestamped data packet\n");2253		return IQ_SEND_FAILED;2254	}2255 2256	if (ndata->reqtype == REQTYPE_NORESP_NET)2257		ndata->reqtype = REQTYPE_RESP_NET;2258	else if (ndata->reqtype == REQTYPE_NORESP_NET_SG)2259		ndata->reqtype = REQTYPE_RESP_NET_SG;2260 2261	sc->callback = handle_timestamp;2262	sc->callback_arg = finfo->skb;2263	sc->iq_no = ndata->q_no;2264 2265	if (OCTEON_CN23XX_PF(oct))2266		len = (u32)((struct octeon_instr_ih3 *)2267			    (&sc->cmd.cmd3.ih3))->dlengsz;2268	else2269		len = (u32)((struct octeon_instr_ih2 *)2270			    (&sc->cmd.cmd2.ih2))->dlengsz;2271 2272	ring_doorbell = !xmit_more;2273 2274	retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd,2275				     sc, len, ndata->reqtype);2276 2277	if (retval == IQ_SEND_FAILED) {2278		dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n",2279			retval);2280		octeon_free_soft_command(oct, sc);2281	} else {2282		netif_info(lio, tx_queued, lio->netdev, "Queued timestamp packet\n");2283	}2284 2285	return retval;2286}2287 2288/**2289 * liquidio_xmit - Transmit networks packets to the Octeon interface2290 * @skb: skbuff struct to be passed to network layer.2291 * @netdev: pointer to network device2292 *2293 * Return: whether the packet was transmitted to the device okay or not2294 *             (NETDEV_TX_OK or NETDEV_TX_BUSY)2295 */2296static netdev_tx_t liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)2297{2298	struct lio *lio;2299	struct octnet_buf_free_info *finfo;2300	union octnic_cmd_setup cmdsetup;2301	struct octnic_data_pkt ndata;2302	struct octeon_device *oct;2303	struct oct_iq_stats *stats;2304	struct octeon_instr_irh *irh;2305	union tx_info *tx_info;2306	int status = 0;2307	int q_idx = 0, iq_no = 0;2308	int j, xmit_more = 0;2309	u64 dptr = 0;2310	u32 tag = 0;2311 2312	lio = GET_LIO(netdev);2313	oct = lio->oct_dev;2314 2315	q_idx = skb_iq(oct, skb);2316	tag = q_idx;2317	iq_no = lio->linfo.txpciq[q_idx].s.q_no;2318 2319	stats = &oct->instr_queue[iq_no]->stats;2320 2321	/* Check for all conditions in which the current packet cannot be2322	 * transmitted.2323	 */2324	if (!(atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) ||2325	    (!lio->linfo.link.s.link_up) ||2326	    (skb->len <= 0)) {2327		netif_info(lio, tx_err, lio->netdev,2328			   "Transmit failed link_status : %d\n",2329			   lio->linfo.link.s.link_up);2330		goto lio_xmit_failed;2331	}2332 2333	/* Use space in skb->cb to store info used to unmap and2334	 * free the buffers.2335	 */2336	finfo = (struct octnet_buf_free_info *)skb->cb;2337	finfo->lio = lio;2338	finfo->skb = skb;2339	finfo->sc = NULL;2340 2341	/* Prepare the attributes for the data to be passed to OSI. */2342	memset(&ndata, 0, sizeof(struct octnic_data_pkt));2343 2344	ndata.buf = (void *)finfo;2345 2346	ndata.q_no = iq_no;2347 2348	if (octnet_iq_is_full(oct, ndata.q_no)) {2349		/* defer sending if queue is full */2350		netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",2351			   ndata.q_no);2352		stats->tx_iq_busy++;2353		return NETDEV_TX_BUSY;2354	}2355 2356	/* pr_info(" XMIT - valid Qs: %d, 1st Q no: %d, cpu:  %d, q_no:%d\n",2357	 *	lio->linfo.num_txpciq, lio->txq, cpu, ndata.q_no);2358	 */2359 2360	ndata.datasize = skb->len;2361 2362	cmdsetup.u64 = 0;2363	cmdsetup.s.iq_no = iq_no;2364 2365	if (skb->ip_summed == CHECKSUM_PARTIAL) {2366		if (skb->encapsulation) {2367			cmdsetup.s.tnl_csum = 1;2368			stats->tx_vxlan++;2369		} else {2370			cmdsetup.s.transport_csum = 1;2371		}2372	}2373	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {2374		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;2375		cmdsetup.s.timestamp = 1;2376	}2377 2378	if (skb_shinfo(skb)->nr_frags == 0) {2379		cmdsetup.s.u.datasize = skb->len;2380		octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);2381 2382		/* Offload checksum calculation for TCP/UDP packets */2383		dptr = dma_map_single(&oct->pci_dev->dev,2384				      skb->data,2385				      skb->len,2386				      DMA_TO_DEVICE);2387		if (dma_mapping_error(&oct->pci_dev->dev, dptr)) {2388			dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n",2389				__func__);2390			stats->tx_dmamap_fail++;2391			return NETDEV_TX_BUSY;2392		}2393 2394		if (OCTEON_CN23XX_PF(oct))2395			ndata.cmd.cmd3.dptr = dptr;2396		else2397			ndata.cmd.cmd2.dptr = dptr;2398		finfo->dptr = dptr;2399		ndata.reqtype = REQTYPE_NORESP_NET;2400 2401	} else {2402		int i, frags;2403		skb_frag_t *frag;2404		struct octnic_gather *g;2405 2406		spin_lock(&lio->glist_lock[q_idx]);2407		g = (struct octnic_gather *)2408			lio_list_delete_head(&lio->glist[q_idx]);2409		spin_unlock(&lio->glist_lock[q_idx]);2410 2411		if (!g) {2412			netif_info(lio, tx_err, lio->netdev,2413				   "Transmit scatter gather: glist null!\n");2414			goto lio_xmit_failed;2415		}2416 2417		cmdsetup.s.gather = 1;2418		cmdsetup.s.u.gatherptrs = (skb_shinfo(skb)->nr_frags + 1);2419		octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);2420 2421		memset(g->sg, 0, g->sg_size);2422 2423		g->sg[0].ptr[0] = dma_map_single(&oct->pci_dev->dev,2424						 skb->data,2425						 (skb->len - skb->data_len),2426						 DMA_TO_DEVICE);2427		if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) {2428			dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n",2429				__func__);2430			stats->tx_dmamap_fail++;2431			return NETDEV_TX_BUSY;2432		}2433		add_sg_size(&g->sg[0], (skb->len - skb->data_len), 0);2434 2435		frags = skb_shinfo(skb)->nr_frags;2436		i = 1;2437		while (frags--) {2438			frag = &skb_shinfo(skb)->frags[i - 1];2439 2440			g->sg[(i >> 2)].ptr[(i & 3)] =2441				skb_frag_dma_map(&oct->pci_dev->dev,2442					         frag, 0, skb_frag_size(frag),2443						 DMA_TO_DEVICE);2444 2445			if (dma_mapping_error(&oct->pci_dev->dev,2446					      g->sg[i >> 2].ptr[i & 3])) {2447				dma_unmap_single(&oct->pci_dev->dev,2448						 g->sg[0].ptr[0],2449						 skb->len - skb->data_len,2450						 DMA_TO_DEVICE);2451				for (j = 1; j < i; j++) {2452					frag = &skb_shinfo(skb)->frags[j - 1];2453					dma_unmap_page(&oct->pci_dev->dev,2454						       g->sg[j >> 2].ptr[j & 3],2455						       skb_frag_size(frag),2456						       DMA_TO_DEVICE);2457				}2458				dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n",2459					__func__);2460				return NETDEV_TX_BUSY;2461			}2462 2463			add_sg_size(&g->sg[(i >> 2)], skb_frag_size(frag),2464				    (i & 3));2465			i++;2466		}2467 2468		dptr = g->sg_dma_ptr;2469 2470		if (OCTEON_CN23XX_PF(oct))2471			ndata.cmd.cmd3.dptr = dptr;2472		else2473			ndata.cmd.cmd2.dptr = dptr;2474		finfo->dptr = dptr;2475		finfo->g = g;2476 2477		ndata.reqtype = REQTYPE_NORESP_NET_SG;2478	}2479 2480	if (OCTEON_CN23XX_PF(oct)) {2481		irh = (struct octeon_instr_irh *)&ndata.cmd.cmd3.irh;2482		tx_info = (union tx_info *)&ndata.cmd.cmd3.ossp[0];2483	} else {2484		irh = (struct octeon_instr_irh *)&ndata.cmd.cmd2.irh;2485		tx_info = (union tx_info *)&ndata.cmd.cmd2.ossp[0];2486	}2487 2488	if (skb_shinfo(skb)->gso_size) {2489		tx_info->s.gso_size = skb_shinfo(skb)->gso_size;2490		tx_info->s.gso_segs = skb_shinfo(skb)->gso_segs;2491		stats->tx_gso++;2492	}2493 2494	/* HW insert VLAN tag */2495	if (skb_vlan_tag_present(skb)) {2496		irh->priority = skb_vlan_tag_get(skb) >> 13;2497		irh->vlan = skb_vlan_tag_get(skb) & 0xfff;2498	}2499 2500	xmit_more = netdev_xmit_more();2501 2502	if (unlikely(cmdsetup.s.timestamp))2503		status = send_nic_timestamp_pkt(oct, &ndata, finfo, xmit_more);2504	else2505		status = octnet_send_nic_data_pkt(oct, &ndata, xmit_more);2506	if (status == IQ_SEND_FAILED)2507		goto lio_xmit_failed;2508 2509	netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n");2510 2511	if (status == IQ_SEND_STOP)2512		netif_stop_subqueue(netdev, q_idx);2513 2514	netif_trans_update(netdev);2515 2516	if (tx_info->s.gso_segs)2517		stats->tx_done += tx_info->s.gso_segs;2518	else2519		stats->tx_done++;2520	stats->tx_tot_bytes += ndata.datasize;2521 2522	return NETDEV_TX_OK;2523 2524lio_xmit_failed:2525	stats->tx_dropped++;2526	netif_info(lio, tx_err, lio->netdev, "IQ%d Transmit dropped:%llu\n",2527		   iq_no, stats->tx_dropped);2528	if (dptr)2529		dma_unmap_single(&oct->pci_dev->dev, dptr,2530				 ndata.datasize, DMA_TO_DEVICE);2531 2532	octeon_ring_doorbell_locked(oct, iq_no);2533 2534	tx_buffer_free(skb);2535	return NETDEV_TX_OK;2536}2537 2538/**2539 * liquidio_tx_timeout - Network device Tx timeout2540 * @netdev:    pointer to network device2541 * @txqueue: index of the hung transmit queue2542 */2543static void liquidio_tx_timeout(struct net_device *netdev, unsigned int txqueue)2544{2545	struct lio *lio;2546 2547	lio = GET_LIO(netdev);2548 2549	netif_info(lio, tx_err, lio->netdev,2550		   "Transmit timeout tx_dropped:%ld, waking up queues now!!\n",2551		   netdev->stats.tx_dropped);2552	netif_trans_update(netdev);2553	wake_txqs(netdev);2554}2555 2556static int liquidio_vlan_rx_add_vid(struct net_device *netdev,2557				    __be16 proto __attribute__((unused)),2558				    u16 vid)2559{2560	struct lio *lio = GET_LIO(netdev);2561	struct octeon_device *oct = lio->oct_dev;2562	struct octnic_ctrl_pkt nctrl;2563	int ret = 0;2564 2565	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));2566 2567	nctrl.ncmd.u64 = 0;2568	nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;2569	nctrl.ncmd.s.param1 = vid;2570	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;2571	nctrl.netpndev = (u64)netdev;2572	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;2573 2574	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);2575	if (ret) {2576		dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",2577			ret);2578		if (ret > 0)2579			ret = -EIO;2580	}2581 2582	return ret;2583}2584 2585static int liquidio_vlan_rx_kill_vid(struct net_device *netdev,2586				     __be16 proto __attribute__((unused)),2587				     u16 vid)2588{2589	struct lio *lio = GET_LIO(netdev);2590	struct octeon_device *oct = lio->oct_dev;2591	struct octnic_ctrl_pkt nctrl;2592	int ret = 0;2593 2594	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));2595 2596	nctrl.ncmd.u64 = 0;2597	nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;2598	nctrl.ncmd.s.param1 = vid;2599	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;2600	nctrl.netpndev = (u64)netdev;2601	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;2602 2603	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);2604	if (ret) {2605		dev_err(&oct->pci_dev->dev, "Del VLAN filter failed in core (ret: 0x%x)\n",2606			ret);2607		if (ret > 0)2608			ret = -EIO;2609	}2610	return ret;2611}2612 2613/**2614 * liquidio_set_rxcsum_command - Sending command to enable/disable RX checksum offload2615 * @netdev:                pointer to network device2616 * @command:               OCTNET_CMD_TNL_RX_CSUM_CTL2617 * @rx_cmd:                OCTNET_CMD_RXCSUM_ENABLE/OCTNET_CMD_RXCSUM_DISABLE2618 * Returns:                SUCCESS or FAILURE2619 */2620static int liquidio_set_rxcsum_command(struct net_device *netdev, int command,2621				       u8 rx_cmd)2622{2623	struct lio *lio = GET_LIO(netdev);2624	struct octeon_device *oct = lio->oct_dev;2625	struct octnic_ctrl_pkt nctrl;2626	int ret = 0;2627 2628	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));2629 2630	nctrl.ncmd.u64 = 0;2631	nctrl.ncmd.s.cmd = command;2632	nctrl.ncmd.s.param1 = rx_cmd;2633	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;2634	nctrl.netpndev = (u64)netdev;2635	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;2636 2637	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);2638	if (ret) {2639		dev_err(&oct->pci_dev->dev,2640			"DEVFLAGS RXCSUM change failed in core(ret:0x%x)\n",2641			ret);2642		if (ret > 0)2643			ret = -EIO;2644	}2645	return ret;2646}2647 2648/**2649 * liquidio_vxlan_port_command - Sending command to add/delete VxLAN UDP port to firmware2650 * @netdev:                pointer to network device2651 * @command:               OCTNET_CMD_VXLAN_PORT_CONFIG2652 * @vxlan_port:            VxLAN port to be added or deleted2653 * @vxlan_cmd_bit:         OCTNET_CMD_VXLAN_PORT_ADD,2654 *                              OCTNET_CMD_VXLAN_PORT_DEL2655 * Return:                     SUCCESS or FAILURE2656 */2657static int liquidio_vxlan_port_command(struct net_device *netdev, int command,2658				       u16 vxlan_port, u8 vxlan_cmd_bit)2659{2660	struct lio *lio = GET_LIO(netdev);2661	struct octeon_device *oct = lio->oct_dev;2662	struct octnic_ctrl_pkt nctrl;2663	int ret = 0;2664 2665	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));2666 2667	nctrl.ncmd.u64 = 0;2668	nctrl.ncmd.s.cmd = command;2669	nctrl.ncmd.s.more = vxlan_cmd_bit;2670	nctrl.ncmd.s.param1 = vxlan_port;2671	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;2672	nctrl.netpndev = (u64)netdev;2673	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;2674 2675	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);2676	if (ret) {2677		dev_err(&oct->pci_dev->dev,2678			"VxLAN port add/delete failed in core (ret:0x%x)\n",2679			ret);2680		if (ret > 0)2681			ret = -EIO;2682	}2683	return ret;2684}2685 2686static int liquidio_udp_tunnel_set_port(struct net_device *netdev,2687					unsigned int table, unsigned int entry,2688					struct udp_tunnel_info *ti)2689{2690	return liquidio_vxlan_port_command(netdev,2691					   OCTNET_CMD_VXLAN_PORT_CONFIG,2692					   htons(ti->port),2693					   OCTNET_CMD_VXLAN_PORT_ADD);2694}2695 2696static int liquidio_udp_tunnel_unset_port(struct net_device *netdev,2697					  unsigned int table,2698					  unsigned int entry,2699					  struct udp_tunnel_info *ti)2700{2701	return liquidio_vxlan_port_command(netdev,2702					   OCTNET_CMD_VXLAN_PORT_CONFIG,2703					   htons(ti->port),2704					   OCTNET_CMD_VXLAN_PORT_DEL);2705}2706 2707static const struct udp_tunnel_nic_info liquidio_udp_tunnels = {2708	.set_port	= liquidio_udp_tunnel_set_port,2709	.unset_port	= liquidio_udp_tunnel_unset_port,2710	.tables		= {2711		{ .n_entries = 1024, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, },2712	},2713};2714 2715/**2716 * liquidio_fix_features - Net device fix features2717 * @netdev:  pointer to network device2718 * @request: features requested2719 * Return: updated features list2720 */2721static netdev_features_t liquidio_fix_features(struct net_device *netdev,2722					       netdev_features_t request)2723{2724	struct lio *lio = netdev_priv(netdev);2725 2726	if ((request & NETIF_F_RXCSUM) &&2727	    !(lio->dev_capability & NETIF_F_RXCSUM))2728		request &= ~NETIF_F_RXCSUM;2729 2730	if ((request & NETIF_F_HW_CSUM) &&2731	    !(lio->dev_capability & NETIF_F_HW_CSUM))2732		request &= ~NETIF_F_HW_CSUM;2733 2734	if ((request & NETIF_F_TSO) && !(lio->dev_capability & NETIF_F_TSO))2735		request &= ~NETIF_F_TSO;2736 2737	if ((request & NETIF_F_TSO6) && !(lio->dev_capability & NETIF_F_TSO6))2738		request &= ~NETIF_F_TSO6;2739 2740	if ((request & NETIF_F_LRO) && !(lio->dev_capability & NETIF_F_LRO))2741		request &= ~NETIF_F_LRO;2742 2743	/*Disable LRO if RXCSUM is off */2744	if (!(request & NETIF_F_RXCSUM) && (netdev->features & NETIF_F_LRO) &&2745	    (lio->dev_capability & NETIF_F_LRO))2746		request &= ~NETIF_F_LRO;2747 2748	if ((request & NETIF_F_HW_VLAN_CTAG_FILTER) &&2749	    !(lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER))2750		request &= ~NETIF_F_HW_VLAN_CTAG_FILTER;2751 2752	return request;2753}2754 2755/**2756 * liquidio_set_features - Net device set features2757 * @netdev:  pointer to network device2758 * @features: features to enable/disable2759 */2760static int liquidio_set_features(struct net_device *netdev,2761				 netdev_features_t features)2762{2763	struct lio *lio = netdev_priv(netdev);2764 2765	if ((features & NETIF_F_LRO) &&2766	    (lio->dev_capability & NETIF_F_LRO) &&2767	    !(netdev->features & NETIF_F_LRO))2768		liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,2769				     OCTNIC_LROIPV4 | OCTNIC_LROIPV6);2770	else if (!(features & NETIF_F_LRO) &&2771		 (lio->dev_capability & NETIF_F_LRO) &&2772		 (netdev->features & NETIF_F_LRO))2773		liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE,2774				     OCTNIC_LROIPV4 | OCTNIC_LROIPV6);2775 2776	/* Sending command to firmware to enable/disable RX checksum2777	 * offload settings using ethtool2778	 */2779	if (!(netdev->features & NETIF_F_RXCSUM) &&2780	    (lio->enc_dev_capability & NETIF_F_RXCSUM) &&2781	    (features & NETIF_F_RXCSUM))2782		liquidio_set_rxcsum_command(netdev,2783					    OCTNET_CMD_TNL_RX_CSUM_CTL,2784					    OCTNET_CMD_RXCSUM_ENABLE);2785	else if ((netdev->features & NETIF_F_RXCSUM) &&2786		 (lio->enc_dev_capability & NETIF_F_RXCSUM) &&2787		 !(features & NETIF_F_RXCSUM))2788		liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,2789					    OCTNET_CMD_RXCSUM_DISABLE);2790 2791	if ((features & NETIF_F_HW_VLAN_CTAG_FILTER) &&2792	    (lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER) &&2793	    !(netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER))2794		liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,2795				     OCTNET_CMD_VLAN_FILTER_ENABLE);2796	else if (!(features & NETIF_F_HW_VLAN_CTAG_FILTER) &&2797		 (lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER) &&2798		 (netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER))2799		liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,2800				     OCTNET_CMD_VLAN_FILTER_DISABLE);2801 2802	return 0;2803}2804 2805static int __liquidio_set_vf_mac(struct net_device *netdev, int vfidx,2806				 u8 *mac, bool is_admin_assigned)2807{2808	struct lio *lio = GET_LIO(netdev);2809	struct octeon_device *oct = lio->oct_dev;2810	struct octnic_ctrl_pkt nctrl;2811	int ret = 0;2812 2813	if (!is_valid_ether_addr(mac))2814		return -EINVAL;2815 2816	if (vfidx < 0 || vfidx >= oct->sriov_info.max_vfs)2817		return -EINVAL;2818 2819	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));2820 2821	nctrl.ncmd.u64 = 0;2822	nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;2823	/* vfidx is 0 based, but vf_num (param1) is 1 based */2824	nctrl.ncmd.s.param1 = vfidx + 1;2825	nctrl.ncmd.s.more = 1;2826	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;2827	nctrl.netpndev = (u64)netdev;2828	if (is_admin_assigned) {2829		nctrl.ncmd.s.param2 = true;2830		nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;2831	}2832 2833	nctrl.udd[0] = 0;2834	/* The MAC Address is presented in network byte order. */2835	ether_addr_copy((u8 *)&nctrl.udd[0] + 2, mac);2836 2837	oct->sriov_info.vf_macaddr[vfidx] = nctrl.udd[0];2838 2839	ret = octnet_send_nic_ctrl_pkt(oct, &nctrl);2840	if (ret > 0)2841		ret = -EIO;2842 2843	return ret;2844}2845 2846static int liquidio_set_vf_mac(struct net_device *netdev, int vfidx, u8 *mac)2847{2848	struct lio *lio = GET_LIO(netdev);2849	struct octeon_device *oct = lio->oct_dev;2850	int retval;2851 2852	if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)2853		return -EINVAL;2854 2855	retval = __liquidio_set_vf_mac(netdev, vfidx, mac, true);2856	if (!retval)2857		cn23xx_tell_vf_its_macaddr_changed(oct, vfidx, mac);2858 2859	return retval;2860}2861 2862static int liquidio_set_vf_spoofchk(struct net_device *netdev, int vfidx,2863				    bool enable)2864{2865	struct lio *lio = GET_LIO(netdev);2866	struct octeon_device *oct = lio->oct_dev;2867	struct octnic_ctrl_pkt nctrl;2868	int retval;2869 2870	if (!(oct->fw_info.app_cap_flags & LIQUIDIO_SPOOFCHK_CAP)) {2871		netif_info(lio, drv, lio->netdev,2872			   "firmware does not support spoofchk\n");2873		return -EOPNOTSUPP;2874	}2875 2876	if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced) {2877		netif_info(lio, drv, lio->netdev, "Invalid vfidx %d\n", vfidx);2878		return -EINVAL;2879	}2880 2881	if (enable) {2882		if (oct->sriov_info.vf_spoofchk[vfidx])2883			return 0;2884	} else {2885		/* Clear */2886		if (!oct->sriov_info.vf_spoofchk[vfidx])2887			return 0;2888	}2889 2890	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));2891	nctrl.ncmd.s.cmdgroup = OCTNET_CMD_GROUP1;2892	nctrl.ncmd.s.cmd = OCTNET_CMD_SET_VF_SPOOFCHK;2893	nctrl.ncmd.s.param1 =2894		vfidx + 1; /* vfidx is 0 based,2895			    * but vf_num (param1) is 1 based2896			    */2897	nctrl.ncmd.s.param2 = enable;2898	nctrl.ncmd.s.more = 0;2899	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;2900	nctrl.cb_fn = NULL;2901 2902	retval = octnet_send_nic_ctrl_pkt(oct, &nctrl);2903 2904	if (retval) {2905		netif_info(lio, drv, lio->netdev,2906			   "Failed to set VF %d spoofchk %s\n", vfidx,2907			enable ? "on" : "off");2908		return -1;2909	}2910 2911	oct->sriov_info.vf_spoofchk[vfidx] = enable;2912	netif_info(lio, drv, lio->netdev, "VF %u spoofchk is %s\n", vfidx,2913		   enable ? "on" : "off");2914 2915	return 0;2916}2917 2918static int liquidio_set_vf_vlan(struct net_device *netdev, int vfidx,2919				u16 vlan, u8 qos, __be16 vlan_proto)2920{2921	struct lio *lio = GET_LIO(netdev);2922	struct octeon_device *oct = lio->oct_dev;2923	struct octnic_ctrl_pkt nctrl;2924	u16 vlantci;2925	int ret = 0;2926 2927	if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)2928		return -EINVAL;2929 2930	if (vlan_proto != htons(ETH_P_8021Q))2931		return -EPROTONOSUPPORT;2932 2933	if (vlan >= VLAN_N_VID || qos > 7)2934		return -EINVAL;2935 2936	if (vlan)2937		vlantci = vlan | (u16)qos << VLAN_PRIO_SHIFT;2938	else2939		vlantci = 0;2940 2941	if (oct->sriov_info.vf_vlantci[vfidx] == vlantci)2942		return 0;2943 2944	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));2945 2946	if (vlan)2947		nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;2948	else2949		nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;2950 2951	nctrl.ncmd.s.param1 = vlantci;2952	nctrl.ncmd.s.param2 =2953	    vfidx + 1; /* vfidx is 0 based, but vf_num (param2) is 1 based */2954	nctrl.ncmd.s.more = 0;2955	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;2956	nctrl.cb_fn = NULL;2957 2958	ret = octnet_send_nic_ctrl_pkt(oct, &nctrl);2959	if (ret) {2960		if (ret > 0)2961			ret = -EIO;2962		return ret;2963	}2964 2965	oct->sriov_info.vf_vlantci[vfidx] = vlantci;2966 2967	return ret;2968}2969 2970static int liquidio_get_vf_config(struct net_device *netdev, int vfidx,2971				  struct ifla_vf_info *ivi)2972{2973	struct lio *lio = GET_LIO(netdev);2974	struct octeon_device *oct = lio->oct_dev;2975	u8 *macaddr;2976 2977	if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)2978		return -EINVAL;2979 2980	memset(ivi, 0, sizeof(struct ifla_vf_info));2981 2982	ivi->vf = vfidx;2983	macaddr = 2 + (u8 *)&oct->sriov_info.vf_macaddr[vfidx];2984	ether_addr_copy(&ivi->mac[0], macaddr);2985	ivi->vlan = oct->sriov_info.vf_vlantci[vfidx] & VLAN_VID_MASK;2986	ivi->qos = oct->sriov_info.vf_vlantci[vfidx] >> VLAN_PRIO_SHIFT;2987	if (oct->sriov_info.trusted_vf.active &&2988	    oct->sriov_info.trusted_vf.id == vfidx)2989		ivi->trusted = true;2990	else2991		ivi->trusted = false;2992	ivi->linkstate = oct->sriov_info.vf_linkstate[vfidx];2993	ivi->spoofchk = oct->sriov_info.vf_spoofchk[vfidx];2994	ivi->max_tx_rate = lio->linfo.link.s.speed;2995	ivi->min_tx_rate = 0;2996 2997	return 0;2998}2999 3000static int liquidio_send_vf_trust_cmd(struct lio *lio, int vfidx, bool trusted)3001{3002	struct octeon_device *oct = lio->oct_dev;3003	struct octeon_soft_command *sc;3004	int retval;3005 3006	sc = octeon_alloc_soft_command(oct, 0, 16, 0);3007	if (!sc)3008		return -ENOMEM;3009 3010	sc->iq_no = lio->linfo.txpciq[0].s.q_no;3011 3012	/* vfidx is 0 based, but vf_num (param1) is 1 based */3013	octeon_prepare_soft_command(oct, sc, OPCODE_NIC,3014				    OPCODE_NIC_SET_TRUSTED_VF, 0, vfidx + 1,3015				    trusted);3016 3017	init_completion(&sc->complete);3018	sc->sc_status = OCTEON_REQUEST_PENDING;3019 3020	retval = octeon_send_soft_command(oct, sc);3021	if (retval == IQ_SEND_FAILED) {3022		octeon_free_soft_command(oct, sc);3023		retval = -1;3024	} else {3025		/* Wait for response or timeout */3026		retval = wait_for_sc_completion_timeout(oct, sc, 0);3027		if (retval)3028			return (retval);3029 3030		WRITE_ONCE(sc->caller_is_done, true);3031	}3032 3033	return retval;3034}3035 3036static int liquidio_set_vf_trust(struct net_device *netdev, int vfidx,3037				 bool setting)3038{3039	struct lio *lio = GET_LIO(netdev);3040	struct octeon_device *oct = lio->oct_dev;3041 3042	if (strcmp(oct->fw_info.liquidio_firmware_version, "1.7.1") < 0) {3043		/* trusted vf is not supported by firmware older than 1.7.1 */3044		return -EOPNOTSUPP;3045	}3046 3047	if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced) {3048		netif_info(lio, drv, lio->netdev, "Invalid vfidx %d\n", vfidx);3049		return -EINVAL;3050	}3051 3052	if (setting) {3053		/* Set */3054 3055		if (oct->sriov_info.trusted_vf.active &&3056		    oct->sriov_info.trusted_vf.id == vfidx)3057			return 0;3058 3059		if (oct->sriov_info.trusted_vf.active) {3060			netif_info(lio, drv, lio->netdev, "More than one trusted VF is not allowed\n");3061			return -EPERM;3062		}3063	} else {3064		/* Clear */3065 3066		if (!oct->sriov_info.trusted_vf.active)3067			return 0;3068	}3069 3070	if (!liquidio_send_vf_trust_cmd(lio, vfidx, setting)) {3071		if (setting) {3072			oct->sriov_info.trusted_vf.id = vfidx;3073			oct->sriov_info.trusted_vf.active = true;3074		} else {3075			oct->sriov_info.trusted_vf.active = false;3076		}3077 3078		netif_info(lio, drv, lio->netdev, "VF %u is %strusted\n", vfidx,3079			   setting ? "" : "not ");3080	} else {3081		netif_info(lio, drv, lio->netdev, "Failed to set VF trusted\n");3082		return -1;3083	}3084 3085	return 0;3086}3087 3088static int liquidio_set_vf_link_state(struct net_device *netdev, int vfidx,3089				      int linkstate)3090{3091	struct lio *lio = GET_LIO(netdev);3092	struct octeon_device *oct = lio->oct_dev;3093	struct octnic_ctrl_pkt nctrl;3094	int ret = 0;3095 3096	if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)3097		return -EINVAL;3098 3099	if (oct->sriov_info.vf_linkstate[vfidx] == linkstate)3100		return 0;3101 3102	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));3103	nctrl.ncmd.s.cmd = OCTNET_CMD_SET_VF_LINKSTATE;3104	nctrl.ncmd.s.param1 =3105	    vfidx + 1; /* vfidx is 0 based, but vf_num (param1) is 1 based */3106	nctrl.ncmd.s.param2 = linkstate;3107	nctrl.ncmd.s.more = 0;3108	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;3109	nctrl.cb_fn = NULL;3110 3111	ret = octnet_send_nic_ctrl_pkt(oct, &nctrl);3112 3113	if (!ret)3114		oct->sriov_info.vf_linkstate[vfidx] = linkstate;3115	else if (ret > 0)3116		ret = -EIO;3117 3118	return ret;3119}3120 3121static int3122liquidio_eswitch_mode_get(struct devlink *devlink, u16 *mode)3123{3124	struct lio_devlink_priv *priv;3125	struct octeon_device *oct;3126 3127	priv = devlink_priv(devlink);3128	oct = priv->oct;3129 3130	*mode = oct->eswitch_mode;3131 3132	return 0;3133}3134 3135static int3136liquidio_eswitch_mode_set(struct devlink *devlink, u16 mode,3137			  struct netlink_ext_ack *extack)3138{3139	struct lio_devlink_priv *priv;3140	struct octeon_device *oct;3141	int ret = 0;3142 3143	priv = devlink_priv(devlink);3144	oct = priv->oct;3145 3146	if (!(oct->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP))3147		return -EINVAL;3148 3149	if (oct->eswitch_mode == mode)3150		return 0;3151 3152	switch (mode) {3153	case DEVLINK_ESWITCH_MODE_SWITCHDEV:3154		oct->eswitch_mode = mode;3155		ret = lio_vf_rep_create(oct);3156		break;3157 3158	case DEVLINK_ESWITCH_MODE_LEGACY:3159		lio_vf_rep_destroy(oct);3160		oct->eswitch_mode = mode;3161		break;3162 3163	default:3164		ret = -EINVAL;3165	}3166 3167	return ret;3168}3169 3170static const struct devlink_ops liquidio_devlink_ops = {3171	.eswitch_mode_get = liquidio_eswitch_mode_get,3172	.eswitch_mode_set = liquidio_eswitch_mode_set,3173};3174 3175static int3176liquidio_get_port_parent_id(struct net_device *dev,3177			    struct netdev_phys_item_id *ppid)3178{3179	struct lio *lio = GET_LIO(dev);3180	struct octeon_device *oct = lio->oct_dev;3181 3182	if (oct->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)3183		return -EOPNOTSUPP;3184 3185	ppid->id_len = ETH_ALEN;3186	ether_addr_copy(ppid->id, (void *)&lio->linfo.hw_addr + 2);3187 3188	return 0;3189}3190 3191static int liquidio_get_vf_stats(struct net_device *netdev, int vfidx,3192				 struct ifla_vf_stats *vf_stats)3193{3194	struct lio *lio = GET_LIO(netdev);3195	struct octeon_device *oct = lio->oct_dev;3196	struct oct_vf_stats stats;3197	int ret;3198 3199	if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)3200		return -EINVAL;3201 3202	memset(&stats, 0, sizeof(struct oct_vf_stats));3203	ret = cn23xx_get_vf_stats(oct, vfidx, &stats);3204	if (!ret) {3205		vf_stats->rx_packets = stats.rx_packets;3206		vf_stats->tx_packets = stats.tx_packets;3207		vf_stats->rx_bytes = stats.rx_bytes;3208		vf_stats->tx_bytes = stats.tx_bytes;3209		vf_stats->broadcast = stats.broadcast;3210		vf_stats->multicast = stats.multicast;3211	}3212 3213	return ret;3214}3215 3216static const struct net_device_ops lionetdevops = {3217	.ndo_open		= liquidio_open,3218	.ndo_stop		= liquidio_stop,3219	.ndo_start_xmit		= liquidio_xmit,3220	.ndo_get_stats64	= liquidio_get_stats64,3221	.ndo_set_mac_address	= liquidio_set_mac,3222	.ndo_set_rx_mode	= liquidio_set_mcast_list,3223	.ndo_tx_timeout		= liquidio_tx_timeout,3224 3225	.ndo_vlan_rx_add_vid    = liquidio_vlan_rx_add_vid,3226	.ndo_vlan_rx_kill_vid   = liquidio_vlan_rx_kill_vid,3227	.ndo_change_mtu		= liquidio_change_mtu,3228	.ndo_eth_ioctl		= liquidio_ioctl,3229	.ndo_fix_features	= liquidio_fix_features,3230	.ndo_set_features	= liquidio_set_features,3231	.ndo_set_vf_mac		= liquidio_set_vf_mac,3232	.ndo_set_vf_vlan	= liquidio_set_vf_vlan,3233	.ndo_get_vf_config	= liquidio_get_vf_config,3234	.ndo_set_vf_spoofchk	= liquidio_set_vf_spoofchk,3235	.ndo_set_vf_trust	= liquidio_set_vf_trust,3236	.ndo_set_vf_link_state  = liquidio_set_vf_link_state,3237	.ndo_get_vf_stats	= liquidio_get_vf_stats,3238	.ndo_get_port_parent_id	= liquidio_get_port_parent_id,3239};3240 3241/**3242 * liquidio_init - Entry point for the liquidio module3243 */3244static int __init liquidio_init(void)3245{3246	int i;3247	struct handshake *hs;3248 3249	init_completion(&first_stage);3250 3251	octeon_init_device_list(OCTEON_CONFIG_TYPE_DEFAULT);3252 3253	if (liquidio_init_pci())3254		return -EINVAL;3255 3256	wait_for_completion_timeout(&first_stage, msecs_to_jiffies(1000));3257 3258	for (i = 0; i < MAX_OCTEON_DEVICES; i++) {3259		hs = &handshake[i];3260		if (hs->pci_dev) {3261			wait_for_completion(&hs->init);3262			if (!hs->init_ok) {3263				/* init handshake failed */3264				dev_err(&hs->pci_dev->dev,3265					"Failed to init device\n");3266				liquidio_deinit_pci();3267				return -EIO;3268			}3269		}3270	}3271 3272	for (i = 0; i < MAX_OCTEON_DEVICES; i++) {3273		hs = &handshake[i];3274		if (hs->pci_dev) {3275			wait_for_completion_timeout(&hs->started,3276						    msecs_to_jiffies(30000));3277			if (!hs->started_ok) {3278				/* starter handshake failed */3279				dev_err(&hs->pci_dev->dev,3280					"Firmware failed to start\n");3281				liquidio_deinit_pci();3282				return -EIO;3283			}3284		}3285	}3286 3287	return 0;3288}3289 3290static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf)3291{3292	struct octeon_device *oct = (struct octeon_device *)buf;3293	struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;3294	int gmxport = 0;3295	union oct_link_status *ls;3296	int i;3297 3298	if (recv_pkt->buffer_size[0] != (sizeof(*ls) + OCT_DROQ_INFO_SIZE)) {3299		dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n",3300			recv_pkt->buffer_size[0],3301			recv_pkt->rh.r_nic_info.gmxport);3302		goto nic_info_err;3303	}3304 3305	gmxport = recv_pkt->rh.r_nic_info.gmxport;3306	ls = (union oct_link_status *)(get_rbd(recv_pkt->buffer_ptr[0]) +3307		OCT_DROQ_INFO_SIZE);3308 3309	octeon_swap_8B_data((u64 *)ls, (sizeof(union oct_link_status)) >> 3);3310	for (i = 0; i < oct->ifcount; i++) {3311		if (oct->props[i].gmxport == gmxport) {3312			update_link_status(oct->props[i].netdev, ls);3313			break;3314		}3315	}3316 3317nic_info_err:3318	for (i = 0; i < recv_pkt->buffer_count; i++)3319		recv_buffer_free(recv_pkt->buffer_ptr[i]);3320	octeon_free_recv_info(recv_info);3321	return 0;3322}3323 3324/**3325 * setup_nic_devices - Setup network interfaces3326 * @octeon_dev:  octeon device3327 *3328 * Called during init time for each device. It assumes the NIC3329 * is already up and running.  The link information for each3330 * interface is passed in link_info.3331 */3332static int setup_nic_devices(struct octeon_device *octeon_dev)3333{3334	struct lio *lio = NULL;3335	struct net_device *netdev;3336	u8 mac[6], i, j, *fw_ver, *micro_ver;3337	unsigned long micro;3338	u32 cur_ver;3339	struct octeon_soft_command *sc;3340	struct liquidio_if_cfg_resp *resp;3341	struct octdev_props *props;3342	int retval, num_iqueues, num_oqueues;3343	int max_num_queues = 0;3344	union oct_nic_if_cfg if_cfg;3345	unsigned int base_queue;3346	unsigned int gmx_port_id;3347	u32 resp_size, data_size;3348	u32 ifidx_or_pfnum;3349	struct lio_version *vdata;3350	struct devlink *devlink;3351	struct lio_devlink_priv *lio_devlink;3352 3353	/* This is to handle link status changes */3354	octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,3355				    OPCODE_NIC_INFO,3356				    lio_nic_info, octeon_dev);3357 3358	/* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions.3359	 * They are handled directly.3360	 */3361	octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET,3362					free_netbuf);3363 3364	octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET_SG,3365					free_netsgbuf);3366 3367	octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_RESP_NET_SG,3368					free_netsgbuf_with_resp);3369 3370	for (i = 0; i < octeon_dev->ifcount; i++) {3371		resp_size = sizeof(struct liquidio_if_cfg_resp);3372		data_size = sizeof(struct lio_version);3373		sc = (struct octeon_soft_command *)3374			octeon_alloc_soft_command(octeon_dev, data_size,3375						  resp_size, 0);3376		resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;3377		vdata = (struct lio_version *)sc->virtdptr;3378 3379		*((u64 *)vdata) = 0;3380		vdata->major = cpu_to_be16(LIQUIDIO_BASE_MAJOR_VERSION);3381		vdata->minor = cpu_to_be16(LIQUIDIO_BASE_MINOR_VERSION);3382		vdata->micro = cpu_to_be16(LIQUIDIO_BASE_MICRO_VERSION);3383 3384		if (OCTEON_CN23XX_PF(octeon_dev)) {3385			num_iqueues = octeon_dev->sriov_info.num_pf_rings;3386			num_oqueues = octeon_dev->sriov_info.num_pf_rings;3387			base_queue = octeon_dev->sriov_info.pf_srn;3388 3389			gmx_port_id = octeon_dev->pf_num;3390			ifidx_or_pfnum = octeon_dev->pf_num;3391		} else {3392			num_iqueues = CFG_GET_NUM_TXQS_NIC_IF(3393						octeon_get_conf(octeon_dev), i);3394			num_oqueues = CFG_GET_NUM_RXQS_NIC_IF(3395						octeon_get_conf(octeon_dev), i);3396			base_queue = CFG_GET_BASE_QUE_NIC_IF(3397						octeon_get_conf(octeon_dev), i);3398			gmx_port_id = CFG_GET_GMXID_NIC_IF(3399						octeon_get_conf(octeon_dev), i);3400			ifidx_or_pfnum = i;3401		}3402 3403		dev_dbg(&octeon_dev->pci_dev->dev,3404			"requesting config for interface %d, iqs %d, oqs %d\n",3405			ifidx_or_pfnum, num_iqueues, num_oqueues);3406 3407		if_cfg.u64 = 0;3408		if_cfg.s.num_iqueues = num_iqueues;3409		if_cfg.s.num_oqueues = num_oqueues;3410		if_cfg.s.base_queue = base_queue;3411		if_cfg.s.gmx_port_id = gmx_port_id;3412 3413		sc->iq_no = 0;3414 3415		octeon_prepare_soft_command(octeon_dev, sc, OPCODE_NIC,3416					    OPCODE_NIC_IF_CFG, 0,3417					    if_cfg.u64, 0);3418 3419		init_completion(&sc->complete);3420		sc->sc_status = OCTEON_REQUEST_PENDING;3421 3422		retval = octeon_send_soft_command(octeon_dev, sc);3423		if (retval == IQ_SEND_FAILED) {3424			dev_err(&octeon_dev->pci_dev->dev,3425				"iq/oq config failed status: %x\n",3426				retval);3427			/* Soft instr is freed by driver in case of failure. */3428			octeon_free_soft_command(octeon_dev, sc);3429			return(-EIO);3430		}3431 3432		/* Sleep on a wait queue till the cond flag indicates that the3433		 * response arrived or timed-out.3434		 */3435		retval = wait_for_sc_completion_timeout(octeon_dev, sc, 0);3436		if (retval)3437			return retval;3438 3439		retval = resp->status;3440		if (retval) {3441			dev_err(&octeon_dev->pci_dev->dev, "iq/oq config failed\n");3442			WRITE_ONCE(sc->caller_is_done, true);3443			goto setup_nic_dev_done;3444		}3445		snprintf(octeon_dev->fw_info.liquidio_firmware_version,3446			 32, "%s",3447			 resp->cfg_info.liquidio_firmware_version);3448 3449		/* Verify f/w version (in case of 'auto' loading from flash) */3450		fw_ver = octeon_dev->fw_info.liquidio_firmware_version;3451		if (memcmp(LIQUIDIO_BASE_VERSION,3452			   fw_ver,3453			   strlen(LIQUIDIO_BASE_VERSION))) {3454			dev_err(&octeon_dev->pci_dev->dev,3455				"Unmatched firmware version. Expected %s.x, got %s.\n",3456				LIQUIDIO_BASE_VERSION, fw_ver);3457			WRITE_ONCE(sc->caller_is_done, true);3458			goto setup_nic_dev_done;3459		} else if (atomic_read(octeon_dev->adapter_fw_state) ==3460			   FW_IS_PRELOADED) {3461			dev_info(&octeon_dev->pci_dev->dev,3462				 "Using auto-loaded firmware version %s.\n",3463				 fw_ver);3464		}3465 3466		/* extract micro version field; point past '<maj>.<min>.' */3467		micro_ver = fw_ver + strlen(LIQUIDIO_BASE_VERSION) + 1;3468		if (kstrtoul(micro_ver, 10, &micro) != 0)3469			micro = 0;3470		octeon_dev->fw_info.ver.maj = LIQUIDIO_BASE_MAJOR_VERSION;3471		octeon_dev->fw_info.ver.min = LIQUIDIO_BASE_MINOR_VERSION;3472		octeon_dev->fw_info.ver.rev = micro;3473 3474		octeon_swap_8B_data((u64 *)(&resp->cfg_info),3475				    (sizeof(struct liquidio_if_cfg_info)) >> 3);3476 3477		num_iqueues = hweight64(resp->cfg_info.iqmask);3478		num_oqueues = hweight64(resp->cfg_info.oqmask);3479 3480		if (!(num_iqueues) || !(num_oqueues)) {3481			dev_err(&octeon_dev->pci_dev->dev,3482				"Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n",3483				resp->cfg_info.iqmask,3484				resp->cfg_info.oqmask);3485			WRITE_ONCE(sc->caller_is_done, true);3486			goto setup_nic_dev_done;3487		}3488 3489		if (OCTEON_CN6XXX(octeon_dev)) {3490			max_num_queues = CFG_GET_IQ_MAX_Q(CHIP_CONF(octeon_dev,3491								    cn6xxx));3492		} else if (OCTEON_CN23XX_PF(octeon_dev)) {3493			max_num_queues = CFG_GET_IQ_MAX_Q(CHIP_CONF(octeon_dev,3494								    cn23xx_pf));3495		}3496 3497		dev_dbg(&octeon_dev->pci_dev->dev,3498			"interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d max_num_queues: %d\n",3499			i, resp->cfg_info.iqmask, resp->cfg_info.oqmask,3500			num_iqueues, num_oqueues, max_num_queues);3501		netdev = alloc_etherdev_mq(LIO_SIZE, max_num_queues);3502 3503		if (!netdev) {3504			dev_err(&octeon_dev->pci_dev->dev, "Device allocation failed\n");3505			WRITE_ONCE(sc->caller_is_done, true);3506			goto setup_nic_dev_done;3507		}3508 3509		SET_NETDEV_DEV(netdev, &octeon_dev->pci_dev->dev);3510 3511		/* Associate the routines that will handle different3512		 * netdev tasks.3513		 */3514		netdev->netdev_ops = &lionetdevops;3515 3516		retval = netif_set_real_num_rx_queues(netdev, num_oqueues);3517		if (retval) {3518			dev_err(&octeon_dev->pci_dev->dev,3519				"setting real number rx failed\n");3520			WRITE_ONCE(sc->caller_is_done, true);3521			goto setup_nic_dev_free;3522		}3523 3524		retval = netif_set_real_num_tx_queues(netdev, num_iqueues);3525		if (retval) {3526			dev_err(&octeon_dev->pci_dev->dev,3527				"setting real number tx failed\n");3528			WRITE_ONCE(sc->caller_is_done, true);3529			goto setup_nic_dev_free;3530		}3531 3532		lio = GET_LIO(netdev);3533 3534		memset(lio, 0, sizeof(struct lio));3535 3536		lio->ifidx = ifidx_or_pfnum;3537 3538		props = &octeon_dev->props[i];3539		props->gmxport = resp->cfg_info.linfo.gmxport;3540		props->netdev = netdev;3541 3542		lio->linfo.num_rxpciq = num_oqueues;3543		lio->linfo.num_txpciq = num_iqueues;3544		for (j = 0; j < num_oqueues; j++) {3545			lio->linfo.rxpciq[j].u64 =3546				resp->cfg_info.linfo.rxpciq[j].u64;3547		}3548		for (j = 0; j < num_iqueues; j++) {3549			lio->linfo.txpciq[j].u64 =3550				resp->cfg_info.linfo.txpciq[j].u64;3551		}3552		lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr;3553		lio->linfo.gmxport = resp->cfg_info.linfo.gmxport;3554		lio->linfo.link.u64 = resp->cfg_info.linfo.link.u64;3555 3556		WRITE_ONCE(sc->caller_is_done, true);3557 3558		lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);3559 3560		if (OCTEON_CN23XX_PF(octeon_dev) ||3561		    OCTEON_CN6XXX(octeon_dev)) {3562			lio->dev_capability = NETIF_F_HIGHDMA3563					      | NETIF_F_IP_CSUM3564					      | NETIF_F_IPV6_CSUM3565					      | NETIF_F_SG | NETIF_F_RXCSUM3566					      | NETIF_F_GRO3567					      | NETIF_F_TSO | NETIF_F_TSO63568					      | NETIF_F_LRO;3569		}3570		netif_set_tso_max_size(netdev, OCTNIC_GSO_MAX_SIZE);3571 3572		/*  Copy of transmit encapsulation capabilities:3573		 *  TSO, TSO6, Checksums for this device3574		 */3575		lio->enc_dev_capability = NETIF_F_IP_CSUM3576					  | NETIF_F_IPV6_CSUM3577					  | NETIF_F_GSO_UDP_TUNNEL3578					  | NETIF_F_HW_CSUM | NETIF_F_SG3579					  | NETIF_F_RXCSUM3580					  | NETIF_F_TSO | NETIF_F_TSO63581					  | NETIF_F_LRO;3582 3583		netdev->hw_enc_features = (lio->enc_dev_capability &3584					   ~NETIF_F_LRO);3585 3586		netdev->udp_tunnel_nic_info = &liquidio_udp_tunnels;3587 3588		lio->dev_capability |= NETIF_F_GSO_UDP_TUNNEL;3589 3590		netdev->vlan_features = lio->dev_capability;3591		/* Add any unchangeable hw features */3592		lio->dev_capability |=  NETIF_F_HW_VLAN_CTAG_FILTER |3593					NETIF_F_HW_VLAN_CTAG_RX |3594					NETIF_F_HW_VLAN_CTAG_TX;3595 3596		netdev->features = (lio->dev_capability & ~NETIF_F_LRO);3597 3598		netdev->hw_features = lio->dev_capability;3599		/*HW_VLAN_RX and HW_VLAN_FILTER is always on*/3600		netdev->hw_features = netdev->hw_features &3601			~NETIF_F_HW_VLAN_CTAG_RX;3602 3603		/* MTU range: 68 - 16000 */3604		netdev->min_mtu = LIO_MIN_MTU_SIZE;3605		netdev->max_mtu = LIO_MAX_MTU_SIZE;3606 3607		/* Point to the  properties for octeon device to which this3608		 * interface belongs.3609		 */3610		lio->oct_dev = octeon_dev;3611		lio->octprops = props;3612		lio->netdev = netdev;3613 3614		dev_dbg(&octeon_dev->pci_dev->dev,3615			"if%d gmx: %d hw_addr: 0x%llx\n", i,3616			lio->linfo.gmxport, CVM_CAST64(lio->linfo.hw_addr));3617 3618		for (j = 0; j < octeon_dev->sriov_info.max_vfs; j++) {3619			u8 vfmac[ETH_ALEN];3620 3621			eth_random_addr(vfmac);3622			if (__liquidio_set_vf_mac(netdev, j, vfmac, false)) {3623				dev_err(&octeon_dev->pci_dev->dev,3624					"Error setting VF%d MAC address\n",3625					j);3626				goto setup_nic_dev_free;3627			}3628		}3629 3630		/* 64-bit swap required on LE machines */3631		octeon_swap_8B_data(&lio->linfo.hw_addr, 1);3632		for (j = 0; j < 6; j++)3633			mac[j] = *((u8 *)(((u8 *)&lio->linfo.hw_addr) + 2 + j));3634 3635		/* Copy MAC Address to OS network device structure */3636 3637		eth_hw_addr_set(netdev, mac);3638 3639		/* By default all interfaces on a single Octeon uses the same3640		 * tx and rx queues3641		 */3642		lio->txq = lio->linfo.txpciq[0].s.q_no;3643		lio->rxq = lio->linfo.rxpciq[0].s.q_no;3644		if (liquidio_setup_io_queues(octeon_dev, i,3645					     lio->linfo.num_txpciq,3646					     lio->linfo.num_rxpciq)) {3647			dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n");3648			goto setup_nic_dev_free;3649		}3650 3651		ifstate_set(lio, LIO_IFSTATE_DROQ_OPS);3652 3653		lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq);3654		lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq);3655 3656		if (lio_setup_glists(octeon_dev, lio, num_iqueues)) {3657			dev_err(&octeon_dev->pci_dev->dev,3658				"Gather list allocation failed\n");3659			goto setup_nic_dev_free;3660		}3661 3662		/* Register ethtool support */3663		liquidio_set_ethtool_ops(netdev);3664		if (lio->oct_dev->chip_id == OCTEON_CN23XX_PF_VID)3665			octeon_dev->priv_flags = OCT_PRIV_FLAG_DEFAULT;3666		else3667			octeon_dev->priv_flags = 0x0;3668 3669		if (netdev->features & NETIF_F_LRO)3670			liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,3671					     OCTNIC_LROIPV4 | OCTNIC_LROIPV6);3672 3673		liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,3674				     OCTNET_CMD_VLAN_FILTER_ENABLE);3675 3676		if ((debug != -1) && (debug & NETIF_MSG_HW))3677			liquidio_set_feature(netdev,3678					     OCTNET_CMD_VERBOSE_ENABLE, 0);3679 3680		if (setup_link_status_change_wq(netdev))3681			goto setup_nic_dev_free;3682 3683		if ((octeon_dev->fw_info.app_cap_flags &3684		     LIQUIDIO_TIME_SYNC_CAP) &&3685		    setup_sync_octeon_time_wq(netdev))3686			goto setup_nic_dev_free;3687 3688		if (setup_rx_oom_poll_fn(netdev))3689			goto setup_nic_dev_free;3690 3691		/* Register the network device with the OS */3692		if (register_netdev(netdev)) {3693			dev_err(&octeon_dev->pci_dev->dev, "Device registration failed\n");3694			goto setup_nic_dev_free;3695		}3696 3697		dev_dbg(&octeon_dev->pci_dev->dev,3698			"Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n",3699			i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);3700		netif_carrier_off(netdev);3701		lio->link_changes++;3702 3703		ifstate_set(lio, LIO_IFSTATE_REGISTERED);3704 3705		/* Sending command to firmware to enable Rx checksum offload3706		 * by default at the time of setup of Liquidio driver for3707		 * this device3708		 */3709		liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,3710					    OCTNET_CMD_RXCSUM_ENABLE);3711		liquidio_set_feature(netdev, OCTNET_CMD_TNL_TX_CSUM_CTL,3712				     OCTNET_CMD_TXCSUM_ENABLE);3713 3714		dev_dbg(&octeon_dev->pci_dev->dev,3715			"NIC ifidx:%d Setup successful\n", i);3716 3717		if (octeon_dev->subsystem_id ==3718			OCTEON_CN2350_25GB_SUBSYS_ID ||3719		    octeon_dev->subsystem_id ==3720			OCTEON_CN2360_25GB_SUBSYS_ID) {3721			cur_ver = OCT_FW_VER(octeon_dev->fw_info.ver.maj,3722					     octeon_dev->fw_info.ver.min,3723					     octeon_dev->fw_info.ver.rev);3724 3725			/* speed control unsupported in f/w older than 1.7.2 */3726			if (cur_ver < OCT_FW_VER(1, 7, 2)) {3727				dev_info(&octeon_dev->pci_dev->dev,3728					 "speed setting not supported by f/w.");3729				octeon_dev->speed_setting = 25;3730				octeon_dev->no_speed_setting = 1;3731			} else {3732				liquidio_get_speed(lio);3733			}3734 3735			if (octeon_dev->speed_setting == 0) {3736				octeon_dev->speed_setting = 25;3737				octeon_dev->no_speed_setting = 1;3738			}3739		} else {3740			octeon_dev->no_speed_setting = 1;3741			octeon_dev->speed_setting = 10;3742		}3743		octeon_dev->speed_boot = octeon_dev->speed_setting;3744 3745		/* don't read FEC setting if unsupported by f/w (see above) */3746		if (octeon_dev->speed_boot == 25 &&3747		    !octeon_dev->no_speed_setting) {3748			liquidio_get_fec(lio);3749			octeon_dev->props[lio->ifidx].fec_boot =3750				octeon_dev->props[lio->ifidx].fec;3751		}3752	}3753 3754	device_lock(&octeon_dev->pci_dev->dev);3755	devlink = devlink_alloc(&liquidio_devlink_ops,3756				sizeof(struct lio_devlink_priv),3757				&octeon_dev->pci_dev->dev);3758	if (!devlink) {3759		device_unlock(&octeon_dev->pci_dev->dev);3760		dev_err(&octeon_dev->pci_dev->dev, "devlink alloc failed\n");3761		goto setup_nic_dev_free;3762	}3763 3764	lio_devlink = devlink_priv(devlink);3765	lio_devlink->oct = octeon_dev;3766 3767	octeon_dev->devlink = devlink;3768	octeon_dev->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY;3769	devlink_register(devlink);3770	device_unlock(&octeon_dev->pci_dev->dev);3771 3772	return 0;3773 3774setup_nic_dev_free:3775 3776	while (i--) {3777		dev_err(&octeon_dev->pci_dev->dev,3778			"NIC ifidx:%d Setup failed\n", i);3779		liquidio_destroy_nic_device(octeon_dev, i);3780	}3781 3782setup_nic_dev_done:3783 3784	return -ENODEV;3785}3786 3787#ifdef CONFIG_PCI_IOV3788static int octeon_enable_sriov(struct octeon_device *oct)3789{3790	unsigned int num_vfs_alloced = oct->sriov_info.num_vfs_alloced;3791	struct pci_dev *vfdev;3792	int err;3793	u32 u;3794 3795	if (OCTEON_CN23XX_PF(oct) && num_vfs_alloced) {3796		err = pci_enable_sriov(oct->pci_dev,3797				       oct->sriov_info.num_vfs_alloced);3798		if (err) {3799			dev_err(&oct->pci_dev->dev,3800				"OCTEON: Failed to enable PCI sriov: %d\n",3801				err);3802			oct->sriov_info.num_vfs_alloced = 0;3803			return err;3804		}3805		oct->sriov_info.sriov_enabled = 1;3806 3807		/* init lookup table that maps DPI ring number to VF pci_dev3808		 * struct pointer3809		 */3810		u = 0;3811		vfdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,3812				       OCTEON_CN23XX_VF_VID, NULL);3813		while (vfdev) {3814			if (vfdev->is_virtfn &&3815			    (vfdev->physfn == oct->pci_dev)) {3816				oct->sriov_info.dpiring_to_vfpcidev_lut[u] =3817					vfdev;3818				u += oct->sriov_info.rings_per_vf;3819			}3820			vfdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,3821					       OCTEON_CN23XX_VF_VID, vfdev);3822		}3823	}3824 3825	return num_vfs_alloced;3826}3827 3828static int lio_pci_sriov_disable(struct octeon_device *oct)3829{3830	int u;3831 3832	if (pci_vfs_assigned(oct->pci_dev)) {3833		dev_err(&oct->pci_dev->dev, "VFs are still assigned to VMs.\n");3834		return -EPERM;3835	}3836 3837	pci_disable_sriov(oct->pci_dev);3838 3839	u = 0;3840	while (u < MAX_POSSIBLE_VFS) {3841		oct->sriov_info.dpiring_to_vfpcidev_lut[u] = NULL;3842		u += oct->sriov_info.rings_per_vf;3843	}3844 3845	oct->sriov_info.num_vfs_alloced = 0;3846	dev_info(&oct->pci_dev->dev, "oct->pf_num:%d disabled VFs\n",3847		 oct->pf_num);3848 3849	return 0;3850}3851 3852static int liquidio_enable_sriov(struct pci_dev *dev, int num_vfs)3853{3854	struct octeon_device *oct = pci_get_drvdata(dev);3855	int ret = 0;3856 3857	if ((num_vfs == oct->sriov_info.num_vfs_alloced) &&3858	    (oct->sriov_info.sriov_enabled)) {3859		dev_info(&oct->pci_dev->dev, "oct->pf_num:%d already enabled num_vfs:%d\n",3860			 oct->pf_num, num_vfs);3861		return 0;3862	}3863 3864	if (!num_vfs) {3865		lio_vf_rep_destroy(oct);3866		ret = lio_pci_sriov_disable(oct);3867	} else if (num_vfs > oct->sriov_info.max_vfs) {3868		dev_err(&oct->pci_dev->dev,3869			"OCTEON: Max allowed VFs:%d user requested:%d",3870			oct->sriov_info.max_vfs, num_vfs);3871		ret = -EPERM;3872	} else {3873		oct->sriov_info.num_vfs_alloced = num_vfs;3874		ret = octeon_enable_sriov(oct);3875		dev_info(&oct->pci_dev->dev, "oct->pf_num:%d num_vfs:%d\n",3876			 oct->pf_num, num_vfs);3877		ret = lio_vf_rep_create(oct);3878		if (ret)3879			dev_info(&oct->pci_dev->dev,3880				 "vf representor create failed");3881	}3882 3883	return ret;3884}3885#endif3886 3887/**3888 * liquidio_init_nic_module - initialize the NIC3889 * @oct: octeon device3890 *3891 * This initialization routine is called once the Octeon device application is3892 * up and running3893 */3894static int liquidio_init_nic_module(struct octeon_device *oct)3895{3896	int i, retval = 0;3897	int num_nic_ports = CFG_GET_NUM_NIC_PORTS(octeon_get_conf(oct));3898 3899	dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n");3900 3901	/* only default iq and oq were initialized3902	 * initialize the rest as well3903	 */3904	/* run port_config command for each port */3905	oct->ifcount = num_nic_ports;3906 3907	memset(oct->props, 0, sizeof(struct octdev_props) * num_nic_ports);3908 3909	for (i = 0; i < MAX_OCTEON_LINKS; i++)3910		oct->props[i].gmxport = -1;3911 3912	retval = setup_nic_devices(oct);3913	if (retval) {3914		dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n");3915		goto octnet_init_failure;3916	}3917 3918	/* Call vf_rep_modinit if the firmware is switchdev capable3919	 * and do it from the first liquidio function probed.3920	 */3921	if (!oct->octeon_id &&3922	    oct->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP) {3923		retval = lio_vf_rep_modinit();3924		if (retval) {3925			liquidio_stop_nic_module(oct);3926			goto octnet_init_failure;3927		}3928	}3929 3930	liquidio_ptp_init(oct);3931 3932	dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n");3933 3934	return retval;3935 3936octnet_init_failure:3937 3938	oct->ifcount = 0;3939 3940	return retval;3941}3942 3943/**3944 * nic_starter - finish init3945 * @work:  work struct work_struct3946 *3947 * starter callback that invokes the remaining initialization work after the NIC is up and running.3948 */3949static void nic_starter(struct work_struct *work)3950{3951	struct octeon_device *oct;3952	struct cavium_wk *wk = (struct cavium_wk *)work;3953 3954	oct = (struct octeon_device *)wk->ctxptr;3955 3956	if (atomic_read(&oct->status) == OCT_DEV_RUNNING)3957		return;3958 3959	/* If the status of the device is CORE_OK, the core3960	 * application has reported its application type. Call3961	 * any registered handlers now and move to the RUNNING3962	 * state.3963	 */3964	if (atomic_read(&oct->status) != OCT_DEV_CORE_OK) {3965		schedule_delayed_work(&oct->nic_poll_work.work,3966				      LIQUIDIO_STARTER_POLL_INTERVAL_MS);3967		return;3968	}3969 3970	atomic_set(&oct->status, OCT_DEV_RUNNING);3971 3972	if (oct->app_mode && oct->app_mode == CVM_DRV_NIC_APP) {3973		dev_dbg(&oct->pci_dev->dev, "Starting NIC module\n");3974 3975		if (liquidio_init_nic_module(oct))3976			dev_err(&oct->pci_dev->dev, "NIC initialization failed\n");3977		else3978			handshake[oct->octeon_id].started_ok = 1;3979	} else {3980		dev_err(&oct->pci_dev->dev,3981			"Unexpected application running on NIC (%d). Check firmware.\n",3982			oct->app_mode);3983	}3984 3985	complete(&handshake[oct->octeon_id].started);3986}3987 3988static int3989octeon_recv_vf_drv_notice(struct octeon_recv_info *recv_info, void *buf)3990{3991	struct octeon_device *oct = (struct octeon_device *)buf;3992	struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;3993	int i, notice, vf_idx;3994	bool cores_crashed;3995	u64 *data, vf_num;3996 3997	notice = recv_pkt->rh.r.ossp;3998	data = (u64 *)(get_rbd(recv_pkt->buffer_ptr[0]) + OCT_DROQ_INFO_SIZE);3999 4000	/* the first 64-bit word of data is the vf_num */4001	vf_num = data[0];4002	octeon_swap_8B_data(&vf_num, 1);4003	vf_idx = (int)vf_num - 1;4004 4005	cores_crashed = READ_ONCE(oct->cores_crashed);4006 4007	if (notice == VF_DRV_LOADED) {4008		if (!(oct->sriov_info.vf_drv_loaded_mask & BIT_ULL(vf_idx))) {4009			oct->sriov_info.vf_drv_loaded_mask |= BIT_ULL(vf_idx);4010			dev_info(&oct->pci_dev->dev,4011				 "driver for VF%d was loaded\n", vf_idx);4012			if (!cores_crashed)4013				try_module_get(THIS_MODULE);4014		}4015	} else if (notice == VF_DRV_REMOVED) {4016		if (oct->sriov_info.vf_drv_loaded_mask & BIT_ULL(vf_idx)) {4017			oct->sriov_info.vf_drv_loaded_mask &= ~BIT_ULL(vf_idx);4018			dev_info(&oct->pci_dev->dev,4019				 "driver for VF%d was removed\n", vf_idx);4020			if (!cores_crashed)4021				module_put(THIS_MODULE);4022		}4023	} else if (notice == VF_DRV_MACADDR_CHANGED) {4024		u8 *b = (u8 *)&data[1];4025 4026		oct->sriov_info.vf_macaddr[vf_idx] = data[1];4027		dev_info(&oct->pci_dev->dev,4028			 "VF driver changed VF%d's MAC address to %pM\n",4029			 vf_idx, b + 2);4030	}4031 4032	for (i = 0; i < recv_pkt->buffer_count; i++)4033		recv_buffer_free(recv_pkt->buffer_ptr[i]);4034	octeon_free_recv_info(recv_info);4035 4036	return 0;4037}4038 4039/**4040 * octeon_device_init - Device initialization for each Octeon device that is probed4041 * @octeon_dev:  octeon device4042 */4043static int octeon_device_init(struct octeon_device *octeon_dev)4044{4045	int j, ret;4046	char bootcmd[] = "\n";4047	char *dbg_enb = NULL;4048	enum lio_fw_state fw_state;4049	struct octeon_device_priv *oct_priv = octeon_dev->priv;4050	atomic_set(&octeon_dev->status, OCT_DEV_BEGIN_STATE);4051 4052	/* Enable access to the octeon device and make its DMA capability4053	 * known to the OS.4054	 */4055	if (octeon_pci_os_setup(octeon_dev))4056		return 1;4057 4058	atomic_set(&octeon_dev->status, OCT_DEV_PCI_ENABLE_DONE);4059 4060	/* Identify the Octeon type and map the BAR address space. */4061	if (octeon_chip_specific_setup(octeon_dev)) {4062		dev_err(&octeon_dev->pci_dev->dev, "Chip specific setup failed\n");4063		return 1;4064	}4065 4066	atomic_set(&octeon_dev->status, OCT_DEV_PCI_MAP_DONE);4067 4068	/* Only add a reference after setting status 'OCT_DEV_PCI_MAP_DONE',4069	 * since that is what is required for the reference to be removed4070	 * during de-initialization (see 'octeon_destroy_resources').4071	 */4072	octeon_register_device(octeon_dev, octeon_dev->pci_dev->bus->number,4073			       PCI_SLOT(octeon_dev->pci_dev->devfn),4074			       PCI_FUNC(octeon_dev->pci_dev->devfn),4075			       true);4076 4077	octeon_dev->app_mode = CVM_DRV_INVALID_APP;4078 4079	/* CN23XX supports preloaded firmware if the following is true:4080	 *4081	 * The adapter indicates that firmware is currently running AND4082	 * 'fw_type' is 'auto'.4083	 *4084	 * (default state is NEEDS_TO_BE_LOADED, override it if appropriate).4085	 */4086	if (OCTEON_CN23XX_PF(octeon_dev) &&4087	    cn23xx_fw_loaded(octeon_dev) && fw_type_is_auto()) {4088		atomic_cmpxchg(octeon_dev->adapter_fw_state,4089			       FW_NEEDS_TO_BE_LOADED, FW_IS_PRELOADED);4090	}4091 4092	/* If loading firmware, only first device of adapter needs to do so. */4093	fw_state = atomic_cmpxchg(octeon_dev->adapter_fw_state,4094				  FW_NEEDS_TO_BE_LOADED,4095				  FW_IS_BEING_LOADED);4096 4097	/* Here, [local variable] 'fw_state' is set to one of:4098	 *4099	 *   FW_IS_PRELOADED:       No firmware is to be loaded (see above)4100	 *   FW_NEEDS_TO_BE_LOADED: The driver's first instance will load4101	 *                          firmware to the adapter.4102	 *   FW_IS_BEING_LOADED:    The driver's second instance will not load4103	 *                          firmware to the adapter.4104	 */4105 4106	/* Prior to f/w load, perform a soft reset of the Octeon device;4107	 * if error resetting, return w/error.4108	 */4109	if (fw_state == FW_NEEDS_TO_BE_LOADED)4110		if (octeon_dev->fn_list.soft_reset(octeon_dev))4111			return 1;4112 4113	/* Initialize the dispatch mechanism used to push packets arriving on4114	 * Octeon Output queues.4115	 */4116	if (octeon_init_dispatch_list(octeon_dev))4117		return 1;4118 4119	octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,4120				    OPCODE_NIC_CORE_DRV_ACTIVE,4121				    octeon_core_drv_init,4122				    octeon_dev);4123 4124	octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,4125				    OPCODE_NIC_VF_DRV_NOTICE,4126				    octeon_recv_vf_drv_notice, octeon_dev);4127	INIT_DELAYED_WORK(&octeon_dev->nic_poll_work.work, nic_starter);4128	octeon_dev->nic_poll_work.ctxptr = (void *)octeon_dev;4129	schedule_delayed_work(&octeon_dev->nic_poll_work.work,4130			      LIQUIDIO_STARTER_POLL_INTERVAL_MS);4131 4132	atomic_set(&octeon_dev->status, OCT_DEV_DISPATCH_INIT_DONE);4133 4134	if (octeon_set_io_queues_off(octeon_dev)) {4135		dev_err(&octeon_dev->pci_dev->dev, "setting io queues off failed\n");4136		return 1;4137	}4138 4139	if (OCTEON_CN23XX_PF(octeon_dev)) {4140		ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);4141		if (ret) {4142			dev_err(&octeon_dev->pci_dev->dev, "OCTEON: Failed to configure device registers\n");4143			return ret;4144		}4145	}4146 4147	/* Initialize soft command buffer pool4148	 */4149	if (octeon_setup_sc_buffer_pool(octeon_dev)) {4150		dev_err(&octeon_dev->pci_dev->dev, "sc buffer pool allocation failed\n");4151		return 1;4152	}4153	atomic_set(&octeon_dev->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE);4154 4155	/*  Setup the data structures that manage this Octeon's Input queues. */4156	if (octeon_setup_instr_queues(octeon_dev)) {4157		dev_err(&octeon_dev->pci_dev->dev,4158			"instruction queue initialization failed\n");4159		return 1;4160	}4161	atomic_set(&octeon_dev->status, OCT_DEV_INSTR_QUEUE_INIT_DONE);4162 4163	/* Initialize lists to manage the requests of different types that4164	 * arrive from user & kernel applications for this octeon device.4165	 */4166	if (octeon_setup_response_list(octeon_dev)) {4167		dev_err(&octeon_dev->pci_dev->dev, "Response list allocation failed\n");4168		return 1;4169	}4170	atomic_set(&octeon_dev->status, OCT_DEV_RESP_LIST_INIT_DONE);4171 4172	if (octeon_setup_output_queues(octeon_dev)) {4173		dev_err(&octeon_dev->pci_dev->dev, "Output queue initialization failed\n");4174		return 1;4175	}4176 4177	atomic_set(&octeon_dev->status, OCT_DEV_DROQ_INIT_DONE);4178 4179	if (OCTEON_CN23XX_PF(octeon_dev)) {4180		if (octeon_dev->fn_list.setup_mbox(octeon_dev)) {4181			dev_err(&octeon_dev->pci_dev->dev, "OCTEON: Mailbox setup failed\n");4182			return 1;4183		}4184		atomic_set(&octeon_dev->status, OCT_DEV_MBOX_SETUP_DONE);4185 4186		if (octeon_allocate_ioq_vector4187				(octeon_dev,4188				 octeon_dev->sriov_info.num_pf_rings)) {4189			dev_err(&octeon_dev->pci_dev->dev, "OCTEON: ioq vector allocation failed\n");4190			return 1;4191		}4192		atomic_set(&octeon_dev->status, OCT_DEV_MSIX_ALLOC_VECTOR_DONE);4193 4194	} else {4195		/* The input and output queue registers were setup earlier (the4196		 * queues were not enabled). Any additional registers4197		 * that need to be programmed should be done now.4198		 */4199		ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);4200		if (ret) {4201			dev_err(&octeon_dev->pci_dev->dev,4202				"Failed to configure device registers\n");4203			return ret;4204		}4205	}4206 4207	/* Initialize the tasklet that handles output queue packet processing.*/4208	dev_dbg(&octeon_dev->pci_dev->dev, "Initializing droq tasklet\n");4209	tasklet_setup(&oct_priv->droq_tasklet, octeon_droq_bh);4210 4211	/* Setup the interrupt handler and record the INT SUM register address4212	 */4213	if (octeon_setup_interrupt(octeon_dev,4214				   octeon_dev->sriov_info.num_pf_rings))4215		return 1;4216 4217	/* Enable Octeon device interrupts */4218	octeon_dev->fn_list.enable_interrupt(octeon_dev, OCTEON_ALL_INTR);4219 4220	atomic_set(&octeon_dev->status, OCT_DEV_INTR_SET_DONE);4221 4222	/* Send Credit for Octeon Output queues. Credits are always sent BEFORE4223	 * the output queue is enabled.4224	 * This ensures that we'll receive the f/w CORE DRV_ACTIVE message in4225	 * case we've configured CN23XX_SLI_GBL_CONTROL[NOPTR_D] = 0.4226	 * Otherwise, it is possible that the DRV_ACTIVE message will be sent4227	 * before any credits have been issued, causing the ring to be reset4228	 * (and the f/w appear to never have started).4229	 */4230	for (j = 0; j < octeon_dev->num_oqs; j++)4231		writel(octeon_dev->droq[j]->max_count,4232		       octeon_dev->droq[j]->pkts_credit_reg);4233 4234	/* Enable the input and output queues for this Octeon device */4235	ret = octeon_dev->fn_list.enable_io_queues(octeon_dev);4236	if (ret) {4237		dev_err(&octeon_dev->pci_dev->dev, "Failed to enable input/output queues");4238		return ret;4239	}4240 4241	atomic_set(&octeon_dev->status, OCT_DEV_IO_QUEUES_DONE);4242 4243	if (fw_state == FW_NEEDS_TO_BE_LOADED) {4244		dev_dbg(&octeon_dev->pci_dev->dev, "Waiting for DDR initialization...\n");4245		if (!ddr_timeout) {4246			dev_info(&octeon_dev->pci_dev->dev,4247				 "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n");4248		}4249 4250		schedule_timeout_uninterruptible(HZ * LIO_RESET_SECS);4251 4252		/* Wait for the octeon to initialize DDR after the soft-reset.*/4253		while (!ddr_timeout) {4254			set_current_state(TASK_INTERRUPTIBLE);4255			if (schedule_timeout(HZ / 10)) {4256				/* user probably pressed Control-C */4257				return 1;4258			}4259		}4260		ret = octeon_wait_for_ddr_init(octeon_dev, &ddr_timeout);4261		if (ret) {4262			dev_err(&octeon_dev->pci_dev->dev,4263				"DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n",4264				ret);4265			return 1;4266		}4267 4268		if (octeon_wait_for_bootloader(octeon_dev, 1000)) {4269			dev_err(&octeon_dev->pci_dev->dev, "Board not responding\n");4270			return 1;4271		}4272 4273		/* Divert uboot to take commands from host instead. */4274		ret = octeon_console_send_cmd(octeon_dev, bootcmd, 50);4275 4276		dev_dbg(&octeon_dev->pci_dev->dev, "Initializing consoles\n");4277		ret = octeon_init_consoles(octeon_dev);4278		if (ret) {4279			dev_err(&octeon_dev->pci_dev->dev, "Could not access board consoles\n");4280			return 1;4281		}4282		/* If console debug enabled, specify empty string to use default4283		 * enablement ELSE specify NULL string for 'disabled'.4284		 */4285		dbg_enb = octeon_console_debug_enabled(0) ? "" : NULL;4286		ret = octeon_add_console(octeon_dev, 0, dbg_enb);4287		if (ret) {4288			dev_err(&octeon_dev->pci_dev->dev, "Could not access board console\n");4289			return 1;4290		} else if (octeon_console_debug_enabled(0)) {4291			/* If console was added AND we're logging console output4292			 * then set our console print function.4293			 */4294			octeon_dev->console[0].print = octeon_dbg_console_print;4295		}4296 4297		atomic_set(&octeon_dev->status, OCT_DEV_CONSOLE_INIT_DONE);4298 4299		dev_dbg(&octeon_dev->pci_dev->dev, "Loading firmware\n");4300		ret = load_firmware(octeon_dev);4301		if (ret) {4302			dev_err(&octeon_dev->pci_dev->dev, "Could not load firmware to board\n");4303			return 1;4304		}4305 4306		atomic_set(octeon_dev->adapter_fw_state, FW_HAS_BEEN_LOADED);4307	}4308 4309	handshake[octeon_dev->octeon_id].init_ok = 1;4310	complete(&handshake[octeon_dev->octeon_id].init);4311 4312	atomic_set(&octeon_dev->status, OCT_DEV_HOST_OK);4313	oct_priv->dev = octeon_dev;4314 4315	return 0;4316}4317 4318/**4319 * octeon_dbg_console_print - Debug console print function4320 * @oct:  octeon device4321 * @console_num: console number4322 * @prefix:      first portion of line to display4323 * @suffix:      second portion of line to display4324 *4325 * The OCTEON debug console outputs entire lines (excluding '\n').4326 * Normally, the line will be passed in the 'prefix' parameter.4327 * However, due to buffering, it is possible for a line to be split into two4328 * parts, in which case they will be passed as the 'prefix' parameter and4329 * 'suffix' parameter.4330 */4331static int octeon_dbg_console_print(struct octeon_device *oct, u32 console_num,4332				    char *prefix, char *suffix)4333{4334	if (prefix && suffix)4335		dev_info(&oct->pci_dev->dev, "%u: %s%s\n", console_num, prefix,4336			 suffix);4337	else if (prefix)4338		dev_info(&oct->pci_dev->dev, "%u: %s\n", console_num, prefix);4339	else if (suffix)4340		dev_info(&oct->pci_dev->dev, "%u: %s\n", console_num, suffix);4341 4342	return 0;4343}4344 4345/**4346 * liquidio_exit - Exits the module4347 */4348static void __exit liquidio_exit(void)4349{4350	liquidio_deinit_pci();4351 4352	pr_info("LiquidIO network module is now unloaded\n");4353}4354 4355module_init(liquidio_init);4356module_exit(liquidio_exit);4357