brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.02/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */3 4#include <linux/ethtool.h>5#include <linux/printk.h>6#include <linux/dynamic_debug.h>7#include <linux/netdevice.h>8#include <linux/etherdevice.h>9#include <linux/if_vlan.h>10#include <linux/rtnetlink.h>11#include <linux/interrupt.h>12#include <linux/pci.h>13#include <linux/cpumask.h>14#include <linux/crash_dump.h>15#include <linux/vmalloc.h>16#include <net/page_pool/helpers.h>17 18#include "ionic.h"19#include "ionic_bus.h"20#include "ionic_dev.h"21#include "ionic_lif.h"22#include "ionic_txrx.h"23#include "ionic_ethtool.h"24#include "ionic_debugfs.h"25 26/* queuetype support level */27static const u8 ionic_qtype_versions[IONIC_QTYPE_MAX] = {28	[IONIC_QTYPE_ADMINQ]  = 0,   /* 0 = Base version with CQ support */29	[IONIC_QTYPE_NOTIFYQ] = 0,   /* 0 = Base version */30	[IONIC_QTYPE_RXQ]     = 2,   /* 0 = Base version with CQ+SG support31				      * 2 =       ... with CMB rings32				      */33	[IONIC_QTYPE_TXQ]     = 3,   /* 0 = Base version with CQ+SG support34				      * 1 =       ... with Tx SG version 135				      * 3 =       ... with CMB rings36				      */37};38 39static void ionic_link_status_check(struct ionic_lif *lif);40static void ionic_lif_handle_fw_down(struct ionic_lif *lif);41static void ionic_lif_handle_fw_up(struct ionic_lif *lif);42static void ionic_lif_set_netdev_info(struct ionic_lif *lif);43 44static void ionic_txrx_deinit(struct ionic_lif *lif);45static int ionic_txrx_init(struct ionic_lif *lif);46static int ionic_start_queues(struct ionic_lif *lif);47static void ionic_stop_queues(struct ionic_lif *lif);48static void ionic_lif_queue_identify(struct ionic_lif *lif);49 50static void ionic_xdp_rxqs_prog_update(struct ionic_lif *lif);51static void ionic_unregister_rxq_info(struct ionic_queue *q);52static int ionic_register_rxq_info(struct ionic_queue *q, unsigned int napi_id);53 54static void ionic_dim_work(struct work_struct *work)55{56	struct dim *dim = container_of(work, struct dim, work);57	struct dim_cq_moder cur_moder;58	struct ionic_intr_info *intr;59	struct ionic_qcq *qcq;60	struct ionic_lif *lif;61	struct ionic_queue *q;62	u32 new_coal;63 64	qcq = container_of(dim, struct ionic_qcq, dim);65	q = &qcq->q;66	if (q->type == IONIC_QTYPE_RXQ)67		cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);68	else69		cur_moder = net_dim_get_tx_moderation(dim->mode, dim->profile_ix);70	lif = q->lif;71	new_coal = ionic_coal_usec_to_hw(lif->ionic, cur_moder.usec);72	new_coal = new_coal ? new_coal : 1;73 74	intr = &qcq->intr;75	if (intr->dim_coal_hw != new_coal) {76		intr->dim_coal_hw = new_coal;77 78		ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,79				     intr->index, intr->dim_coal_hw);80	}81 82	dim->state = DIM_START_MEASURE;83}84 85static void ionic_lif_deferred_work(struct work_struct *work)86{87	struct ionic_lif *lif = container_of(work, struct ionic_lif, deferred.work);88	struct ionic_deferred *def = &lif->deferred;89	struct ionic_deferred_work *w = NULL;90 91	do {92		spin_lock_bh(&def->lock);93		if (!list_empty(&def->list)) {94			w = list_first_entry(&def->list,95					     struct ionic_deferred_work, list);96			list_del(&w->list);97		}98		spin_unlock_bh(&def->lock);99 100		if (!w)101			break;102 103		switch (w->type) {104		case IONIC_DW_TYPE_RX_MODE:105			ionic_lif_rx_mode(lif);106			break;107		case IONIC_DW_TYPE_LINK_STATUS:108			ionic_link_status_check(lif);109			break;110		case IONIC_DW_TYPE_LIF_RESET:111			if (w->fw_status) {112				ionic_lif_handle_fw_up(lif);113			} else {114				ionic_lif_handle_fw_down(lif);115 116				/* Fire off another watchdog to see117				 * if the FW is already back rather than118				 * waiting another whole cycle119				 */120				mod_timer(&lif->ionic->watchdog_timer, jiffies + 1);121			}122			break;123		default:124			break;125		}126		kfree(w);127		w = NULL;128	} while (true);129}130 131void ionic_lif_deferred_enqueue(struct ionic_lif *lif,132				struct ionic_deferred_work *work)133{134	spin_lock_bh(&lif->deferred.lock);135	list_add_tail(&work->list, &lif->deferred.list);136	spin_unlock_bh(&lif->deferred.lock);137	queue_work(lif->ionic->wq, &lif->deferred.work);138}139 140static void ionic_link_status_check(struct ionic_lif *lif)141{142	struct net_device *netdev = lif->netdev;143	u16 link_status;144	bool link_up;145 146	if (!test_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))147		return;148 149	/* Don't put carrier back up if we're in a broken state */150	if (test_bit(IONIC_LIF_F_BROKEN, lif->state)) {151		clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);152		return;153	}154 155	link_status = le16_to_cpu(lif->info->status.link_status);156	link_up = link_status == IONIC_PORT_OPER_STATUS_UP;157 158	if (link_up) {159		int err = 0;160 161		if (netdev->flags & IFF_UP && netif_running(netdev)) {162			mutex_lock(&lif->queue_lock);163			err = ionic_start_queues(lif);164			if (err && err != -EBUSY) {165				netdev_err(netdev,166					   "Failed to start queues: %d\n", err);167				set_bit(IONIC_LIF_F_BROKEN, lif->state);168				netif_carrier_off(lif->netdev);169			}170			mutex_unlock(&lif->queue_lock);171		}172 173		if (!err && !netif_carrier_ok(netdev)) {174			ionic_port_identify(lif->ionic);175			netdev_info(netdev, "Link up - %d Gbps\n",176				    le32_to_cpu(lif->info->status.link_speed) / 1000);177			netif_carrier_on(netdev);178		}179	} else {180		if (netif_carrier_ok(netdev)) {181			lif->link_down_count++;182			netdev_info(netdev, "Link down\n");183			netif_carrier_off(netdev);184		}185 186		if (netdev->flags & IFF_UP && netif_running(netdev)) {187			mutex_lock(&lif->queue_lock);188			ionic_stop_queues(lif);189			mutex_unlock(&lif->queue_lock);190		}191	}192 193	clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);194}195 196void ionic_link_status_check_request(struct ionic_lif *lif, bool can_sleep)197{198	struct ionic_deferred_work *work;199 200	/* we only need one request outstanding at a time */201	if (test_and_set_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))202		return;203 204	if (!can_sleep) {205		work = kzalloc(sizeof(*work), GFP_ATOMIC);206		if (!work) {207			clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);208			return;209		}210 211		work->type = IONIC_DW_TYPE_LINK_STATUS;212		ionic_lif_deferred_enqueue(lif, work);213	} else {214		ionic_link_status_check(lif);215	}216}217 218static irqreturn_t ionic_isr(int irq, void *data)219{220	struct napi_struct *napi = data;221 222	napi_schedule_irqoff(napi);223 224	return IRQ_HANDLED;225}226 227static int ionic_request_irq(struct ionic_lif *lif, struct ionic_qcq *qcq)228{229	struct ionic_intr_info *intr = &qcq->intr;230	struct device *dev = lif->ionic->dev;231	struct ionic_queue *q = &qcq->q;232	const char *name;233 234	if (lif->registered)235		name = netdev_name(lif->netdev);236	else237		name = dev_name(dev);238 239	snprintf(intr->name, sizeof(intr->name),240		 "%.5s-%.16s-%.8s", IONIC_DRV_NAME, name, q->name);241 242	return devm_request_irq(dev, intr->vector, ionic_isr,243				0, intr->name, &qcq->napi);244}245 246static int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)247{248	struct ionic *ionic = lif->ionic;249	int index;250 251	index = find_first_zero_bit(ionic->intrs, ionic->nintrs);252	if (index == ionic->nintrs) {253		netdev_warn(lif->netdev, "%s: no intr, index=%d nintrs=%d\n",254			    __func__, index, ionic->nintrs);255		return -ENOSPC;256	}257 258	set_bit(index, ionic->intrs);259	ionic_intr_init(&ionic->idev, intr, index);260 261	return 0;262}263 264static void ionic_intr_free(struct ionic *ionic, int index)265{266	if (index != IONIC_INTR_INDEX_NOT_ASSIGNED && index < ionic->nintrs)267		clear_bit(index, ionic->intrs);268}269 270static void ionic_irq_aff_notify(struct irq_affinity_notify *notify,271				 const cpumask_t *mask)272{273	struct ionic_intr_info *intr = container_of(notify, struct ionic_intr_info, aff_notify);274 275	cpumask_copy(*intr->affinity_mask, mask);276}277 278static void ionic_irq_aff_release(struct kref __always_unused *ref)279{280}281 282static int ionic_qcq_enable(struct ionic_qcq *qcq)283{284	struct ionic_queue *q = &qcq->q;285	struct ionic_lif *lif = q->lif;286	struct ionic_dev *idev;287	struct device *dev;288 289	struct ionic_admin_ctx ctx = {290		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),291		.cmd.q_control = {292			.opcode = IONIC_CMD_Q_CONTROL,293			.lif_index = cpu_to_le16(lif->index),294			.type = q->type,295			.index = cpu_to_le32(q->index),296			.oper = IONIC_Q_ENABLE,297		},298	};299	int ret;300 301	idev = &lif->ionic->idev;302	dev = lif->ionic->dev;303 304	dev_dbg(dev, "q_enable.index %d q_enable.qtype %d\n",305		ctx.cmd.q_control.index, ctx.cmd.q_control.type);306 307	if (qcq->flags & IONIC_QCQ_F_INTR)308		ionic_intr_clean(idev->intr_ctrl, qcq->intr.index);309 310	ret = ionic_adminq_post_wait(lif, &ctx);311	if (ret)312		return ret;313 314	if (qcq->flags & IONIC_QCQ_F_INTR) {315		napi_enable(&qcq->napi);316		irq_set_affinity_notifier(qcq->intr.vector,317					  &qcq->intr.aff_notify);318		irq_set_affinity_hint(qcq->intr.vector,319				      *qcq->intr.affinity_mask);320		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,321				IONIC_INTR_MASK_CLEAR);322	}323 324	return 0;325}326 327static int ionic_qcq_disable(struct ionic_lif *lif, struct ionic_qcq *qcq, int fw_err)328{329	struct ionic_queue *q;330 331	struct ionic_admin_ctx ctx = {332		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),333		.cmd.q_control = {334			.opcode = IONIC_CMD_Q_CONTROL,335			.oper = IONIC_Q_DISABLE,336		},337	};338 339	if (!qcq) {340		netdev_err(lif->netdev, "%s: bad qcq\n", __func__);341		return -ENXIO;342	}343 344	q = &qcq->q;345 346	if (qcq->flags & IONIC_QCQ_F_INTR) {347		struct ionic_dev *idev = &lif->ionic->idev;348 349		if (lif->doorbell_wa)350			cancel_work_sync(&qcq->doorbell_napi_work);351		cancel_work_sync(&qcq->dim.work);352		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,353				IONIC_INTR_MASK_SET);354		synchronize_irq(qcq->intr.vector);355		irq_set_affinity_notifier(qcq->intr.vector, NULL);356		irq_set_affinity_hint(qcq->intr.vector, NULL);357		napi_disable(&qcq->napi);358	}359 360	/* If there was a previous fw communcation error, don't bother with361	 * sending the adminq command and just return the same error value.362	 */363	if (fw_err == -ETIMEDOUT || fw_err == -ENXIO)364		return fw_err;365 366	ctx.cmd.q_control.lif_index = cpu_to_le16(lif->index);367	ctx.cmd.q_control.type = q->type;368	ctx.cmd.q_control.index = cpu_to_le32(q->index);369	dev_dbg(lif->ionic->dev, "q_disable.index %d q_disable.qtype %d\n",370		ctx.cmd.q_control.index, ctx.cmd.q_control.type);371 372	return ionic_adminq_post_wait(lif, &ctx);373}374 375static void ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)376{377	struct ionic_dev *idev = &lif->ionic->idev;378 379	if (!qcq)380		return;381 382	if (!(qcq->flags & IONIC_QCQ_F_INITED))383		return;384 385	ionic_unregister_rxq_info(&qcq->q);386	if (qcq->flags & IONIC_QCQ_F_INTR) {387		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,388				IONIC_INTR_MASK_SET);389		netif_napi_del(&qcq->napi);390	}391 392	qcq->flags &= ~IONIC_QCQ_F_INITED;393}394 395static void ionic_qcq_intr_free(struct ionic_lif *lif, struct ionic_qcq *qcq)396{397	if (!(qcq->flags & IONIC_QCQ_F_INTR) || qcq->intr.vector == 0)398		return;399 400	irq_set_affinity_hint(qcq->intr.vector, NULL);401	devm_free_irq(lif->ionic->dev, qcq->intr.vector, &qcq->napi);402	qcq->intr.vector = 0;403	ionic_intr_free(lif->ionic, qcq->intr.index);404	qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;405}406 407static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq)408{409	struct device *dev = lif->ionic->dev;410 411	if (!qcq)412		return;413 414	ionic_debugfs_del_qcq(qcq);415 416	if (qcq->q_base) {417		dma_free_coherent(dev, qcq->q_size, qcq->q_base, qcq->q_base_pa);418		qcq->q_base = NULL;419		qcq->q_base_pa = 0;420	}421 422	if (qcq->cmb_q_base) {423		iounmap(qcq->cmb_q_base);424		ionic_put_cmb(lif, qcq->cmb_pgid, qcq->cmb_order);425		qcq->cmb_pgid = 0;426		qcq->cmb_order = 0;427		qcq->cmb_q_base = NULL;428		qcq->cmb_q_base_pa = 0;429	}430 431	if (qcq->cq_base) {432		dma_free_coherent(dev, qcq->cq_size, qcq->cq_base, qcq->cq_base_pa);433		qcq->cq_base = NULL;434		qcq->cq_base_pa = 0;435	}436 437	if (qcq->sg_base) {438		dma_free_coherent(dev, qcq->sg_size, qcq->sg_base, qcq->sg_base_pa);439		qcq->sg_base = NULL;440		qcq->sg_base_pa = 0;441	}442 443	page_pool_destroy(qcq->q.page_pool);444	qcq->q.page_pool = NULL;445 446	ionic_qcq_intr_free(lif, qcq);447	vfree(qcq->q.info);448	qcq->q.info = NULL;449}450 451void ionic_qcqs_free(struct ionic_lif *lif)452{453	struct device *dev = lif->ionic->dev;454	struct ionic_qcq *adminqcq;455	unsigned long irqflags;456 457	if (lif->notifyqcq) {458		ionic_qcq_free(lif, lif->notifyqcq);459		devm_kfree(dev, lif->notifyqcq);460		lif->notifyqcq = NULL;461	}462 463	if (lif->adminqcq) {464		spin_lock_irqsave(&lif->adminq_lock, irqflags);465		adminqcq = READ_ONCE(lif->adminqcq);466		lif->adminqcq = NULL;467		spin_unlock_irqrestore(&lif->adminq_lock, irqflags);468		if (adminqcq) {469			ionic_qcq_free(lif, adminqcq);470			devm_kfree(dev, adminqcq);471		}472	}473 474	if (lif->rxqcqs) {475		devm_kfree(dev, lif->rxqstats);476		lif->rxqstats = NULL;477		devm_kfree(dev, lif->rxqcqs);478		lif->rxqcqs = NULL;479	}480 481	if (lif->txqcqs) {482		devm_kfree(dev, lif->txqstats);483		lif->txqstats = NULL;484		devm_kfree(dev, lif->txqcqs);485		lif->txqcqs = NULL;486	}487}488 489static void ionic_link_qcq_interrupts(struct ionic_qcq *src_qcq,490				      struct ionic_qcq *n_qcq)491{492	n_qcq->intr.vector = src_qcq->intr.vector;493	n_qcq->intr.index = src_qcq->intr.index;494}495 496static int ionic_alloc_qcq_interrupt(struct ionic_lif *lif, struct ionic_qcq *qcq)497{498	cpumask_var_t *affinity_mask;499	int err;500 501	if (!(qcq->flags & IONIC_QCQ_F_INTR)) {502		qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;503		return 0;504	}505 506	err = ionic_intr_alloc(lif, &qcq->intr);507	if (err) {508		netdev_warn(lif->netdev, "no intr for %s: %d\n",509			    qcq->q.name, err);510		goto err_out;511	}512 513	err = ionic_bus_get_irq(lif->ionic, qcq->intr.index);514	if (err < 0) {515		netdev_warn(lif->netdev, "no vector for %s: %d\n",516			    qcq->q.name, err);517		goto err_out_free_intr;518	}519	qcq->intr.vector = err;520	ionic_intr_mask_assert(lif->ionic->idev.intr_ctrl, qcq->intr.index,521			       IONIC_INTR_MASK_SET);522 523	err = ionic_request_irq(lif, qcq);524	if (err) {525		netdev_warn(lif->netdev, "irq request failed %d\n", err);526		goto err_out_free_intr;527	}528 529	/* try to get the irq on the local numa node first */530	affinity_mask = &lif->ionic->affinity_masks[qcq->intr.index];531	if (cpumask_empty(*affinity_mask)) {532		unsigned int cpu;533 534		cpu = cpumask_local_spread(qcq->intr.index,535					   dev_to_node(lif->ionic->dev));536		if (cpu != -1)537			cpumask_set_cpu(cpu, *affinity_mask);538	}539 540	qcq->intr.affinity_mask = affinity_mask;541	qcq->intr.aff_notify.notify = ionic_irq_aff_notify;542	qcq->intr.aff_notify.release = ionic_irq_aff_release;543 544	netdev_dbg(lif->netdev, "%s: Interrupt index %d\n", qcq->q.name, qcq->intr.index);545	return 0;546 547err_out_free_intr:548	ionic_intr_free(lif->ionic, qcq->intr.index);549err_out:550	return err;551}552 553static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type,554			   unsigned int index,555			   const char *name, unsigned int flags,556			   unsigned int num_descs, unsigned int desc_size,557			   unsigned int cq_desc_size,558			   unsigned int sg_desc_size,559			   unsigned int desc_info_size,560			   unsigned int pid, struct bpf_prog *xdp_prog,561			   struct ionic_qcq **qcq)562{563	struct ionic_dev *idev = &lif->ionic->idev;564	struct device *dev = lif->ionic->dev;565	struct ionic_qcq *new;566	int err;567 568	*qcq = NULL;569 570	new = devm_kzalloc(dev, sizeof(*new), GFP_KERNEL);571	if (!new) {572		netdev_err(lif->netdev, "Cannot allocate queue structure\n");573		err = -ENOMEM;574		goto err_out;575	}576 577	new->q.dev = dev;578	new->flags = flags;579 580	new->q.info = vcalloc(num_descs, desc_info_size);581	if (!new->q.info) {582		netdev_err(lif->netdev, "Cannot allocate queue info\n");583		err = -ENOMEM;584		goto err_out_free_qcq;585	}586 587	if (type == IONIC_QTYPE_RXQ) {588		struct page_pool_params pp_params = {589			.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,590			.order = 0,591			.pool_size = num_descs,592			.nid = NUMA_NO_NODE,593			.dev = lif->ionic->dev,594			.napi = &new->napi,595			.dma_dir = DMA_FROM_DEVICE,596			.max_len = PAGE_SIZE,597			.netdev = lif->netdev,598		};599 600		if (xdp_prog)601			pp_params.dma_dir = DMA_BIDIRECTIONAL;602 603		new->q.page_pool = page_pool_create(&pp_params);604		if (IS_ERR(new->q.page_pool)) {605			netdev_err(lif->netdev, "Cannot create page_pool\n");606			err = PTR_ERR(new->q.page_pool);607			new->q.page_pool = NULL;608			goto err_out_free_q_info;609		}610	}611 612	new->q.type = type;613	new->q.max_sg_elems = lif->qtype_info[type].max_sg_elems;614 615	err = ionic_q_init(lif, idev, &new->q, index, name, num_descs,616			   desc_size, sg_desc_size, pid);617	if (err) {618		netdev_err(lif->netdev, "Cannot initialize queue\n");619		goto err_out_free_page_pool;620	}621 622	err = ionic_alloc_qcq_interrupt(lif, new);623	if (err)624		goto err_out_free_page_pool;625 626	err = ionic_cq_init(lif, &new->cq, &new->intr, num_descs, cq_desc_size);627	if (err) {628		netdev_err(lif->netdev, "Cannot initialize completion queue\n");629		goto err_out_free_irq;630	}631 632	if (flags & IONIC_QCQ_F_NOTIFYQ) {633		int q_size;634 635		/* q & cq need to be contiguous in NotifyQ, so alloc it all in q636		 * and don't alloc qc.  We leave new->qc_size and new->qc_base637		 * as 0 to be sure we don't try to free it later.638		 */639		q_size = ALIGN(num_descs * desc_size, PAGE_SIZE);640		new->q_size = PAGE_SIZE + q_size +641			      ALIGN(num_descs * cq_desc_size, PAGE_SIZE);642		new->q_base = dma_alloc_coherent(dev, new->q_size,643						 &new->q_base_pa, GFP_KERNEL);644		if (!new->q_base) {645			netdev_err(lif->netdev, "Cannot allocate qcq DMA memory\n");646			err = -ENOMEM;647			goto err_out_free_irq;648		}649		new->q.base = PTR_ALIGN(new->q_base, PAGE_SIZE);650		new->q.base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);651 652		/* Base the NotifyQ cq.base off of the ALIGNed q.base */653		new->cq.base = PTR_ALIGN(new->q.base + q_size, PAGE_SIZE);654		new->cq.base_pa = ALIGN(new->q_base_pa + q_size, PAGE_SIZE);655		new->cq.bound_q = &new->q;656	} else {657		/* regular DMA q descriptors */658		new->q_size = PAGE_SIZE + (num_descs * desc_size);659		new->q_base = dma_alloc_coherent(dev, new->q_size, &new->q_base_pa,660						 GFP_KERNEL);661		if (!new->q_base) {662			netdev_err(lif->netdev, "Cannot allocate queue DMA memory\n");663			err = -ENOMEM;664			goto err_out_free_irq;665		}666		new->q.base = PTR_ALIGN(new->q_base, PAGE_SIZE);667		new->q.base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);668 669		if (flags & IONIC_QCQ_F_CMB_RINGS) {670			/* on-chip CMB q descriptors */671			new->cmb_q_size = num_descs * desc_size;672			new->cmb_order = order_base_2(new->cmb_q_size / PAGE_SIZE);673 674			err = ionic_get_cmb(lif, &new->cmb_pgid, &new->cmb_q_base_pa,675					    new->cmb_order);676			if (err) {677				netdev_err(lif->netdev,678					   "Cannot allocate queue order %d from cmb: err %d\n",679					   new->cmb_order, err);680				goto err_out_free_q;681			}682 683			new->cmb_q_base = ioremap_wc(new->cmb_q_base_pa, new->cmb_q_size);684			if (!new->cmb_q_base) {685				netdev_err(lif->netdev, "Cannot map queue from cmb\n");686				ionic_put_cmb(lif, new->cmb_pgid, new->cmb_order);687				err = -ENOMEM;688				goto err_out_free_q;689			}690 691			new->cmb_q_base_pa -= idev->phy_cmb_pages;692			new->q.cmb_base = new->cmb_q_base;693			new->q.cmb_base_pa = new->cmb_q_base_pa;694		}695 696		/* cq DMA descriptors */697		new->cq_size = PAGE_SIZE + (num_descs * cq_desc_size);698		new->cq_base = dma_alloc_coherent(dev, new->cq_size, &new->cq_base_pa,699						  GFP_KERNEL);700		if (!new->cq_base) {701			netdev_err(lif->netdev, "Cannot allocate cq DMA memory\n");702			err = -ENOMEM;703			goto err_out_free_q;704		}705		new->cq.base = PTR_ALIGN(new->cq_base, PAGE_SIZE);706		new->cq.base_pa = ALIGN(new->cq_base_pa, PAGE_SIZE);707		new->cq.bound_q = &new->q;708	}709 710	if (flags & IONIC_QCQ_F_SG) {711		new->sg_size = PAGE_SIZE + (num_descs * sg_desc_size);712		new->sg_base = dma_alloc_coherent(dev, new->sg_size, &new->sg_base_pa,713						  GFP_KERNEL);714		if (!new->sg_base) {715			netdev_err(lif->netdev, "Cannot allocate sg DMA memory\n");716			err = -ENOMEM;717			goto err_out_free_cq;718		}719		new->q.sg_base = PTR_ALIGN(new->sg_base, PAGE_SIZE);720		new->q.sg_base_pa = ALIGN(new->sg_base_pa, PAGE_SIZE);721	}722 723	INIT_WORK(&new->dim.work, ionic_dim_work);724	new->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE;725	if (lif->doorbell_wa)726		INIT_WORK(&new->doorbell_napi_work, ionic_doorbell_napi_work);727 728	*qcq = new;729 730	return 0;731 732err_out_free_cq:733	dma_free_coherent(dev, new->cq_size, new->cq_base, new->cq_base_pa);734err_out_free_q:735	if (new->cmb_q_base) {736		iounmap(new->cmb_q_base);737		ionic_put_cmb(lif, new->cmb_pgid, new->cmb_order);738	}739	dma_free_coherent(dev, new->q_size, new->q_base, new->q_base_pa);740err_out_free_irq:741	if (flags & IONIC_QCQ_F_INTR) {742		devm_free_irq(dev, new->intr.vector, &new->napi);743		ionic_intr_free(lif->ionic, new->intr.index);744	}745err_out_free_page_pool:746	page_pool_destroy(new->q.page_pool);747err_out_free_q_info:748	vfree(new->q.info);749err_out_free_qcq:750	devm_kfree(dev, new);751err_out:752	dev_err(dev, "qcq alloc of %s%d failed %d\n", name, index, err);753	return err;754}755 756static int ionic_qcqs_alloc(struct ionic_lif *lif)757{758	struct device *dev = lif->ionic->dev;759	unsigned int flags;760	int err;761 762	flags = IONIC_QCQ_F_INTR;763	err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,764			      IONIC_ADMINQ_LENGTH,765			      sizeof(struct ionic_admin_cmd),766			      sizeof(struct ionic_admin_comp),767			      0,768			      sizeof(struct ionic_admin_desc_info),769			      lif->kern_pid, NULL, &lif->adminqcq);770	if (err)771		return err;772	ionic_debugfs_add_qcq(lif, lif->adminqcq);773 774	if (lif->ionic->nnqs_per_lif) {775		flags = IONIC_QCQ_F_NOTIFYQ;776		err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notifyq",777				      flags, IONIC_NOTIFYQ_LENGTH,778				      sizeof(struct ionic_notifyq_cmd),779				      sizeof(union ionic_notifyq_comp),780				      0,781				      sizeof(struct ionic_admin_desc_info),782				      lif->kern_pid, NULL, &lif->notifyqcq);783		if (err)784			goto err_out;785		ionic_debugfs_add_qcq(lif, lif->notifyqcq);786 787		/* Let the notifyq ride on the adminq interrupt */788		ionic_link_qcq_interrupts(lif->adminqcq, lif->notifyqcq);789	}790 791	err = -ENOMEM;792	lif->txqcqs = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif,793				   sizeof(*lif->txqcqs), GFP_KERNEL);794	if (!lif->txqcqs)795		goto err_out;796	lif->rxqcqs = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif,797				   sizeof(*lif->rxqcqs), GFP_KERNEL);798	if (!lif->rxqcqs)799		goto err_out;800 801	lif->txqstats = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif + 1,802				     sizeof(*lif->txqstats), GFP_KERNEL);803	if (!lif->txqstats)804		goto err_out;805	lif->rxqstats = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif + 1,806				     sizeof(*lif->rxqstats), GFP_KERNEL);807	if (!lif->rxqstats)808		goto err_out;809 810	return 0;811 812err_out:813	ionic_qcqs_free(lif);814	return err;815}816 817static void ionic_qcq_sanitize(struct ionic_qcq *qcq)818{819	qcq->q.tail_idx = 0;820	qcq->q.head_idx = 0;821	qcq->cq.tail_idx = 0;822	qcq->cq.done_color = 1;823	memset(qcq->q_base, 0, qcq->q_size);824	if (qcq->cmb_q_base)825		memset_io(qcq->cmb_q_base, 0, qcq->cmb_q_size);826	memset(qcq->cq_base, 0, qcq->cq_size);827	memset(qcq->sg_base, 0, qcq->sg_size);828}829 830static int ionic_lif_txq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)831{832	struct device *dev = lif->ionic->dev;833	struct ionic_queue *q = &qcq->q;834	struct ionic_cq *cq = &qcq->cq;835	struct ionic_admin_ctx ctx = {836		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),837		.cmd.q_init = {838			.opcode = IONIC_CMD_Q_INIT,839			.lif_index = cpu_to_le16(lif->index),840			.type = q->type,841			.ver = lif->qtype_info[q->type].version,842			.index = cpu_to_le32(q->index),843			.flags = cpu_to_le16(IONIC_QINIT_F_IRQ |844					     IONIC_QINIT_F_SG),845			.intr_index = cpu_to_le16(qcq->intr.index),846			.pid = cpu_to_le16(q->pid),847			.ring_size = ilog2(q->num_descs),848			.ring_base = cpu_to_le64(q->base_pa),849			.cq_ring_base = cpu_to_le64(cq->base_pa),850			.sg_ring_base = cpu_to_le64(q->sg_base_pa),851			.features = cpu_to_le64(q->features),852		},853	};854	int err;855 856	if (qcq->flags & IONIC_QCQ_F_CMB_RINGS) {857		ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_CMB);858		ctx.cmd.q_init.ring_base = cpu_to_le64(qcq->cmb_q_base_pa);859	}860 861	dev_dbg(dev, "txq_init.pid %d\n", ctx.cmd.q_init.pid);862	dev_dbg(dev, "txq_init.index %d\n", ctx.cmd.q_init.index);863	dev_dbg(dev, "txq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);864	dev_dbg(dev, "txq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);865	dev_dbg(dev, "txq_init.cq_ring_base 0x%llx\n", ctx.cmd.q_init.cq_ring_base);866	dev_dbg(dev, "txq_init.sg_ring_base 0x%llx\n", ctx.cmd.q_init.sg_ring_base);867	dev_dbg(dev, "txq_init.flags 0x%x\n", ctx.cmd.q_init.flags);868	dev_dbg(dev, "txq_init.ver %d\n", ctx.cmd.q_init.ver);869	dev_dbg(dev, "txq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);870 871	ionic_qcq_sanitize(qcq);872 873	err = ionic_adminq_post_wait(lif, &ctx);874	if (err)875		return err;876 877	q->hw_type = ctx.comp.q_init.hw_type;878	q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);879	q->dbval = IONIC_DBELL_QID(q->hw_index);880 881	dev_dbg(dev, "txq->hw_type %d\n", q->hw_type);882	dev_dbg(dev, "txq->hw_index %d\n", q->hw_index);883 884	q->dbell_deadline = IONIC_TX_DOORBELL_DEADLINE;885	q->dbell_jiffies = jiffies;886 887	if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))888		netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi);889 890	qcq->flags |= IONIC_QCQ_F_INITED;891 892	return 0;893}894 895static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)896{897	struct device *dev = lif->ionic->dev;898	struct ionic_queue *q = &qcq->q;899	struct ionic_cq *cq = &qcq->cq;900	struct ionic_admin_ctx ctx = {901		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),902		.cmd.q_init = {903			.opcode = IONIC_CMD_Q_INIT,904			.lif_index = cpu_to_le16(lif->index),905			.type = q->type,906			.ver = lif->qtype_info[q->type].version,907			.index = cpu_to_le32(q->index),908			.flags = cpu_to_le16(IONIC_QINIT_F_IRQ),909			.intr_index = cpu_to_le16(cq->bound_intr->index),910			.pid = cpu_to_le16(q->pid),911			.ring_size = ilog2(q->num_descs),912			.ring_base = cpu_to_le64(q->base_pa),913			.cq_ring_base = cpu_to_le64(cq->base_pa),914			.sg_ring_base = cpu_to_le64(q->sg_base_pa),915			.features = cpu_to_le64(q->features),916		},917	};918	int err;919 920	q->partner = &lif->txqcqs[q->index]->q;921	q->partner->partner = q;922 923	if (!lif->xdp_prog ||924	    (lif->xdp_prog->aux && lif->xdp_prog->aux->xdp_has_frags))925		ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_SG);926 927	if (qcq->flags & IONIC_QCQ_F_CMB_RINGS) {928		ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_CMB);929		ctx.cmd.q_init.ring_base = cpu_to_le64(qcq->cmb_q_base_pa);930	}931 932	dev_dbg(dev, "rxq_init.pid %d\n", ctx.cmd.q_init.pid);933	dev_dbg(dev, "rxq_init.index %d\n", ctx.cmd.q_init.index);934	dev_dbg(dev, "rxq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);935	dev_dbg(dev, "rxq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);936	dev_dbg(dev, "rxq_init.flags 0x%x\n", ctx.cmd.q_init.flags);937	dev_dbg(dev, "rxq_init.ver %d\n", ctx.cmd.q_init.ver);938	dev_dbg(dev, "rxq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);939 940	ionic_qcq_sanitize(qcq);941 942	err = ionic_adminq_post_wait(lif, &ctx);943	if (err)944		return err;945 946	q->hw_type = ctx.comp.q_init.hw_type;947	q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);948	q->dbval = IONIC_DBELL_QID(q->hw_index);949 950	dev_dbg(dev, "rxq->hw_type %d\n", q->hw_type);951	dev_dbg(dev, "rxq->hw_index %d\n", q->hw_index);952 953	q->dbell_deadline = IONIC_RX_MIN_DOORBELL_DEADLINE;954	q->dbell_jiffies = jiffies;955 956	if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))957		netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi);958	else959		netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi);960	err = ionic_register_rxq_info(q, qcq->napi.napi_id);961	if (err) {962		netif_napi_del(&qcq->napi);963		return err;964	}965 966	qcq->flags |= IONIC_QCQ_F_INITED;967 968	return 0;969}970 971int ionic_lif_create_hwstamp_txq(struct ionic_lif *lif)972{973	unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;974	unsigned int txq_i, flags;975	struct ionic_qcq *txq;976	u64 features;977	int err;978 979	if (lif->hwstamp_txq)980		return 0;981 982	features = IONIC_Q_F_2X_CQ_DESC | IONIC_TXQ_F_HWSTAMP;983 984	num_desc = IONIC_MIN_TXRX_DESC;985	desc_sz = sizeof(struct ionic_txq_desc);986	comp_sz = 2 * sizeof(struct ionic_txq_comp);987 988	if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&989	    lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz == sizeof(struct ionic_txq_sg_desc_v1))990		sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);991	else992		sg_desc_sz = sizeof(struct ionic_txq_sg_desc);993 994	txq_i = lif->ionic->ntxqs_per_lif;995	flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;996 997	err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, txq_i, "hwstamp_tx", flags,998			      num_desc, desc_sz, comp_sz, sg_desc_sz,999			      sizeof(struct ionic_tx_desc_info),1000			      lif->kern_pid, NULL, &txq);1001	if (err)1002		goto err_qcq_alloc;1003 1004	txq->q.features = features;1005 1006	ionic_link_qcq_interrupts(lif->adminqcq, txq);1007	ionic_debugfs_add_qcq(lif, txq);1008 1009	lif->hwstamp_txq = txq;1010 1011	if (netif_running(lif->netdev)) {1012		err = ionic_lif_txq_init(lif, txq);1013		if (err)1014			goto err_qcq_init;1015 1016		if (test_bit(IONIC_LIF_F_UP, lif->state)) {1017			err = ionic_qcq_enable(txq);1018			if (err)1019				goto err_qcq_enable;1020		}1021	}1022 1023	return 0;1024 1025err_qcq_enable:1026	ionic_lif_qcq_deinit(lif, txq);1027err_qcq_init:1028	lif->hwstamp_txq = NULL;1029	ionic_debugfs_del_qcq(txq);1030	ionic_qcq_free(lif, txq);1031	devm_kfree(lif->ionic->dev, txq);1032err_qcq_alloc:1033	return err;1034}1035 1036int ionic_lif_create_hwstamp_rxq(struct ionic_lif *lif)1037{1038	unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;1039	unsigned int rxq_i, flags;1040	struct ionic_qcq *rxq;1041	u64 features;1042	int err;1043 1044	if (lif->hwstamp_rxq)1045		return 0;1046 1047	features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;1048 1049	num_desc = IONIC_MIN_TXRX_DESC;1050	desc_sz = sizeof(struct ionic_rxq_desc);1051	comp_sz = 2 * sizeof(struct ionic_rxq_comp);1052	sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);1053 1054	rxq_i = lif->ionic->nrxqs_per_lif;1055	flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG;1056 1057	err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, rxq_i, "hwstamp_rx", flags,1058			      num_desc, desc_sz, comp_sz, sg_desc_sz,1059			      sizeof(struct ionic_rx_desc_info),1060			      lif->kern_pid, NULL, &rxq);1061	if (err)1062		goto err_qcq_alloc;1063 1064	rxq->q.features = features;1065 1066	ionic_link_qcq_interrupts(lif->adminqcq, rxq);1067	ionic_debugfs_add_qcq(lif, rxq);1068 1069	lif->hwstamp_rxq = rxq;1070 1071	if (netif_running(lif->netdev)) {1072		err = ionic_lif_rxq_init(lif, rxq);1073		if (err)1074			goto err_qcq_init;1075 1076		if (test_bit(IONIC_LIF_F_UP, lif->state)) {1077			ionic_rx_fill(&rxq->q, NULL);1078			err = ionic_qcq_enable(rxq);1079			if (err)1080				goto err_qcq_enable;1081		}1082	}1083 1084	return 0;1085 1086err_qcq_enable:1087	ionic_lif_qcq_deinit(lif, rxq);1088err_qcq_init:1089	lif->hwstamp_rxq = NULL;1090	ionic_debugfs_del_qcq(rxq);1091	ionic_qcq_free(lif, rxq);1092	devm_kfree(lif->ionic->dev, rxq);1093err_qcq_alloc:1094	return err;1095}1096 1097int ionic_lif_config_hwstamp_rxq_all(struct ionic_lif *lif, bool rx_all)1098{1099	struct ionic_queue_params qparam;1100 1101	ionic_init_queue_params(lif, &qparam);1102 1103	if (rx_all)1104		qparam.rxq_features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;1105	else1106		qparam.rxq_features = 0;1107 1108	/* if we're not running, just set the values and return */1109	if (!netif_running(lif->netdev)) {1110		lif->rxq_features = qparam.rxq_features;1111		return 0;1112	}1113 1114	return ionic_reconfigure_queues(lif, &qparam);1115}1116 1117int ionic_lif_set_hwstamp_txmode(struct ionic_lif *lif, u16 txstamp_mode)1118{1119	struct ionic_admin_ctx ctx = {1120		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),1121		.cmd.lif_setattr = {1122			.opcode = IONIC_CMD_LIF_SETATTR,1123			.index = cpu_to_le16(lif->index),1124			.attr = IONIC_LIF_ATTR_TXSTAMP,1125			.txstamp_mode = cpu_to_le16(txstamp_mode),1126		},1127	};1128 1129	return ionic_adminq_post_wait(lif, &ctx);1130}1131 1132static void ionic_lif_del_hwstamp_rxfilt(struct ionic_lif *lif)1133{1134	struct ionic_admin_ctx ctx = {1135		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),1136		.cmd.rx_filter_del = {1137			.opcode = IONIC_CMD_RX_FILTER_DEL,1138			.lif_index = cpu_to_le16(lif->index),1139		},1140	};1141	struct ionic_rx_filter *f;1142	u32 filter_id;1143	int err;1144 1145	spin_lock_bh(&lif->rx_filters.lock);1146 1147	f = ionic_rx_filter_rxsteer(lif);1148	if (!f) {1149		spin_unlock_bh(&lif->rx_filters.lock);1150		return;1151	}1152 1153	filter_id = f->filter_id;1154	ionic_rx_filter_free(lif, f);1155 1156	spin_unlock_bh(&lif->rx_filters.lock);1157 1158	netdev_dbg(lif->netdev, "rx_filter del RXSTEER (id %d)\n", filter_id);1159 1160	ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(filter_id);1161 1162	err = ionic_adminq_post_wait(lif, &ctx);1163	if (err && err != -EEXIST)1164		netdev_dbg(lif->netdev, "failed to delete rx_filter RXSTEER (id %d)\n", filter_id);1165}1166 1167static int ionic_lif_add_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)1168{1169	struct ionic_admin_ctx ctx = {1170		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),1171		.cmd.rx_filter_add = {1172			.opcode = IONIC_CMD_RX_FILTER_ADD,1173			.lif_index = cpu_to_le16(lif->index),1174			.match = cpu_to_le16(IONIC_RX_FILTER_STEER_PKTCLASS),1175			.pkt_class = cpu_to_le64(pkt_class),1176		},1177	};1178	u8 qtype;1179	u32 qid;1180	int err;1181 1182	if (!lif->hwstamp_rxq)1183		return -EINVAL;1184 1185	qtype = lif->hwstamp_rxq->q.type;1186	ctx.cmd.rx_filter_add.qtype = qtype;1187 1188	qid = lif->hwstamp_rxq->q.index;1189	ctx.cmd.rx_filter_add.qid = cpu_to_le32(qid);1190 1191	netdev_dbg(lif->netdev, "rx_filter add RXSTEER\n");1192	err = ionic_adminq_post_wait(lif, &ctx);1193	if (err && err != -EEXIST)1194		return err;1195 1196	spin_lock_bh(&lif->rx_filters.lock);1197	err = ionic_rx_filter_save(lif, 0, qid, 0, &ctx, IONIC_FILTER_STATE_SYNCED);1198	spin_unlock_bh(&lif->rx_filters.lock);1199 1200	return err;1201}1202 1203int ionic_lif_set_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)1204{1205	ionic_lif_del_hwstamp_rxfilt(lif);1206 1207	if (!pkt_class)1208		return 0;1209 1210	return ionic_lif_add_hwstamp_rxfilt(lif, pkt_class);1211}1212 1213static int ionic_adminq_napi(struct napi_struct *napi, int budget)1214{1215	struct ionic_intr_info *intr = napi_to_cq(napi)->bound_intr;1216	struct ionic_lif *lif = napi_to_cq(napi)->lif;1217	struct ionic_dev *idev = &lif->ionic->idev;1218	unsigned long irqflags;1219	unsigned int flags = 0;1220	int rx_work = 0;1221	int tx_work = 0;1222	int n_work = 0;1223	int a_work = 0;1224	int work_done;1225	int credits;1226 1227	if (lif->notifyqcq && lif->notifyqcq->flags & IONIC_QCQ_F_INITED)1228		n_work = ionic_cq_service(&lif->notifyqcq->cq, budget,1229					  ionic_notifyq_service, NULL, NULL);1230 1231	spin_lock_irqsave(&lif->adminq_lock, irqflags);1232	if (lif->adminqcq && lif->adminqcq->flags & IONIC_QCQ_F_INITED)1233		a_work = ionic_cq_service(&lif->adminqcq->cq, budget,1234					  ionic_adminq_service, NULL, NULL);1235 1236	spin_unlock_irqrestore(&lif->adminq_lock, irqflags);1237 1238	if (lif->hwstamp_rxq)1239		rx_work = ionic_cq_service(&lif->hwstamp_rxq->cq, budget,1240					   ionic_rx_service, NULL, NULL);1241 1242	if (lif->hwstamp_txq)1243		tx_work = ionic_tx_cq_service(&lif->hwstamp_txq->cq, budget, !!budget);1244 1245	work_done = max(max(n_work, a_work), max(rx_work, tx_work));1246	if (work_done < budget && napi_complete_done(napi, work_done)) {1247		flags |= IONIC_INTR_CRED_UNMASK;1248		intr->rearm_count++;1249	}1250 1251	if (work_done || flags) {1252		flags |= IONIC_INTR_CRED_RESET_COALESCE;1253		credits = n_work + a_work + rx_work + tx_work;1254		ionic_intr_credits(idev->intr_ctrl, intr->index, credits, flags);1255	}1256 1257	if (lif->doorbell_wa) {1258		if (!a_work)1259			ionic_adminq_poke_doorbell(&lif->adminqcq->q);1260		if (lif->hwstamp_rxq && !rx_work)1261			ionic_rxq_poke_doorbell(&lif->hwstamp_rxq->q);1262		if (lif->hwstamp_txq && !tx_work)1263			ionic_txq_poke_doorbell(&lif->hwstamp_txq->q);1264	}1265 1266	return work_done;1267}1268 1269void ionic_get_stats64(struct net_device *netdev,1270		       struct rtnl_link_stats64 *ns)1271{1272	struct ionic_lif *lif = netdev_priv(netdev);1273	struct ionic_lif_stats *ls;1274 1275	memset(ns, 0, sizeof(*ns));1276	ls = &lif->info->stats;1277 1278	ns->rx_packets = le64_to_cpu(ls->rx_ucast_packets) +1279			 le64_to_cpu(ls->rx_mcast_packets) +1280			 le64_to_cpu(ls->rx_bcast_packets);1281 1282	ns->tx_packets = le64_to_cpu(ls->tx_ucast_packets) +1283			 le64_to_cpu(ls->tx_mcast_packets) +1284			 le64_to_cpu(ls->tx_bcast_packets);1285 1286	ns->rx_bytes = le64_to_cpu(ls->rx_ucast_bytes) +1287		       le64_to_cpu(ls->rx_mcast_bytes) +1288		       le64_to_cpu(ls->rx_bcast_bytes);1289 1290	ns->tx_bytes = le64_to_cpu(ls->tx_ucast_bytes) +1291		       le64_to_cpu(ls->tx_mcast_bytes) +1292		       le64_to_cpu(ls->tx_bcast_bytes);1293 1294	ns->rx_dropped = le64_to_cpu(ls->rx_ucast_drop_packets) +1295			 le64_to_cpu(ls->rx_mcast_drop_packets) +1296			 le64_to_cpu(ls->rx_bcast_drop_packets);1297 1298	ns->tx_dropped = le64_to_cpu(ls->tx_ucast_drop_packets) +1299			 le64_to_cpu(ls->tx_mcast_drop_packets) +1300			 le64_to_cpu(ls->tx_bcast_drop_packets);1301 1302	ns->multicast = le64_to_cpu(ls->rx_mcast_packets);1303 1304	ns->rx_over_errors = le64_to_cpu(ls->rx_queue_empty);1305 1306	ns->rx_missed_errors = le64_to_cpu(ls->rx_dma_error) +1307			       le64_to_cpu(ls->rx_queue_disabled) +1308			       le64_to_cpu(ls->rx_desc_fetch_error) +1309			       le64_to_cpu(ls->rx_desc_data_error);1310 1311	ns->tx_aborted_errors = le64_to_cpu(ls->tx_dma_error) +1312				le64_to_cpu(ls->tx_queue_disabled) +1313				le64_to_cpu(ls->tx_desc_fetch_error) +1314				le64_to_cpu(ls->tx_desc_data_error);1315 1316	ns->rx_errors = ns->rx_over_errors +1317			ns->rx_missed_errors;1318 1319	ns->tx_errors = ns->tx_aborted_errors;1320}1321 1322static int ionic_addr_add(struct net_device *netdev, const u8 *addr)1323{1324	return ionic_lif_list_addr(netdev_priv(netdev), addr, ADD_ADDR);1325}1326 1327static int ionic_addr_del(struct net_device *netdev, const u8 *addr)1328{1329	/* Don't delete our own address from the uc list */1330	if (ether_addr_equal(addr, netdev->dev_addr))1331		return 0;1332 1333	return ionic_lif_list_addr(netdev_priv(netdev), addr, DEL_ADDR);1334}1335 1336void ionic_lif_rx_mode(struct ionic_lif *lif)1337{1338	struct net_device *netdev = lif->netdev;1339	unsigned int nfilters;1340	unsigned int nd_flags;1341	char buf[128];1342	u16 rx_mode;1343	int i;1344#define REMAIN(__x) (sizeof(buf) - (__x))1345 1346	mutex_lock(&lif->config_lock);1347 1348	/* grab the flags once for local use */1349	nd_flags = netdev->flags;1350 1351	rx_mode = IONIC_RX_MODE_F_UNICAST;1352	rx_mode |= (nd_flags & IFF_MULTICAST) ? IONIC_RX_MODE_F_MULTICAST : 0;1353	rx_mode |= (nd_flags & IFF_BROADCAST) ? IONIC_RX_MODE_F_BROADCAST : 0;1354	rx_mode |= (nd_flags & IFF_PROMISC) ? IONIC_RX_MODE_F_PROMISC : 0;1355	rx_mode |= (nd_flags & IFF_ALLMULTI) ? IONIC_RX_MODE_F_ALLMULTI : 0;1356 1357	/* sync the filters */1358	ionic_rx_filter_sync(lif);1359 1360	/* check for overflow state1361	 *    if so, we track that we overflowed and enable NIC PROMISC1362	 *    else if the overflow is set and not needed1363	 *       we remove our overflow flag and check the netdev flags1364	 *       to see if we can disable NIC PROMISC1365	 */1366	nfilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);1367 1368	if (((lif->nucast + lif->nmcast) >= nfilters) ||1369	    (lif->max_vlans && lif->nvlans >= lif->max_vlans)) {1370		rx_mode |= IONIC_RX_MODE_F_PROMISC;1371		rx_mode |= IONIC_RX_MODE_F_ALLMULTI;1372	} else {1373		if (!(nd_flags & IFF_PROMISC))1374			rx_mode &= ~IONIC_RX_MODE_F_PROMISC;1375		if (!(nd_flags & IFF_ALLMULTI))1376			rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;1377	}1378 1379	i = scnprintf(buf, sizeof(buf), "rx_mode 0x%04x -> 0x%04x:",1380		      lif->rx_mode, rx_mode);1381	if (rx_mode & IONIC_RX_MODE_F_UNICAST)1382		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_UNICAST");1383	if (rx_mode & IONIC_RX_MODE_F_MULTICAST)1384		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_MULTICAST");1385	if (rx_mode & IONIC_RX_MODE_F_BROADCAST)1386		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_BROADCAST");1387	if (rx_mode & IONIC_RX_MODE_F_PROMISC)1388		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_PROMISC");1389	if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)1390		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_ALLMULTI");1391	if (rx_mode & IONIC_RX_MODE_F_RDMA_SNIFFER)1392		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_RDMA_SNIFFER");1393	netdev_dbg(netdev, "lif%d %s\n", lif->index, buf);1394 1395	if (lif->rx_mode != rx_mode) {1396		struct ionic_admin_ctx ctx = {1397			.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),1398			.cmd.rx_mode_set = {1399				.opcode = IONIC_CMD_RX_MODE_SET,1400				.lif_index = cpu_to_le16(lif->index),1401			},1402		};1403		int err;1404 1405		ctx.cmd.rx_mode_set.rx_mode = cpu_to_le16(rx_mode);1406		err = ionic_adminq_post_wait(lif, &ctx);1407		if (err)1408			netdev_warn(netdev, "set rx_mode 0x%04x failed: %d\n",1409				    rx_mode, err);1410		else1411			lif->rx_mode = rx_mode;1412	}1413 1414	mutex_unlock(&lif->config_lock);1415}1416 1417static void ionic_ndo_set_rx_mode(struct net_device *netdev)1418{1419	struct ionic_lif *lif = netdev_priv(netdev);1420	struct ionic_deferred_work *work;1421 1422	/* Sync the kernel filter list with the driver filter list */1423	__dev_uc_sync(netdev, ionic_addr_add, ionic_addr_del);1424	__dev_mc_sync(netdev, ionic_addr_add, ionic_addr_del);1425 1426	/* Shove off the rest of the rxmode work to the work task1427	 * which will include syncing the filters to the firmware.1428	 */1429	work = kzalloc(sizeof(*work), GFP_ATOMIC);1430	if (!work) {1431		netdev_err(lif->netdev, "rxmode change dropped\n");1432		return;1433	}1434	work->type = IONIC_DW_TYPE_RX_MODE;1435	netdev_dbg(lif->netdev, "deferred: rx_mode\n");1436	ionic_lif_deferred_enqueue(lif, work);1437}1438 1439static __le64 ionic_netdev_features_to_nic(netdev_features_t features)1440{1441	u64 wanted = 0;1442 1443	if (features & NETIF_F_HW_VLAN_CTAG_TX)1444		wanted |= IONIC_ETH_HW_VLAN_TX_TAG;1445	if (features & NETIF_F_HW_VLAN_CTAG_RX)1446		wanted |= IONIC_ETH_HW_VLAN_RX_STRIP;1447	if (features & NETIF_F_HW_VLAN_CTAG_FILTER)1448		wanted |= IONIC_ETH_HW_VLAN_RX_FILTER;1449	if (features & NETIF_F_RXHASH)1450		wanted |= IONIC_ETH_HW_RX_HASH;1451	if (features & NETIF_F_RXCSUM)1452		wanted |= IONIC_ETH_HW_RX_CSUM;1453	if (features & NETIF_F_SG)1454		wanted |= IONIC_ETH_HW_TX_SG;1455	if (features & NETIF_F_HW_CSUM)1456		wanted |= IONIC_ETH_HW_TX_CSUM;1457	if (features & NETIF_F_TSO)1458		wanted |= IONIC_ETH_HW_TSO;1459	if (features & NETIF_F_TSO6)1460		wanted |= IONIC_ETH_HW_TSO_IPV6;1461	if (features & NETIF_F_TSO_ECN)1462		wanted |= IONIC_ETH_HW_TSO_ECN;1463	if (features & NETIF_F_GSO_GRE)1464		wanted |= IONIC_ETH_HW_TSO_GRE;1465	if (features & NETIF_F_GSO_GRE_CSUM)1466		wanted |= IONIC_ETH_HW_TSO_GRE_CSUM;1467	if (features & NETIF_F_GSO_IPXIP4)1468		wanted |= IONIC_ETH_HW_TSO_IPXIP4;1469	if (features & NETIF_F_GSO_IPXIP6)1470		wanted |= IONIC_ETH_HW_TSO_IPXIP6;1471	if (features & NETIF_F_GSO_UDP_TUNNEL)1472		wanted |= IONIC_ETH_HW_TSO_UDP;1473	if (features & NETIF_F_GSO_UDP_TUNNEL_CSUM)1474		wanted |= IONIC_ETH_HW_TSO_UDP_CSUM;1475 1476	return cpu_to_le64(wanted);1477}1478 1479static int ionic_set_nic_features(struct ionic_lif *lif,1480				  netdev_features_t features)1481{1482	struct device *dev = lif->ionic->dev;1483	struct ionic_admin_ctx ctx = {1484		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),1485		.cmd.lif_setattr = {1486			.opcode = IONIC_CMD_LIF_SETATTR,1487			.index = cpu_to_le16(lif->index),1488			.attr = IONIC_LIF_ATTR_FEATURES,1489		},1490	};1491	u64 vlan_flags = IONIC_ETH_HW_VLAN_TX_TAG |1492			 IONIC_ETH_HW_VLAN_RX_STRIP |1493			 IONIC_ETH_HW_VLAN_RX_FILTER;1494	u64 old_hw_features;1495	int err;1496 1497	ctx.cmd.lif_setattr.features = ionic_netdev_features_to_nic(features);1498 1499	if (lif->phc)1500		ctx.cmd.lif_setattr.features |= cpu_to_le64(IONIC_ETH_HW_TIMESTAMP);1501 1502	err = ionic_adminq_post_wait(lif, &ctx);1503	if (err)1504		return err;1505 1506	old_hw_features = lif->hw_features;1507	lif->hw_features = le64_to_cpu(ctx.cmd.lif_setattr.features &1508				       ctx.comp.lif_setattr.features);1509 1510	if ((old_hw_features ^ lif->hw_features) & IONIC_ETH_HW_RX_HASH)1511		ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);1512 1513	if ((vlan_flags & le64_to_cpu(ctx.cmd.lif_setattr.features)) &&1514	    !(vlan_flags & le64_to_cpu(ctx.comp.lif_setattr.features)))1515		dev_info_once(lif->ionic->dev, "NIC is not supporting vlan offload, likely in SmartNIC mode\n");1516 1517	if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)1518		dev_dbg(dev, "feature ETH_HW_VLAN_TX_TAG\n");1519	if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)1520		dev_dbg(dev, "feature ETH_HW_VLAN_RX_STRIP\n");1521	if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)1522		dev_dbg(dev, "feature ETH_HW_VLAN_RX_FILTER\n");1523	if (lif->hw_features & IONIC_ETH_HW_RX_HASH)1524		dev_dbg(dev, "feature ETH_HW_RX_HASH\n");1525	if (lif->hw_features & IONIC_ETH_HW_TX_SG)1526		dev_dbg(dev, "feature ETH_HW_TX_SG\n");1527	if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)1528		dev_dbg(dev, "feature ETH_HW_TX_CSUM\n");1529	if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)1530		dev_dbg(dev, "feature ETH_HW_RX_CSUM\n");1531	if (lif->hw_features & IONIC_ETH_HW_TSO)1532		dev_dbg(dev, "feature ETH_HW_TSO\n");1533	if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)1534		dev_dbg(dev, "feature ETH_HW_TSO_IPV6\n");1535	if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)1536		dev_dbg(dev, "feature ETH_HW_TSO_ECN\n");1537	if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)1538		dev_dbg(dev, "feature ETH_HW_TSO_GRE\n");1539	if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)1540		dev_dbg(dev, "feature ETH_HW_TSO_GRE_CSUM\n");1541	if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)1542		dev_dbg(dev, "feature ETH_HW_TSO_IPXIP4\n");1543	if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)1544		dev_dbg(dev, "feature ETH_HW_TSO_IPXIP6\n");1545	if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)1546		dev_dbg(dev, "feature ETH_HW_TSO_UDP\n");1547	if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)1548		dev_dbg(dev, "feature ETH_HW_TSO_UDP_CSUM\n");1549	if (lif->hw_features & IONIC_ETH_HW_TIMESTAMP)1550		dev_dbg(dev, "feature ETH_HW_TIMESTAMP\n");1551 1552	return 0;1553}1554 1555static int ionic_init_nic_features(struct ionic_lif *lif)1556{1557	struct net_device *netdev = lif->netdev;1558	netdev_features_t features;1559	int err;1560 1561	/* set up what we expect to support by default */1562	features = NETIF_F_HW_VLAN_CTAG_TX |1563		   NETIF_F_HW_VLAN_CTAG_RX |1564		   NETIF_F_HW_VLAN_CTAG_FILTER |1565		   NETIF_F_SG |1566		   NETIF_F_HW_CSUM |1567		   NETIF_F_RXCSUM |1568		   NETIF_F_TSO |1569		   NETIF_F_TSO6 |1570		   NETIF_F_TSO_ECN |1571		   NETIF_F_GSO_GRE |1572		   NETIF_F_GSO_GRE_CSUM |1573		   NETIF_F_GSO_IPXIP4 |1574		   NETIF_F_GSO_IPXIP6 |1575		   NETIF_F_GSO_UDP_TUNNEL |1576		   NETIF_F_GSO_UDP_TUNNEL_CSUM;1577 1578	if (lif->nxqs > 1)1579		features |= NETIF_F_RXHASH;1580 1581	err = ionic_set_nic_features(lif, features);1582	if (err)1583		return err;1584 1585	/* tell the netdev what we actually can support */1586	netdev->features |= NETIF_F_HIGHDMA;1587 1588	if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)1589		netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;1590	if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)1591		netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;1592	if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)1593		netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;1594	if (lif->hw_features & IONIC_ETH_HW_RX_HASH)1595		netdev->hw_features |= NETIF_F_RXHASH;1596	if (lif->hw_features & IONIC_ETH_HW_TX_SG)1597		netdev->hw_features |= NETIF_F_SG;1598 1599	if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)1600		netdev->hw_enc_features |= NETIF_F_HW_CSUM;1601	if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)1602		netdev->hw_enc_features |= NETIF_F_RXCSUM;1603	if (lif->hw_features & IONIC_ETH_HW_TSO)1604		netdev->hw_enc_features |= NETIF_F_TSO;1605	if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)1606		netdev->hw_enc_features |= NETIF_F_TSO6;1607	if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)1608		netdev->hw_enc_features |= NETIF_F_TSO_ECN;1609	if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)1610		netdev->hw_enc_features |= NETIF_F_GSO_GRE;1611	if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)1612		netdev->hw_enc_features |= NETIF_F_GSO_GRE_CSUM;1613	if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)1614		netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4;1615	if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)1616		netdev->hw_enc_features |= NETIF_F_GSO_IPXIP6;1617	if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)1618		netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;1619	if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)1620		netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;1621 1622	netdev->hw_features |= netdev->hw_enc_features;1623	netdev->features |= netdev->hw_features;1624	netdev->vlan_features |= netdev->features & ~NETIF_F_VLAN_FEATURES;1625 1626	netdev->priv_flags |= IFF_UNICAST_FLT |1627			      IFF_LIVE_ADDR_CHANGE;1628 1629	netdev->xdp_features = NETDEV_XDP_ACT_BASIC    |1630			       NETDEV_XDP_ACT_REDIRECT |1631			       NETDEV_XDP_ACT_RX_SG    |1632			       NETDEV_XDP_ACT_NDO_XMIT |1633			       NETDEV_XDP_ACT_NDO_XMIT_SG;1634 1635	return 0;1636}1637 1638static int ionic_set_features(struct net_device *netdev,1639			      netdev_features_t features)1640{1641	struct ionic_lif *lif = netdev_priv(netdev);1642	int err;1643 1644	netdev_dbg(netdev, "%s: lif->features=0x%08llx new_features=0x%08llx\n",1645		   __func__, (u64)lif->netdev->features, (u64)features);1646 1647	err = ionic_set_nic_features(lif, features);1648 1649	return err;1650}1651 1652static int ionic_set_attr_mac(struct ionic_lif *lif, u8 *mac)1653{1654	struct ionic_admin_ctx ctx = {1655		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),1656		.cmd.lif_setattr = {1657			.opcode = IONIC_CMD_LIF_SETATTR,1658			.index = cpu_to_le16(lif->index),1659			.attr = IONIC_LIF_ATTR_MAC,1660		},1661	};1662 1663	ether_addr_copy(ctx.cmd.lif_setattr.mac, mac);1664	return ionic_adminq_post_wait(lif, &ctx);1665}1666 1667static int ionic_get_attr_mac(struct ionic_lif *lif, u8 *mac_addr)1668{1669	struct ionic_admin_ctx ctx = {1670		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),1671		.cmd.lif_getattr = {1672			.opcode = IONIC_CMD_LIF_GETATTR,1673			.index = cpu_to_le16(lif->index),1674			.attr = IONIC_LIF_ATTR_MAC,1675		},1676	};1677	int err;1678 1679	err = ionic_adminq_post_wait(lif, &ctx);1680	if (err)1681		return err;1682 1683	ether_addr_copy(mac_addr, ctx.comp.lif_getattr.mac);1684	return 0;1685}1686 1687static int ionic_program_mac(struct ionic_lif *lif, u8 *mac)1688{1689	u8  get_mac[ETH_ALEN];1690	int err;1691 1692	err = ionic_set_attr_mac(lif, mac);1693	if (err)1694		return err;1695 1696	err = ionic_get_attr_mac(lif, get_mac);1697	if (err)1698		return err;1699 1700	/* To deal with older firmware that silently ignores the set attr mac:1701	 * doesn't actually change the mac and doesn't return an error, so we1702	 * do the get attr to verify whether or not the set actually happened1703	 */1704	if (!ether_addr_equal(get_mac, mac))1705		return 1;1706 1707	return 0;1708}1709 1710static int ionic_set_mac_address(struct net_device *netdev, void *sa)1711{1712	struct ionic_lif *lif = netdev_priv(netdev);1713	struct sockaddr *addr = sa;1714	u8 *mac;1715	int err;1716 1717	mac = (u8 *)addr->sa_data;1718	if (ether_addr_equal(netdev->dev_addr, mac))1719		return 0;1720 1721	err = ionic_program_mac(lif, mac);1722	if (err < 0)1723		return err;1724 1725	if (err > 0)1726		netdev_dbg(netdev, "%s: SET and GET ATTR Mac are not equal-due to old FW running\n",1727			   __func__);1728 1729	err = eth_prepare_mac_addr_change(netdev, addr);1730	if (err)1731		return err;1732 1733	if (!is_zero_ether_addr(netdev->dev_addr)) {1734		netdev_info(netdev, "deleting mac addr %pM\n",1735			    netdev->dev_addr);1736		ionic_lif_addr_del(netdev_priv(netdev), netdev->dev_addr);1737	}1738 1739	eth_commit_mac_addr_change(netdev, addr);1740	netdev_info(netdev, "updating mac addr %pM\n", mac);1741 1742	return ionic_lif_addr_add(netdev_priv(netdev), mac);1743}1744 1745void ionic_stop_queues_reconfig(struct ionic_lif *lif)1746{1747	/* Stop and clean the queues before reconfiguration */1748	netif_device_detach(lif->netdev);1749	ionic_stop_queues(lif);1750	ionic_txrx_deinit(lif);1751}1752 1753static int ionic_start_queues_reconfig(struct ionic_lif *lif)1754{1755	int err;1756 1757	/* Re-init the queues after reconfiguration */1758 1759	/* The only way txrx_init can fail here is if communication1760	 * with FW is suddenly broken.  There's not much we can do1761	 * at this point - error messages have already been printed,1762	 * so we can continue on and the user can eventually do a1763	 * DOWN and UP to try to reset and clear the issue.1764	 */1765	err = ionic_txrx_init(lif);1766	ionic_link_status_check_request(lif, CAN_NOT_SLEEP);1767	netif_device_attach(lif->netdev);1768 1769	return err;1770}1771 1772static bool ionic_xdp_is_valid_mtu(struct ionic_lif *lif, u32 mtu,1773				   struct bpf_prog *xdp_prog)1774{1775	if (!xdp_prog)1776		return true;1777 1778	if (mtu <= IONIC_XDP_MAX_LINEAR_MTU)1779		return true;1780 1781	if (xdp_prog->aux && xdp_prog->aux->xdp_has_frags)1782		return true;1783 1784	return false;1785}1786 1787static int ionic_change_mtu(struct net_device *netdev, int new_mtu)1788{1789	struct ionic_lif *lif = netdev_priv(netdev);1790	struct ionic_admin_ctx ctx = {1791		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),1792		.cmd.lif_setattr = {1793			.opcode = IONIC_CMD_LIF_SETATTR,1794			.index = cpu_to_le16(lif->index),1795			.attr = IONIC_LIF_ATTR_MTU,1796			.mtu = cpu_to_le32(new_mtu),1797		},1798	};1799	struct bpf_prog *xdp_prog;1800	int err;1801 1802	xdp_prog = READ_ONCE(lif->xdp_prog);1803	if (!ionic_xdp_is_valid_mtu(lif, new_mtu, xdp_prog))1804		return -EINVAL;1805 1806	err = ionic_adminq_post_wait(lif, &ctx);1807	if (err)1808		return err;1809 1810	/* if we're not running, nothing more to do */1811	if (!netif_running(netdev)) {1812		WRITE_ONCE(netdev->mtu, new_mtu);1813		return 0;1814	}1815 1816	mutex_lock(&lif->queue_lock);1817	ionic_stop_queues_reconfig(lif);1818	WRITE_ONCE(netdev->mtu, new_mtu);1819	err = ionic_start_queues_reconfig(lif);1820	mutex_unlock(&lif->queue_lock);1821 1822	return err;1823}1824 1825static void ionic_tx_timeout_work(struct work_struct *ws)1826{1827	struct ionic_lif *lif = container_of(ws, struct ionic_lif, tx_timeout_work);1828	int err;1829 1830	if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))1831		return;1832 1833	/* if we were stopped before this scheduled job was launched,1834	 * don't bother the queues as they are already stopped.1835	 */1836	if (!netif_running(lif->netdev))1837		return;1838 1839	mutex_lock(&lif->queue_lock);1840	ionic_stop_queues_reconfig(lif);1841	err = ionic_start_queues_reconfig(lif);1842	mutex_unlock(&lif->queue_lock);1843 1844	if (err)1845		dev_err(lif->ionic->dev, "%s: Restarting queues failed\n", __func__);1846}1847 1848static void ionic_tx_timeout(struct net_device *netdev, unsigned int txqueue)1849{1850	struct ionic_lif *lif = netdev_priv(netdev);1851 1852	netdev_info(lif->netdev, "Tx Timeout triggered - txq %d\n", txqueue);1853	schedule_work(&lif->tx_timeout_work);1854}1855 1856static int ionic_vlan_rx_add_vid(struct net_device *netdev, __be16 proto,1857				 u16 vid)1858{1859	struct ionic_lif *lif = netdev_priv(netdev);1860	int err;1861 1862	err = ionic_lif_vlan_add(lif, vid);1863	if (err)1864		return err;1865 1866	ionic_lif_rx_mode(lif);1867 1868	return 0;1869}1870 1871static int ionic_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto,1872				  u16 vid)1873{1874	struct ionic_lif *lif = netdev_priv(netdev);1875	int err;1876 1877	err = ionic_lif_vlan_del(lif, vid);1878	if (err)1879		return err;1880 1881	ionic_lif_rx_mode(lif);1882 1883	return 0;1884}1885 1886int ionic_lif_rss_config(struct ionic_lif *lif, const u16 types,1887			 const u8 *key, const u32 *indir)1888{1889	struct ionic_admin_ctx ctx = {1890		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),1891		.cmd.lif_setattr = {1892			.opcode = IONIC_CMD_LIF_SETATTR,1893			.attr = IONIC_LIF_ATTR_RSS,1894			.rss.addr = cpu_to_le64(lif->rss_ind_tbl_pa),1895		},1896	};1897	unsigned int i, tbl_sz;1898 1899	if (lif->hw_features & IONIC_ETH_HW_RX_HASH) {1900		lif->rss_types = types;1901		ctx.cmd.lif_setattr.rss.types = cpu_to_le16(types);1902	}1903 1904	if (key)1905		memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);1906 1907	if (indir) {1908		tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);1909		for (i = 0; i < tbl_sz; i++)1910			lif->rss_ind_tbl[i] = indir[i];1911	}1912 1913	memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,1914	       IONIC_RSS_HASH_KEY_SIZE);1915 1916	return ionic_adminq_post_wait(lif, &ctx);1917}1918 1919static int ionic_lif_rss_init(struct ionic_lif *lif)1920{1921	unsigned int tbl_sz;1922	unsigned int i;1923 1924	lif->rss_types = IONIC_RSS_TYPE_IPV4     |1925			 IONIC_RSS_TYPE_IPV4_TCP |1926			 IONIC_RSS_TYPE_IPV4_UDP |1927			 IONIC_RSS_TYPE_IPV6     |1928			 IONIC_RSS_TYPE_IPV6_TCP |1929			 IONIC_RSS_TYPE_IPV6_UDP;1930 1931	/* Fill indirection table with 'default' values */1932	tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);1933	for (i = 0; i < tbl_sz; i++)1934		lif->rss_ind_tbl[i] = ethtool_rxfh_indir_default(i, lif->nxqs);1935 1936	return ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);1937}1938 1939static void ionic_lif_rss_deinit(struct ionic_lif *lif)1940{1941	int tbl_sz;1942 1943	tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);1944	memset(lif->rss_ind_tbl, 0, tbl_sz);1945	memset(lif->rss_hash_key, 0, IONIC_RSS_HASH_KEY_SIZE);1946 1947	ionic_lif_rss_config(lif, 0x0, NULL, NULL);1948}1949 1950static void ionic_lif_quiesce(struct ionic_lif *lif)1951{1952	struct ionic_admin_ctx ctx = {1953		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),1954		.cmd.lif_setattr = {1955			.opcode = IONIC_CMD_LIF_SETATTR,1956			.index = cpu_to_le16(lif->index),1957			.attr = IONIC_LIF_ATTR_STATE,1958			.state = IONIC_LIF_QUIESCE,1959		},1960	};1961	int err;1962 1963	err = ionic_adminq_post_wait(lif, &ctx);1964	if (err)1965		netdev_dbg(lif->netdev, "lif quiesce failed %d\n", err);1966}1967 1968static void ionic_txrx_disable(struct ionic_lif *lif)1969{1970	unsigned int i;1971	int err = 0;1972 1973	if (lif->txqcqs) {1974		for (i = 0; i < lif->nxqs; i++)1975			err = ionic_qcq_disable(lif, lif->txqcqs[i], err);1976	}1977 1978	if (lif->hwstamp_txq)1979		err = ionic_qcq_disable(lif, lif->hwstamp_txq, err);1980 1981	if (lif->rxqcqs) {1982		for (i = 0; i < lif->nxqs; i++)1983			err = ionic_qcq_disable(lif, lif->rxqcqs[i], err);1984	}1985 1986	if (lif->hwstamp_rxq)1987		err = ionic_qcq_disable(lif, lif->hwstamp_rxq, err);1988 1989	ionic_lif_quiesce(lif);1990}1991 1992static void ionic_txrx_deinit(struct ionic_lif *lif)1993{1994	unsigned int i;1995 1996	if (lif->txqcqs) {1997		for (i = 0; i < lif->nxqs && lif->txqcqs[i]; i++) {1998			ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);1999			ionic_tx_flush(&lif->txqcqs[i]->cq);2000			ionic_tx_empty(&lif->txqcqs[i]->q);2001		}2002	}2003 2004	if (lif->rxqcqs) {2005		for (i = 0; i < lif->nxqs && lif->rxqcqs[i]; i++) {2006			ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);2007			ionic_rx_empty(&lif->rxqcqs[i]->q);2008		}2009	}2010	lif->rx_mode = 0;2011 2012	if (lif->hwstamp_txq) {2013		ionic_lif_qcq_deinit(lif, lif->hwstamp_txq);2014		ionic_tx_flush(&lif->hwstamp_txq->cq);2015		ionic_tx_empty(&lif->hwstamp_txq->q);2016	}2017 2018	if (lif->hwstamp_rxq) {2019		ionic_lif_qcq_deinit(lif, lif->hwstamp_rxq);2020		ionic_rx_empty(&lif->hwstamp_rxq->q);2021	}2022}2023 2024void ionic_txrx_free(struct ionic_lif *lif)2025{2026	unsigned int i;2027 2028	if (lif->txqcqs) {2029		for (i = 0; i < lif->ionic->ntxqs_per_lif && lif->txqcqs[i]; i++) {2030			ionic_qcq_free(lif, lif->txqcqs[i]);2031			devm_kfree(lif->ionic->dev, lif->txqcqs[i]);2032			lif->txqcqs[i] = NULL;2033		}2034	}2035 2036	if (lif->rxqcqs) {2037		for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {2038			ionic_qcq_free(lif, lif->rxqcqs[i]);2039			devm_kfree(lif->ionic->dev, lif->rxqcqs[i]);2040			lif->rxqcqs[i] = NULL;2041		}2042	}2043 2044	if (lif->hwstamp_txq) {2045		ionic_qcq_free(lif, lif->hwstamp_txq);2046		devm_kfree(lif->ionic->dev, lif->hwstamp_txq);2047		lif->hwstamp_txq = NULL;2048	}2049 2050	if (lif->hwstamp_rxq) {2051		ionic_qcq_free(lif, lif->hwstamp_rxq);2052		devm_kfree(lif->ionic->dev, lif->hwstamp_rxq);2053		lif->hwstamp_rxq = NULL;2054	}2055}2056 2057static int ionic_txrx_alloc(struct ionic_lif *lif)2058{2059	unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;2060	unsigned int flags, i;2061	int err = 0;2062 2063	num_desc = lif->ntxq_descs;2064	desc_sz = sizeof(struct ionic_txq_desc);2065	comp_sz = sizeof(struct ionic_txq_comp);2066 2067	if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&2068	    lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==2069					  sizeof(struct ionic_txq_sg_desc_v1))2070		sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);2071	else2072		sg_desc_sz = sizeof(struct ionic_txq_sg_desc);2073 2074	flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;2075 2076	if (test_bit(IONIC_LIF_F_CMB_TX_RINGS, lif->state))2077		flags |= IONIC_QCQ_F_CMB_RINGS;2078 2079	if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))2080		flags |= IONIC_QCQ_F_INTR;2081 2082	for (i = 0; i < lif->nxqs; i++) {2083		err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,2084				      num_desc, desc_sz, comp_sz, sg_desc_sz,2085				      sizeof(struct ionic_tx_desc_info),2086				      lif->kern_pid, NULL, &lif->txqcqs[i]);2087		if (err)2088			goto err_out;2089 2090		if (flags & IONIC_QCQ_F_INTR) {2091			ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,2092					     lif->txqcqs[i]->intr.index,2093					     lif->tx_coalesce_hw);2094			if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))2095				lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;2096		}2097 2098		ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);2099	}2100 2101	flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG | IONIC_QCQ_F_INTR;2102 2103	if (test_bit(IONIC_LIF_F_CMB_RX_RINGS, lif->state))2104		flags |= IONIC_QCQ_F_CMB_RINGS;2105 2106	num_desc = lif->nrxq_descs;2107	desc_sz = sizeof(struct ionic_rxq_desc);2108	comp_sz = sizeof(struct ionic_rxq_comp);2109	sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);2110 2111	if (lif->rxq_features & IONIC_Q_F_2X_CQ_DESC)2112		comp_sz *= 2;2113 2114	for (i = 0; i < lif->nxqs; i++) {2115		err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,2116				      num_desc, desc_sz, comp_sz, sg_desc_sz,2117				      sizeof(struct ionic_rx_desc_info),2118				      lif->kern_pid, lif->xdp_prog,2119				      &lif->rxqcqs[i]);2120		if (err)2121			goto err_out;2122 2123		lif->rxqcqs[i]->q.features = lif->rxq_features;2124 2125		ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,2126				     lif->rxqcqs[i]->intr.index,2127				     lif->rx_coalesce_hw);2128		if (test_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state))2129			lif->rxqcqs[i]->intr.dim_coal_hw = lif->rx_coalesce_hw;2130 2131		if (!test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))2132			ionic_link_qcq_interrupts(lif->rxqcqs[i],2133						  lif->txqcqs[i]);2134 2135		ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);2136	}2137 2138	return 0;2139 2140err_out:2141	ionic_txrx_free(lif);2142 2143	return err;2144}2145 2146static int ionic_txrx_init(struct ionic_lif *lif)2147{2148	unsigned int i;2149	int err;2150 2151	for (i = 0; i < lif->nxqs; i++) {2152		err = ionic_lif_txq_init(lif, lif->txqcqs[i]);2153		if (err)2154			goto err_out;2155 2156		err = ionic_lif_rxq_init(lif, lif->rxqcqs[i]);2157		if (err) {2158			ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);2159			goto err_out;2160		}2161	}2162 2163	if (lif->netdev->features & NETIF_F_RXHASH)2164		ionic_lif_rss_init(lif);2165 2166	ionic_lif_rx_mode(lif);2167 2168	return 0;2169 2170err_out:2171	while (i--) {2172		ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);2173		ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);2174	}2175 2176	return err;2177}2178 2179static int ionic_txrx_enable(struct ionic_lif *lif)2180{2181	int derr = 0;2182	int i, err;2183 2184	ionic_xdp_rxqs_prog_update(lif);2185 2186	for (i = 0; i < lif->nxqs; i++) {2187		if (!(lif->rxqcqs[i] && lif->txqcqs[i])) {2188			dev_err(lif->ionic->dev, "%s: bad qcq %d\n", __func__, i);2189			err = -ENXIO;2190			goto err_out;2191		}2192 2193		ionic_rx_fill(&lif->rxqcqs[i]->q,2194			      READ_ONCE(lif->rxqcqs[i]->q.xdp_prog));2195		err = ionic_qcq_enable(lif->rxqcqs[i]);2196		if (err)2197			goto err_out;2198 2199		err = ionic_qcq_enable(lif->txqcqs[i]);2200		if (err) {2201			derr = ionic_qcq_disable(lif, lif->rxqcqs[i], err);2202			goto err_out;2203		}2204	}2205 2206	if (lif->hwstamp_rxq) {2207		ionic_rx_fill(&lif->hwstamp_rxq->q, NULL);2208		err = ionic_qcq_enable(lif->hwstamp_rxq);2209		if (err)2210			goto err_out_hwstamp_rx;2211	}2212 2213	if (lif->hwstamp_txq) {2214		err = ionic_qcq_enable(lif->hwstamp_txq);2215		if (err)2216			goto err_out_hwstamp_tx;2217	}2218 2219	return 0;2220 2221err_out_hwstamp_tx:2222	if (lif->hwstamp_rxq)2223		derr = ionic_qcq_disable(lif, lif->hwstamp_rxq, derr);2224err_out_hwstamp_rx:2225	i = lif->nxqs;2226err_out:2227	while (i--) {2228		derr = ionic_qcq_disable(lif, lif->txqcqs[i], derr);2229		derr = ionic_qcq_disable(lif, lif->rxqcqs[i], derr);2230	}2231 2232	ionic_xdp_rxqs_prog_update(lif);2233 2234	return err;2235}2236 2237static int ionic_start_queues(struct ionic_lif *lif)2238{2239	int err;2240 2241	if (test_bit(IONIC_LIF_F_BROKEN, lif->state))2242		return -EIO;2243 2244	if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))2245		return -EBUSY;2246 2247	if (test_and_set_bit(IONIC_LIF_F_UP, lif->state))2248		return 0;2249 2250	err = ionic_txrx_enable(lif);2251	if (err) {2252		clear_bit(IONIC_LIF_F_UP, lif->state);2253		return err;2254	}2255	netif_tx_wake_all_queues(lif->netdev);2256 2257	return 0;2258}2259 2260static int ionic_open(struct net_device *netdev)2261{2262	struct ionic_lif *lif = netdev_priv(netdev);2263	int err;2264 2265	/* If recovering from a broken state, clear the bit and we'll try again */2266	if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))2267		netdev_info(netdev, "clearing broken state\n");2268 2269	mutex_lock(&lif->queue_lock);2270 2271	err = ionic_txrx_alloc(lif);2272	if (err)2273		goto err_unlock;2274 2275	err = ionic_txrx_init(lif);2276	if (err)2277		goto err_txrx_free;2278 2279	err = netif_set_real_num_tx_queues(netdev, lif->nxqs);2280	if (err)2281		goto err_txrx_deinit;2282 2283	err = netif_set_real_num_rx_queues(netdev, lif->nxqs);2284	if (err)2285		goto err_txrx_deinit;2286 2287	/* don't start the queues until we have link */2288	if (netif_carrier_ok(netdev)) {2289		err = ionic_start_queues(lif);2290		if (err)2291			goto err_txrx_deinit;2292	}2293 2294	/* If hardware timestamping is enabled, but the queues were freed by2295	 * ionic_stop, those need to be reallocated and initialized, too.2296	 */2297	ionic_lif_hwstamp_recreate_queues(lif);2298 2299	mutex_unlock(&lif->queue_lock);2300 2301	return 0;2302 2303err_txrx_deinit:2304	ionic_txrx_deinit(lif);2305err_txrx_free:2306	ionic_txrx_free(lif);2307err_unlock:2308	mutex_unlock(&lif->queue_lock);2309	return err;2310}2311 2312static void ionic_stop_queues(struct ionic_lif *lif)2313{2314	if (!test_and_clear_bit(IONIC_LIF_F_UP, lif->state))2315		return;2316 2317	netif_tx_disable(lif->netdev);2318	ionic_txrx_disable(lif);2319}2320 2321static int ionic_stop(struct net_device *netdev)2322{2323	struct ionic_lif *lif = netdev_priv(netdev);2324 2325	if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))2326		return 0;2327 2328	mutex_lock(&lif->queue_lock);2329	ionic_stop_queues(lif);2330	ionic_txrx_deinit(lif);2331	ionic_txrx_free(lif);2332	mutex_unlock(&lif->queue_lock);2333 2334	return 0;2335}2336 2337static int ionic_eth_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)2338{2339	struct ionic_lif *lif = netdev_priv(netdev);2340 2341	switch (cmd) {2342	case SIOCSHWTSTAMP:2343		return ionic_lif_hwstamp_set(lif, ifr);2344	case SIOCGHWTSTAMP:2345		return ionic_lif_hwstamp_get(lif, ifr);2346	default:2347		return -EOPNOTSUPP;2348	}2349}2350 2351static int ionic_get_vf_config(struct net_device *netdev,2352			       int vf, struct ifla_vf_info *ivf)2353{2354	struct ionic_lif *lif = netdev_priv(netdev);2355	struct ionic *ionic = lif->ionic;2356	int ret = 0;2357 2358	if (!netif_device_present(netdev))2359		return -EBUSY;2360 2361	down_read(&ionic->vf_op_lock);2362 2363	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {2364		ret = -EINVAL;2365	} else {2366		struct ionic_vf *vfdata = &ionic->vfs[vf];2367 2368		ivf->vf		  = vf;2369		ivf->qos	  = 0;2370		ivf->vlan         = le16_to_cpu(vfdata->vlanid);2371		ivf->spoofchk     = vfdata->spoofchk;2372		ivf->linkstate    = vfdata->linkstate;2373		ivf->max_tx_rate  = le32_to_cpu(vfdata->maxrate);2374		ivf->trusted      = vfdata->trusted;2375		ether_addr_copy(ivf->mac, vfdata->macaddr);2376	}2377 2378	up_read(&ionic->vf_op_lock);2379	return ret;2380}2381 2382static int ionic_get_vf_stats(struct net_device *netdev, int vf,2383			      struct ifla_vf_stats *vf_stats)2384{2385	struct ionic_lif *lif = netdev_priv(netdev);2386	struct ionic *ionic = lif->ionic;2387	struct ionic_lif_stats *vs;2388	int ret = 0;2389 2390	if (!netif_device_present(netdev))2391		return -EBUSY;2392 2393	down_read(&ionic->vf_op_lock);2394 2395	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {2396		ret = -EINVAL;2397	} else {2398		memset(vf_stats, 0, sizeof(*vf_stats));2399		vs = &ionic->vfs[vf].stats;2400 2401		vf_stats->rx_packets = le64_to_cpu(vs->rx_ucast_packets);2402		vf_stats->tx_packets = le64_to_cpu(vs->tx_ucast_packets);2403		vf_stats->rx_bytes   = le64_to_cpu(vs->rx_ucast_bytes);2404		vf_stats->tx_bytes   = le64_to_cpu(vs->tx_ucast_bytes);2405		vf_stats->broadcast  = le64_to_cpu(vs->rx_bcast_packets);2406		vf_stats->multicast  = le64_to_cpu(vs->rx_mcast_packets);2407		vf_stats->rx_dropped = le64_to_cpu(vs->rx_ucast_drop_packets) +2408				       le64_to_cpu(vs->rx_mcast_drop_packets) +2409				       le64_to_cpu(vs->rx_bcast_drop_packets);2410		vf_stats->tx_dropped = le64_to_cpu(vs->tx_ucast_drop_packets) +2411				       le64_to_cpu(vs->tx_mcast_drop_packets) +2412				       le64_to_cpu(vs->tx_bcast_drop_packets);2413	}2414 2415	up_read(&ionic->vf_op_lock);2416	return ret;2417}2418 2419static int ionic_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)2420{2421	struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_MAC };2422	struct ionic_lif *lif = netdev_priv(netdev);2423	struct ionic *ionic = lif->ionic;2424	int ret;2425 2426	if (!(is_zero_ether_addr(mac) || is_valid_ether_addr(mac)))2427		return -EINVAL;2428 2429	if (!netif_device_present(netdev))2430		return -EBUSY;2431 2432	down_write(&ionic->vf_op_lock);2433 2434	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {2435		ret = -EINVAL;2436	} else {2437		ether_addr_copy(vfc.macaddr, mac);2438		dev_dbg(ionic->dev, "%s: vf %d macaddr %pM\n",2439			__func__, vf, vfc.macaddr);2440 2441		ret = ionic_set_vf_config(ionic, vf, &vfc);2442		if (!ret)2443			ether_addr_copy(ionic->vfs[vf].macaddr, mac);2444	}2445 2446	up_write(&ionic->vf_op_lock);2447	return ret;2448}2449 2450static int ionic_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,2451			     u8 qos, __be16 proto)2452{2453	struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_VLAN };2454	struct ionic_lif *lif = netdev_priv(netdev);2455	struct ionic *ionic = lif->ionic;2456	int ret;2457 2458	/* until someday when we support qos */2459	if (qos)2460		return -EINVAL;2461 2462	if (vlan > 4095)2463		return -EINVAL;2464 2465	if (proto != htons(ETH_P_8021Q))2466		return -EPROTONOSUPPORT;2467 2468	if (!netif_device_present(netdev))2469		return -EBUSY;2470 2471	down_write(&ionic->vf_op_lock);2472 2473	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {2474		ret = -EINVAL;2475	} else {2476		vfc.vlanid = cpu_to_le16(vlan);2477		dev_dbg(ionic->dev, "%s: vf %d vlan %d\n",2478			__func__, vf, le16_to_cpu(vfc.vlanid));2479 2480		ret = ionic_set_vf_config(ionic, vf, &vfc);2481		if (!ret)2482			ionic->vfs[vf].vlanid = cpu_to_le16(vlan);2483	}2484 2485	up_write(&ionic->vf_op_lock);2486	return ret;2487}2488 2489static int ionic_set_vf_rate(struct net_device *netdev, int vf,2490			     int tx_min, int tx_max)2491{2492	struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_RATE };2493	struct ionic_lif *lif = netdev_priv(netdev);2494	struct ionic *ionic = lif->ionic;2495	int ret;2496 2497	/* setting the min just seems silly */2498	if (tx_min)2499		return -EINVAL;2500 2501	if (!netif_device_present(netdev))2502		return -EBUSY;2503 2504	down_write(&ionic->vf_op_lock);2505 2506	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {2507		ret = -EINVAL;2508	} else {2509		vfc.maxrate = cpu_to_le32(tx_max);2510		dev_dbg(ionic->dev, "%s: vf %d maxrate %d\n",2511			__func__, vf, le32_to_cpu(vfc.maxrate));2512 2513		ret = ionic_set_vf_config(ionic, vf, &vfc);2514		if (!ret)2515			ionic->vfs[vf].maxrate = cpu_to_le32(tx_max);2516	}2517 2518	up_write(&ionic->vf_op_lock);2519	return ret;2520}2521 2522static int ionic_set_vf_spoofchk(struct net_device *netdev, int vf, bool set)2523{2524	struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_SPOOFCHK };2525	struct ionic_lif *lif = netdev_priv(netdev);2526	struct ionic *ionic = lif->ionic;2527	int ret;2528 2529	if (!netif_device_present(netdev))2530		return -EBUSY;2531 2532	down_write(&ionic->vf_op_lock);2533 2534	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {2535		ret = -EINVAL;2536	} else {2537		vfc.spoofchk = set;2538		dev_dbg(ionic->dev, "%s: vf %d spoof %d\n",2539			__func__, vf, vfc.spoofchk);2540 2541		ret = ionic_set_vf_config(ionic, vf, &vfc);2542		if (!ret)2543			ionic->vfs[vf].spoofchk = set;2544	}2545 2546	up_write(&ionic->vf_op_lock);2547	return ret;2548}2549 2550static int ionic_set_vf_trust(struct net_device *netdev, int vf, bool set)2551{2552	struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_TRUST };2553	struct ionic_lif *lif = netdev_priv(netdev);2554	struct ionic *ionic = lif->ionic;2555	int ret;2556 2557	if (!netif_device_present(netdev))2558		return -EBUSY;2559 2560	down_write(&ionic->vf_op_lock);2561 2562	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {2563		ret = -EINVAL;2564	} else {2565		vfc.trust = set;2566		dev_dbg(ionic->dev, "%s: vf %d trust %d\n",2567			__func__, vf, vfc.trust);2568 2569		ret = ionic_set_vf_config(ionic, vf, &vfc);2570		if (!ret)2571			ionic->vfs[vf].trusted = set;2572	}2573 2574	up_write(&ionic->vf_op_lock);2575	return ret;2576}2577 2578static int ionic_set_vf_link_state(struct net_device *netdev, int vf, int set)2579{2580	struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_LINKSTATE };2581	struct ionic_lif *lif = netdev_priv(netdev);2582	struct ionic *ionic = lif->ionic;2583	u8 vfls;2584	int ret;2585 2586	switch (set) {2587	case IFLA_VF_LINK_STATE_ENABLE:2588		vfls = IONIC_VF_LINK_STATUS_UP;2589		break;2590	case IFLA_VF_LINK_STATE_DISABLE:2591		vfls = IONIC_VF_LINK_STATUS_DOWN;2592		break;2593	case IFLA_VF_LINK_STATE_AUTO:2594		vfls = IONIC_VF_LINK_STATUS_AUTO;2595		break;2596	default:2597		return -EINVAL;2598	}2599 2600	if (!netif_device_present(netdev))2601		return -EBUSY;2602 2603	down_write(&ionic->vf_op_lock);2604 2605	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {2606		ret = -EINVAL;2607	} else {2608		vfc.linkstate = vfls;2609		dev_dbg(ionic->dev, "%s: vf %d linkstate %d\n",2610			__func__, vf, vfc.linkstate);2611 2612		ret = ionic_set_vf_config(ionic, vf, &vfc);2613		if (!ret)2614			ionic->vfs[vf].linkstate = set;2615	}2616 2617	up_write(&ionic->vf_op_lock);2618	return ret;2619}2620 2621static void ionic_vf_attr_replay(struct ionic_lif *lif)2622{2623	struct ionic_vf_setattr_cmd vfc = { };2624	struct ionic *ionic = lif->ionic;2625	struct ionic_vf *v;2626	int i;2627 2628	if (!ionic->vfs)2629		return;2630 2631	down_read(&ionic->vf_op_lock);2632 2633	for (i = 0; i < ionic->num_vfs; i++) {2634		v = &ionic->vfs[i];2635 2636		if (v->stats_pa) {2637			vfc.attr = IONIC_VF_ATTR_STATSADDR;2638			vfc.stats_pa = cpu_to_le64(v->stats_pa);2639			ionic_set_vf_config(ionic, i, &vfc);2640			vfc.stats_pa = 0;2641		}2642 2643		if (!is_zero_ether_addr(v->macaddr)) {2644			vfc.attr = IONIC_VF_ATTR_MAC;2645			ether_addr_copy(vfc.macaddr, v->macaddr);2646			ionic_set_vf_config(ionic, i, &vfc);2647			eth_zero_addr(vfc.macaddr);2648		}2649 2650		if (v->vlanid) {2651			vfc.attr = IONIC_VF_ATTR_VLAN;2652			vfc.vlanid = v->vlanid;2653			ionic_set_vf_config(ionic, i, &vfc);2654			vfc.vlanid = 0;2655		}2656 2657		if (v->maxrate) {2658			vfc.attr = IONIC_VF_ATTR_RATE;2659			vfc.maxrate = v->maxrate;2660			ionic_set_vf_config(ionic, i, &vfc);2661			vfc.maxrate = 0;2662		}2663 2664		if (v->spoofchk) {2665			vfc.attr = IONIC_VF_ATTR_SPOOFCHK;2666			vfc.spoofchk = v->spoofchk;2667			ionic_set_vf_config(ionic, i, &vfc);2668			vfc.spoofchk = 0;2669		}2670 2671		if (v->trusted) {2672			vfc.attr = IONIC_VF_ATTR_TRUST;2673			vfc.trust = v->trusted;2674			ionic_set_vf_config(ionic, i, &vfc);2675			vfc.trust = 0;2676		}2677 2678		if (v->linkstate) {2679			vfc.attr = IONIC_VF_ATTR_LINKSTATE;2680			vfc.linkstate = v->linkstate;2681			ionic_set_vf_config(ionic, i, &vfc);2682			vfc.linkstate = 0;2683		}2684	}2685 2686	up_read(&ionic->vf_op_lock);2687 2688	ionic_vf_start(ionic);2689}2690 2691static void ionic_unregister_rxq_info(struct ionic_queue *q)2692{2693	struct xdp_rxq_info *xi;2694 2695	if (!q->xdp_rxq_info)2696		return;2697 2698	xi = q->xdp_rxq_info;2699	q->xdp_rxq_info = NULL;2700 2701	xdp_rxq_info_unreg(xi);2702	kfree(xi);2703}2704 2705static int ionic_register_rxq_info(struct ionic_queue *q, unsigned int napi_id)2706{2707	struct xdp_rxq_info *rxq_info;2708	int err;2709 2710	rxq_info = kzalloc(sizeof(*rxq_info), GFP_KERNEL);2711	if (!rxq_info)2712		return -ENOMEM;2713 2714	err = xdp_rxq_info_reg(rxq_info, q->lif->netdev, q->index, napi_id);2715	if (err) {2716		netdev_err(q->lif->netdev, "q%d xdp_rxq_info_reg failed, err %d\n",2717			   q->index, err);2718		goto err_out;2719	}2720 2721	err = xdp_rxq_info_reg_mem_model(rxq_info, MEM_TYPE_PAGE_POOL, q->page_pool);2722	if (err) {2723		netdev_err(q->lif->netdev, "q%d xdp_rxq_info_reg_mem_model failed, err %d\n",2724			   q->index, err);2725		xdp_rxq_info_unreg(rxq_info);2726		goto err_out;2727	}2728 2729	q->xdp_rxq_info = rxq_info;2730 2731	return 0;2732 2733err_out:2734	kfree(rxq_info);2735	return err;2736}2737 2738static void ionic_xdp_rxqs_prog_update(struct ionic_lif *lif)2739{2740	struct bpf_prog *xdp_prog;2741	unsigned int i;2742 2743	if (!lif->rxqcqs)2744		return;2745 2746	xdp_prog = READ_ONCE(lif->xdp_prog);2747	for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {2748		struct ionic_queue *q = &lif->rxqcqs[i]->q;2749 2750		WRITE_ONCE(q->xdp_prog, xdp_prog);2751	}2752}2753 2754static int ionic_xdp_config(struct net_device *netdev, struct netdev_bpf *bpf)2755{2756	struct ionic_lif *lif = netdev_priv(netdev);2757	struct bpf_prog *old_prog;2758	u32 maxfs;2759 2760	if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state)) {2761#define XDP_ERR_SPLIT "XDP not available with split Tx/Rx interrupts"2762		NL_SET_ERR_MSG_MOD(bpf->extack, XDP_ERR_SPLIT);2763		netdev_info(lif->netdev, XDP_ERR_SPLIT);2764		return -EOPNOTSUPP;2765	}2766 2767	if (!ionic_xdp_is_valid_mtu(lif, netdev->mtu, bpf->prog)) {2768#define XDP_ERR_MTU "MTU is too large for XDP without frags support"2769		NL_SET_ERR_MSG_MOD(bpf->extack, XDP_ERR_MTU);2770		netdev_info(lif->netdev, XDP_ERR_MTU);2771		return -EINVAL;2772	}2773 2774	maxfs = __le32_to_cpu(lif->identity->eth.max_frame_size) - VLAN_ETH_HLEN;2775	if (bpf->prog && !(bpf->prog->aux && bpf->prog->aux->xdp_has_frags))2776		maxfs = min_t(u32, maxfs, IONIC_XDP_MAX_LINEAR_MTU);2777	netdev->max_mtu = maxfs;2778 2779	if (!netif_running(netdev)) {2780		old_prog = xchg(&lif->xdp_prog, bpf->prog);2781	} else if (lif->xdp_prog && bpf->prog) {2782		old_prog = xchg(&lif->xdp_prog, bpf->prog);2783		ionic_xdp_rxqs_prog_update(lif);2784	} else {2785		struct ionic_queue_params qparams;2786 2787		ionic_init_queue_params(lif, &qparams);2788		qparams.xdp_prog = bpf->prog;2789		mutex_lock(&lif->queue_lock);2790		ionic_reconfigure_queues(lif, &qparams);2791		old_prog = xchg(&lif->xdp_prog, bpf->prog);2792		mutex_unlock(&lif->queue_lock);2793	}2794 2795	if (old_prog)2796		bpf_prog_put(old_prog);2797 2798	return 0;2799}2800 2801static int ionic_xdp(struct net_device *netdev, struct netdev_bpf *bpf)2802{2803	switch (bpf->command) {2804	case XDP_SETUP_PROG:2805		return ionic_xdp_config(netdev, bpf);2806	default:2807		return -EINVAL;2808	}2809}2810 2811static const struct net_device_ops ionic_netdev_ops = {2812	.ndo_open               = ionic_open,2813	.ndo_stop               = ionic_stop,2814	.ndo_eth_ioctl		= ionic_eth_ioctl,2815	.ndo_start_xmit		= ionic_start_xmit,2816	.ndo_bpf		= ionic_xdp,2817	.ndo_xdp_xmit		= ionic_xdp_xmit,2818	.ndo_get_stats64	= ionic_get_stats64,2819	.ndo_set_rx_mode	= ionic_ndo_set_rx_mode,2820	.ndo_set_features	= ionic_set_features,2821	.ndo_set_mac_address	= ionic_set_mac_address,2822	.ndo_validate_addr	= eth_validate_addr,2823	.ndo_tx_timeout         = ionic_tx_timeout,2824	.ndo_change_mtu         = ionic_change_mtu,2825	.ndo_vlan_rx_add_vid    = ionic_vlan_rx_add_vid,2826	.ndo_vlan_rx_kill_vid   = ionic_vlan_rx_kill_vid,2827	.ndo_set_vf_vlan	= ionic_set_vf_vlan,2828	.ndo_set_vf_trust	= ionic_set_vf_trust,2829	.ndo_set_vf_mac		= ionic_set_vf_mac,2830	.ndo_set_vf_rate	= ionic_set_vf_rate,2831	.ndo_set_vf_spoofchk	= ionic_set_vf_spoofchk,2832	.ndo_get_vf_config	= ionic_get_vf_config,2833	.ndo_set_vf_link_state	= ionic_set_vf_link_state,2834	.ndo_get_vf_stats       = ionic_get_vf_stats,2835};2836 2837static int ionic_cmb_reconfig(struct ionic_lif *lif,2838			      struct ionic_queue_params *qparam)2839{2840	struct ionic_queue_params start_qparams;2841	int err = 0;2842 2843	/* When changing CMB queue parameters, we're using limited2844	 * on-device memory and don't have extra memory to use for2845	 * duplicate allocations, so we free it all first then2846	 * re-allocate with the new parameters.2847	 */2848 2849	/* Checkpoint for possible unwind */2850	ionic_init_queue_params(lif, &start_qparams);2851 2852	/* Stop and free the queues */2853	ionic_stop_queues_reconfig(lif);2854	ionic_txrx_free(lif);2855 2856	/* Set up new qparams */2857	ionic_set_queue_params(lif, qparam);2858 2859	if (netif_running(lif->netdev)) {2860		/* Alloc and start the new configuration */2861		err = ionic_txrx_alloc(lif);2862		if (err) {2863			dev_warn(lif->ionic->dev,2864				 "CMB reconfig failed, restoring values: %d\n", err);2865 2866			/* Back out the changes */2867			ionic_set_queue_params(lif, &start_qparams);2868			err = ionic_txrx_alloc(lif);2869			if (err) {2870				dev_err(lif->ionic->dev,2871					"CMB restore failed: %d\n", err);2872				goto err_out;2873			}2874		}2875 2876		err = ionic_start_queues_reconfig(lif);2877		if (err) {2878			dev_err(lif->ionic->dev,2879				"CMB reconfig failed: %d\n", err);2880			goto err_out;2881		}2882	}2883 2884err_out:2885	/* This was detached in ionic_stop_queues_reconfig() */2886	netif_device_attach(lif->netdev);2887 2888	return err;2889}2890 2891static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b)2892{2893	/* only swapping the queues and napi, not flags or other stuff */2894	swap(a->napi,         b->napi);2895 2896	if (a->q.type == IONIC_QTYPE_RXQ) {2897		swap(a->q.page_pool, b->q.page_pool);2898		a->q.page_pool->p.napi = &a->napi;2899		if (b->q.page_pool)  /* is NULL when increasing queue count */2900			b->q.page_pool->p.napi = &b->napi;2901	}2902 2903	swap(a->q.features,   b->q.features);2904	swap(a->q.num_descs,  b->q.num_descs);2905	swap(a->q.desc_size,  b->q.desc_size);2906	swap(a->q.base,       b->q.base);2907	swap(a->q.base_pa,    b->q.base_pa);2908	swap(a->q.info,       b->q.info);2909	swap(a->q.xdp_prog,   b->q.xdp_prog);2910	swap(a->q.xdp_rxq_info, b->q.xdp_rxq_info);2911	swap(a->q.partner,    b->q.partner);2912	swap(a->q_base,       b->q_base);2913	swap(a->q_base_pa,    b->q_base_pa);2914	swap(a->q_size,       b->q_size);2915 2916	swap(a->q.sg_desc_size, b->q.sg_desc_size);2917	swap(a->q.sg_base,    b->q.sg_base);2918	swap(a->q.sg_base_pa, b->q.sg_base_pa);2919	swap(a->sg_base,      b->sg_base);2920	swap(a->sg_base_pa,   b->sg_base_pa);2921	swap(a->sg_size,      b->sg_size);2922 2923	swap(a->cq.num_descs, b->cq.num_descs);2924	swap(a->cq.desc_size, b->cq.desc_size);2925	swap(a->cq.base,      b->cq.base);2926	swap(a->cq.base_pa,   b->cq.base_pa);2927	swap(a->cq_base,      b->cq_base);2928	swap(a->cq_base_pa,   b->cq_base_pa);2929	swap(a->cq_size,      b->cq_size);2930 2931	ionic_debugfs_del_qcq(a);2932	ionic_debugfs_add_qcq(a->q.lif, a);2933}2934 2935int ionic_reconfigure_queues(struct ionic_lif *lif,2936			     struct ionic_queue_params *qparam)2937{2938	unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;2939	struct ionic_qcq **tx_qcqs = NULL;2940	struct ionic_qcq **rx_qcqs = NULL;2941	unsigned int flags, i;2942	int err = 0;2943 2944	/* Are we changing q params while CMB is on */2945	if ((test_bit(IONIC_LIF_F_CMB_TX_RINGS, lif->state) && qparam->cmb_tx) ||2946	    (test_bit(IONIC_LIF_F_CMB_RX_RINGS, lif->state) && qparam->cmb_rx))2947		return ionic_cmb_reconfig(lif, qparam);2948 2949	/* allocate temporary qcq arrays to hold new queue structs */2950	if (qparam->nxqs != lif->nxqs || qparam->ntxq_descs != lif->ntxq_descs) {2951		tx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->ntxqs_per_lif,2952				       sizeof(struct ionic_qcq *), GFP_KERNEL);2953		if (!tx_qcqs) {2954			err = -ENOMEM;2955			goto err_out;2956		}2957	}2958	if (qparam->nxqs != lif->nxqs ||2959	    qparam->nrxq_descs != lif->nrxq_descs ||2960	    qparam->rxq_features != lif->rxq_features ||2961	    qparam->xdp_prog != lif->xdp_prog) {2962		rx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->nrxqs_per_lif,2963				       sizeof(struct ionic_qcq *), GFP_KERNEL);2964		if (!rx_qcqs) {2965			err = -ENOMEM;2966			goto err_out;2967		}2968	}2969 2970	/* allocate new desc_info and rings, but leave the interrupt setup2971	 * until later so as to not mess with the still-running queues2972	 */2973	if (tx_qcqs) {2974		num_desc = qparam->ntxq_descs;2975		desc_sz = sizeof(struct ionic_txq_desc);2976		comp_sz = sizeof(struct ionic_txq_comp);2977 2978		if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&2979		    lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==2980		    sizeof(struct ionic_txq_sg_desc_v1))2981			sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);2982		else2983			sg_desc_sz = sizeof(struct ionic_txq_sg_desc);2984 2985		for (i = 0; i < qparam->nxqs; i++) {2986			/* If missing, short placeholder qcq needed for swap */2987			if (!lif->txqcqs[i]) {2988				flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;2989				err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,2990						      4, desc_sz, comp_sz, sg_desc_sz,2991						      sizeof(struct ionic_tx_desc_info),2992						      lif->kern_pid, NULL, &lif->txqcqs[i]);2993				if (err)2994					goto err_out;2995			}2996 2997			flags = lif->txqcqs[i]->flags & ~IONIC_QCQ_F_INTR;2998			err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,2999					      num_desc, desc_sz, comp_sz, sg_desc_sz,3000					      sizeof(struct ionic_tx_desc_info),3001					      lif->kern_pid, NULL, &tx_qcqs[i]);3002			if (err)3003				goto err_out;3004		}3005	}3006 3007	if (rx_qcqs) {3008		num_desc = qparam->nrxq_descs;3009		desc_sz = sizeof(struct ionic_rxq_desc);3010		comp_sz = sizeof(struct ionic_rxq_comp);3011		sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);3012 3013		if (qparam->rxq_features & IONIC_Q_F_2X_CQ_DESC)3014			comp_sz *= 2;3015 3016		for (i = 0; i < qparam->nxqs; i++) {3017			/* If missing, short placeholder qcq needed for swap */3018			if (!lif->rxqcqs[i]) {3019				flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG;3020				err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,3021						      4, desc_sz, comp_sz, sg_desc_sz,3022						      sizeof(struct ionic_rx_desc_info),3023						      lif->kern_pid, NULL, &lif->rxqcqs[i]);3024				if (err)3025					goto err_out;3026			}3027 3028			flags = lif->rxqcqs[i]->flags & ~IONIC_QCQ_F_INTR;3029			err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,3030					      num_desc, desc_sz, comp_sz, sg_desc_sz,3031					      sizeof(struct ionic_rx_desc_info),3032					      lif->kern_pid, qparam->xdp_prog, &rx_qcqs[i]);3033			if (err)3034				goto err_out;3035 3036			rx_qcqs[i]->q.features = qparam->rxq_features;3037			rx_qcqs[i]->q.xdp_prog = qparam->xdp_prog;3038		}3039	}3040 3041	/* stop and clean the queues */3042	ionic_stop_queues_reconfig(lif);3043 3044	if (qparam->nxqs != lif->nxqs) {3045		err = netif_set_real_num_tx_queues(lif->netdev, qparam->nxqs);3046		if (err)3047			goto err_out_reinit_unlock;3048		err = netif_set_real_num_rx_queues(lif->netdev, qparam->nxqs);3049		if (err) {3050			netif_set_real_num_tx_queues(lif->netdev, lif->nxqs);3051			goto err_out_reinit_unlock;3052		}3053	}3054 3055	/* swap new desc_info and rings, keeping existing interrupt config */3056	if (tx_qcqs) {3057		lif->ntxq_descs = qparam->ntxq_descs;3058		for (i = 0; i < qparam->nxqs; i++)3059			ionic_swap_queues(lif->txqcqs[i], tx_qcqs[i]);3060	}3061 3062	if (rx_qcqs) {3063		lif->nrxq_descs = qparam->nrxq_descs;3064		for (i = 0; i < qparam->nxqs; i++)3065			ionic_swap_queues(lif->rxqcqs[i], rx_qcqs[i]);3066	}3067 3068	/* if we need to change the interrupt layout, this is the time */3069	if (qparam->intr_split != test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state) ||3070	    qparam->nxqs != lif->nxqs) {3071		if (qparam->intr_split) {3072			set_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);3073		} else {3074			clear_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);3075			lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;3076			lif->tx_coalesce_hw = lif->rx_coalesce_hw;3077		}3078 3079		/* Clear existing interrupt assignments.  We check for NULL here3080		 * because we're checking the whole array for potential qcqs, not3081		 * just those qcqs that have just been set up.3082		 */3083		for (i = 0; i < lif->ionic->ntxqs_per_lif; i++) {3084			if (lif->txqcqs[i])3085				ionic_qcq_intr_free(lif, lif->txqcqs[i]);3086			if (lif->rxqcqs[i])3087				ionic_qcq_intr_free(lif, lif->rxqcqs[i]);3088		}3089 3090		/* re-assign the interrupts */3091		for (i = 0; i < qparam->nxqs; i++) {3092			lif->rxqcqs[i]->flags |= IONIC_QCQ_F_INTR;3093			err = ionic_alloc_qcq_interrupt(lif, lif->rxqcqs[i]);3094			ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,3095					     lif->rxqcqs[i]->intr.index,3096					     lif->rx_coalesce_hw);3097 3098			if (qparam->intr_split) {3099				lif->txqcqs[i]->flags |= IONIC_QCQ_F_INTR;3100				err = ionic_alloc_qcq_interrupt(lif, lif->txqcqs[i]);3101				ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,3102						     lif->txqcqs[i]->intr.index,3103						     lif->tx_coalesce_hw);3104				if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))3105					lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;3106			} else {3107				lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;3108				ionic_link_qcq_interrupts(lif->rxqcqs[i], lif->txqcqs[i]);3109			}3110		}3111	}3112 3113	/* now we can rework the debugfs mappings */3114	if (tx_qcqs) {3115		for (i = 0; i < qparam->nxqs; i++) {3116			ionic_debugfs_del_qcq(lif->txqcqs[i]);3117			ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);3118		}3119	}3120 3121	if (rx_qcqs) {3122		for (i = 0; i < qparam->nxqs; i++) {3123			ionic_debugfs_del_qcq(lif->rxqcqs[i]);3124			ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);3125		}3126	}3127 3128	swap(lif->nxqs, qparam->nxqs);3129	swap(lif->rxq_features, qparam->rxq_features);3130 3131err_out_reinit_unlock:3132	/* re-init the queues, but don't lose an error code */3133	if (err)3134		ionic_start_queues_reconfig(lif);3135	else3136		err = ionic_start_queues_reconfig(lif);3137 3138err_out:3139	/* free old allocs without cleaning intr */3140	for (i = 0; i < qparam->nxqs; i++) {3141		if (tx_qcqs && tx_qcqs[i]) {3142			tx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;3143			ionic_qcq_free(lif, tx_qcqs[i]);3144			devm_kfree(lif->ionic->dev, tx_qcqs[i]);3145			tx_qcqs[i] = NULL;3146		}3147		if (rx_qcqs && rx_qcqs[i]) {3148			rx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;3149			ionic_qcq_free(lif, rx_qcqs[i]);3150			devm_kfree(lif->ionic->dev, rx_qcqs[i]);3151			rx_qcqs[i] = NULL;3152		}3153	}3154 3155	/* free q array */3156	if (rx_qcqs) {3157		devm_kfree(lif->ionic->dev, rx_qcqs);3158		rx_qcqs = NULL;3159	}3160	if (tx_qcqs) {3161		devm_kfree(lif->ionic->dev, tx_qcqs);3162		tx_qcqs = NULL;3163	}3164 3165	/* clean the unused dma and info allocations when new set is smaller3166	 * than the full array, but leave the qcq shells in place3167	 */3168	for (i = lif->nxqs; i < lif->ionic->ntxqs_per_lif; i++) {3169		if (lif->txqcqs && lif->txqcqs[i]) {3170			lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;3171			ionic_qcq_free(lif, lif->txqcqs[i]);3172		}3173 3174		if (lif->rxqcqs && lif->rxqcqs[i]) {3175			lif->rxqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;3176			ionic_qcq_free(lif, lif->rxqcqs[i]);3177		}3178	}3179 3180	if (err)3181		netdev_info(lif->netdev, "%s: failed %d\n", __func__, err);3182 3183	return err;3184}3185 3186static int ionic_affinity_masks_alloc(struct ionic *ionic)3187{3188	cpumask_var_t *affinity_masks;3189	int nintrs = ionic->nintrs;3190	int i;3191 3192	affinity_masks = kcalloc(nintrs, sizeof(cpumask_var_t), GFP_KERNEL);3193	if (!affinity_masks)3194		return -ENOMEM;3195 3196	for (i = 0; i < nintrs; i++) {3197		if (!zalloc_cpumask_var_node(&affinity_masks[i], GFP_KERNEL,3198					     dev_to_node(ionic->dev)))3199			goto err_out;3200	}3201 3202	ionic->affinity_masks = affinity_masks;3203 3204	return 0;3205 3206err_out:3207	for (--i; i >= 0; i--)3208		free_cpumask_var(affinity_masks[i]);3209	kfree(affinity_masks);3210 3211	return -ENOMEM;3212}3213 3214static void ionic_affinity_masks_free(struct ionic *ionic)3215{3216	int i;3217 3218	for (i = 0; i < ionic->nintrs; i++)3219		free_cpumask_var(ionic->affinity_masks[i]);3220	kfree(ionic->affinity_masks);3221	ionic->affinity_masks = NULL;3222}3223 3224int ionic_lif_alloc(struct ionic *ionic)3225{3226	struct device *dev = ionic->dev;3227	union ionic_lif_identity *lid;3228	struct net_device *netdev;3229	struct ionic_lif *lif;3230	int tbl_sz;3231	int err;3232 3233	lid = kzalloc(sizeof(*lid), GFP_KERNEL);3234	if (!lid)3235		return -ENOMEM;3236 3237	netdev = alloc_etherdev_mqs(sizeof(*lif),3238				    ionic->ntxqs_per_lif, ionic->ntxqs_per_lif);3239	if (!netdev) {3240		dev_err(dev, "Cannot allocate netdev, aborting\n");3241		err = -ENOMEM;3242		goto err_out_free_lid;3243	}3244 3245	SET_NETDEV_DEV(netdev, dev);3246 3247	lif = netdev_priv(netdev);3248	lif->netdev = netdev;3249	ionic->lif = lif;3250	lif->ionic = ionic;3251	netdev->netdev_ops = &ionic_netdev_ops;3252	ionic_ethtool_set_ops(netdev);3253 3254	netdev->watchdog_timeo = 5 * HZ;3255	netif_carrier_off(netdev);3256 3257	lif->identity = lid;3258	lif->lif_type = IONIC_LIF_TYPE_CLASSIC;3259	err = ionic_lif_identify(ionic, lif->lif_type, lif->identity);3260	if (err) {3261		dev_err(ionic->dev, "Cannot identify type %d: %d\n",3262			lif->lif_type, err);3263		goto err_out_free_netdev;3264	}3265	lif->netdev->min_mtu = max_t(unsigned int, ETH_MIN_MTU,3266				     le32_to_cpu(lif->identity->eth.min_frame_size));3267	lif->netdev->max_mtu =3268		le32_to_cpu(lif->identity->eth.max_frame_size) - ETH_HLEN - VLAN_HLEN;3269 3270	lif->neqs = ionic->neqs_per_lif;3271	lif->nxqs = ionic->ntxqs_per_lif;3272 3273	lif->index = 0;3274 3275	if (is_kdump_kernel()) {3276		lif->ntxq_descs = IONIC_MIN_TXRX_DESC;3277		lif->nrxq_descs = IONIC_MIN_TXRX_DESC;3278	} else {3279		lif->ntxq_descs = IONIC_DEF_TXRX_DESC;3280		lif->nrxq_descs = IONIC_DEF_TXRX_DESC;3281	}3282 3283	/* Convert the default coalesce value to actual hw resolution */3284	lif->rx_coalesce_usecs = IONIC_ITR_COAL_USEC_DEFAULT;3285	lif->rx_coalesce_hw = ionic_coal_usec_to_hw(lif->ionic,3286						    lif->rx_coalesce_usecs);3287	lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;3288	lif->tx_coalesce_hw = lif->rx_coalesce_hw;3289	set_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state);3290	set_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state);3291 3292	snprintf(lif->name, sizeof(lif->name), "lif%u", lif->index);3293 3294	mutex_init(&lif->queue_lock);3295	mutex_init(&lif->config_lock);3296 3297	spin_lock_init(&lif->adminq_lock);3298 3299	spin_lock_init(&lif->deferred.lock);3300	INIT_LIST_HEAD(&lif->deferred.list);3301	INIT_WORK(&lif->deferred.work, ionic_lif_deferred_work);3302 3303	/* allocate lif info */3304	lif->info_sz = ALIGN(sizeof(*lif->info), PAGE_SIZE);3305	lif->info = dma_alloc_coherent(dev, lif->info_sz,3306				       &lif->info_pa, GFP_KERNEL);3307	if (!lif->info) {3308		dev_err(dev, "Failed to allocate lif info, aborting\n");3309		err = -ENOMEM;3310		goto err_out_free_mutex;3311	}3312 3313	ionic_debugfs_add_lif(lif);3314 3315	err = ionic_affinity_masks_alloc(ionic);3316	if (err)3317		goto err_out_free_lif_info;3318 3319	/* allocate control queues and txrx queue arrays */3320	ionic_lif_queue_identify(lif);3321	err = ionic_qcqs_alloc(lif);3322	if (err)3323		goto err_out_free_affinity_masks;3324 3325	/* allocate rss indirection table */3326	tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);3327	lif->rss_ind_tbl_sz = sizeof(*lif->rss_ind_tbl) * tbl_sz;3328	lif->rss_ind_tbl = dma_alloc_coherent(dev, lif->rss_ind_tbl_sz,3329					      &lif->rss_ind_tbl_pa,3330					      GFP_KERNEL);3331 3332	if (!lif->rss_ind_tbl) {3333		err = -ENOMEM;3334		dev_err(dev, "Failed to allocate rss indirection table, aborting\n");3335		goto err_out_free_qcqs;3336	}3337	netdev_rss_key_fill(lif->rss_hash_key, IONIC_RSS_HASH_KEY_SIZE);3338 3339	ionic_lif_alloc_phc(lif);3340 3341	return 0;3342 3343err_out_free_qcqs:3344	ionic_qcqs_free(lif);3345err_out_free_affinity_masks:3346	ionic_affinity_masks_free(lif->ionic);3347err_out_free_lif_info:3348	dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);3349	lif->info = NULL;3350	lif->info_pa = 0;3351err_out_free_mutex:3352	mutex_destroy(&lif->config_lock);3353	mutex_destroy(&lif->queue_lock);3354err_out_free_netdev:3355	free_netdev(lif->netdev);3356	lif = NULL;3357err_out_free_lid:3358	kfree(lid);3359 3360	return err;3361}3362 3363static void ionic_lif_reset(struct ionic_lif *lif)3364{3365	struct ionic_dev *idev = &lif->ionic->idev;3366 3367	if (!ionic_is_fw_running(idev))3368		return;3369 3370	mutex_lock(&lif->ionic->dev_cmd_lock);3371	ionic_dev_cmd_lif_reset(idev, lif->index);3372	ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);3373	mutex_unlock(&lif->ionic->dev_cmd_lock);3374}3375 3376static void ionic_lif_handle_fw_down(struct ionic_lif *lif)3377{3378	struct ionic *ionic = lif->ionic;3379 3380	if (test_and_set_bit(IONIC_LIF_F_FW_RESET, lif->state))3381		return;3382 3383	dev_info(ionic->dev, "FW Down: Stopping LIFs\n");3384 3385	netif_device_detach(lif->netdev);3386 3387	mutex_lock(&lif->queue_lock);3388	if (test_bit(IONIC_LIF_F_UP, lif->state)) {3389		dev_info(ionic->dev, "Surprise FW stop, stopping queues\n");3390		ionic_stop_queues(lif);3391	}3392 3393	if (netif_running(lif->netdev)) {3394		ionic_txrx_deinit(lif);3395		ionic_txrx_free(lif);3396	}3397	ionic_lif_deinit(lif);3398	ionic_reset(ionic);3399	ionic_qcqs_free(lif);3400 3401	mutex_unlock(&lif->queue_lock);3402 3403	clear_bit(IONIC_LIF_F_FW_STOPPING, lif->state);3404	dev_info(ionic->dev, "FW Down: LIFs stopped\n");3405}3406 3407int ionic_restart_lif(struct ionic_lif *lif)3408{3409	struct ionic *ionic = lif->ionic;3410	int err;3411 3412	mutex_lock(&lif->queue_lock);3413 3414	if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))3415		dev_info(ionic->dev, "FW Up: clearing broken state\n");3416 3417	err = ionic_qcqs_alloc(lif);3418	if (err)3419		goto err_unlock;3420 3421	err = ionic_lif_init(lif);3422	if (err)3423		goto err_qcqs_free;3424 3425	ionic_vf_attr_replay(lif);3426 3427	if (lif->registered)3428		ionic_lif_set_netdev_info(lif);3429 3430	ionic_rx_filter_replay(lif);3431 3432	if (netif_running(lif->netdev)) {3433		err = ionic_txrx_alloc(lif);3434		if (err)3435			goto err_lifs_deinit;3436 3437		err = ionic_txrx_init(lif);3438		if (err)3439			goto err_txrx_free;3440	}3441 3442	mutex_unlock(&lif->queue_lock);3443 3444	clear_bit(IONIC_LIF_F_FW_RESET, lif->state);3445	ionic_link_status_check_request(lif, CAN_SLEEP);3446	netif_device_attach(lif->netdev);3447	ionic_queue_doorbell_check(ionic, IONIC_NAPI_DEADLINE);3448 3449	return 0;3450 3451err_txrx_free:3452	ionic_txrx_free(lif);3453err_lifs_deinit:3454	ionic_lif_deinit(lif);3455err_qcqs_free:3456	ionic_qcqs_free(lif);3457err_unlock:3458	mutex_unlock(&lif->queue_lock);3459 3460	return err;3461}3462 3463static void ionic_lif_handle_fw_up(struct ionic_lif *lif)3464{3465	struct ionic *ionic = lif->ionic;3466	int err;3467 3468	if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))3469		return;3470 3471	dev_info(ionic->dev, "FW Up: restarting LIFs\n");3472 3473	/* This is a little different from what happens at3474	 * probe time because the LIF already exists so we3475	 * just need to reanimate it.3476	 */3477	ionic_init_devinfo(ionic);3478	ionic_reset(ionic);3479	err = ionic_identify(ionic);3480	if (err)3481		goto err_out;3482	err = ionic_port_identify(ionic);3483	if (err)3484		goto err_out;3485	err = ionic_port_init(ionic);3486	if (err)3487		goto err_out;3488 3489	err = ionic_restart_lif(lif);3490	if (err)3491		goto err_out;3492 3493	dev_info(ionic->dev, "FW Up: LIFs restarted\n");3494 3495	/* restore the hardware timestamping queues */3496	ionic_lif_hwstamp_replay(lif);3497 3498	return;3499 3500err_out:3501	dev_err(ionic->dev, "FW Up: LIFs restart failed - err %d\n", err);3502}3503 3504void ionic_lif_free(struct ionic_lif *lif)3505{3506	struct device *dev = lif->ionic->dev;3507 3508	ionic_lif_free_phc(lif);3509 3510	/* free rss indirection table */3511	dma_free_coherent(dev, lif->rss_ind_tbl_sz, lif->rss_ind_tbl,3512			  lif->rss_ind_tbl_pa);3513	lif->rss_ind_tbl = NULL;3514	lif->rss_ind_tbl_pa = 0;3515 3516	/* free queues */3517	ionic_qcqs_free(lif);3518	if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))3519		ionic_lif_reset(lif);3520 3521	ionic_affinity_masks_free(lif->ionic);3522 3523	/* free lif info */3524	kfree(lif->identity);3525	dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);3526	lif->info = NULL;3527	lif->info_pa = 0;3528 3529	/* unmap doorbell page */3530	ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);3531	lif->kern_dbpage = NULL;3532 3533	mutex_destroy(&lif->config_lock);3534	mutex_destroy(&lif->queue_lock);3535 3536	/* free netdev & lif */3537	ionic_debugfs_del_lif(lif);3538	free_netdev(lif->netdev);3539}3540 3541void ionic_lif_deinit(struct ionic_lif *lif)3542{3543	if (!test_and_clear_bit(IONIC_LIF_F_INITED, lif->state))3544		return;3545 3546	if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {3547		cancel_work_sync(&lif->deferred.work);3548		cancel_work_sync(&lif->tx_timeout_work);3549		ionic_rx_filters_deinit(lif);3550		if (lif->netdev->features & NETIF_F_RXHASH)3551			ionic_lif_rss_deinit(lif);3552	}3553 3554	napi_disable(&lif->adminqcq->napi);3555	ionic_lif_qcq_deinit(lif, lif->notifyqcq);3556	ionic_lif_qcq_deinit(lif, lif->adminqcq);3557 3558	ionic_lif_reset(lif);3559}3560 3561static int ionic_lif_adminq_init(struct ionic_lif *lif)3562{3563	struct device *dev = lif->ionic->dev;3564	struct ionic_q_init_comp comp;3565	struct ionic_dev *idev;3566	struct ionic_qcq *qcq;3567	struct ionic_queue *q;3568	int err;3569 3570	idev = &lif->ionic->idev;3571	qcq = lif->adminqcq;3572	q = &qcq->q;3573 3574	mutex_lock(&lif->ionic->dev_cmd_lock);3575	ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);3576	err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);3577	ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);3578	mutex_unlock(&lif->ionic->dev_cmd_lock);3579	if (err) {3580		netdev_err(lif->netdev, "adminq init failed %d\n", err);3581		return err;3582	}3583 3584	q->hw_type = comp.hw_type;3585	q->hw_index = le32_to_cpu(comp.hw_index);3586	q->dbval = IONIC_DBELL_QID(q->hw_index);3587 3588	dev_dbg(dev, "adminq->hw_type %d\n", q->hw_type);3589	dev_dbg(dev, "adminq->hw_index %d\n", q->hw_index);3590 3591	q->dbell_deadline = IONIC_ADMIN_DOORBELL_DEADLINE;3592	q->dbell_jiffies = jiffies;3593 3594	netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi);3595 3596	napi_enable(&qcq->napi);3597 3598	if (qcq->flags & IONIC_QCQ_F_INTR) {3599		irq_set_affinity_hint(qcq->intr.vector,3600				      *qcq->intr.affinity_mask);3601		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,3602				IONIC_INTR_MASK_CLEAR);3603	}3604 3605	qcq->flags |= IONIC_QCQ_F_INITED;3606 3607	return 0;3608}3609 3610static int ionic_lif_notifyq_init(struct ionic_lif *lif)3611{3612	struct ionic_qcq *qcq = lif->notifyqcq;3613	struct device *dev = lif->ionic->dev;3614	struct ionic_queue *q = &qcq->q;3615	int err;3616 3617	struct ionic_admin_ctx ctx = {3618		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),3619		.cmd.q_init = {3620			.opcode = IONIC_CMD_Q_INIT,3621			.lif_index = cpu_to_le16(lif->index),3622			.type = q->type,3623			.ver = lif->qtype_info[q->type].version,3624			.index = cpu_to_le32(q->index),3625			.flags = cpu_to_le16(IONIC_QINIT_F_IRQ |3626					     IONIC_QINIT_F_ENA),3627			.intr_index = cpu_to_le16(lif->adminqcq->intr.index),3628			.pid = cpu_to_le16(q->pid),3629			.ring_size = ilog2(q->num_descs),3630			.ring_base = cpu_to_le64(q->base_pa),3631		}3632	};3633 3634	dev_dbg(dev, "notifyq_init.pid %d\n", ctx.cmd.q_init.pid);3635	dev_dbg(dev, "notifyq_init.index %d\n", ctx.cmd.q_init.index);3636	dev_dbg(dev, "notifyq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);3637	dev_dbg(dev, "notifyq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);3638 3639	err = ionic_adminq_post_wait(lif, &ctx);3640	if (err)3641		return err;3642 3643	lif->last_eid = 0;3644	q->hw_type = ctx.comp.q_init.hw_type;3645	q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);3646	q->dbval = IONIC_DBELL_QID(q->hw_index);3647 3648	dev_dbg(dev, "notifyq->hw_type %d\n", q->hw_type);3649	dev_dbg(dev, "notifyq->hw_index %d\n", q->hw_index);3650 3651	/* preset the callback info */3652	q->admin_info[0].ctx = lif;3653 3654	qcq->flags |= IONIC_QCQ_F_INITED;3655 3656	return 0;3657}3658 3659static int ionic_station_set(struct ionic_lif *lif)3660{3661	struct net_device *netdev = lif->netdev;3662	struct ionic_admin_ctx ctx = {3663		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),3664		.cmd.lif_getattr = {3665			.opcode = IONIC_CMD_LIF_GETATTR,3666			.index = cpu_to_le16(lif->index),3667			.attr = IONIC_LIF_ATTR_MAC,3668		},3669	};3670	u8 mac_address[ETH_ALEN];3671	struct sockaddr addr;3672	int err;3673 3674	err = ionic_adminq_post_wait(lif, &ctx);3675	if (err)3676		return err;3677	netdev_dbg(lif->netdev, "found initial MAC addr %pM\n",3678		   ctx.comp.lif_getattr.mac);3679	ether_addr_copy(mac_address, ctx.comp.lif_getattr.mac);3680 3681	if (is_zero_ether_addr(mac_address)) {3682		eth_hw_addr_random(netdev);3683		netdev_dbg(netdev, "Random Mac generated: %pM\n", netdev->dev_addr);3684		ether_addr_copy(mac_address, netdev->dev_addr);3685 3686		err = ionic_program_mac(lif, mac_address);3687		if (err < 0)3688			return err;3689 3690		if (err > 0) {3691			netdev_dbg(netdev, "%s:SET/GET ATTR Mac are not same-due to old FW running\n",3692				   __func__);3693			return 0;3694		}3695	}3696 3697	if (!is_zero_ether_addr(netdev->dev_addr)) {3698		/* If the netdev mac is non-zero and doesn't match the default3699		 * device address, it was set by something earlier and we're3700		 * likely here again after a fw-upgrade reset.  We need to be3701		 * sure the netdev mac is in our filter list.3702		 */3703		if (!ether_addr_equal(mac_address, netdev->dev_addr))3704			ionic_lif_addr_add(lif, netdev->dev_addr);3705	} else {3706		/* Update the netdev mac with the device's mac */3707		ether_addr_copy(addr.sa_data, mac_address);3708		addr.sa_family = AF_INET;3709		err = eth_prepare_mac_addr_change(netdev, &addr);3710		if (err) {3711			netdev_warn(lif->netdev, "ignoring bad MAC addr from NIC %pM - err %d\n",3712				    addr.sa_data, err);3713			return 0;3714		}3715 3716		eth_commit_mac_addr_change(netdev, &addr);3717	}3718 3719	netdev_dbg(lif->netdev, "adding station MAC addr %pM\n",3720		   netdev->dev_addr);3721	ionic_lif_addr_add(lif, netdev->dev_addr);3722 3723	return 0;3724}3725 3726int ionic_lif_init(struct ionic_lif *lif)3727{3728	struct ionic_dev *idev = &lif->ionic->idev;3729	struct device *dev = lif->ionic->dev;3730	struct ionic_lif_init_comp comp;3731	int dbpage_num;3732	int err;3733 3734	mutex_lock(&lif->ionic->dev_cmd_lock);3735	ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);3736	err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);3737	ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);3738	mutex_unlock(&lif->ionic->dev_cmd_lock);3739	if (err)3740		return err;3741 3742	lif->hw_index = le16_to_cpu(comp.hw_index);3743 3744	/* now that we have the hw_index we can figure out our doorbell page */3745	lif->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif);3746	if (!lif->dbid_count) {3747		dev_err(dev, "No doorbell pages, aborting\n");3748		return -EINVAL;3749	}3750 3751	lif->kern_pid = 0;3752	dbpage_num = ionic_db_page_num(lif, lif->kern_pid);3753	lif->kern_dbpage = ionic_bus_map_dbpage(lif->ionic, dbpage_num);3754	if (!lif->kern_dbpage) {3755		dev_err(dev, "Cannot map dbpage, aborting\n");3756		return -ENOMEM;3757	}3758 3759	err = ionic_lif_adminq_init(lif);3760	if (err)3761		goto err_out_adminq_deinit;3762 3763	if (lif->ionic->nnqs_per_lif) {3764		err = ionic_lif_notifyq_init(lif);3765		if (err)3766			goto err_out_notifyq_deinit;3767	}3768 3769	if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))3770		err = ionic_set_nic_features(lif, lif->netdev->features);3771	else3772		err = ionic_init_nic_features(lif);3773	if (err)3774		goto err_out_notifyq_deinit;3775 3776	if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {3777		err = ionic_rx_filters_init(lif);3778		if (err)3779			goto err_out_notifyq_deinit;3780	}3781 3782	err = ionic_station_set(lif);3783	if (err)3784		goto err_out_notifyq_deinit;3785 3786	lif->rx_copybreak = IONIC_RX_COPYBREAK_DEFAULT;3787	lif->doorbell_wa = ionic_doorbell_wa(lif->ionic);3788 3789	set_bit(IONIC_LIF_F_INITED, lif->state);3790 3791	INIT_WORK(&lif->tx_timeout_work, ionic_tx_timeout_work);3792 3793	return 0;3794 3795err_out_notifyq_deinit:3796	napi_disable(&lif->adminqcq->napi);3797	ionic_lif_qcq_deinit(lif, lif->notifyqcq);3798err_out_adminq_deinit:3799	ionic_lif_qcq_deinit(lif, lif->adminqcq);3800	ionic_lif_reset(lif);3801	ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);3802	lif->kern_dbpage = NULL;3803 3804	return err;3805}3806 3807static void ionic_lif_notify_work(struct work_struct *ws)3808{3809}3810 3811static void ionic_lif_set_netdev_info(struct ionic_lif *lif)3812{3813	struct ionic_admin_ctx ctx = {3814		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),3815		.cmd.lif_setattr = {3816			.opcode = IONIC_CMD_LIF_SETATTR,3817			.index = cpu_to_le16(lif->index),3818			.attr = IONIC_LIF_ATTR_NAME,3819		},3820	};3821 3822	strscpy(ctx.cmd.lif_setattr.name, netdev_name(lif->netdev),3823		sizeof(ctx.cmd.lif_setattr.name));3824 3825	ionic_adminq_post_wait(lif, &ctx);3826}3827 3828static struct ionic_lif *ionic_netdev_lif(struct net_device *netdev)3829{3830	if (!netdev || netdev->netdev_ops->ndo_start_xmit != ionic_start_xmit)3831		return NULL;3832 3833	return netdev_priv(netdev);3834}3835 3836static int ionic_lif_notify(struct notifier_block *nb,3837			    unsigned long event, void *info)3838{3839	struct net_device *ndev = netdev_notifier_info_to_dev(info);3840	struct ionic *ionic = container_of(nb, struct ionic, nb);3841	struct ionic_lif *lif = ionic_netdev_lif(ndev);3842 3843	if (!lif || lif->ionic != ionic)3844		return NOTIFY_DONE;3845 3846	switch (event) {3847	case NETDEV_CHANGENAME:3848		ionic_lif_set_netdev_info(lif);3849		break;3850	}3851 3852	return NOTIFY_DONE;3853}3854 3855int ionic_lif_register(struct ionic_lif *lif)3856{3857	int err;3858 3859	ionic_lif_register_phc(lif);3860 3861	INIT_WORK(&lif->ionic->nb_work, ionic_lif_notify_work);3862 3863	lif->ionic->nb.notifier_call = ionic_lif_notify;3864 3865	err = register_netdevice_notifier(&lif->ionic->nb);3866	if (err)3867		lif->ionic->nb.notifier_call = NULL;3868 3869	/* only register LIF0 for now */3870	err = register_netdev(lif->netdev);3871	if (err) {3872		dev_err(lif->ionic->dev, "Cannot register net device, aborting\n");3873		ionic_lif_unregister_phc(lif);3874		return err;3875	}3876 3877	ionic_link_status_check_request(lif, CAN_SLEEP);3878	lif->registered = true;3879	ionic_lif_set_netdev_info(lif);3880 3881	return 0;3882}3883 3884void ionic_lif_unregister(struct ionic_lif *lif)3885{3886	if (lif->ionic->nb.notifier_call) {3887		unregister_netdevice_notifier(&lif->ionic->nb);3888		cancel_work_sync(&lif->ionic->nb_work);3889		lif->ionic->nb.notifier_call = NULL;3890	}3891 3892	if (lif->netdev->reg_state == NETREG_REGISTERED)3893		unregister_netdev(lif->netdev);3894 3895	ionic_lif_unregister_phc(lif);3896 3897	lif->registered = false;3898}3899 3900static void ionic_lif_queue_identify(struct ionic_lif *lif)3901{3902	union ionic_q_identity __iomem *q_ident;3903	struct ionic *ionic = lif->ionic;3904	struct ionic_dev *idev;3905	u16 max_frags;3906	int qtype;3907	int err;3908 3909	idev = &lif->ionic->idev;3910	q_ident = (union ionic_q_identity __iomem *)&idev->dev_cmd_regs->data;3911 3912	for (qtype = 0; qtype < ARRAY_SIZE(ionic_qtype_versions); qtype++) {3913		struct ionic_qtype_info *qti = &lif->qtype_info[qtype];3914 3915		/* filter out the ones we know about */3916		switch (qtype) {3917		case IONIC_QTYPE_ADMINQ:3918		case IONIC_QTYPE_NOTIFYQ:3919		case IONIC_QTYPE_RXQ:3920		case IONIC_QTYPE_TXQ:3921			break;3922		default:3923			continue;3924		}3925 3926		memset(qti, 0, sizeof(*qti));3927 3928		mutex_lock(&ionic->dev_cmd_lock);3929		ionic_dev_cmd_queue_identify(idev, lif->lif_type, qtype,3930					     ionic_qtype_versions[qtype]);3931		err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);3932		if (!err) {3933			qti->version   = readb(&q_ident->version);3934			qti->supported = readb(&q_ident->supported);3935			qti->features  = readq(&q_ident->features);3936			qti->desc_sz   = readw(&q_ident->desc_sz);3937			qti->comp_sz   = readw(&q_ident->comp_sz);3938			qti->sg_desc_sz   = readw(&q_ident->sg_desc_sz);3939			qti->max_sg_elems = readw(&q_ident->max_sg_elems);3940			qti->sg_desc_stride = readw(&q_ident->sg_desc_stride);3941		}3942		mutex_unlock(&ionic->dev_cmd_lock);3943 3944		if (err == -EINVAL) {3945			dev_err(ionic->dev, "qtype %d not supported\n", qtype);3946			continue;3947		} else if (err == -EIO) {3948			dev_err(ionic->dev, "q_ident failed, not supported on older FW\n");3949			return;3950		} else if (err) {3951			dev_err(ionic->dev, "q_ident failed, qtype %d: %d\n",3952				qtype, err);3953			return;3954		}3955 3956		dev_dbg(ionic->dev, " qtype[%d].version = %d\n",3957			qtype, qti->version);3958		dev_dbg(ionic->dev, " qtype[%d].supported = 0x%02x\n",3959			qtype, qti->supported);3960		dev_dbg(ionic->dev, " qtype[%d].features = 0x%04llx\n",3961			qtype, qti->features);3962		dev_dbg(ionic->dev, " qtype[%d].desc_sz = %d\n",3963			qtype, qti->desc_sz);3964		dev_dbg(ionic->dev, " qtype[%d].comp_sz = %d\n",3965			qtype, qti->comp_sz);3966		dev_dbg(ionic->dev, " qtype[%d].sg_desc_sz = %d\n",3967			qtype, qti->sg_desc_sz);3968		dev_dbg(ionic->dev, " qtype[%d].max_sg_elems = %d\n",3969			qtype, qti->max_sg_elems);3970		dev_dbg(ionic->dev, " qtype[%d].sg_desc_stride = %d\n",3971			qtype, qti->sg_desc_stride);3972 3973		if (qtype == IONIC_QTYPE_TXQ)3974			max_frags = IONIC_TX_MAX_FRAGS;3975		else if (qtype == IONIC_QTYPE_RXQ)3976			max_frags = IONIC_RX_MAX_FRAGS;3977		else3978			max_frags = 1;3979 3980		qti->max_sg_elems = min_t(u16, max_frags - 1, MAX_SKB_FRAGS);3981		dev_dbg(ionic->dev, "qtype %d max_sg_elems %d\n",3982			qtype, qti->max_sg_elems);3983	}3984}3985 3986int ionic_lif_identify(struct ionic *ionic, u8 lif_type,3987		       union ionic_lif_identity *lid)3988{3989	struct ionic_dev *idev = &ionic->idev;3990	size_t sz;3991	int err;3992 3993	sz = min(sizeof(*lid), sizeof(idev->dev_cmd_regs->data));3994 3995	mutex_lock(&ionic->dev_cmd_lock);3996	ionic_dev_cmd_lif_identify(idev, lif_type, IONIC_IDENTITY_VERSION_1);3997	err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);3998	memcpy_fromio(lid, &idev->dev_cmd_regs->data, sz);3999	mutex_unlock(&ionic->dev_cmd_lock);4000	if (err)4001		return (err);4002 4003	dev_dbg(ionic->dev, "capabilities 0x%llx\n",4004		le64_to_cpu(lid->capabilities));4005 4006	dev_dbg(ionic->dev, "eth.max_ucast_filters %d\n",4007		le32_to_cpu(lid->eth.max_ucast_filters));4008	dev_dbg(ionic->dev, "eth.max_mcast_filters %d\n",4009		le32_to_cpu(lid->eth.max_mcast_filters));4010	dev_dbg(ionic->dev, "eth.features 0x%llx\n",4011		le64_to_cpu(lid->eth.config.features));4012	dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_ADMINQ] %d\n",4013		le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_ADMINQ]));4014	dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] %d\n",4015		le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]));4016	dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_RXQ] %d\n",4017		le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_RXQ]));4018	dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_TXQ] %d\n",4019		le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_TXQ]));4020	dev_dbg(ionic->dev, "eth.config.name %s\n", lid->eth.config.name);4021	dev_dbg(ionic->dev, "eth.config.mac %pM\n", lid->eth.config.mac);4022	dev_dbg(ionic->dev, "eth.config.mtu %d\n",4023		le32_to_cpu(lid->eth.config.mtu));4024 4025	return 0;4026}4027 4028int ionic_lif_size(struct ionic *ionic)4029{4030	struct ionic_identity *ident = &ionic->ident;4031	unsigned int nintrs, dev_nintrs;4032	union ionic_lif_config *lc;4033	unsigned int ntxqs_per_lif;4034	unsigned int nrxqs_per_lif;4035	unsigned int neqs_per_lif;4036	unsigned int nnqs_per_lif;4037	unsigned int nxqs, neqs;4038	unsigned int min_intrs;4039	int err;4040 4041	/* retrieve basic values from FW */4042	lc = &ident->lif.eth.config;4043	dev_nintrs = le32_to_cpu(ident->dev.nintrs);4044	neqs_per_lif = le32_to_cpu(ident->lif.rdma.eq_qtype.qid_count);4045	nnqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_NOTIFYQ]);4046	ntxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_TXQ]);4047	nrxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_RXQ]);4048 4049	/* limit values to play nice with kdump */4050	if (is_kdump_kernel()) {4051		dev_nintrs = 2;4052		neqs_per_lif = 0;4053		nnqs_per_lif = 0;4054		ntxqs_per_lif = 1;4055		nrxqs_per_lif = 1;4056	}4057 4058	/* reserve last queue id for hardware timestamping */4059	if (lc->features & cpu_to_le64(IONIC_ETH_HW_TIMESTAMP)) {4060		if (ntxqs_per_lif <= 1 || nrxqs_per_lif <= 1) {4061			lc->features &= cpu_to_le64(~IONIC_ETH_HW_TIMESTAMP);4062		} else {4063			ntxqs_per_lif -= 1;4064			nrxqs_per_lif -= 1;4065		}4066	}4067 4068	nxqs = min(ntxqs_per_lif, nrxqs_per_lif);4069	nxqs = min(nxqs, num_online_cpus());4070	neqs = min(neqs_per_lif, num_online_cpus());4071 4072try_again:4073	/* interrupt usage:4074	 *    1 for master lif adminq/notifyq4075	 *    1 for each CPU for master lif TxRx queue pairs4076	 *    whatever's left is for RDMA queues4077	 */4078	nintrs = 1 + nxqs + neqs;4079	min_intrs = 2;  /* adminq + 1 TxRx queue pair */4080 4081	if (nintrs > dev_nintrs)4082		goto try_fewer;4083 4084	err = ionic_bus_alloc_irq_vectors(ionic, nintrs);4085	if (err < 0 && err != -ENOSPC) {4086		dev_err(ionic->dev, "Can't get intrs from OS: %d\n", err);4087		return err;4088	}4089	if (err == -ENOSPC)4090		goto try_fewer;4091 4092	if (err != nintrs) {4093		ionic_bus_free_irq_vectors(ionic);4094		goto try_fewer;4095	}4096 4097	ionic->nnqs_per_lif = nnqs_per_lif;4098	ionic->neqs_per_lif = neqs;4099	ionic->ntxqs_per_lif = nxqs;4100	ionic->nrxqs_per_lif = nxqs;4101	ionic->nintrs = nintrs;4102 4103	ionic_debugfs_add_sizes(ionic);4104 4105	return 0;4106 4107try_fewer:4108	if (nnqs_per_lif > 1) {4109		nnqs_per_lif >>= 1;4110		goto try_again;4111	}4112	if (neqs > 1) {4113		neqs >>= 1;4114		goto try_again;4115	}4116	if (nxqs > 1) {4117		nxqs >>= 1;4118		goto try_again;4119	}4120	dev_err(ionic->dev, "Can't get minimum %d intrs from OS\n", min_intrs);4121	return -ENOSPC;4122}4123