brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.02/* Marvell RVU Admin Function driver3 *4 * Copyright (C) 2018 Marvell.5 *6 */7 8#include <linux/module.h>9#include <linux/interrupt.h>10#include <linux/delay.h>11#include <linux/irq.h>12#include <linux/pci.h>13#include <linux/sysfs.h>14 15#include "cgx.h"16#include "rvu.h"17#include "rvu_reg.h"18#include "ptp.h"19#include "mcs.h"20 21#include "rvu_trace.h"22#include "rvu_npc_hash.h"23 24#define DRV_NAME	"rvu_af"25#define DRV_STRING      "Marvell OcteonTX2 RVU Admin Function Driver"26 27static void rvu_set_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,28				struct rvu_block *block, int lf);29static void rvu_clear_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,30				  struct rvu_block *block, int lf);31static void __rvu_flr_handler(struct rvu *rvu, u16 pcifunc);32 33static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw,34			 int type, int num,35			 void (mbox_handler)(struct work_struct *),36			 void (mbox_up_handler)(struct work_struct *));37enum {38	TYPE_AFVF,39	TYPE_AFPF,40};41 42/* Supported devices */43static const struct pci_device_id rvu_id_table[] = {44	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_RVU_AF) },45	{ 0, }  /* end of table */46};47 48MODULE_AUTHOR("Sunil Goutham <sgoutham@marvell.com>");49MODULE_DESCRIPTION(DRV_STRING);50MODULE_LICENSE("GPL v2");51MODULE_DEVICE_TABLE(pci, rvu_id_table);52 53static char *mkex_profile; /* MKEX profile name */54module_param(mkex_profile, charp, 0000);55MODULE_PARM_DESC(mkex_profile, "MKEX profile name string");56 57static char *kpu_profile; /* KPU profile name */58module_param(kpu_profile, charp, 0000);59MODULE_PARM_DESC(kpu_profile, "KPU profile name string");60 61static void rvu_setup_hw_capabilities(struct rvu *rvu)62{63	struct rvu_hwinfo *hw = rvu->hw;64 65	hw->cap.nix_tx_aggr_lvl = NIX_TXSCH_LVL_TL1;66	hw->cap.nix_fixed_txschq_mapping = false;67	hw->cap.nix_shaping = true;68	hw->cap.nix_tx_link_bp = true;69	hw->cap.nix_rx_multicast = true;70	hw->cap.nix_shaper_toggle_wait = false;71	hw->cap.npc_hash_extract = false;72	hw->cap.npc_exact_match_enabled = false;73	hw->rvu = rvu;74 75	if (is_rvu_pre_96xx_C0(rvu)) {76		hw->cap.nix_fixed_txschq_mapping = true;77		hw->cap.nix_txsch_per_cgx_lmac = 4;78		hw->cap.nix_txsch_per_lbk_lmac = 132;79		hw->cap.nix_txsch_per_sdp_lmac = 76;80		hw->cap.nix_shaping = false;81		hw->cap.nix_tx_link_bp = false;82		if (is_rvu_96xx_A0(rvu) || is_rvu_95xx_A0(rvu))83			hw->cap.nix_rx_multicast = false;84	}85	if (!is_rvu_pre_96xx_C0(rvu))86		hw->cap.nix_shaper_toggle_wait = true;87 88	if (!is_rvu_otx2(rvu))89		hw->cap.per_pf_mbox_regs = true;90 91	if (is_rvu_npc_hash_extract_en(rvu))92		hw->cap.npc_hash_extract = true;93}94 95/* Poll a RVU block's register 'offset', for a 'zero'96 * or 'nonzero' at bits specified by 'mask'97 */98int rvu_poll_reg(struct rvu *rvu, u64 block, u64 offset, u64 mask, bool zero)99{100	unsigned long timeout = jiffies + usecs_to_jiffies(20000);101	bool twice = false;102	void __iomem *reg;103	u64 reg_val;104 105	reg = rvu->afreg_base + ((block << 28) | offset);106again:107	reg_val = readq(reg);108	if (zero && !(reg_val & mask))109		return 0;110	if (!zero && (reg_val & mask))111		return 0;112	if (time_before(jiffies, timeout)) {113		usleep_range(1, 5);114		goto again;115	}116	/* In scenarios where CPU is scheduled out before checking117	 * 'time_before' (above) and gets scheduled in such that118	 * jiffies are beyond timeout value, then check again if HW is119	 * done with the operation in the meantime.120	 */121	if (!twice) {122		twice = true;123		goto again;124	}125	return -EBUSY;126}127 128int rvu_alloc_rsrc(struct rsrc_bmap *rsrc)129{130	int id;131 132	if (!rsrc->bmap)133		return -EINVAL;134 135	id = find_first_zero_bit(rsrc->bmap, rsrc->max);136	if (id >= rsrc->max)137		return -ENOSPC;138 139	__set_bit(id, rsrc->bmap);140 141	return id;142}143 144int rvu_alloc_rsrc_contig(struct rsrc_bmap *rsrc, int nrsrc)145{146	int start;147 148	if (!rsrc->bmap)149		return -EINVAL;150 151	start = bitmap_find_next_zero_area(rsrc->bmap, rsrc->max, 0, nrsrc, 0);152	if (start >= rsrc->max)153		return -ENOSPC;154 155	bitmap_set(rsrc->bmap, start, nrsrc);156	return start;157}158 159void rvu_free_rsrc_contig(struct rsrc_bmap *rsrc, int nrsrc, int start)160{161	if (!rsrc->bmap)162		return;163	if (start >= rsrc->max)164		return;165 166	bitmap_clear(rsrc->bmap, start, nrsrc);167}168 169bool rvu_rsrc_check_contig(struct rsrc_bmap *rsrc, int nrsrc)170{171	int start;172 173	if (!rsrc->bmap)174		return false;175 176	start = bitmap_find_next_zero_area(rsrc->bmap, rsrc->max, 0, nrsrc, 0);177	if (start >= rsrc->max)178		return false;179 180	return true;181}182 183void rvu_free_rsrc(struct rsrc_bmap *rsrc, int id)184{185	if (!rsrc->bmap)186		return;187 188	__clear_bit(id, rsrc->bmap);189}190 191int rvu_rsrc_free_count(struct rsrc_bmap *rsrc)192{193	int used;194 195	if (!rsrc->bmap)196		return 0;197 198	used = bitmap_weight(rsrc->bmap, rsrc->max);199	return (rsrc->max - used);200}201 202bool is_rsrc_free(struct rsrc_bmap *rsrc, int id)203{204	if (!rsrc->bmap)205		return false;206 207	return !test_bit(id, rsrc->bmap);208}209 210int rvu_alloc_bitmap(struct rsrc_bmap *rsrc)211{212	rsrc->bmap = kcalloc(BITS_TO_LONGS(rsrc->max),213			     sizeof(long), GFP_KERNEL);214	if (!rsrc->bmap)215		return -ENOMEM;216	return 0;217}218 219void rvu_free_bitmap(struct rsrc_bmap *rsrc)220{221	kfree(rsrc->bmap);222}223 224/* Get block LF's HW index from a PF_FUNC's block slot number */225int rvu_get_lf(struct rvu *rvu, struct rvu_block *block, u16 pcifunc, u16 slot)226{227	u16 match = 0;228	int lf;229 230	mutex_lock(&rvu->rsrc_lock);231	for (lf = 0; lf < block->lf.max; lf++) {232		if (block->fn_map[lf] == pcifunc) {233			if (slot == match) {234				mutex_unlock(&rvu->rsrc_lock);235				return lf;236			}237			match++;238		}239	}240	mutex_unlock(&rvu->rsrc_lock);241	return -ENODEV;242}243 244/* Convert BLOCK_TYPE_E to a BLOCK_ADDR_E.245 * Some silicon variants of OcteonTX2 supports246 * multiple blocks of same type.247 *248 * @pcifunc has to be zero when no LF is yet attached.249 *250 * For a pcifunc if LFs are attached from multiple blocks of same type, then251 * return blkaddr of first encountered block.252 */253int rvu_get_blkaddr(struct rvu *rvu, int blktype, u16 pcifunc)254{255	int devnum, blkaddr = -ENODEV;256	u64 cfg, reg;257	bool is_pf;258 259	switch (blktype) {260	case BLKTYPE_NPC:261		blkaddr = BLKADDR_NPC;262		goto exit;263	case BLKTYPE_NPA:264		blkaddr = BLKADDR_NPA;265		goto exit;266	case BLKTYPE_NIX:267		/* For now assume NIX0 */268		if (!pcifunc) {269			blkaddr = BLKADDR_NIX0;270			goto exit;271		}272		break;273	case BLKTYPE_SSO:274		blkaddr = BLKADDR_SSO;275		goto exit;276	case BLKTYPE_SSOW:277		blkaddr = BLKADDR_SSOW;278		goto exit;279	case BLKTYPE_TIM:280		blkaddr = BLKADDR_TIM;281		goto exit;282	case BLKTYPE_CPT:283		/* For now assume CPT0 */284		if (!pcifunc) {285			blkaddr = BLKADDR_CPT0;286			goto exit;287		}288		break;289	}290 291	/* Check if this is a RVU PF or VF */292	if (pcifunc & RVU_PFVF_FUNC_MASK) {293		is_pf = false;294		devnum = rvu_get_hwvf(rvu, pcifunc);295	} else {296		is_pf = true;297		devnum = rvu_get_pf(pcifunc);298	}299 300	/* Check if the 'pcifunc' has a NIX LF from 'BLKADDR_NIX0' or301	 * 'BLKADDR_NIX1'.302	 */303	if (blktype == BLKTYPE_NIX) {304		reg = is_pf ? RVU_PRIV_PFX_NIXX_CFG(0) :305			RVU_PRIV_HWVFX_NIXX_CFG(0);306		cfg = rvu_read64(rvu, BLKADDR_RVUM, reg | (devnum << 16));307		if (cfg) {308			blkaddr = BLKADDR_NIX0;309			goto exit;310		}311 312		reg = is_pf ? RVU_PRIV_PFX_NIXX_CFG(1) :313			RVU_PRIV_HWVFX_NIXX_CFG(1);314		cfg = rvu_read64(rvu, BLKADDR_RVUM, reg | (devnum << 16));315		if (cfg)316			blkaddr = BLKADDR_NIX1;317	}318 319	if (blktype == BLKTYPE_CPT) {320		reg = is_pf ? RVU_PRIV_PFX_CPTX_CFG(0) :321			RVU_PRIV_HWVFX_CPTX_CFG(0);322		cfg = rvu_read64(rvu, BLKADDR_RVUM, reg | (devnum << 16));323		if (cfg) {324			blkaddr = BLKADDR_CPT0;325			goto exit;326		}327 328		reg = is_pf ? RVU_PRIV_PFX_CPTX_CFG(1) :329			RVU_PRIV_HWVFX_CPTX_CFG(1);330		cfg = rvu_read64(rvu, BLKADDR_RVUM, reg | (devnum << 16));331		if (cfg)332			blkaddr = BLKADDR_CPT1;333	}334 335exit:336	if (is_block_implemented(rvu->hw, blkaddr))337		return blkaddr;338	return -ENODEV;339}340 341static void rvu_update_rsrc_map(struct rvu *rvu, struct rvu_pfvf *pfvf,342				struct rvu_block *block, u16 pcifunc,343				u16 lf, bool attach)344{345	int devnum, num_lfs = 0;346	bool is_pf;347	u64 reg;348 349	if (lf >= block->lf.max) {350		dev_err(&rvu->pdev->dev,351			"%s: FATAL: LF %d is >= %s's max lfs i.e %d\n",352			__func__, lf, block->name, block->lf.max);353		return;354	}355 356	/* Check if this is for a RVU PF or VF */357	if (pcifunc & RVU_PFVF_FUNC_MASK) {358		is_pf = false;359		devnum = rvu_get_hwvf(rvu, pcifunc);360	} else {361		is_pf = true;362		devnum = rvu_get_pf(pcifunc);363	}364 365	block->fn_map[lf] = attach ? pcifunc : 0;366 367	switch (block->addr) {368	case BLKADDR_NPA:369		pfvf->npalf = attach ? true : false;370		num_lfs = pfvf->npalf;371		break;372	case BLKADDR_NIX0:373	case BLKADDR_NIX1:374		pfvf->nixlf = attach ? true : false;375		num_lfs = pfvf->nixlf;376		break;377	case BLKADDR_SSO:378		attach ? pfvf->sso++ : pfvf->sso--;379		num_lfs = pfvf->sso;380		break;381	case BLKADDR_SSOW:382		attach ? pfvf->ssow++ : pfvf->ssow--;383		num_lfs = pfvf->ssow;384		break;385	case BLKADDR_TIM:386		attach ? pfvf->timlfs++ : pfvf->timlfs--;387		num_lfs = pfvf->timlfs;388		break;389	case BLKADDR_CPT0:390		attach ? pfvf->cptlfs++ : pfvf->cptlfs--;391		num_lfs = pfvf->cptlfs;392		break;393	case BLKADDR_CPT1:394		attach ? pfvf->cpt1_lfs++ : pfvf->cpt1_lfs--;395		num_lfs = pfvf->cpt1_lfs;396		break;397	}398 399	reg = is_pf ? block->pf_lfcnt_reg : block->vf_lfcnt_reg;400	rvu_write64(rvu, BLKADDR_RVUM, reg | (devnum << 16), num_lfs);401}402 403inline int rvu_get_pf(u16 pcifunc)404{405	return (pcifunc >> RVU_PFVF_PF_SHIFT) & RVU_PFVF_PF_MASK;406}407 408void rvu_get_pf_numvfs(struct rvu *rvu, int pf, int *numvfs, int *hwvf)409{410	u64 cfg;411 412	/* Get numVFs attached to this PF and first HWVF */413	cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));414	if (numvfs)415		*numvfs = (cfg >> 12) & 0xFF;416	if (hwvf)417		*hwvf = cfg & 0xFFF;418}419 420int rvu_get_hwvf(struct rvu *rvu, int pcifunc)421{422	int pf, func;423	u64 cfg;424 425	pf = rvu_get_pf(pcifunc);426	func = pcifunc & RVU_PFVF_FUNC_MASK;427 428	/* Get first HWVF attached to this PF */429	cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));430 431	return ((cfg & 0xFFF) + func - 1);432}433 434struct rvu_pfvf *rvu_get_pfvf(struct rvu *rvu, int pcifunc)435{436	/* Check if it is a PF or VF */437	if (pcifunc & RVU_PFVF_FUNC_MASK)438		return &rvu->hwvf[rvu_get_hwvf(rvu, pcifunc)];439	else440		return &rvu->pf[rvu_get_pf(pcifunc)];441}442 443static bool is_pf_func_valid(struct rvu *rvu, u16 pcifunc)444{445	int pf, vf, nvfs;446	u64 cfg;447 448	pf = rvu_get_pf(pcifunc);449	if (pf >= rvu->hw->total_pfs)450		return false;451 452	if (!(pcifunc & RVU_PFVF_FUNC_MASK))453		return true;454 455	/* Check if VF is within number of VFs attached to this PF */456	vf = (pcifunc & RVU_PFVF_FUNC_MASK) - 1;457	cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));458	nvfs = (cfg >> 12) & 0xFF;459	if (vf >= nvfs)460		return false;461 462	return true;463}464 465bool is_block_implemented(struct rvu_hwinfo *hw, int blkaddr)466{467	struct rvu_block *block;468 469	if (blkaddr < BLKADDR_RVUM || blkaddr >= BLK_COUNT)470		return false;471 472	block = &hw->block[blkaddr];473	return block->implemented;474}475 476static void rvu_check_block_implemented(struct rvu *rvu)477{478	struct rvu_hwinfo *hw = rvu->hw;479	struct rvu_block *block;480	int blkid;481	u64 cfg;482 483	/* For each block check if 'implemented' bit is set */484	for (blkid = 0; blkid < BLK_COUNT; blkid++) {485		block = &hw->block[blkid];486		cfg = rvupf_read64(rvu, RVU_PF_BLOCK_ADDRX_DISC(blkid));487		if (cfg & BIT_ULL(11))488			block->implemented = true;489	}490}491 492static void rvu_setup_rvum_blk_revid(struct rvu *rvu)493{494	rvu_write64(rvu, BLKADDR_RVUM,495		    RVU_PRIV_BLOCK_TYPEX_REV(BLKTYPE_RVUM),496		    RVU_BLK_RVUM_REVID);497}498 499static void rvu_clear_rvum_blk_revid(struct rvu *rvu)500{501	rvu_write64(rvu, BLKADDR_RVUM,502		    RVU_PRIV_BLOCK_TYPEX_REV(BLKTYPE_RVUM), 0x00);503}504 505int rvu_lf_reset(struct rvu *rvu, struct rvu_block *block, int lf)506{507	int err;508 509	if (!block->implemented)510		return 0;511 512	rvu_write64(rvu, block->addr, block->lfreset_reg, lf | BIT_ULL(12));513	err = rvu_poll_reg(rvu, block->addr, block->lfreset_reg, BIT_ULL(12),514			   true);515	return err;516}517 518static void rvu_block_reset(struct rvu *rvu, int blkaddr, u64 rst_reg)519{520	struct rvu_block *block = &rvu->hw->block[blkaddr];521	int err;522 523	if (!block->implemented)524		return;525 526	rvu_write64(rvu, blkaddr, rst_reg, BIT_ULL(0));527	err = rvu_poll_reg(rvu, blkaddr, rst_reg, BIT_ULL(63), true);528	if (err) {529		dev_err(rvu->dev, "HW block:%d reset timeout retrying again\n", blkaddr);530		while (rvu_poll_reg(rvu, blkaddr, rst_reg, BIT_ULL(63), true) == -EBUSY)531			;532	}533}534 535static void rvu_reset_all_blocks(struct rvu *rvu)536{537	/* Do a HW reset of all RVU blocks */538	rvu_block_reset(rvu, BLKADDR_NPA, NPA_AF_BLK_RST);539	rvu_block_reset(rvu, BLKADDR_NIX0, NIX_AF_BLK_RST);540	rvu_block_reset(rvu, BLKADDR_NIX1, NIX_AF_BLK_RST);541	rvu_block_reset(rvu, BLKADDR_NPC, NPC_AF_BLK_RST);542	rvu_block_reset(rvu, BLKADDR_SSO, SSO_AF_BLK_RST);543	rvu_block_reset(rvu, BLKADDR_TIM, TIM_AF_BLK_RST);544	rvu_block_reset(rvu, BLKADDR_CPT0, CPT_AF_BLK_RST);545	rvu_block_reset(rvu, BLKADDR_CPT1, CPT_AF_BLK_RST);546	rvu_block_reset(rvu, BLKADDR_NDC_NIX0_RX, NDC_AF_BLK_RST);547	rvu_block_reset(rvu, BLKADDR_NDC_NIX0_TX, NDC_AF_BLK_RST);548	rvu_block_reset(rvu, BLKADDR_NDC_NIX1_RX, NDC_AF_BLK_RST);549	rvu_block_reset(rvu, BLKADDR_NDC_NIX1_TX, NDC_AF_BLK_RST);550	rvu_block_reset(rvu, BLKADDR_NDC_NPA0, NDC_AF_BLK_RST);551}552 553static void rvu_scan_block(struct rvu *rvu, struct rvu_block *block)554{555	struct rvu_pfvf *pfvf;556	u64 cfg;557	int lf;558 559	for (lf = 0; lf < block->lf.max; lf++) {560		cfg = rvu_read64(rvu, block->addr,561				 block->lfcfg_reg | (lf << block->lfshift));562		if (!(cfg & BIT_ULL(63)))563			continue;564 565		/* Set this resource as being used */566		__set_bit(lf, block->lf.bmap);567 568		/* Get, to whom this LF is attached */569		pfvf = rvu_get_pfvf(rvu, (cfg >> 8) & 0xFFFF);570		rvu_update_rsrc_map(rvu, pfvf, block,571				    (cfg >> 8) & 0xFFFF, lf, true);572 573		/* Set start MSIX vector for this LF within this PF/VF */574		rvu_set_msix_offset(rvu, pfvf, block, lf);575	}576}577 578static void rvu_check_min_msix_vec(struct rvu *rvu, int nvecs, int pf, int vf)579{580	int min_vecs;581 582	if (!vf)583		goto check_pf;584 585	if (!nvecs) {586		dev_warn(rvu->dev,587			 "PF%d:VF%d is configured with zero msix vectors, %d\n",588			 pf, vf - 1, nvecs);589	}590	return;591 592check_pf:593	if (pf == 0)594		min_vecs = RVU_AF_INT_VEC_CNT + RVU_PF_INT_VEC_CNT;595	else596		min_vecs = RVU_PF_INT_VEC_CNT;597 598	if (!(nvecs < min_vecs))599		return;600	dev_warn(rvu->dev,601		 "PF%d is configured with too few vectors, %d, min is %d\n",602		 pf, nvecs, min_vecs);603}604 605static int rvu_setup_msix_resources(struct rvu *rvu)606{607	struct rvu_hwinfo *hw = rvu->hw;608	int pf, vf, numvfs, hwvf, err;609	int nvecs, offset, max_msix;610	struct rvu_pfvf *pfvf;611	u64 cfg, phy_addr;612	dma_addr_t iova;613 614	for (pf = 0; pf < hw->total_pfs; pf++) {615		cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));616		/* If PF is not enabled, nothing to do */617		if (!((cfg >> 20) & 0x01))618			continue;619 620		rvu_get_pf_numvfs(rvu, pf, &numvfs, &hwvf);621 622		pfvf = &rvu->pf[pf];623		/* Get num of MSIX vectors attached to this PF */624		cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_MSIX_CFG(pf));625		pfvf->msix.max = ((cfg >> 32) & 0xFFF) + 1;626		rvu_check_min_msix_vec(rvu, pfvf->msix.max, pf, 0);627 628		/* Alloc msix bitmap for this PF */629		err = rvu_alloc_bitmap(&pfvf->msix);630		if (err)631			return err;632 633		/* Allocate memory for MSIX vector to RVU block LF mapping */634		pfvf->msix_lfmap = devm_kcalloc(rvu->dev, pfvf->msix.max,635						sizeof(u16), GFP_KERNEL);636		if (!pfvf->msix_lfmap)637			return -ENOMEM;638 639		/* For PF0 (AF) firmware will set msix vector offsets for640		 * AF, block AF and PF0_INT vectors, so jump to VFs.641		 */642		if (!pf)643			goto setup_vfmsix;644 645		/* Set MSIX offset for PF's 'RVU_PF_INT_VEC' vectors.646		 * These are allocated on driver init and never freed,647		 * so no need to set 'msix_lfmap' for these.648		 */649		cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_INT_CFG(pf));650		nvecs = (cfg >> 12) & 0xFF;651		cfg &= ~0x7FFULL;652		offset = rvu_alloc_rsrc_contig(&pfvf->msix, nvecs);653		rvu_write64(rvu, BLKADDR_RVUM,654			    RVU_PRIV_PFX_INT_CFG(pf), cfg | offset);655setup_vfmsix:656		/* Alloc msix bitmap for VFs */657		for (vf = 0; vf < numvfs; vf++) {658			pfvf =  &rvu->hwvf[hwvf + vf];659			/* Get num of MSIX vectors attached to this VF */660			cfg = rvu_read64(rvu, BLKADDR_RVUM,661					 RVU_PRIV_PFX_MSIX_CFG(pf));662			pfvf->msix.max = (cfg & 0xFFF) + 1;663			rvu_check_min_msix_vec(rvu, pfvf->msix.max, pf, vf + 1);664 665			/* Alloc msix bitmap for this VF */666			err = rvu_alloc_bitmap(&pfvf->msix);667			if (err)668				return err;669 670			pfvf->msix_lfmap =671				devm_kcalloc(rvu->dev, pfvf->msix.max,672					     sizeof(u16), GFP_KERNEL);673			if (!pfvf->msix_lfmap)674				return -ENOMEM;675 676			/* Set MSIX offset for HWVF's 'RVU_VF_INT_VEC' vectors.677			 * These are allocated on driver init and never freed,678			 * so no need to set 'msix_lfmap' for these.679			 */680			cfg = rvu_read64(rvu, BLKADDR_RVUM,681					 RVU_PRIV_HWVFX_INT_CFG(hwvf + vf));682			nvecs = (cfg >> 12) & 0xFF;683			cfg &= ~0x7FFULL;684			offset = rvu_alloc_rsrc_contig(&pfvf->msix, nvecs);685			rvu_write64(rvu, BLKADDR_RVUM,686				    RVU_PRIV_HWVFX_INT_CFG(hwvf + vf),687				    cfg | offset);688		}689	}690 691	/* HW interprets RVU_AF_MSIXTR_BASE address as an IOVA, hence692	 * create an IOMMU mapping for the physical address configured by693	 * firmware and reconfig RVU_AF_MSIXTR_BASE with IOVA.694	 */695	cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_CONST);696	max_msix = cfg & 0xFFFFF;697	if (rvu->fwdata && rvu->fwdata->msixtr_base)698		phy_addr = rvu->fwdata->msixtr_base;699	else700		phy_addr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_MSIXTR_BASE);701 702	iova = dma_map_resource(rvu->dev, phy_addr,703				max_msix * PCI_MSIX_ENTRY_SIZE,704				DMA_BIDIRECTIONAL, 0);705 706	if (dma_mapping_error(rvu->dev, iova))707		return -ENOMEM;708 709	rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_MSIXTR_BASE, (u64)iova);710	rvu->msix_base_iova = iova;711	rvu->msixtr_base_phy = phy_addr;712 713	return 0;714}715 716static void rvu_reset_msix(struct rvu *rvu)717{718	/* Restore msixtr base register */719	rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_MSIXTR_BASE,720		    rvu->msixtr_base_phy);721}722 723static void rvu_free_hw_resources(struct rvu *rvu)724{725	struct rvu_hwinfo *hw = rvu->hw;726	struct rvu_block *block;727	struct rvu_pfvf  *pfvf;728	int id, max_msix;729	u64 cfg;730 731	rvu_npa_freemem(rvu);732	rvu_npc_freemem(rvu);733	rvu_nix_freemem(rvu);734 735	/* Free block LF bitmaps */736	for (id = 0; id < BLK_COUNT; id++) {737		block = &hw->block[id];738		kfree(block->lf.bmap);739	}740 741	/* Free MSIX bitmaps */742	for (id = 0; id < hw->total_pfs; id++) {743		pfvf = &rvu->pf[id];744		kfree(pfvf->msix.bmap);745	}746 747	for (id = 0; id < hw->total_vfs; id++) {748		pfvf = &rvu->hwvf[id];749		kfree(pfvf->msix.bmap);750	}751 752	/* Unmap MSIX vector base IOVA mapping */753	if (!rvu->msix_base_iova)754		return;755	cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_CONST);756	max_msix = cfg & 0xFFFFF;757	dma_unmap_resource(rvu->dev, rvu->msix_base_iova,758			   max_msix * PCI_MSIX_ENTRY_SIZE,759			   DMA_BIDIRECTIONAL, 0);760 761	rvu_reset_msix(rvu);762	mutex_destroy(&rvu->rsrc_lock);763}764 765static void rvu_setup_pfvf_macaddress(struct rvu *rvu)766{767	struct rvu_hwinfo *hw = rvu->hw;768	int pf, vf, numvfs, hwvf;769	struct rvu_pfvf *pfvf;770	u64 *mac;771 772	for (pf = 0; pf < hw->total_pfs; pf++) {773		/* For PF0(AF), Assign MAC address to only VFs (LBKVFs) */774		if (!pf)775			goto lbkvf;776 777		if (!is_pf_cgxmapped(rvu, pf))778			continue;779		/* Assign MAC address to PF */780		pfvf = &rvu->pf[pf];781		if (rvu->fwdata && pf < PF_MACNUM_MAX) {782			mac = &rvu->fwdata->pf_macs[pf];783			if (*mac)784				u64_to_ether_addr(*mac, pfvf->mac_addr);785			else786				eth_random_addr(pfvf->mac_addr);787		} else {788			eth_random_addr(pfvf->mac_addr);789		}790		ether_addr_copy(pfvf->default_mac, pfvf->mac_addr);791 792lbkvf:793		/* Assign MAC address to VFs*/794		rvu_get_pf_numvfs(rvu, pf, &numvfs, &hwvf);795		for (vf = 0; vf < numvfs; vf++, hwvf++) {796			pfvf = &rvu->hwvf[hwvf];797			if (rvu->fwdata && hwvf < VF_MACNUM_MAX) {798				mac = &rvu->fwdata->vf_macs[hwvf];799				if (*mac)800					u64_to_ether_addr(*mac, pfvf->mac_addr);801				else802					eth_random_addr(pfvf->mac_addr);803			} else {804				eth_random_addr(pfvf->mac_addr);805			}806			ether_addr_copy(pfvf->default_mac, pfvf->mac_addr);807		}808	}809}810 811static int rvu_fwdata_init(struct rvu *rvu)812{813	u64 fwdbase;814	int err;815 816	/* Get firmware data base address */817	err = cgx_get_fwdata_base(&fwdbase);818	if (err)819		goto fail;820 821	BUILD_BUG_ON(offsetof(struct rvu_fwdata, cgx_fw_data) > FWDATA_CGX_LMAC_OFFSET);822	rvu->fwdata = ioremap_wc(fwdbase, sizeof(struct rvu_fwdata));823	if (!rvu->fwdata)824		goto fail;825	if (!is_rvu_fwdata_valid(rvu)) {826		dev_err(rvu->dev,827			"Mismatch in 'fwdata' struct btw kernel and firmware\n");828		iounmap(rvu->fwdata);829		rvu->fwdata = NULL;830		return -EINVAL;831	}832	return 0;833fail:834	dev_info(rvu->dev, "Unable to fetch 'fwdata' from firmware\n");835	return -EIO;836}837 838static void rvu_fwdata_exit(struct rvu *rvu)839{840	if (rvu->fwdata)841		iounmap(rvu->fwdata);842}843 844static int rvu_setup_nix_hw_resource(struct rvu *rvu, int blkaddr)845{846	struct rvu_hwinfo *hw = rvu->hw;847	struct rvu_block *block;848	int blkid;849	u64 cfg;850 851	/* Init NIX LF's bitmap */852	block = &hw->block[blkaddr];853	if (!block->implemented)854		return 0;855	blkid = (blkaddr == BLKADDR_NIX0) ? 0 : 1;856	cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2);857	block->lf.max = cfg & 0xFFF;858	block->addr = blkaddr;859	block->type = BLKTYPE_NIX;860	block->lfshift = 8;861	block->lookup_reg = NIX_AF_RVU_LF_CFG_DEBUG;862	block->pf_lfcnt_reg = RVU_PRIV_PFX_NIXX_CFG(blkid);863	block->vf_lfcnt_reg = RVU_PRIV_HWVFX_NIXX_CFG(blkid);864	block->lfcfg_reg = NIX_PRIV_LFX_CFG;865	block->msixcfg_reg = NIX_PRIV_LFX_INT_CFG;866	block->lfreset_reg = NIX_AF_LF_RST;867	block->rvu = rvu;868	sprintf(block->name, "NIX%d", blkid);869	rvu->nix_blkaddr[blkid] = blkaddr;870	return rvu_alloc_bitmap(&block->lf);871}872 873static int rvu_setup_cpt_hw_resource(struct rvu *rvu, int blkaddr)874{875	struct rvu_hwinfo *hw = rvu->hw;876	struct rvu_block *block;877	int blkid;878	u64 cfg;879 880	/* Init CPT LF's bitmap */881	block = &hw->block[blkaddr];882	if (!block->implemented)883		return 0;884	blkid = (blkaddr == BLKADDR_CPT0) ? 0 : 1;885	cfg = rvu_read64(rvu, blkaddr, CPT_AF_CONSTANTS0);886	block->lf.max = cfg & 0xFF;887	block->addr = blkaddr;888	block->type = BLKTYPE_CPT;889	block->multislot = true;890	block->lfshift = 3;891	block->lookup_reg = CPT_AF_RVU_LF_CFG_DEBUG;892	block->pf_lfcnt_reg = RVU_PRIV_PFX_CPTX_CFG(blkid);893	block->vf_lfcnt_reg = RVU_PRIV_HWVFX_CPTX_CFG(blkid);894	block->lfcfg_reg = CPT_PRIV_LFX_CFG;895	block->msixcfg_reg = CPT_PRIV_LFX_INT_CFG;896	block->lfreset_reg = CPT_AF_LF_RST;897	block->rvu = rvu;898	sprintf(block->name, "CPT%d", blkid);899	return rvu_alloc_bitmap(&block->lf);900}901 902static void rvu_get_lbk_bufsize(struct rvu *rvu)903{904	struct pci_dev *pdev = NULL;905	void __iomem *base;906	u64 lbk_const;907 908	pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,909			      PCI_DEVID_OCTEONTX2_LBK, pdev);910	if (!pdev)911		return;912 913	base = pci_ioremap_bar(pdev, 0);914	if (!base)915		goto err_put;916 917	lbk_const = readq(base + LBK_CONST);918 919	/* cache fifo size */920	rvu->hw->lbk_bufsize = FIELD_GET(LBK_CONST_BUF_SIZE, lbk_const);921 922	iounmap(base);923err_put:924	pci_dev_put(pdev);925}926 927static int rvu_setup_hw_resources(struct rvu *rvu)928{929	struct rvu_hwinfo *hw = rvu->hw;930	struct rvu_block *block;931	int blkid, err;932	u64 cfg;933 934	/* Get HW supported max RVU PF & VF count */935	cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_CONST);936	hw->total_pfs = (cfg >> 32) & 0xFF;937	hw->total_vfs = (cfg >> 20) & 0xFFF;938	hw->max_vfs_per_pf = (cfg >> 40) & 0xFF;939 940	if (!is_rvu_otx2(rvu))941		rvu_apr_block_cn10k_init(rvu);942 943	/* Init NPA LF's bitmap */944	block = &hw->block[BLKADDR_NPA];945	if (!block->implemented)946		goto nix;947	cfg = rvu_read64(rvu, BLKADDR_NPA, NPA_AF_CONST);948	block->lf.max = (cfg >> 16) & 0xFFF;949	block->addr = BLKADDR_NPA;950	block->type = BLKTYPE_NPA;951	block->lfshift = 8;952	block->lookup_reg = NPA_AF_RVU_LF_CFG_DEBUG;953	block->pf_lfcnt_reg = RVU_PRIV_PFX_NPA_CFG;954	block->vf_lfcnt_reg = RVU_PRIV_HWVFX_NPA_CFG;955	block->lfcfg_reg = NPA_PRIV_LFX_CFG;956	block->msixcfg_reg = NPA_PRIV_LFX_INT_CFG;957	block->lfreset_reg = NPA_AF_LF_RST;958	block->rvu = rvu;959	sprintf(block->name, "NPA");960	err = rvu_alloc_bitmap(&block->lf);961	if (err) {962		dev_err(rvu->dev,963			"%s: Failed to allocate NPA LF bitmap\n", __func__);964		return err;965	}966 967nix:968	err = rvu_setup_nix_hw_resource(rvu, BLKADDR_NIX0);969	if (err) {970		dev_err(rvu->dev,971			"%s: Failed to allocate NIX0 LFs bitmap\n", __func__);972		return err;973	}974 975	err = rvu_setup_nix_hw_resource(rvu, BLKADDR_NIX1);976	if (err) {977		dev_err(rvu->dev,978			"%s: Failed to allocate NIX1 LFs bitmap\n", __func__);979		return err;980	}981 982	/* Init SSO group's bitmap */983	block = &hw->block[BLKADDR_SSO];984	if (!block->implemented)985		goto ssow;986	cfg = rvu_read64(rvu, BLKADDR_SSO, SSO_AF_CONST);987	block->lf.max = cfg & 0xFFFF;988	block->addr = BLKADDR_SSO;989	block->type = BLKTYPE_SSO;990	block->multislot = true;991	block->lfshift = 3;992	block->lookup_reg = SSO_AF_RVU_LF_CFG_DEBUG;993	block->pf_lfcnt_reg = RVU_PRIV_PFX_SSO_CFG;994	block->vf_lfcnt_reg = RVU_PRIV_HWVFX_SSO_CFG;995	block->lfcfg_reg = SSO_PRIV_LFX_HWGRP_CFG;996	block->msixcfg_reg = SSO_PRIV_LFX_HWGRP_INT_CFG;997	block->lfreset_reg = SSO_AF_LF_HWGRP_RST;998	block->rvu = rvu;999	sprintf(block->name, "SSO GROUP");1000	err = rvu_alloc_bitmap(&block->lf);1001	if (err) {1002		dev_err(rvu->dev,1003			"%s: Failed to allocate SSO LF bitmap\n", __func__);1004		return err;1005	}1006 1007ssow:1008	/* Init SSO workslot's bitmap */1009	block = &hw->block[BLKADDR_SSOW];1010	if (!block->implemented)1011		goto tim;1012	block->lf.max = (cfg >> 56) & 0xFF;1013	block->addr = BLKADDR_SSOW;1014	block->type = BLKTYPE_SSOW;1015	block->multislot = true;1016	block->lfshift = 3;1017	block->lookup_reg = SSOW_AF_RVU_LF_HWS_CFG_DEBUG;1018	block->pf_lfcnt_reg = RVU_PRIV_PFX_SSOW_CFG;1019	block->vf_lfcnt_reg = RVU_PRIV_HWVFX_SSOW_CFG;1020	block->lfcfg_reg = SSOW_PRIV_LFX_HWS_CFG;1021	block->msixcfg_reg = SSOW_PRIV_LFX_HWS_INT_CFG;1022	block->lfreset_reg = SSOW_AF_LF_HWS_RST;1023	block->rvu = rvu;1024	sprintf(block->name, "SSOWS");1025	err = rvu_alloc_bitmap(&block->lf);1026	if (err) {1027		dev_err(rvu->dev,1028			"%s: Failed to allocate SSOW LF bitmap\n", __func__);1029		return err;1030	}1031 1032tim:1033	/* Init TIM LF's bitmap */1034	block = &hw->block[BLKADDR_TIM];1035	if (!block->implemented)1036		goto cpt;1037	cfg = rvu_read64(rvu, BLKADDR_TIM, TIM_AF_CONST);1038	block->lf.max = cfg & 0xFFFF;1039	block->addr = BLKADDR_TIM;1040	block->type = BLKTYPE_TIM;1041	block->multislot = true;1042	block->lfshift = 3;1043	block->lookup_reg = TIM_AF_RVU_LF_CFG_DEBUG;1044	block->pf_lfcnt_reg = RVU_PRIV_PFX_TIM_CFG;1045	block->vf_lfcnt_reg = RVU_PRIV_HWVFX_TIM_CFG;1046	block->lfcfg_reg = TIM_PRIV_LFX_CFG;1047	block->msixcfg_reg = TIM_PRIV_LFX_INT_CFG;1048	block->lfreset_reg = TIM_AF_LF_RST;1049	block->rvu = rvu;1050	sprintf(block->name, "TIM");1051	err = rvu_alloc_bitmap(&block->lf);1052	if (err) {1053		dev_err(rvu->dev,1054			"%s: Failed to allocate TIM LF bitmap\n", __func__);1055		return err;1056	}1057 1058cpt:1059	err = rvu_setup_cpt_hw_resource(rvu, BLKADDR_CPT0);1060	if (err) {1061		dev_err(rvu->dev,1062			"%s: Failed to allocate CPT0 LF bitmap\n", __func__);1063		return err;1064	}1065	err = rvu_setup_cpt_hw_resource(rvu, BLKADDR_CPT1);1066	if (err) {1067		dev_err(rvu->dev,1068			"%s: Failed to allocate CPT1 LF bitmap\n", __func__);1069		return err;1070	}1071 1072	/* Allocate memory for PFVF data */1073	rvu->pf = devm_kcalloc(rvu->dev, hw->total_pfs,1074			       sizeof(struct rvu_pfvf), GFP_KERNEL);1075	if (!rvu->pf) {1076		dev_err(rvu->dev,1077			"%s: Failed to allocate memory for PF's rvu_pfvf struct\n", __func__);1078		return -ENOMEM;1079	}1080 1081	rvu->hwvf = devm_kcalloc(rvu->dev, hw->total_vfs,1082				 sizeof(struct rvu_pfvf), GFP_KERNEL);1083	if (!rvu->hwvf) {1084		dev_err(rvu->dev,1085			"%s: Failed to allocate memory for VF's rvu_pfvf struct\n", __func__);1086		return -ENOMEM;1087	}1088 1089	mutex_init(&rvu->rsrc_lock);1090 1091	rvu_fwdata_init(rvu);1092 1093	err = rvu_setup_msix_resources(rvu);1094	if (err) {1095		dev_err(rvu->dev,1096			"%s: Failed to setup MSIX resources\n", __func__);1097		return err;1098	}1099 1100	for (blkid = 0; blkid < BLK_COUNT; blkid++) {1101		block = &hw->block[blkid];1102		if (!block->lf.bmap)1103			continue;1104 1105		/* Allocate memory for block LF/slot to pcifunc mapping info */1106		block->fn_map = devm_kcalloc(rvu->dev, block->lf.max,1107					     sizeof(u16), GFP_KERNEL);1108		if (!block->fn_map) {1109			err = -ENOMEM;1110			goto msix_err;1111		}1112 1113		/* Scan all blocks to check if low level firmware has1114		 * already provisioned any of the resources to a PF/VF.1115		 */1116		rvu_scan_block(rvu, block);1117	}1118 1119	err = rvu_set_channels_base(rvu);1120	if (err)1121		goto msix_err;1122 1123	err = rvu_npc_init(rvu);1124	if (err) {1125		dev_err(rvu->dev, "%s: Failed to initialize npc\n", __func__);1126		goto npc_err;1127	}1128 1129	err = rvu_cgx_init(rvu);1130	if (err) {1131		dev_err(rvu->dev, "%s: Failed to initialize cgx\n", __func__);1132		goto cgx_err;1133	}1134 1135	err = rvu_npc_exact_init(rvu);1136	if (err) {1137		dev_err(rvu->dev, "failed to initialize exact match table\n");1138		return err;1139	}1140 1141	/* Assign MACs for CGX mapped functions */1142	rvu_setup_pfvf_macaddress(rvu);1143 1144	err = rvu_npa_init(rvu);1145	if (err) {1146		dev_err(rvu->dev, "%s: Failed to initialize npa\n", __func__);1147		goto npa_err;1148	}1149 1150	rvu_get_lbk_bufsize(rvu);1151 1152	err = rvu_nix_init(rvu);1153	if (err) {1154		dev_err(rvu->dev, "%s: Failed to initialize nix\n", __func__);1155		goto nix_err;1156	}1157 1158	err = rvu_sdp_init(rvu);1159	if (err) {1160		dev_err(rvu->dev, "%s: Failed to initialize sdp\n", __func__);1161		goto nix_err;1162	}1163 1164	rvu_program_channels(rvu);1165 1166	err = rvu_mcs_init(rvu);1167	if (err) {1168		dev_err(rvu->dev, "%s: Failed to initialize mcs\n", __func__);1169		goto nix_err;1170	}1171 1172	err = rvu_cpt_init(rvu);1173	if (err) {1174		dev_err(rvu->dev, "%s: Failed to initialize cpt\n", __func__);1175		goto mcs_err;1176	}1177 1178	return 0;1179 1180mcs_err:1181	rvu_mcs_exit(rvu);1182nix_err:1183	rvu_nix_freemem(rvu);1184npa_err:1185	rvu_npa_freemem(rvu);1186cgx_err:1187	rvu_cgx_exit(rvu);1188npc_err:1189	rvu_npc_freemem(rvu);1190	rvu_fwdata_exit(rvu);1191msix_err:1192	rvu_reset_msix(rvu);1193	return err;1194}1195 1196/* NPA and NIX admin queue APIs */1197void rvu_aq_free(struct rvu *rvu, struct admin_queue *aq)1198{1199	if (!aq)1200		return;1201 1202	qmem_free(rvu->dev, aq->inst);1203	qmem_free(rvu->dev, aq->res);1204	devm_kfree(rvu->dev, aq);1205}1206 1207int rvu_aq_alloc(struct rvu *rvu, struct admin_queue **ad_queue,1208		 int qsize, int inst_size, int res_size)1209{1210	struct admin_queue *aq;1211	int err;1212 1213	*ad_queue = devm_kzalloc(rvu->dev, sizeof(*aq), GFP_KERNEL);1214	if (!*ad_queue)1215		return -ENOMEM;1216	aq = *ad_queue;1217 1218	/* Alloc memory for instructions i.e AQ */1219	err = qmem_alloc(rvu->dev, &aq->inst, qsize, inst_size);1220	if (err) {1221		devm_kfree(rvu->dev, aq);1222		return err;1223	}1224 1225	/* Alloc memory for results */1226	err = qmem_alloc(rvu->dev, &aq->res, qsize, res_size);1227	if (err) {1228		rvu_aq_free(rvu, aq);1229		return err;1230	}1231 1232	spin_lock_init(&aq->lock);1233	return 0;1234}1235 1236int rvu_mbox_handler_ready(struct rvu *rvu, struct msg_req *req,1237			   struct ready_msg_rsp *rsp)1238{1239	if (rvu->fwdata) {1240		rsp->rclk_freq = rvu->fwdata->rclk;1241		rsp->sclk_freq = rvu->fwdata->sclk;1242	}1243	return 0;1244}1245 1246/* Get current count of a RVU block's LF/slots1247 * provisioned to a given RVU func.1248 */1249u16 rvu_get_rsrc_mapcount(struct rvu_pfvf *pfvf, int blkaddr)1250{1251	switch (blkaddr) {1252	case BLKADDR_NPA:1253		return pfvf->npalf ? 1 : 0;1254	case BLKADDR_NIX0:1255	case BLKADDR_NIX1:1256		return pfvf->nixlf ? 1 : 0;1257	case BLKADDR_SSO:1258		return pfvf->sso;1259	case BLKADDR_SSOW:1260		return pfvf->ssow;1261	case BLKADDR_TIM:1262		return pfvf->timlfs;1263	case BLKADDR_CPT0:1264		return pfvf->cptlfs;1265	case BLKADDR_CPT1:1266		return pfvf->cpt1_lfs;1267	}1268	return 0;1269}1270 1271/* Return true if LFs of block type are attached to pcifunc */1272static bool is_blktype_attached(struct rvu_pfvf *pfvf, int blktype)1273{1274	switch (blktype) {1275	case BLKTYPE_NPA:1276		return pfvf->npalf ? 1 : 0;1277	case BLKTYPE_NIX:1278		return pfvf->nixlf ? 1 : 0;1279	case BLKTYPE_SSO:1280		return !!pfvf->sso;1281	case BLKTYPE_SSOW:1282		return !!pfvf->ssow;1283	case BLKTYPE_TIM:1284		return !!pfvf->timlfs;1285	case BLKTYPE_CPT:1286		return pfvf->cptlfs || pfvf->cpt1_lfs;1287	}1288 1289	return false;1290}1291 1292bool is_pffunc_map_valid(struct rvu *rvu, u16 pcifunc, int blktype)1293{1294	struct rvu_pfvf *pfvf;1295 1296	if (!is_pf_func_valid(rvu, pcifunc))1297		return false;1298 1299	pfvf = rvu_get_pfvf(rvu, pcifunc);1300 1301	/* Check if this PFFUNC has a LF of type blktype attached */1302	if (!is_blktype_attached(pfvf, blktype))1303		return false;1304 1305	return true;1306}1307 1308static int rvu_lookup_rsrc(struct rvu *rvu, struct rvu_block *block,1309			   int pcifunc, int slot)1310{1311	u64 val;1312 1313	val = ((u64)pcifunc << 24) | (slot << 16) | (1ULL << 13);1314	rvu_write64(rvu, block->addr, block->lookup_reg, val);1315	/* Wait for the lookup to finish */1316	/* TODO: put some timeout here */1317	while (rvu_read64(rvu, block->addr, block->lookup_reg) & (1ULL << 13))1318		;1319 1320	val = rvu_read64(rvu, block->addr, block->lookup_reg);1321 1322	/* Check LF valid bit */1323	if (!(val & (1ULL << 12)))1324		return -1;1325 1326	return (val & 0xFFF);1327}1328 1329int rvu_get_blkaddr_from_slot(struct rvu *rvu, int blktype, u16 pcifunc,1330			      u16 global_slot, u16 *slot_in_block)1331{1332	struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);1333	int numlfs, total_lfs = 0, nr_blocks = 0;1334	int i, num_blkaddr[BLK_COUNT] = { 0 };1335	struct rvu_block *block;1336	int blkaddr;1337	u16 start_slot;1338 1339	if (!is_blktype_attached(pfvf, blktype))1340		return -ENODEV;1341 1342	/* Get all the block addresses from which LFs are attached to1343	 * the given pcifunc in num_blkaddr[].1344	 */1345	for (blkaddr = BLKADDR_RVUM; blkaddr < BLK_COUNT; blkaddr++) {1346		block = &rvu->hw->block[blkaddr];1347		if (block->type != blktype)1348			continue;1349		if (!is_block_implemented(rvu->hw, blkaddr))1350			continue;1351 1352		numlfs = rvu_get_rsrc_mapcount(pfvf, blkaddr);1353		if (numlfs) {1354			total_lfs += numlfs;1355			num_blkaddr[nr_blocks] = blkaddr;1356			nr_blocks++;1357		}1358	}1359 1360	if (global_slot >= total_lfs)1361		return -ENODEV;1362 1363	/* Based on the given global slot number retrieve the1364	 * correct block address out of all attached block1365	 * addresses and slot number in that block.1366	 */1367	total_lfs = 0;1368	blkaddr = -ENODEV;1369	for (i = 0; i < nr_blocks; i++) {1370		numlfs = rvu_get_rsrc_mapcount(pfvf, num_blkaddr[i]);1371		total_lfs += numlfs;1372		if (global_slot < total_lfs) {1373			blkaddr = num_blkaddr[i];1374			start_slot = total_lfs - numlfs;1375			*slot_in_block = global_slot - start_slot;1376			break;1377		}1378	}1379 1380	return blkaddr;1381}1382 1383static void rvu_detach_block(struct rvu *rvu, int pcifunc, int blktype)1384{1385	struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);1386	struct rvu_hwinfo *hw = rvu->hw;1387	struct rvu_block *block;1388	int slot, lf, num_lfs;1389	int blkaddr;1390 1391	blkaddr = rvu_get_blkaddr(rvu, blktype, pcifunc);1392	if (blkaddr < 0)1393		return;1394 1395	if (blktype == BLKTYPE_NIX)1396		rvu_nix_reset_mac(pfvf, pcifunc);1397 1398	block = &hw->block[blkaddr];1399 1400	num_lfs = rvu_get_rsrc_mapcount(pfvf, block->addr);1401	if (!num_lfs)1402		return;1403 1404	for (slot = 0; slot < num_lfs; slot++) {1405		lf = rvu_lookup_rsrc(rvu, block, pcifunc, slot);1406		if (lf < 0) /* This should never happen */1407			continue;1408 1409		/* Disable the LF */1410		rvu_write64(rvu, blkaddr, block->lfcfg_reg |1411			    (lf << block->lfshift), 0x00ULL);1412 1413		/* Update SW maintained mapping info as well */1414		rvu_update_rsrc_map(rvu, pfvf, block,1415				    pcifunc, lf, false);1416 1417		/* Free the resource */1418		rvu_free_rsrc(&block->lf, lf);1419 1420		/* Clear MSIX vector offset for this LF */1421		rvu_clear_msix_offset(rvu, pfvf, block, lf);1422	}1423}1424 1425static int rvu_detach_rsrcs(struct rvu *rvu, struct rsrc_detach *detach,1426			    u16 pcifunc)1427{1428	struct rvu_hwinfo *hw = rvu->hw;1429	bool detach_all = true;1430	struct rvu_block *block;1431	int blkid;1432 1433	mutex_lock(&rvu->rsrc_lock);1434 1435	/* Check for partial resource detach */1436	if (detach && detach->partial)1437		detach_all = false;1438 1439	/* Check for RVU block's LFs attached to this func,1440	 * if so, detach them.1441	 */1442	for (blkid = 0; blkid < BLK_COUNT; blkid++) {1443		block = &hw->block[blkid];1444		if (!block->lf.bmap)1445			continue;1446		if (!detach_all && detach) {1447			if (blkid == BLKADDR_NPA && !detach->npalf)1448				continue;1449			else if ((blkid == BLKADDR_NIX0) && !detach->nixlf)1450				continue;1451			else if ((blkid == BLKADDR_NIX1) && !detach->nixlf)1452				continue;1453			else if ((blkid == BLKADDR_SSO) && !detach->sso)1454				continue;1455			else if ((blkid == BLKADDR_SSOW) && !detach->ssow)1456				continue;1457			else if ((blkid == BLKADDR_TIM) && !detach->timlfs)1458				continue;1459			else if ((blkid == BLKADDR_CPT0) && !detach->cptlfs)1460				continue;1461			else if ((blkid == BLKADDR_CPT1) && !detach->cptlfs)1462				continue;1463		}1464		rvu_detach_block(rvu, pcifunc, block->type);1465	}1466 1467	mutex_unlock(&rvu->rsrc_lock);1468	return 0;1469}1470 1471int rvu_mbox_handler_detach_resources(struct rvu *rvu,1472				      struct rsrc_detach *detach,1473				      struct msg_rsp *rsp)1474{1475	return rvu_detach_rsrcs(rvu, detach, detach->hdr.pcifunc);1476}1477 1478int rvu_get_nix_blkaddr(struct rvu *rvu, u16 pcifunc)1479{1480	struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);1481	int blkaddr = BLKADDR_NIX0, vf;1482	struct rvu_pfvf *pf;1483 1484	pf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK);1485 1486	/* All CGX mapped PFs are set with assigned NIX block during init */1487	if (is_pf_cgxmapped(rvu, rvu_get_pf(pcifunc))) {1488		blkaddr = pf->nix_blkaddr;1489	} else if (is_lbk_vf(rvu, pcifunc)) {1490		vf = pcifunc - 1;1491		/* Assign NIX based on VF number. All even numbered VFs get1492		 * NIX0 and odd numbered gets NIX11493		 */1494		blkaddr = (vf & 1) ? BLKADDR_NIX1 : BLKADDR_NIX0;1495		/* NIX1 is not present on all silicons */1496		if (!is_block_implemented(rvu->hw, BLKADDR_NIX1))1497			blkaddr = BLKADDR_NIX0;1498	}1499 1500	/* if SDP1 then the blkaddr is NIX1 */1501	if (is_sdp_pfvf(pcifunc) && pf->sdp_info->node_id == 1)1502		blkaddr = BLKADDR_NIX1;1503 1504	switch (blkaddr) {1505	case BLKADDR_NIX1:1506		pfvf->nix_blkaddr = BLKADDR_NIX1;1507		pfvf->nix_rx_intf = NIX_INTFX_RX(1);1508		pfvf->nix_tx_intf = NIX_INTFX_TX(1);1509		break;1510	case BLKADDR_NIX0:1511	default:1512		pfvf->nix_blkaddr = BLKADDR_NIX0;1513		pfvf->nix_rx_intf = NIX_INTFX_RX(0);1514		pfvf->nix_tx_intf = NIX_INTFX_TX(0);1515		break;1516	}1517 1518	return pfvf->nix_blkaddr;1519}1520 1521static int rvu_get_attach_blkaddr(struct rvu *rvu, int blktype,1522				  u16 pcifunc, struct rsrc_attach *attach)1523{1524	int blkaddr;1525 1526	switch (blktype) {1527	case BLKTYPE_NIX:1528		blkaddr = rvu_get_nix_blkaddr(rvu, pcifunc);1529		break;1530	case BLKTYPE_CPT:1531		if (attach->hdr.ver < RVU_MULTI_BLK_VER)1532			return rvu_get_blkaddr(rvu, blktype, 0);1533		blkaddr = attach->cpt_blkaddr ? attach->cpt_blkaddr :1534			  BLKADDR_CPT0;1535		if (blkaddr != BLKADDR_CPT0 && blkaddr != BLKADDR_CPT1)1536			return -ENODEV;1537		break;1538	default:1539		return rvu_get_blkaddr(rvu, blktype, 0);1540	}1541 1542	if (is_block_implemented(rvu->hw, blkaddr))1543		return blkaddr;1544 1545	return -ENODEV;1546}1547 1548static void rvu_attach_block(struct rvu *rvu, int pcifunc, int blktype,1549			     int num_lfs, struct rsrc_attach *attach)1550{1551	struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);1552	struct rvu_hwinfo *hw = rvu->hw;1553	struct rvu_block *block;1554	int slot, lf;1555	int blkaddr;1556	u64 cfg;1557 1558	if (!num_lfs)1559		return;1560 1561	blkaddr = rvu_get_attach_blkaddr(rvu, blktype, pcifunc, attach);1562	if (blkaddr < 0)1563		return;1564 1565	block = &hw->block[blkaddr];1566	if (!block->lf.bmap)1567		return;1568 1569	for (slot = 0; slot < num_lfs; slot++) {1570		/* Allocate the resource */1571		lf = rvu_alloc_rsrc(&block->lf);1572		if (lf < 0)1573			return;1574 1575		cfg = (1ULL << 63) | (pcifunc << 8) | slot;1576		rvu_write64(rvu, blkaddr, block->lfcfg_reg |1577			    (lf << block->lfshift), cfg);1578		rvu_update_rsrc_map(rvu, pfvf, block,1579				    pcifunc, lf, true);1580 1581		/* Set start MSIX vector for this LF within this PF/VF */1582		rvu_set_msix_offset(rvu, pfvf, block, lf);1583	}1584}1585 1586static int rvu_check_rsrc_availability(struct rvu *rvu,1587				       struct rsrc_attach *req, u16 pcifunc)1588{1589	struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);1590	int free_lfs, mappedlfs, blkaddr;1591	struct rvu_hwinfo *hw = rvu->hw;1592	struct rvu_block *block;1593 1594	/* Only one NPA LF can be attached */1595	if (req->npalf && !is_blktype_attached(pfvf, BLKTYPE_NPA)) {1596		block = &hw->block[BLKADDR_NPA];1597		free_lfs = rvu_rsrc_free_count(&block->lf);1598		if (!free_lfs)1599			goto fail;1600	} else if (req->npalf) {1601		dev_err(&rvu->pdev->dev,1602			"Func 0x%x: Invalid req, already has NPA\n",1603			 pcifunc);1604		return -EINVAL;1605	}1606 1607	/* Only one NIX LF can be attached */1608	if (req->nixlf && !is_blktype_attached(pfvf, BLKTYPE_NIX)) {1609		blkaddr = rvu_get_attach_blkaddr(rvu, BLKTYPE_NIX,1610						 pcifunc, req);1611		if (blkaddr < 0)1612			return blkaddr;1613		block = &hw->block[blkaddr];1614		free_lfs = rvu_rsrc_free_count(&block->lf);1615		if (!free_lfs)1616			goto fail;1617	} else if (req->nixlf) {1618		dev_err(&rvu->pdev->dev,1619			"Func 0x%x: Invalid req, already has NIX\n",1620			pcifunc);1621		return -EINVAL;1622	}1623 1624	if (req->sso) {1625		block = &hw->block[BLKADDR_SSO];1626		/* Is request within limits ? */1627		if (req->sso > block->lf.max) {1628			dev_err(&rvu->pdev->dev,1629				"Func 0x%x: Invalid SSO req, %d > max %d\n",1630				 pcifunc, req->sso, block->lf.max);1631			return -EINVAL;1632		}1633		mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->addr);1634		free_lfs = rvu_rsrc_free_count(&block->lf);1635		/* Check if additional resources are available */1636		if (req->sso > mappedlfs &&1637		    ((req->sso - mappedlfs) > free_lfs))1638			goto fail;1639	}1640 1641	if (req->ssow) {1642		block = &hw->block[BLKADDR_SSOW];1643		if (req->ssow > block->lf.max) {1644			dev_err(&rvu->pdev->dev,1645				"Func 0x%x: Invalid SSOW req, %d > max %d\n",1646				 pcifunc, req->ssow, block->lf.max);1647			return -EINVAL;1648		}1649		mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->addr);1650		free_lfs = rvu_rsrc_free_count(&block->lf);1651		if (req->ssow > mappedlfs &&1652		    ((req->ssow - mappedlfs) > free_lfs))1653			goto fail;1654	}1655 1656	if (req->timlfs) {1657		block = &hw->block[BLKADDR_TIM];1658		if (req->timlfs > block->lf.max) {1659			dev_err(&rvu->pdev->dev,1660				"Func 0x%x: Invalid TIMLF req, %d > max %d\n",1661				 pcifunc, req->timlfs, block->lf.max);1662			return -EINVAL;1663		}1664		mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->addr);1665		free_lfs = rvu_rsrc_free_count(&block->lf);1666		if (req->timlfs > mappedlfs &&1667		    ((req->timlfs - mappedlfs) > free_lfs))1668			goto fail;1669	}1670 1671	if (req->cptlfs) {1672		blkaddr = rvu_get_attach_blkaddr(rvu, BLKTYPE_CPT,1673						 pcifunc, req);1674		if (blkaddr < 0)1675			return blkaddr;1676		block = &hw->block[blkaddr];1677		if (req->cptlfs > block->lf.max) {1678			dev_err(&rvu->pdev->dev,1679				"Func 0x%x: Invalid CPTLF req, %d > max %d\n",1680				 pcifunc, req->cptlfs, block->lf.max);1681			return -EINVAL;1682		}1683		mappedlfs = rvu_get_rsrc_mapcount(pfvf, block->addr);1684		free_lfs = rvu_rsrc_free_count(&block->lf);1685		if (req->cptlfs > mappedlfs &&1686		    ((req->cptlfs - mappedlfs) > free_lfs))1687			goto fail;1688	}1689 1690	return 0;1691 1692fail:1693	dev_info(rvu->dev, "Request for %s failed\n", block->name);1694	return -ENOSPC;1695}1696 1697static bool rvu_attach_from_same_block(struct rvu *rvu, int blktype,1698				       struct rsrc_attach *attach)1699{1700	int blkaddr, num_lfs;1701 1702	blkaddr = rvu_get_attach_blkaddr(rvu, blktype,1703					 attach->hdr.pcifunc, attach);1704	if (blkaddr < 0)1705		return false;1706 1707	num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, attach->hdr.pcifunc),1708					blkaddr);1709	/* Requester already has LFs from given block ? */1710	return !!num_lfs;1711}1712 1713int rvu_mbox_handler_attach_resources(struct rvu *rvu,1714				      struct rsrc_attach *attach,1715				      struct msg_rsp *rsp)1716{1717	u16 pcifunc = attach->hdr.pcifunc;1718	int err;1719 1720	/* If first request, detach all existing attached resources */1721	if (!attach->modify)1722		rvu_detach_rsrcs(rvu, NULL, pcifunc);1723 1724	mutex_lock(&rvu->rsrc_lock);1725 1726	/* Check if the request can be accommodated */1727	err = rvu_check_rsrc_availability(rvu, attach, pcifunc);1728	if (err)1729		goto exit;1730 1731	/* Now attach the requested resources */1732	if (attach->npalf)1733		rvu_attach_block(rvu, pcifunc, BLKTYPE_NPA, 1, attach);1734 1735	if (attach->nixlf)1736		rvu_attach_block(rvu, pcifunc, BLKTYPE_NIX, 1, attach);1737 1738	if (attach->sso) {1739		/* RVU func doesn't know which exact LF or slot is attached1740		 * to it, it always sees as slot 0,1,2. So for a 'modify'1741		 * request, simply detach all existing attached LFs/slots1742		 * and attach a fresh.1743		 */1744		if (attach->modify)1745			rvu_detach_block(rvu, pcifunc, BLKTYPE_SSO);1746		rvu_attach_block(rvu, pcifunc, BLKTYPE_SSO,1747				 attach->sso, attach);1748	}1749 1750	if (attach->ssow) {1751		if (attach->modify)1752			rvu_detach_block(rvu, pcifunc, BLKTYPE_SSOW);1753		rvu_attach_block(rvu, pcifunc, BLKTYPE_SSOW,1754				 attach->ssow, attach);1755	}1756 1757	if (attach->timlfs) {1758		if (attach->modify)1759			rvu_detach_block(rvu, pcifunc, BLKTYPE_TIM);1760		rvu_attach_block(rvu, pcifunc, BLKTYPE_TIM,1761				 attach->timlfs, attach);1762	}1763 1764	if (attach->cptlfs) {1765		if (attach->modify &&1766		    rvu_attach_from_same_block(rvu, BLKTYPE_CPT, attach))1767			rvu_detach_block(rvu, pcifunc, BLKTYPE_CPT);1768		rvu_attach_block(rvu, pcifunc, BLKTYPE_CPT,1769				 attach->cptlfs, attach);1770	}1771 1772exit:1773	mutex_unlock(&rvu->rsrc_lock);1774	return err;1775}1776 1777static u16 rvu_get_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,1778			       int blkaddr, int lf)1779{1780	u16 vec;1781 1782	if (lf < 0)1783		return MSIX_VECTOR_INVALID;1784 1785	for (vec = 0; vec < pfvf->msix.max; vec++) {1786		if (pfvf->msix_lfmap[vec] == MSIX_BLKLF(blkaddr, lf))1787			return vec;1788	}1789	return MSIX_VECTOR_INVALID;1790}1791 1792static void rvu_set_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,1793				struct rvu_block *block, int lf)1794{1795	u16 nvecs, vec, offset;1796	u64 cfg;1797 1798	cfg = rvu_read64(rvu, block->addr, block->msixcfg_reg |1799			 (lf << block->lfshift));1800	nvecs = (cfg >> 12) & 0xFF;1801 1802	/* Check and alloc MSIX vectors, must be contiguous */1803	if (!rvu_rsrc_check_contig(&pfvf->msix, nvecs))1804		return;1805 1806	offset = rvu_alloc_rsrc_contig(&pfvf->msix, nvecs);1807 1808	/* Config MSIX offset in LF */1809	rvu_write64(rvu, block->addr, block->msixcfg_reg |1810		    (lf << block->lfshift), (cfg & ~0x7FFULL) | offset);1811 1812	/* Update the bitmap as well */1813	for (vec = 0; vec < nvecs; vec++)1814		pfvf->msix_lfmap[offset + vec] = MSIX_BLKLF(block->addr, lf);1815}1816 1817static void rvu_clear_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,1818				  struct rvu_block *block, int lf)1819{1820	u16 nvecs, vec, offset;1821	u64 cfg;1822 1823	cfg = rvu_read64(rvu, block->addr, block->msixcfg_reg |1824			 (lf << block->lfshift));1825	nvecs = (cfg >> 12) & 0xFF;1826 1827	/* Clear MSIX offset in LF */1828	rvu_write64(rvu, block->addr, block->msixcfg_reg |1829		    (lf << block->lfshift), cfg & ~0x7FFULL);1830 1831	offset = rvu_get_msix_offset(rvu, pfvf, block->addr, lf);1832 1833	/* Update the mapping */1834	for (vec = 0; vec < nvecs; vec++)1835		pfvf->msix_lfmap[offset + vec] = 0;1836 1837	/* Free the same in MSIX bitmap */1838	rvu_free_rsrc_contig(&pfvf->msix, nvecs, offset);1839}1840 1841int rvu_mbox_handler_msix_offset(struct rvu *rvu, struct msg_req *req,1842				 struct msix_offset_rsp *rsp)1843{1844	struct rvu_hwinfo *hw = rvu->hw;1845	u16 pcifunc = req->hdr.pcifunc;1846	struct rvu_pfvf *pfvf;1847	int lf, slot, blkaddr;1848 1849	pfvf = rvu_get_pfvf(rvu, pcifunc);1850	if (!pfvf->msix.bmap)1851		return 0;1852 1853	/* Set MSIX offsets for each block's LFs attached to this PF/VF */1854	lf = rvu_get_lf(rvu, &hw->block[BLKADDR_NPA], pcifunc, 0);1855	rsp->npa_msixoff = rvu_get_msix_offset(rvu, pfvf, BLKADDR_NPA, lf);1856 1857	/* Get BLKADDR from which LFs are attached to pcifunc */1858	blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);1859	if (blkaddr < 0) {1860		rsp->nix_msixoff = MSIX_VECTOR_INVALID;1861	} else {1862		lf = rvu_get_lf(rvu, &hw->block[blkaddr], pcifunc, 0);1863		rsp->nix_msixoff = rvu_get_msix_offset(rvu, pfvf, blkaddr, lf);1864	}1865 1866	rsp->sso = pfvf->sso;1867	for (slot = 0; slot < rsp->sso; slot++) {1868		lf = rvu_get_lf(rvu, &hw->block[BLKADDR_SSO], pcifunc, slot);1869		rsp->sso_msixoff[slot] =1870			rvu_get_msix_offset(rvu, pfvf, BLKADDR_SSO, lf);1871	}1872 1873	rsp->ssow = pfvf->ssow;1874	for (slot = 0; slot < rsp->ssow; slot++) {1875		lf = rvu_get_lf(rvu, &hw->block[BLKADDR_SSOW], pcifunc, slot);1876		rsp->ssow_msixoff[slot] =1877			rvu_get_msix_offset(rvu, pfvf, BLKADDR_SSOW, lf);1878	}1879 1880	rsp->timlfs = pfvf->timlfs;1881	for (slot = 0; slot < rsp->timlfs; slot++) {1882		lf = rvu_get_lf(rvu, &hw->block[BLKADDR_TIM], pcifunc, slot);1883		rsp->timlf_msixoff[slot] =1884			rvu_get_msix_offset(rvu, pfvf, BLKADDR_TIM, lf);1885	}1886 1887	rsp->cptlfs = pfvf->cptlfs;1888	for (slot = 0; slot < rsp->cptlfs; slot++) {1889		lf = rvu_get_lf(rvu, &hw->block[BLKADDR_CPT0], pcifunc, slot);1890		rsp->cptlf_msixoff[slot] =1891			rvu_get_msix_offset(rvu, pfvf, BLKADDR_CPT0, lf);1892	}1893 1894	rsp->cpt1_lfs = pfvf->cpt1_lfs;1895	for (slot = 0; slot < rsp->cpt1_lfs; slot++) {1896		lf = rvu_get_lf(rvu, &hw->block[BLKADDR_CPT1], pcifunc, slot);1897		rsp->cpt1_lf_msixoff[slot] =1898			rvu_get_msix_offset(rvu, pfvf, BLKADDR_CPT1, lf);1899	}1900 1901	return 0;1902}1903 1904int rvu_mbox_handler_free_rsrc_cnt(struct rvu *rvu, struct msg_req *req,1905				   struct free_rsrcs_rsp *rsp)1906{1907	struct rvu_hwinfo *hw = rvu->hw;1908	struct rvu_block *block;1909	struct nix_txsch *txsch;1910	struct nix_hw *nix_hw;1911 1912	mutex_lock(&rvu->rsrc_lock);1913 1914	block = &hw->block[BLKADDR_NPA];1915	rsp->npa = rvu_rsrc_free_count(&block->lf);1916 1917	block = &hw->block[BLKADDR_NIX0];1918	rsp->nix = rvu_rsrc_free_count(&block->lf);1919 1920	block = &hw->block[BLKADDR_NIX1];1921	rsp->nix1 = rvu_rsrc_free_count(&block->lf);1922 1923	block = &hw->block[BLKADDR_SSO];1924	rsp->sso = rvu_rsrc_free_count(&block->lf);1925 1926	block = &hw->block[BLKADDR_SSOW];1927	rsp->ssow = rvu_rsrc_free_count(&block->lf);1928 1929	block = &hw->block[BLKADDR_TIM];1930	rsp->tim = rvu_rsrc_free_count(&block->lf);1931 1932	block = &hw->block[BLKADDR_CPT0];1933	rsp->cpt = rvu_rsrc_free_count(&block->lf);1934 1935	block = &hw->block[BLKADDR_CPT1];1936	rsp->cpt1 = rvu_rsrc_free_count(&block->lf);1937 1938	if (rvu->hw->cap.nix_fixed_txschq_mapping) {1939		rsp->schq[NIX_TXSCH_LVL_SMQ] = 1;1940		rsp->schq[NIX_TXSCH_LVL_TL4] = 1;1941		rsp->schq[NIX_TXSCH_LVL_TL3] = 1;1942		rsp->schq[NIX_TXSCH_LVL_TL2] = 1;1943		/* NIX1 */1944		if (!is_block_implemented(rvu->hw, BLKADDR_NIX1))1945			goto out;1946		rsp->schq_nix1[NIX_TXSCH_LVL_SMQ] = 1;1947		rsp->schq_nix1[NIX_TXSCH_LVL_TL4] = 1;1948		rsp->schq_nix1[NIX_TXSCH_LVL_TL3] = 1;1949		rsp->schq_nix1[NIX_TXSCH_LVL_TL2] = 1;1950	} else {1951		nix_hw = get_nix_hw(hw, BLKADDR_NIX0);1952		txsch = &nix_hw->txsch[NIX_TXSCH_LVL_SMQ];1953		rsp->schq[NIX_TXSCH_LVL_SMQ] =1954				rvu_rsrc_free_count(&txsch->schq);1955 1956		txsch = &nix_hw->txsch[NIX_TXSCH_LVL_TL4];1957		rsp->schq[NIX_TXSCH_LVL_TL4] =1958				rvu_rsrc_free_count(&txsch->schq);1959 1960		txsch = &nix_hw->txsch[NIX_TXSCH_LVL_TL3];1961		rsp->schq[NIX_TXSCH_LVL_TL3] =1962				rvu_rsrc_free_count(&txsch->schq);1963 1964		txsch = &nix_hw->txsch[NIX_TXSCH_LVL_TL2];1965		rsp->schq[NIX_TXSCH_LVL_TL2] =1966				rvu_rsrc_free_count(&txsch->schq);1967 1968		if (!is_block_implemented(rvu->hw, BLKADDR_NIX1))1969			goto out;1970 1971		nix_hw = get_nix_hw(hw, BLKADDR_NIX1);1972		txsch = &nix_hw->txsch[NIX_TXSCH_LVL_SMQ];1973		rsp->schq_nix1[NIX_TXSCH_LVL_SMQ] =1974				rvu_rsrc_free_count(&txsch->schq);1975 1976		txsch = &nix_hw->txsch[NIX_TXSCH_LVL_TL4];1977		rsp->schq_nix1[NIX_TXSCH_LVL_TL4] =1978				rvu_rsrc_free_count(&txsch->schq);1979 1980		txsch = &nix_hw->txsch[NIX_TXSCH_LVL_TL3];1981		rsp->schq_nix1[NIX_TXSCH_LVL_TL3] =1982				rvu_rsrc_free_count(&txsch->schq);1983 1984		txsch = &nix_hw->txsch[NIX_TXSCH_LVL_TL2];1985		rsp->schq_nix1[NIX_TXSCH_LVL_TL2] =1986				rvu_rsrc_free_count(&txsch->schq);1987	}1988 1989	rsp->schq_nix1[NIX_TXSCH_LVL_TL1] = 1;1990out:1991	rsp->schq[NIX_TXSCH_LVL_TL1] = 1;1992	mutex_unlock(&rvu->rsrc_lock);1993 1994	return 0;1995}1996 1997int rvu_mbox_handler_vf_flr(struct rvu *rvu, struct msg_req *req,1998			    struct msg_rsp *rsp)1999{2000	u16 pcifunc = req->hdr.pcifunc;2001	u16 vf, numvfs;2002	u64 cfg;2003 2004	vf = pcifunc & RVU_PFVF_FUNC_MASK;2005	cfg = rvu_read64(rvu, BLKADDR_RVUM,2006			 RVU_PRIV_PFX_CFG(rvu_get_pf(pcifunc)));2007	numvfs = (cfg >> 12) & 0xFF;2008 2009	if (vf && vf <= numvfs)2010		__rvu_flr_handler(rvu, pcifunc);2011	else2012		return RVU_INVALID_VF_ID;2013 2014	return 0;2015}2016 2017int rvu_ndc_sync(struct rvu *rvu, int lfblkaddr, int lfidx, u64 lfoffset)2018{2019	/* Sync cached info for this LF in NDC to LLC/DRAM */2020	rvu_write64(rvu, lfblkaddr, lfoffset, BIT_ULL(12) | lfidx);2021	return rvu_poll_reg(rvu, lfblkaddr, lfoffset, BIT_ULL(12), true);2022}2023 2024int rvu_mbox_handler_get_hw_cap(struct rvu *rvu, struct msg_req *req,2025				struct get_hw_cap_rsp *rsp)2026{2027	struct rvu_hwinfo *hw = rvu->hw;2028 2029	rsp->nix_fixed_txschq_mapping = hw->cap.nix_fixed_txschq_mapping;2030	rsp->nix_shaping = hw->cap.nix_shaping;2031	rsp->npc_hash_extract = hw->cap.npc_hash_extract;2032 2033	return 0;2034}2035 2036int rvu_mbox_handler_set_vf_perm(struct rvu *rvu, struct set_vf_perm *req,2037				 struct msg_rsp *rsp)2038{2039	struct rvu_hwinfo *hw = rvu->hw;2040	u16 pcifunc = req->hdr.pcifunc;2041	struct rvu_pfvf *pfvf;2042	int blkaddr, nixlf;2043	u16 target;2044 2045	/* Only PF can add VF permissions */2046	if ((pcifunc & RVU_PFVF_FUNC_MASK) || is_lbk_vf(rvu, pcifunc))2047		return -EOPNOTSUPP;2048 2049	target = (pcifunc & ~RVU_PFVF_FUNC_MASK) | (req->vf + 1);2050	pfvf = rvu_get_pfvf(rvu, target);2051 2052	if (req->flags & RESET_VF_PERM) {2053		pfvf->flags &= RVU_CLEAR_VF_PERM;2054	} else if (test_bit(PF_SET_VF_TRUSTED, &pfvf->flags) ^2055		 (req->flags & VF_TRUSTED)) {2056		change_bit(PF_SET_VF_TRUSTED, &pfvf->flags);2057		/* disable multicast and promisc entries */2058		if (!test_bit(PF_SET_VF_TRUSTED, &pfvf->flags)) {2059			blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, target);2060			if (blkaddr < 0)2061				return 0;2062			nixlf = rvu_get_lf(rvu, &hw->block[blkaddr],2063					   target, 0);2064			if (nixlf < 0)2065				return 0;2066			npc_enadis_default_mce_entry(rvu, target, nixlf,2067						     NIXLF_ALLMULTI_ENTRY,2068						     false);2069			npc_enadis_default_mce_entry(rvu, target, nixlf,2070						     NIXLF_PROMISC_ENTRY,2071						     false);2072		}2073	}2074 2075	return 0;2076}2077 2078int rvu_mbox_handler_ndc_sync_op(struct rvu *rvu,2079				 struct ndc_sync_op *req,2080				 struct msg_rsp *rsp)2081{2082	struct rvu_hwinfo *hw = rvu->hw;2083	u16 pcifunc = req->hdr.pcifunc;2084	int err, lfidx, lfblkaddr;2085 2086	if (req->npa_lf_sync) {2087		/* Get NPA LF data */2088		lfblkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, pcifunc);2089		if (lfblkaddr < 0)2090			return NPA_AF_ERR_AF_LF_INVALID;2091 2092		lfidx = rvu_get_lf(rvu, &hw->block[lfblkaddr], pcifunc, 0);2093		if (lfidx < 0)2094			return NPA_AF_ERR_AF_LF_INVALID;2095 2096		/* Sync NPA NDC */2097		err = rvu_ndc_sync(rvu, lfblkaddr,2098				   lfidx, NPA_AF_NDC_SYNC);2099		if (err)2100			dev_err(rvu->dev,2101				"NDC-NPA sync failed for LF %u\n", lfidx);2102	}2103 2104	if (!req->nix_lf_tx_sync && !req->nix_lf_rx_sync)2105		return 0;2106 2107	/* Get NIX LF data */2108	lfblkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);2109	if (lfblkaddr < 0)2110		return NIX_AF_ERR_AF_LF_INVALID;2111 2112	lfidx = rvu_get_lf(rvu, &hw->block[lfblkaddr], pcifunc, 0);2113	if (lfidx < 0)2114		return NIX_AF_ERR_AF_LF_INVALID;2115 2116	if (req->nix_lf_tx_sync) {2117		/* Sync NIX TX NDC */2118		err = rvu_ndc_sync(rvu, lfblkaddr,2119				   lfidx, NIX_AF_NDC_TX_SYNC);2120		if (err)2121			dev_err(rvu->dev,2122				"NDC-NIX-TX sync fail for LF %u\n", lfidx);2123	}2124 2125	if (req->nix_lf_rx_sync) {2126		/* Sync NIX RX NDC */2127		err = rvu_ndc_sync(rvu, lfblkaddr,2128				   lfidx, NIX_AF_NDC_RX_SYNC);2129		if (err)2130			dev_err(rvu->dev,2131				"NDC-NIX-RX sync failed for LF %u\n", lfidx);2132	}2133 2134	return 0;2135}2136 2137static int rvu_process_mbox_msg(struct otx2_mbox *mbox, int devid,2138				struct mbox_msghdr *req)2139{2140	struct rvu *rvu = pci_get_drvdata(mbox->pdev);2141 2142	/* Check if valid, if not reply with a invalid msg */2143	if (req->sig != OTX2_MBOX_REQ_SIG)2144		goto bad_message;2145 2146	switch (req->id) {2147#define M(_name, _id, _fn_name, _req_type, _rsp_type)			\2148	case _id: {							\2149		struct _rsp_type *rsp;					\2150		int err;						\2151									\2152		rsp = (struct _rsp_type *)otx2_mbox_alloc_msg(		\2153			mbox, devid,					\2154			sizeof(struct _rsp_type));			\2155		/* some handlers should complete even if reply */	\2156		/* could not be allocated */				\2157		if (!rsp &&						\2158		    _id != MBOX_MSG_DETACH_RESOURCES &&			\2159		    _id != MBOX_MSG_NIX_TXSCH_FREE &&			\2160		    _id != MBOX_MSG_VF_FLR)				\2161			return -ENOMEM;					\2162		if (rsp) {						\2163			rsp->hdr.id = _id;				\2164			rsp->hdr.sig = OTX2_MBOX_RSP_SIG;		\2165			rsp->hdr.pcifunc = req->pcifunc;		\2166			rsp->hdr.rc = 0;				\2167		}							\2168									\2169		err = rvu_mbox_handler_ ## _fn_name(rvu,		\2170						    (struct _req_type *)req, \2171						    rsp);		\2172		if (rsp && err)						\2173			rsp->hdr.rc = err;				\2174									\2175		trace_otx2_msg_process(mbox->pdev, _id, err);		\2176		return rsp ? err : -ENOMEM;				\2177	}2178MBOX_MESSAGES2179#undef M2180 2181bad_message:2182	default:2183		otx2_reply_invalid_msg(mbox, devid, req->pcifunc, req->id);2184		return -ENODEV;2185	}2186}2187 2188static void __rvu_mbox_handler(struct rvu_work *mwork, int type, bool poll)2189{2190	struct rvu *rvu = mwork->rvu;2191	int offset, err, id, devid;2192	struct otx2_mbox_dev *mdev;2193	struct mbox_hdr *req_hdr;2194	struct mbox_msghdr *msg;2195	struct mbox_wq_info *mw;2196	struct otx2_mbox *mbox;2197 2198	switch (type) {2199	case TYPE_AFPF:2200		mw = &rvu->afpf_wq_info;2201		break;2202	case TYPE_AFVF:2203		mw = &rvu->afvf_wq_info;2204		break;2205	default:2206		return;2207	}2208 2209	devid = mwork - mw->mbox_wrk;2210	mbox = &mw->mbox;2211	mdev = &mbox->dev[devid];2212 2213	/* Process received mbox messages */2214	req_hdr = mdev->mbase + mbox->rx_start;2215	if (mw->mbox_wrk[devid].num_msgs == 0)2216		return;2217 2218	offset = mbox->rx_start + ALIGN(sizeof(*req_hdr), MBOX_MSG_ALIGN);2219 2220	for (id = 0; id < mw->mbox_wrk[devid].num_msgs; id++) {2221		msg = mdev->mbase + offset;2222 2223		/* Set which PF/VF sent this message based on mbox IRQ */2224		switch (type) {2225		case TYPE_AFPF:2226			msg->pcifunc &=2227				~(RVU_PFVF_PF_MASK << RVU_PFVF_PF_SHIFT);2228			msg->pcifunc |= (devid << RVU_PFVF_PF_SHIFT);2229			break;2230		case TYPE_AFVF:2231			msg->pcifunc &=2232				~(RVU_PFVF_FUNC_MASK << RVU_PFVF_FUNC_SHIFT);2233			msg->pcifunc |= (devid << RVU_PFVF_FUNC_SHIFT) + 1;2234			break;2235		}2236 2237		err = rvu_process_mbox_msg(mbox, devid, msg);2238		if (!err) {2239			offset = mbox->rx_start + msg->next_msgoff;2240			continue;2241		}2242 2243		if (msg->pcifunc & RVU_PFVF_FUNC_MASK)2244			dev_warn(rvu->dev, "Error %d when processing message %s (0x%x) from PF%d:VF%d\n",2245				 err, otx2_mbox_id2name(msg->id),2246				 msg->id, rvu_get_pf(msg->pcifunc),2247				 (msg->pcifunc & RVU_PFVF_FUNC_MASK) - 1);2248		else2249			dev_warn(rvu->dev, "Error %d when processing message %s (0x%x) from PF%d\n",2250				 err, otx2_mbox_id2name(msg->id),2251				 msg->id, devid);2252	}2253	mw->mbox_wrk[devid].num_msgs = 0;2254 2255	if (poll)2256		otx2_mbox_wait_for_zero(mbox, devid);2257 2258	/* Send mbox responses to VF/PF */2259	otx2_mbox_msg_send(mbox, devid);2260}2261 2262static inline void rvu_afpf_mbox_handler(struct work_struct *work)2263{2264	struct rvu_work *mwork = container_of(work, struct rvu_work, work);2265	struct rvu *rvu = mwork->rvu;2266 2267	mutex_lock(&rvu->mbox_lock);2268	__rvu_mbox_handler(mwork, TYPE_AFPF, true);2269	mutex_unlock(&rvu->mbox_lock);2270}2271 2272static inline void rvu_afvf_mbox_handler(struct work_struct *work)2273{2274	struct rvu_work *mwork = container_of(work, struct rvu_work, work);2275 2276	__rvu_mbox_handler(mwork, TYPE_AFVF, false);2277}2278 2279static void __rvu_mbox_up_handler(struct rvu_work *mwork, int type)2280{2281	struct rvu *rvu = mwork->rvu;2282	struct otx2_mbox_dev *mdev;2283	struct mbox_hdr *rsp_hdr;2284	struct mbox_msghdr *msg;2285	struct mbox_wq_info *mw;2286	struct otx2_mbox *mbox;2287	int offset, id, devid;2288 2289	switch (type) {2290	case TYPE_AFPF:2291		mw = &rvu->afpf_wq_info;2292		break;2293	case TYPE_AFVF:2294		mw = &rvu->afvf_wq_info;2295		break;2296	default:2297		return;2298	}2299 2300	devid = mwork - mw->mbox_wrk_up;2301	mbox = &mw->mbox_up;2302	mdev = &mbox->dev[devid];2303 2304	rsp_hdr = mdev->mbase + mbox->rx_start;2305	if (mw->mbox_wrk_up[devid].up_num_msgs == 0) {2306		dev_warn(rvu->dev, "mbox up handler: num_msgs = 0\n");2307		return;2308	}2309 2310	offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN);2311 2312	for (id = 0; id < mw->mbox_wrk_up[devid].up_num_msgs; id++) {2313		msg = mdev->mbase + offset;2314 2315		if (msg->id >= MBOX_MSG_MAX) {2316			dev_err(rvu->dev,2317				"Mbox msg with unknown ID 0x%x\n", msg->id);2318			goto end;2319		}2320 2321		if (msg->sig != OTX2_MBOX_RSP_SIG) {2322			dev_err(rvu->dev,2323				"Mbox msg with wrong signature %x, ID 0x%x\n",2324				msg->sig, msg->id);2325			goto end;2326		}2327 2328		switch (msg->id) {2329		case MBOX_MSG_CGX_LINK_EVENT:2330			break;2331		default:2332			if (msg->rc)2333				dev_err(rvu->dev,2334					"Mbox msg response has err %d, ID 0x%x\n",2335					msg->rc, msg->id);2336			break;2337		}2338end:2339		offset = mbox->rx_start + msg->next_msgoff;2340		mdev->msgs_acked++;2341	}2342	mw->mbox_wrk_up[devid].up_num_msgs = 0;2343 2344	otx2_mbox_reset(mbox, devid);2345}2346 2347static inline void rvu_afpf_mbox_up_handler(struct work_struct *work)2348{2349	struct rvu_work *mwork = container_of(work, struct rvu_work, work);2350 2351	__rvu_mbox_up_handler(mwork, TYPE_AFPF);2352}2353 2354static inline void rvu_afvf_mbox_up_handler(struct work_struct *work)2355{2356	struct rvu_work *mwork = container_of(work, struct rvu_work, work);2357 2358	__rvu_mbox_up_handler(mwork, TYPE_AFVF);2359}2360 2361static int rvu_get_mbox_regions(struct rvu *rvu, void **mbox_addr,2362				int num, int type, unsigned long *pf_bmap)2363{2364	struct rvu_hwinfo *hw = rvu->hw;2365	int region;2366	u64 bar4;2367 2368	/* For cn10k platform VF mailbox regions of a PF follows after the2369	 * PF <-> AF mailbox region. Whereas for Octeontx2 it is read from2370	 * RVU_PF_VF_BAR4_ADDR register.2371	 */2372	if (type == TYPE_AFVF) {2373		for (region = 0; region < num; region++) {2374			if (!test_bit(region, pf_bmap))2375				continue;2376 2377			if (hw->cap.per_pf_mbox_regs) {2378				bar4 = rvu_read64(rvu, BLKADDR_RVUM,2379						  RVU_AF_PFX_BAR4_ADDR(0)) +2380						  MBOX_SIZE;2381				bar4 += region * MBOX_SIZE;2382			} else {2383				bar4 = rvupf_read64(rvu, RVU_PF_VF_BAR4_ADDR);2384				bar4 += region * MBOX_SIZE;2385			}2386			mbox_addr[region] = (void *)ioremap_wc(bar4, MBOX_SIZE);2387			if (!mbox_addr[region])2388				goto error;2389		}2390		return 0;2391	}2392 2393	/* For cn10k platform AF <-> PF mailbox region of a PF is read from per2394	 * PF registers. Whereas for Octeontx2 it is read from2395	 * RVU_AF_PF_BAR4_ADDR register.2396	 */2397	for (region = 0; region < num; region++) {2398		if (!test_bit(region, pf_bmap))2399			continue;2400 2401		if (hw->cap.per_pf_mbox_regs) {2402			bar4 = rvu_read64(rvu, BLKADDR_RVUM,2403					  RVU_AF_PFX_BAR4_ADDR(region));2404		} else {2405			bar4 = rvu_read64(rvu, BLKADDR_RVUM,2406					  RVU_AF_PF_BAR4_ADDR);2407			bar4 += region * MBOX_SIZE;2408		}2409		mbox_addr[region] = (void *)ioremap_wc(bar4, MBOX_SIZE);2410		if (!mbox_addr[region])2411			goto error;2412	}2413	return 0;2414 2415error:2416	while (region--)2417		iounmap((void __iomem *)mbox_addr[region]);2418	return -ENOMEM;2419}2420 2421static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw,2422			 int type, int num,2423			 void (mbox_handler)(struct work_struct *),2424			 void (mbox_up_handler)(struct work_struct *))2425{2426	int err = -EINVAL, i, dir, dir_up;2427	void __iomem *reg_base;2428	struct rvu_work *mwork;2429	unsigned long *pf_bmap;2430	void **mbox_regions;2431	const char *name;2432	u64 cfg;2433 2434	pf_bmap = bitmap_zalloc(num, GFP_KERNEL);2435	if (!pf_bmap)2436		return -ENOMEM;2437 2438	/* RVU VFs */2439	if (type == TYPE_AFVF)2440		bitmap_set(pf_bmap, 0, num);2441 2442	if (type == TYPE_AFPF) {2443		/* Mark enabled PFs in bitmap */2444		for (i = 0; i < num; i++) {2445			cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(i));2446			if (cfg & BIT_ULL(20))2447				set_bit(i, pf_bmap);2448		}2449	}2450 2451	mutex_init(&rvu->mbox_lock);2452 2453	mbox_regions = kcalloc(num, sizeof(void *), GFP_KERNEL);2454	if (!mbox_regions) {2455		err = -ENOMEM;2456		goto free_bitmap;2457	}2458 2459	switch (type) {2460	case TYPE_AFPF:2461		name = "rvu_afpf_mailbox";2462		dir = MBOX_DIR_AFPF;2463		dir_up = MBOX_DIR_AFPF_UP;2464		reg_base = rvu->afreg_base;2465		err = rvu_get_mbox_regions(rvu, mbox_regions, num, TYPE_AFPF, pf_bmap);2466		if (err)2467			goto free_regions;2468		break;2469	case TYPE_AFVF:2470		name = "rvu_afvf_mailbox";2471		dir = MBOX_DIR_PFVF;2472		dir_up = MBOX_DIR_PFVF_UP;2473		reg_base = rvu->pfreg_base;2474		err = rvu_get_mbox_regions(rvu, mbox_regions, num, TYPE_AFVF, pf_bmap);2475		if (err)2476			goto free_regions;2477		break;2478	default:2479		goto free_regions;2480	}2481 2482	mw->mbox_wq = alloc_workqueue("%s",2483				      WQ_UNBOUND | WQ_HIGHPRI | WQ_MEM_RECLAIM,2484				      num, name);2485	if (!mw->mbox_wq) {2486		err = -ENOMEM;2487		goto unmap_regions;2488	}2489 2490	mw->mbox_wrk = devm_kcalloc(rvu->dev, num,2491				    sizeof(struct rvu_work), GFP_KERNEL);2492	if (!mw->mbox_wrk) {2493		err = -ENOMEM;2494		goto exit;2495	}2496 2497	mw->mbox_wrk_up = devm_kcalloc(rvu->dev, num,2498				       sizeof(struct rvu_work), GFP_KERNEL);2499	if (!mw->mbox_wrk_up) {2500		err = -ENOMEM;2501		goto exit;2502	}2503 2504	err = otx2_mbox_regions_init(&mw->mbox, mbox_regions, rvu->pdev,2505				     reg_base, dir, num, pf_bmap);2506	if (err)2507		goto exit;2508 2509	err = otx2_mbox_regions_init(&mw->mbox_up, mbox_regions, rvu->pdev,2510				     reg_base, dir_up, num, pf_bmap);2511	if (err)2512		goto exit;2513 2514	for (i = 0; i < num; i++) {2515		if (!test_bit(i, pf_bmap))2516			continue;2517 2518		mwork = &mw->mbox_wrk[i];2519		mwork->rvu = rvu;2520		INIT_WORK(&mwork->work, mbox_handler);2521 2522		mwork = &mw->mbox_wrk_up[i];2523		mwork->rvu = rvu;2524		INIT_WORK(&mwork->work, mbox_up_handler);2525	}2526	goto free_regions;2527 2528exit:2529	destroy_workqueue(mw->mbox_wq);2530unmap_regions:2531	while (num--)2532		iounmap((void __iomem *)mbox_regions[num]);2533free_regions:2534	kfree(mbox_regions);2535free_bitmap:2536	bitmap_free(pf_bmap);2537	return err;2538}2539 2540static void rvu_mbox_destroy(struct mbox_wq_info *mw)2541{2542	struct otx2_mbox *mbox = &mw->mbox;2543	struct otx2_mbox_dev *mdev;2544	int devid;2545 2546	if (mw->mbox_wq) {2547		destroy_workqueue(mw->mbox_wq);2548		mw->mbox_wq = NULL;2549	}2550 2551	for (devid = 0; devid < mbox->ndevs; devid++) {2552		mdev = &mbox->dev[devid];2553		if (mdev->hwbase)2554			iounmap((void __iomem *)mdev->hwbase);2555	}2556 2557	otx2_mbox_destroy(&mw->mbox);2558	otx2_mbox_destroy(&mw->mbox_up);2559}2560 2561static void rvu_queue_work(struct mbox_wq_info *mw, int first,2562			   int mdevs, u64 intr)2563{2564	struct otx2_mbox_dev *mdev;2565	struct otx2_mbox *mbox;2566	struct mbox_hdr *hdr;2567	int i;2568 2569	for (i = first; i < mdevs; i++) {2570		/* start from 0 */2571		if (!(intr & BIT_ULL(i - first)))2572			continue;2573 2574		mbox = &mw->mbox;2575		mdev = &mbox->dev[i];2576		hdr = mdev->mbase + mbox->rx_start;2577 2578		/*The hdr->num_msgs is set to zero immediately in the interrupt2579		 * handler to  ensure that it holds a correct value next time2580		 * when the interrupt handler is called.2581		 * pf->mbox.num_msgs holds the data for use in pfaf_mbox_handler2582		 * pf>mbox.up_num_msgs holds the data for use in2583		 * pfaf_mbox_up_handler.2584		 */2585 2586		if (hdr->num_msgs) {2587			mw->mbox_wrk[i].num_msgs = hdr->num_msgs;2588			hdr->num_msgs = 0;2589			queue_work(mw->mbox_wq, &mw->mbox_wrk[i].work);2590		}2591		mbox = &mw->mbox_up;2592		mdev = &mbox->dev[i];2593		hdr = mdev->mbase + mbox->rx_start;2594		if (hdr->num_msgs) {2595			mw->mbox_wrk_up[i].up_num_msgs = hdr->num_msgs;2596			hdr->num_msgs = 0;2597			queue_work(mw->mbox_wq, &mw->mbox_wrk_up[i].work);2598		}2599	}2600}2601 2602static irqreturn_t rvu_mbox_pf_intr_handler(int irq, void *rvu_irq)2603{2604	struct rvu *rvu = (struct rvu *)rvu_irq;2605	u64 intr;2606 2607	intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT);2608	/* Clear interrupts */2609	rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT, intr);2610	if (intr)2611		trace_otx2_msg_interrupt(rvu->pdev, "PF(s) to AF", intr);2612 2613	/* Sync with mbox memory region */2614	rmb();2615 2616	rvu_queue_work(&rvu->afpf_wq_info, 0, rvu->hw->total_pfs, intr);2617 2618	return IRQ_HANDLED;2619}2620 2621static irqreturn_t rvu_mbox_intr_handler(int irq, void *rvu_irq)2622{2623	struct rvu *rvu = (struct rvu *)rvu_irq;2624	int vfs = rvu->vfs;2625	u64 intr;2626 2627	/* Sync with mbox memory region */2628	rmb();2629 2630	/* Handle VF interrupts */2631	if (vfs > 64) {2632		intr = rvupf_read64(rvu, RVU_PF_VFPF_MBOX_INTX(1));2633		rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(1), intr);2634 2635		rvu_queue_work(&rvu->afvf_wq_info, 64, vfs, intr);2636		vfs -= 64;2637	}2638 2639	intr = rvupf_read64(rvu, RVU_PF_VFPF_MBOX_INTX(0));2640	rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(0), intr);2641	if (intr)2642		trace_otx2_msg_interrupt(rvu->pdev, "VF(s) to AF", intr);2643 2644	rvu_queue_work(&rvu->afvf_wq_info, 0, vfs, intr);2645 2646	return IRQ_HANDLED;2647}2648 2649static void rvu_enable_mbox_intr(struct rvu *rvu)2650{2651	struct rvu_hwinfo *hw = rvu->hw;2652 2653	/* Clear spurious irqs, if any */2654	rvu_write64(rvu, BLKADDR_RVUM,2655		    RVU_AF_PFAF_MBOX_INT, INTR_MASK(hw->total_pfs));2656 2657	/* Enable mailbox interrupt for all PFs except PF0 i.e AF itself */2658	rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT_ENA_W1S,2659		    INTR_MASK(hw->total_pfs) & ~1ULL);2660}2661 2662static void rvu_blklf_teardown(struct rvu *rvu, u16 pcifunc, u8 blkaddr)2663{2664	struct rvu_block *block;2665	int slot, lf, num_lfs;2666	int err;2667 2668	block = &rvu->hw->block[blkaddr];2669	num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc),2670					block->addr);2671	if (!num_lfs)2672		return;2673	for (slot = 0; slot < num_lfs; slot++) {2674		lf = rvu_get_lf(rvu, block, pcifunc, slot);2675		if (lf < 0)2676			continue;2677 2678		/* Cleanup LF and reset it */2679		if (block->addr == BLKADDR_NIX0 || block->addr == BLKADDR_NIX1)2680			rvu_nix_lf_teardown(rvu, pcifunc, block->addr, lf);2681		else if (block->addr == BLKADDR_NPA)2682			rvu_npa_lf_teardown(rvu, pcifunc, lf);2683		else if ((block->addr == BLKADDR_CPT0) ||2684			 (block->addr == BLKADDR_CPT1))2685			rvu_cpt_lf_teardown(rvu, pcifunc, block->addr, lf,2686					    slot);2687 2688		err = rvu_lf_reset(rvu, block, lf);2689		if (err) {2690			dev_err(rvu->dev, "Failed to reset blkaddr %d LF%d\n",2691				block->addr, lf);2692		}2693	}2694}2695 2696static void __rvu_flr_handler(struct rvu *rvu, u16 pcifunc)2697{2698	if (rvu_npc_exact_has_match_table(rvu))2699		rvu_npc_exact_reset(rvu, pcifunc);2700 2701	mutex_lock(&rvu->flr_lock);2702	/* Reset order should reflect inter-block dependencies:2703	 * 1. Reset any packet/work sources (NIX, CPT, TIM)2704	 * 2. Flush and reset SSO/SSOW2705	 * 3. Cleanup pools (NPA)2706	 */2707 2708	/* Free allocated BPIDs */2709	rvu_nix_flr_free_bpids(rvu, pcifunc);2710 2711	/* Free multicast/mirror node associated with the 'pcifunc' */2712	rvu_nix_mcast_flr_free_entries(rvu, pcifunc);2713 2714	rvu_blklf_teardown(rvu, pcifunc, BLKADDR_NIX0);2715	rvu_blklf_teardown(rvu, pcifunc, BLKADDR_NIX1);2716	rvu_blklf_teardown(rvu, pcifunc, BLKADDR_CPT0);2717	rvu_blklf_teardown(rvu, pcifunc, BLKADDR_CPT1);2718	rvu_blklf_teardown(rvu, pcifunc, BLKADDR_TIM);2719	rvu_blklf_teardown(rvu, pcifunc, BLKADDR_SSOW);2720	rvu_blklf_teardown(rvu, pcifunc, BLKADDR_SSO);2721	rvu_blklf_teardown(rvu, pcifunc, BLKADDR_NPA);2722	rvu_reset_lmt_map_tbl(rvu, pcifunc);2723	rvu_detach_rsrcs(rvu, NULL, pcifunc);2724	/* In scenarios where PF/VF drivers detach NIXLF without freeing MCAM2725	 * entries, check and free the MCAM entries explicitly to avoid leak.2726	 * Since LF is detached use LF number as -1.2727	 */2728	rvu_npc_free_mcam_entries(rvu, pcifunc, -1);2729	rvu_mac_reset(rvu, pcifunc);2730 2731	if (rvu->mcs_blk_cnt)2732		rvu_mcs_flr_handler(rvu, pcifunc);2733 2734	mutex_unlock(&rvu->flr_lock);2735}2736 2737static void rvu_afvf_flr_handler(struct rvu *rvu, int vf)2738{2739	int reg = 0;2740 2741	/* pcifunc = 0(PF0) | (vf + 1) */2742	__rvu_flr_handler(rvu, vf + 1);2743 2744	if (vf >= 64) {2745		reg = 1;2746		vf = vf - 64;2747	}2748 2749	/* Signal FLR finish and enable IRQ */2750	rvupf_write64(rvu, RVU_PF_VFTRPENDX(reg), BIT_ULL(vf));2751	rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1SX(reg), BIT_ULL(vf));2752}2753 2754static void rvu_flr_handler(struct work_struct *work)2755{2756	struct rvu_work *flrwork = container_of(work, struct rvu_work, work);2757	struct rvu *rvu = flrwork->rvu;2758	u16 pcifunc, numvfs, vf;2759	u64 cfg;2760	int pf;2761 2762	pf = flrwork - rvu->flr_wrk;2763	if (pf >= rvu->hw->total_pfs) {2764		rvu_afvf_flr_handler(rvu, pf - rvu->hw->total_pfs);2765		return;2766	}2767 2768	cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));2769	numvfs = (cfg >> 12) & 0xFF;2770	pcifunc  = pf << RVU_PFVF_PF_SHIFT;2771 2772	for (vf = 0; vf < numvfs; vf++)2773		__rvu_flr_handler(rvu, (pcifunc | (vf + 1)));2774 2775	__rvu_flr_handler(rvu, pcifunc);2776 2777	/* Signal FLR finish */2778	rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFTRPEND, BIT_ULL(pf));2779 2780	/* Enable interrupt */2781	rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1S,  BIT_ULL(pf));2782}2783 2784static void rvu_afvf_queue_flr_work(struct rvu *rvu, int start_vf, int numvfs)2785{2786	int dev, vf, reg = 0;2787	u64 intr;2788 2789	if (start_vf >= 64)2790		reg = 1;2791 2792	intr = rvupf_read64(rvu, RVU_PF_VFFLR_INTX(reg));2793	if (!intr)2794		return;2795 2796	for (vf = 0; vf < numvfs; vf++) {2797		if (!(intr & BIT_ULL(vf)))2798			continue;2799		/* Clear and disable the interrupt */2800		rvupf_write64(rvu, RVU_PF_VFFLR_INTX(reg), BIT_ULL(vf));2801		rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1CX(reg), BIT_ULL(vf));2802 2803		dev = vf + start_vf + rvu->hw->total_pfs;2804		queue_work(rvu->flr_wq, &rvu->flr_wrk[dev].work);2805	}2806}2807 2808static irqreturn_t rvu_flr_intr_handler(int irq, void *rvu_irq)2809{2810	struct rvu *rvu = (struct rvu *)rvu_irq;2811	u64 intr;2812	u8  pf;2813 2814	intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT);2815	if (!intr)2816		goto afvf_flr;2817 2818	for (pf = 0; pf < rvu->hw->total_pfs; pf++) {2819		if (intr & (1ULL << pf)) {2820			/* clear interrupt */2821			rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT,2822				    BIT_ULL(pf));2823			/* Disable the interrupt */2824			rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1C,2825				    BIT_ULL(pf));2826			/* PF is already dead do only AF related operations */2827			queue_work(rvu->flr_wq, &rvu->flr_wrk[pf].work);2828		}2829	}2830 2831afvf_flr:2832	rvu_afvf_queue_flr_work(rvu, 0, 64);2833	if (rvu->vfs > 64)2834		rvu_afvf_queue_flr_work(rvu, 64, rvu->vfs - 64);2835 2836	return IRQ_HANDLED;2837}2838 2839static void rvu_me_handle_vfset(struct rvu *rvu, int idx, u64 intr)2840{2841	int vf;2842 2843	/* Nothing to be done here other than clearing the2844	 * TRPEND bit.2845	 */2846	for (vf = 0; vf < 64; vf++) {2847		if (intr & (1ULL << vf)) {2848			/* clear the trpend due to ME(master enable) */2849			rvupf_write64(rvu, RVU_PF_VFTRPENDX(idx), BIT_ULL(vf));2850			/* clear interrupt */2851			rvupf_write64(rvu, RVU_PF_VFME_INTX(idx), BIT_ULL(vf));2852		}2853	}2854}2855 2856/* Handles ME interrupts from VFs of AF */2857static irqreturn_t rvu_me_vf_intr_handler(int irq, void *rvu_irq)2858{2859	struct rvu *rvu = (struct rvu *)rvu_irq;2860	int vfset;2861	u64 intr;2862 2863	intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT);2864 2865	for (vfset = 0; vfset <= 1; vfset++) {2866		intr = rvupf_read64(rvu, RVU_PF_VFME_INTX(vfset));2867		if (intr)2868			rvu_me_handle_vfset(rvu, vfset, intr);2869	}2870 2871	return IRQ_HANDLED;2872}2873 2874/* Handles ME interrupts from PFs */2875static irqreturn_t rvu_me_pf_intr_handler(int irq, void *rvu_irq)2876{2877	struct rvu *rvu = (struct rvu *)rvu_irq;2878	u64 intr;2879	u8  pf;2880 2881	intr = rvu_read64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT);2882 2883	/* Nothing to be done here other than clearing the2884	 * TRPEND bit.2885	 */2886	for (pf = 0; pf < rvu->hw->total_pfs; pf++) {2887		if (intr & (1ULL << pf)) {2888			/* clear the trpend due to ME(master enable) */2889			rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFTRPEND,2890				    BIT_ULL(pf));2891			/* clear interrupt */2892			rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT,2893				    BIT_ULL(pf));2894		}2895	}2896 2897	return IRQ_HANDLED;2898}2899 2900static void rvu_unregister_interrupts(struct rvu *rvu)2901{2902	int irq;2903 2904	rvu_cpt_unregister_interrupts(rvu);2905 2906	/* Disable the Mbox interrupt */2907	rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFAF_MBOX_INT_ENA_W1C,2908		    INTR_MASK(rvu->hw->total_pfs) & ~1ULL);2909 2910	/* Disable the PF FLR interrupt */2911	rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1C,2912		    INTR_MASK(rvu->hw->total_pfs) & ~1ULL);2913 2914	/* Disable the PF ME interrupt */2915	rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT_ENA_W1C,2916		    INTR_MASK(rvu->hw->total_pfs) & ~1ULL);2917 2918	for (irq = 0; irq < rvu->num_vec; irq++) {2919		if (rvu->irq_allocated[irq]) {2920			free_irq(pci_irq_vector(rvu->pdev, irq), rvu);2921			rvu->irq_allocated[irq] = false;2922		}2923	}2924 2925	pci_free_irq_vectors(rvu->pdev);2926	rvu->num_vec = 0;2927}2928 2929static int rvu_afvf_msix_vectors_num_ok(struct rvu *rvu)2930{2931	struct rvu_pfvf *pfvf = &rvu->pf[0];2932	int offset;2933 2934	pfvf = &rvu->pf[0];2935	offset = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_INT_CFG(0)) & 0x3ff;2936 2937	/* Make sure there are enough MSIX vectors configured so that2938	 * VF interrupts can be handled. Offset equal to zero means2939	 * that PF vectors are not configured and overlapping AF vectors.2940	 */2941	return (pfvf->msix.max >= RVU_AF_INT_VEC_CNT + RVU_PF_INT_VEC_CNT) &&2942	       offset;2943}2944 2945static int rvu_register_interrupts(struct rvu *rvu)2946{2947	int ret, offset, pf_vec_start;2948 2949	rvu->num_vec = pci_msix_vec_count(rvu->pdev);2950 2951	rvu->irq_name = devm_kmalloc_array(rvu->dev, rvu->num_vec,2952					   NAME_SIZE, GFP_KERNEL);2953	if (!rvu->irq_name)2954		return -ENOMEM;2955 2956	rvu->irq_allocated = devm_kcalloc(rvu->dev, rvu->num_vec,2957					  sizeof(bool), GFP_KERNEL);2958	if (!rvu->irq_allocated)2959		return -ENOMEM;2960 2961	/* Enable MSI-X */2962	ret = pci_alloc_irq_vectors(rvu->pdev, rvu->num_vec,2963				    rvu->num_vec, PCI_IRQ_MSIX);2964	if (ret < 0) {2965		dev_err(rvu->dev,2966			"RVUAF: Request for %d msix vectors failed, ret %d\n",2967			rvu->num_vec, ret);2968		return ret;2969	}2970 2971	/* Register mailbox interrupt handler */2972	sprintf(&rvu->irq_name[RVU_AF_INT_VEC_MBOX * NAME_SIZE], "RVUAF Mbox");2973	ret = request_irq(pci_irq_vector(rvu->pdev, RVU_AF_INT_VEC_MBOX),2974			  rvu_mbox_pf_intr_handler, 0,2975			  &rvu->irq_name[RVU_AF_INT_VEC_MBOX * NAME_SIZE], rvu);2976	if (ret) {2977		dev_err(rvu->dev,2978			"RVUAF: IRQ registration failed for mbox irq\n");2979		goto fail;2980	}2981 2982	rvu->irq_allocated[RVU_AF_INT_VEC_MBOX] = true;2983 2984	/* Enable mailbox interrupts from all PFs */2985	rvu_enable_mbox_intr(rvu);2986 2987	/* Register FLR interrupt handler */2988	sprintf(&rvu->irq_name[RVU_AF_INT_VEC_PFFLR * NAME_SIZE],2989		"RVUAF FLR");2990	ret = request_irq(pci_irq_vector(rvu->pdev, RVU_AF_INT_VEC_PFFLR),2991			  rvu_flr_intr_handler, 0,2992			  &rvu->irq_name[RVU_AF_INT_VEC_PFFLR * NAME_SIZE],2993			  rvu);2994	if (ret) {2995		dev_err(rvu->dev,2996			"RVUAF: IRQ registration failed for FLR\n");2997		goto fail;2998	}2999	rvu->irq_allocated[RVU_AF_INT_VEC_PFFLR] = true;3000 3001	/* Enable FLR interrupt for all PFs*/3002	rvu_write64(rvu, BLKADDR_RVUM,3003		    RVU_AF_PFFLR_INT, INTR_MASK(rvu->hw->total_pfs));3004 3005	rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFFLR_INT_ENA_W1S,3006		    INTR_MASK(rvu->hw->total_pfs) & ~1ULL);3007 3008	/* Register ME interrupt handler */3009	sprintf(&rvu->irq_name[RVU_AF_INT_VEC_PFME * NAME_SIZE],3010		"RVUAF ME");3011	ret = request_irq(pci_irq_vector(rvu->pdev, RVU_AF_INT_VEC_PFME),3012			  rvu_me_pf_intr_handler, 0,3013			  &rvu->irq_name[RVU_AF_INT_VEC_PFME * NAME_SIZE],3014			  rvu);3015	if (ret) {3016		dev_err(rvu->dev,3017			"RVUAF: IRQ registration failed for ME\n");3018	}3019	rvu->irq_allocated[RVU_AF_INT_VEC_PFME] = true;3020 3021	/* Clear TRPEND bit for all PF */3022	rvu_write64(rvu, BLKADDR_RVUM,3023		    RVU_AF_PFTRPEND, INTR_MASK(rvu->hw->total_pfs));3024	/* Enable ME interrupt for all PFs*/3025	rvu_write64(rvu, BLKADDR_RVUM,3026		    RVU_AF_PFME_INT, INTR_MASK(rvu->hw->total_pfs));3027 3028	rvu_write64(rvu, BLKADDR_RVUM, RVU_AF_PFME_INT_ENA_W1S,3029		    INTR_MASK(rvu->hw->total_pfs) & ~1ULL);3030 3031	if (!rvu_afvf_msix_vectors_num_ok(rvu))3032		return 0;3033 3034	/* Get PF MSIX vectors offset. */3035	pf_vec_start = rvu_read64(rvu, BLKADDR_RVUM,3036				  RVU_PRIV_PFX_INT_CFG(0)) & 0x3ff;3037 3038	/* Register MBOX0 interrupt. */3039	offset = pf_vec_start + RVU_PF_INT_VEC_VFPF_MBOX0;3040	sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF Mbox0");3041	ret = request_irq(pci_irq_vector(rvu->pdev, offset),3042			  rvu_mbox_intr_handler, 0,3043			  &rvu->irq_name[offset * NAME_SIZE],3044			  rvu);3045	if (ret)3046		dev_err(rvu->dev,3047			"RVUAF: IRQ registration failed for Mbox0\n");3048 3049	rvu->irq_allocated[offset] = true;3050 3051	/* Register MBOX1 interrupt. MBOX1 IRQ number follows MBOX0 so3052	 * simply increment current offset by 1.3053	 */3054	offset = pf_vec_start + RVU_PF_INT_VEC_VFPF_MBOX1;3055	sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF Mbox1");3056	ret = request_irq(pci_irq_vector(rvu->pdev, offset),3057			  rvu_mbox_intr_handler, 0,3058			  &rvu->irq_name[offset * NAME_SIZE],3059			  rvu);3060	if (ret)3061		dev_err(rvu->dev,3062			"RVUAF: IRQ registration failed for Mbox1\n");3063 3064	rvu->irq_allocated[offset] = true;3065 3066	/* Register FLR interrupt handler for AF's VFs */3067	offset = pf_vec_start + RVU_PF_INT_VEC_VFFLR0;3068	sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF FLR0");3069	ret = request_irq(pci_irq_vector(rvu->pdev, offset),3070			  rvu_flr_intr_handler, 0,3071			  &rvu->irq_name[offset * NAME_SIZE], rvu);3072	if (ret) {3073		dev_err(rvu->dev,3074			"RVUAF: IRQ registration failed for RVUAFVF FLR0\n");3075		goto fail;3076	}3077	rvu->irq_allocated[offset] = true;3078 3079	offset = pf_vec_start + RVU_PF_INT_VEC_VFFLR1;3080	sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF FLR1");3081	ret = request_irq(pci_irq_vector(rvu->pdev, offset),3082			  rvu_flr_intr_handler, 0,3083			  &rvu->irq_name[offset * NAME_SIZE], rvu);3084	if (ret) {3085		dev_err(rvu->dev,3086			"RVUAF: IRQ registration failed for RVUAFVF FLR1\n");3087		goto fail;3088	}3089	rvu->irq_allocated[offset] = true;3090 3091	/* Register ME interrupt handler for AF's VFs */3092	offset = pf_vec_start + RVU_PF_INT_VEC_VFME0;3093	sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF ME0");3094	ret = request_irq(pci_irq_vector(rvu->pdev, offset),3095			  rvu_me_vf_intr_handler, 0,3096			  &rvu->irq_name[offset * NAME_SIZE], rvu);3097	if (ret) {3098		dev_err(rvu->dev,3099			"RVUAF: IRQ registration failed for RVUAFVF ME0\n");3100		goto fail;3101	}3102	rvu->irq_allocated[offset] = true;3103 3104	offset = pf_vec_start + RVU_PF_INT_VEC_VFME1;3105	sprintf(&rvu->irq_name[offset * NAME_SIZE], "RVUAFVF ME1");3106	ret = request_irq(pci_irq_vector(rvu->pdev, offset),3107			  rvu_me_vf_intr_handler, 0,3108			  &rvu->irq_name[offset * NAME_SIZE], rvu);3109	if (ret) {3110		dev_err(rvu->dev,3111			"RVUAF: IRQ registration failed for RVUAFVF ME1\n");3112		goto fail;3113	}3114	rvu->irq_allocated[offset] = true;3115 3116	ret = rvu_cpt_register_interrupts(rvu);3117	if (ret)3118		goto fail;3119 3120	return 0;3121 3122fail:3123	rvu_unregister_interrupts(rvu);3124	return ret;3125}3126 3127static void rvu_flr_wq_destroy(struct rvu *rvu)3128{3129	if (rvu->flr_wq) {3130		destroy_workqueue(rvu->flr_wq);3131		rvu->flr_wq = NULL;3132	}3133}3134 3135static int rvu_flr_init(struct rvu *rvu)3136{3137	int dev, num_devs;3138	u64 cfg;3139	int pf;3140 3141	/* Enable FLR for all PFs*/3142	for (pf = 0; pf < rvu->hw->total_pfs; pf++) {3143		cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));3144		rvu_write64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf),3145			    cfg | BIT_ULL(22));3146	}3147 3148	rvu->flr_wq = alloc_ordered_workqueue("rvu_afpf_flr",3149					      WQ_HIGHPRI | WQ_MEM_RECLAIM);3150	if (!rvu->flr_wq)3151		return -ENOMEM;3152 3153	num_devs = rvu->hw->total_pfs + pci_sriov_get_totalvfs(rvu->pdev);3154	rvu->flr_wrk = devm_kcalloc(rvu->dev, num_devs,3155				    sizeof(struct rvu_work), GFP_KERNEL);3156	if (!rvu->flr_wrk) {3157		destroy_workqueue(rvu->flr_wq);3158		return -ENOMEM;3159	}3160 3161	for (dev = 0; dev < num_devs; dev++) {3162		rvu->flr_wrk[dev].rvu = rvu;3163		INIT_WORK(&rvu->flr_wrk[dev].work, rvu_flr_handler);3164	}3165 3166	mutex_init(&rvu->flr_lock);3167 3168	return 0;3169}3170 3171static void rvu_disable_afvf_intr(struct rvu *rvu)3172{3173	int vfs = rvu->vfs;3174 3175	rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(0), INTR_MASK(vfs));3176	rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1CX(0), INTR_MASK(vfs));3177	rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1CX(0), INTR_MASK(vfs));3178	if (vfs <= 64)3179		return;3180 3181	rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(1),3182		      INTR_MASK(vfs - 64));3183	rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1CX(1), INTR_MASK(vfs - 64));3184	rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1CX(1), INTR_MASK(vfs - 64));3185}3186 3187static void rvu_enable_afvf_intr(struct rvu *rvu)3188{3189	int vfs = rvu->vfs;3190 3191	/* Clear any pending interrupts and enable AF VF interrupts for3192	 * the first 64 VFs.3193	 */3194	/* Mbox */3195	rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(0), INTR_MASK(vfs));3196	rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(0), INTR_MASK(vfs));3197 3198	/* FLR */3199	rvupf_write64(rvu, RVU_PF_VFFLR_INTX(0), INTR_MASK(vfs));3200	rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1SX(0), INTR_MASK(vfs));3201	rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1SX(0), INTR_MASK(vfs));3202 3203	/* Same for remaining VFs, if any. */3204	if (vfs <= 64)3205		return;3206 3207	rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INTX(1), INTR_MASK(vfs - 64));3208	rvupf_write64(rvu, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(1),3209		      INTR_MASK(vfs - 64));3210 3211	rvupf_write64(rvu, RVU_PF_VFFLR_INTX(1), INTR_MASK(vfs - 64));3212	rvupf_write64(rvu, RVU_PF_VFFLR_INT_ENA_W1SX(1), INTR_MASK(vfs - 64));3213	rvupf_write64(rvu, RVU_PF_VFME_INT_ENA_W1SX(1), INTR_MASK(vfs - 64));3214}3215 3216int rvu_get_num_lbk_chans(void)3217{3218	struct pci_dev *pdev;3219	void __iomem *base;3220	int ret = -EIO;3221 3222	pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_LBK,3223			      NULL);3224	if (!pdev)3225		goto err;3226 3227	base = pci_ioremap_bar(pdev, 0);3228	if (!base)3229		goto err_put;3230 3231	/* Read number of available LBK channels from LBK(0)_CONST register. */3232	ret = (readq(base + 0x10) >> 32) & 0xffff;3233	iounmap(base);3234err_put:3235	pci_dev_put(pdev);3236err:3237	return ret;3238}3239 3240static int rvu_enable_sriov(struct rvu *rvu)3241{3242	struct pci_dev *pdev = rvu->pdev;3243	int err, chans, vfs;3244	int pos = 0;3245 3246	if (!rvu_afvf_msix_vectors_num_ok(rvu)) {3247		dev_warn(&pdev->dev,3248			 "Skipping SRIOV enablement since not enough IRQs are available\n");3249		return 0;3250	}3251 3252	/* Get RVU VFs device id */3253	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);3254	if (!pos)3255		return 0;3256	pci_read_config_word(pdev, pos + PCI_SRIOV_VF_DID, &rvu->vf_devid);3257 3258	chans = rvu_get_num_lbk_chans();3259	if (chans < 0)3260		return chans;3261 3262	vfs = pci_sriov_get_totalvfs(pdev);3263 3264	/* Limit VFs in case we have more VFs than LBK channels available. */3265	if (vfs > chans)3266		vfs = chans;3267 3268	if (!vfs)3269		return 0;3270 3271	/* LBK channel number 63 is used for switching packets between3272	 * CGX mapped VFs. Hence limit LBK pairs till 62 only.3273	 */3274	if (vfs > 62)3275		vfs = 62;3276 3277	/* Save VFs number for reference in VF interrupts handlers.3278	 * Since interrupts might start arriving during SRIOV enablement3279	 * ordinary API cannot be used to get number of enabled VFs.3280	 */3281	rvu->vfs = vfs;3282 3283	err = rvu_mbox_init(rvu, &rvu->afvf_wq_info, TYPE_AFVF, vfs,3284			    rvu_afvf_mbox_handler, rvu_afvf_mbox_up_handler);3285	if (err)3286		return err;3287 3288	rvu_enable_afvf_intr(rvu);3289	/* Make sure IRQs are enabled before SRIOV. */3290	mb();3291 3292	err = pci_enable_sriov(pdev, vfs);3293	if (err) {3294		rvu_disable_afvf_intr(rvu);3295		rvu_mbox_destroy(&rvu->afvf_wq_info);3296		return err;3297	}3298 3299	return 0;3300}3301 3302static void rvu_disable_sriov(struct rvu *rvu)3303{3304	rvu_disable_afvf_intr(rvu);3305	rvu_mbox_destroy(&rvu->afvf_wq_info);3306	pci_disable_sriov(rvu->pdev);3307}3308 3309static void rvu_update_module_params(struct rvu *rvu)3310{3311	const char *default_pfl_name = "default";3312 3313	strscpy(rvu->mkex_pfl_name,3314		mkex_profile ? mkex_profile : default_pfl_name, MKEX_NAME_LEN);3315	strscpy(rvu->kpu_pfl_name,3316		kpu_profile ? kpu_profile : default_pfl_name, KPU_NAME_LEN);3317}3318 3319static int rvu_probe(struct pci_dev *pdev, const struct pci_device_id *id)3320{3321	struct device *dev = &pdev->dev;3322	struct rvu *rvu;3323	int    err;3324 3325	rvu = devm_kzalloc(dev, sizeof(*rvu), GFP_KERNEL);3326	if (!rvu)3327		return -ENOMEM;3328 3329	rvu->hw = devm_kzalloc(dev, sizeof(struct rvu_hwinfo), GFP_KERNEL);3330	if (!rvu->hw) {3331		devm_kfree(dev, rvu);3332		return -ENOMEM;3333	}3334 3335	pci_set_drvdata(pdev, rvu);3336	rvu->pdev = pdev;3337	rvu->dev = &pdev->dev;3338 3339	err = pci_enable_device(pdev);3340	if (err) {3341		dev_err(dev, "Failed to enable PCI device\n");3342		goto err_freemem;3343	}3344 3345	err = pci_request_regions(pdev, DRV_NAME);3346	if (err) {3347		dev_err(dev, "PCI request regions failed 0x%x\n", err);3348		goto err_disable_device;3349	}3350 3351	err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));3352	if (err) {3353		dev_err(dev, "DMA mask config failed, abort\n");3354		goto err_release_regions;3355	}3356 3357	pci_set_master(pdev);3358 3359	rvu->ptp = ptp_get();3360	if (IS_ERR(rvu->ptp)) {3361		err = PTR_ERR(rvu->ptp);3362		if (err)3363			goto err_release_regions;3364		rvu->ptp = NULL;3365	}3366 3367	/* Map Admin function CSRs */3368	rvu->afreg_base = pcim_iomap(pdev, PCI_AF_REG_BAR_NUM, 0);3369	rvu->pfreg_base = pcim_iomap(pdev, PCI_PF_REG_BAR_NUM, 0);3370	if (!rvu->afreg_base || !rvu->pfreg_base) {3371		dev_err(dev, "Unable to map admin function CSRs, aborting\n");3372		err = -ENOMEM;3373		goto err_put_ptp;3374	}3375 3376	/* Store module params in rvu structure */3377	rvu_update_module_params(rvu);3378 3379	/* Check which blocks the HW supports */3380	rvu_check_block_implemented(rvu);3381 3382	rvu_reset_all_blocks(rvu);3383 3384	rvu_setup_hw_capabilities(rvu);3385 3386	err = rvu_setup_hw_resources(rvu);3387	if (err)3388		goto err_put_ptp;3389 3390	/* Init mailbox btw AF and PFs */3391	err = rvu_mbox_init(rvu, &rvu->afpf_wq_info, TYPE_AFPF,3392			    rvu->hw->total_pfs, rvu_afpf_mbox_handler,3393			    rvu_afpf_mbox_up_handler);3394	if (err) {3395		dev_err(dev, "%s: Failed to initialize mbox\n", __func__);3396		goto err_hwsetup;3397	}3398 3399	err = rvu_flr_init(rvu);3400	if (err) {3401		dev_err(dev, "%s: Failed to initialize flr\n", __func__);3402		goto err_mbox;3403	}3404 3405	err = rvu_register_interrupts(rvu);3406	if (err) {3407		dev_err(dev, "%s: Failed to register interrupts\n", __func__);3408		goto err_flr;3409	}3410 3411	err = rvu_register_dl(rvu);3412	if (err) {3413		dev_err(dev, "%s: Failed to register devlink\n", __func__);3414		goto err_irq;3415	}3416 3417	rvu_setup_rvum_blk_revid(rvu);3418 3419	/* Enable AF's VFs (if any) */3420	err = rvu_enable_sriov(rvu);3421	if (err) {3422		dev_err(dev, "%s: Failed to enable sriov\n", __func__);3423		goto err_dl;3424	}3425 3426	/* Initialize debugfs */3427	rvu_dbg_init(rvu);3428 3429	mutex_init(&rvu->rswitch.switch_lock);3430 3431	if (rvu->fwdata)3432		ptp_start(rvu, rvu->fwdata->sclk, rvu->fwdata->ptp_ext_clk_rate,3433			  rvu->fwdata->ptp_ext_tstamp);3434 3435	return 0;3436err_dl:3437	rvu_unregister_dl(rvu);3438err_irq:3439	rvu_unregister_interrupts(rvu);3440err_flr:3441	rvu_flr_wq_destroy(rvu);3442err_mbox:3443	rvu_mbox_destroy(&rvu->afpf_wq_info);3444err_hwsetup:3445	rvu_cgx_exit(rvu);3446	rvu_fwdata_exit(rvu);3447	rvu_mcs_exit(rvu);3448	rvu_reset_all_blocks(rvu);3449	rvu_free_hw_resources(rvu);3450	rvu_clear_rvum_blk_revid(rvu);3451err_put_ptp:3452	ptp_put(rvu->ptp);3453err_release_regions:3454	pci_release_regions(pdev);3455err_disable_device:3456	pci_disable_device(pdev);3457err_freemem:3458	pci_set_drvdata(pdev, NULL);3459	devm_kfree(&pdev->dev, rvu->hw);3460	devm_kfree(dev, rvu);3461	return err;3462}3463 3464static void rvu_remove(struct pci_dev *pdev)3465{3466	struct rvu *rvu = pci_get_drvdata(pdev);3467 3468	rvu_dbg_exit(rvu);3469	rvu_unregister_dl(rvu);3470	rvu_unregister_interrupts(rvu);3471	rvu_flr_wq_destroy(rvu);3472	rvu_cgx_exit(rvu);3473	rvu_fwdata_exit(rvu);3474	rvu_mcs_exit(rvu);3475	rvu_mbox_destroy(&rvu->afpf_wq_info);3476	rvu_disable_sriov(rvu);3477	rvu_reset_all_blocks(rvu);3478	rvu_free_hw_resources(rvu);3479	rvu_clear_rvum_blk_revid(rvu);3480	ptp_put(rvu->ptp);3481	pci_release_regions(pdev);3482	pci_disable_device(pdev);3483	pci_set_drvdata(pdev, NULL);3484 3485	devm_kfree(&pdev->dev, rvu->hw);3486	devm_kfree(&pdev->dev, rvu);3487}3488 3489static struct pci_driver rvu_driver = {3490	.name = DRV_NAME,3491	.id_table = rvu_id_table,3492	.probe = rvu_probe,3493	.remove = rvu_remove,3494};3495 3496static int __init rvu_init_module(void)3497{3498	int err;3499 3500	pr_info("%s: %s\n", DRV_NAME, DRV_STRING);3501 3502	err = pci_register_driver(&cgx_driver);3503	if (err < 0)3504		return err;3505 3506	err = pci_register_driver(&ptp_driver);3507	if (err < 0)3508		goto ptp_err;3509 3510	err = pci_register_driver(&mcs_driver);3511	if (err < 0)3512		goto mcs_err;3513 3514	err =  pci_register_driver(&rvu_driver);3515	if (err < 0)3516		goto rvu_err;3517 3518	return 0;3519rvu_err:3520	pci_unregister_driver(&mcs_driver);3521mcs_err:3522	pci_unregister_driver(&ptp_driver);3523ptp_err:3524	pci_unregister_driver(&cgx_driver);3525 3526	return err;3527}3528 3529static void __exit rvu_cleanup_module(void)3530{3531	pci_unregister_driver(&rvu_driver);3532	pci_unregister_driver(&mcs_driver);3533	pci_unregister_driver(&ptp_driver);3534	pci_unregister_driver(&cgx_driver);3535}3536 3537module_init(rvu_init_module);3538module_exit(rvu_cleanup_module);3539