1320 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/* Marvell RVU Admin Function driver3 *4 * Copyright (C) 2020 Marvell.5 *6 */7 8#include <linux/bitfield.h>9#include <linux/pci.h>10#include "rvu_struct.h"11#include "rvu_reg.h"12#include "mbox.h"13#include "rvu.h"14 15/* CPT PF device id */16#define PCI_DEVID_OTX2_CPT_PF 0xA0FD17#define PCI_DEVID_OTX2_CPT10K_PF 0xA0F218 19/* Length of initial context fetch in 128 byte words */20#define CPT_CTX_ILEN 1ULL21 22/* Interrupt vector count of CPT RVU and RAS interrupts */23#define CPT_10K_AF_RVU_RAS_INT_VEC_CNT 224 25/* Default CPT_AF_RXC_CFG1:max_rxc_icb_cnt */26#define CPT_DFLT_MAX_RXC_ICB_CNT 0xC0ULL27 28#define cpt_get_eng_sts(e_min, e_max, rsp, etype) \29({ \30 u64 free_sts = 0, busy_sts = 0; \31 typeof(rsp) _rsp = rsp; \32 u32 e, i; \33 \34 for (e = (e_min), i = 0; e < (e_max); e++, i++) { \35 reg = rvu_read64(rvu, blkaddr, CPT_AF_EXEX_STS(e)); \36 if (reg & 0x1) \37 busy_sts |= 1ULL << i; \38 \39 if (reg & 0x2) \40 free_sts |= 1ULL << i; \41 } \42 (_rsp)->busy_sts_##etype = busy_sts; \43 (_rsp)->free_sts_##etype = free_sts; \44})45 46#define MAX_AE GENMASK_ULL(47, 32)47#define MAX_IE GENMASK_ULL(31, 16)48#define MAX_SE GENMASK_ULL(15, 0)49 50static u16 cpt_max_engines_get(struct rvu *rvu)51{52 u16 max_ses, max_ies, max_aes;53 u64 reg;54 55 reg = rvu_read64(rvu, BLKADDR_CPT0, CPT_AF_CONSTANTS1);56 max_ses = FIELD_GET(MAX_SE, reg);57 max_ies = FIELD_GET(MAX_IE, reg);58 max_aes = FIELD_GET(MAX_AE, reg);59 60 return max_ses + max_ies + max_aes;61}62 63/* Number of flt interrupt vectors are depends on number of engines that the64 * chip has. Each flt vector represents 64 engines.65 */66static int cpt_10k_flt_nvecs_get(struct rvu *rvu, u16 max_engs)67{68 int flt_vecs;69 70 flt_vecs = DIV_ROUND_UP(max_engs, 64);71 72 if (flt_vecs > CPT_10K_AF_INT_VEC_FLT_MAX) {73 dev_warn_once(rvu->dev, "flt_vecs:%d exceeds the max vectors:%d\n",74 flt_vecs, CPT_10K_AF_INT_VEC_FLT_MAX);75 flt_vecs = CPT_10K_AF_INT_VEC_FLT_MAX;76 }77 78 return flt_vecs;79}80 81static irqreturn_t cpt_af_flt_intr_handler(int vec, void *ptr)82{83 struct rvu_block *block = ptr;84 struct rvu *rvu = block->rvu;85 int blkaddr = block->addr;86 u64 reg, val;87 int i, eng;88 u8 grp;89 90 reg = rvu_read64(rvu, blkaddr, CPT_AF_FLTX_INT(vec));91 dev_err_ratelimited(rvu->dev, "Received CPTAF FLT%d irq : 0x%llx", vec, reg);92 93 i = -1;94 while ((i = find_next_bit((unsigned long *)®, 64, i + 1)) < 64) {95 switch (vec) {96 case 0:97 eng = i;98 break;99 case 1:100 eng = i + 64;101 break;102 case 2:103 eng = i + 128;104 break;105 }106 grp = rvu_read64(rvu, blkaddr, CPT_AF_EXEX_CTL2(eng)) & 0xFF;107 /* Disable and enable the engine which triggers fault */108 rvu_write64(rvu, blkaddr, CPT_AF_EXEX_CTL2(eng), 0x0);109 val = rvu_read64(rvu, blkaddr, CPT_AF_EXEX_CTL(eng));110 rvu_write64(rvu, blkaddr, CPT_AF_EXEX_CTL(eng), val & ~1ULL);111 112 rvu_write64(rvu, blkaddr, CPT_AF_EXEX_CTL2(eng), grp);113 rvu_write64(rvu, blkaddr, CPT_AF_EXEX_CTL(eng), val | 1ULL);114 115 spin_lock(&rvu->cpt_intr_lock);116 block->cpt_flt_eng_map[vec] |= BIT_ULL(i);117 val = rvu_read64(rvu, blkaddr, CPT_AF_EXEX_STS(eng));118 val = val & 0x3;119 if (val == 0x1 || val == 0x2)120 block->cpt_rcvrd_eng_map[vec] |= BIT_ULL(i);121 spin_unlock(&rvu->cpt_intr_lock);122 }123 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT(vec), reg);124 125 return IRQ_HANDLED;126}127 128static irqreturn_t rvu_cpt_af_flt0_intr_handler(int irq, void *ptr)129{130 return cpt_af_flt_intr_handler(CPT_AF_INT_VEC_FLT0, ptr);131}132 133static irqreturn_t rvu_cpt_af_flt1_intr_handler(int irq, void *ptr)134{135 return cpt_af_flt_intr_handler(CPT_AF_INT_VEC_FLT1, ptr);136}137 138static irqreturn_t rvu_cpt_af_flt2_intr_handler(int irq, void *ptr)139{140 return cpt_af_flt_intr_handler(CPT_10K_AF_INT_VEC_FLT2, ptr);141}142 143static irqreturn_t rvu_cpt_af_rvu_intr_handler(int irq, void *ptr)144{145 struct rvu_block *block = ptr;146 struct rvu *rvu = block->rvu;147 int blkaddr = block->addr;148 u64 reg;149 150 reg = rvu_read64(rvu, blkaddr, CPT_AF_RVU_INT);151 dev_err_ratelimited(rvu->dev, "Received CPTAF RVU irq : 0x%llx", reg);152 153 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT, reg);154 return IRQ_HANDLED;155}156 157static irqreturn_t rvu_cpt_af_ras_intr_handler(int irq, void *ptr)158{159 struct rvu_block *block = ptr;160 struct rvu *rvu = block->rvu;161 int blkaddr = block->addr;162 u64 reg;163 164 reg = rvu_read64(rvu, blkaddr, CPT_AF_RAS_INT);165 dev_err_ratelimited(rvu->dev, "Received CPTAF RAS irq : 0x%llx", reg);166 167 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT, reg);168 return IRQ_HANDLED;169}170 171static int rvu_cpt_do_register_interrupt(struct rvu_block *block, int irq_offs,172 irq_handler_t handler,173 const char *name)174{175 struct rvu *rvu = block->rvu;176 int ret;177 178 ret = request_irq(pci_irq_vector(rvu->pdev, irq_offs), handler, 0,179 name, block);180 if (ret) {181 dev_err(rvu->dev, "RVUAF: %s irq registration failed", name);182 return ret;183 }184 185 WARN_ON(rvu->irq_allocated[irq_offs]);186 rvu->irq_allocated[irq_offs] = true;187 return 0;188}189 190static void cpt_10k_unregister_interrupts(struct rvu_block *block, int off)191{192 struct rvu *rvu = block->rvu;193 int blkaddr = block->addr;194 int i, flt_vecs;195 u16 max_engs;196 u8 nr;197 198 max_engs = cpt_max_engines_get(rvu);199 flt_vecs = cpt_10k_flt_nvecs_get(rvu, max_engs);200 201 /* Disable all CPT AF interrupts */202 for (i = CPT_10K_AF_INT_VEC_FLT0; i < flt_vecs; i++) {203 nr = (max_engs > 64) ? 64 : max_engs;204 max_engs -= nr;205 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1C(i),206 INTR_MASK(nr));207 }208 209 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1C, 0x1);210 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1C, 0x1);211 212 /* CPT AF interrupt vectors are flt_int, rvu_int and ras_int. */213 for (i = 0; i < flt_vecs + CPT_10K_AF_RVU_RAS_INT_VEC_CNT; i++)214 if (rvu->irq_allocated[off + i]) {215 free_irq(pci_irq_vector(rvu->pdev, off + i), block);216 rvu->irq_allocated[off + i] = false;217 }218}219 220static void cpt_unregister_interrupts(struct rvu *rvu, int blkaddr)221{222 struct rvu_hwinfo *hw = rvu->hw;223 struct rvu_block *block;224 int i, offs;225 226 if (!is_block_implemented(rvu->hw, blkaddr))227 return;228 offs = rvu_read64(rvu, blkaddr, CPT_PRIV_AF_INT_CFG) & 0x7FF;229 if (!offs) {230 dev_warn(rvu->dev,231 "Failed to get CPT_AF_INT vector offsets\n");232 return;233 }234 block = &hw->block[blkaddr];235 if (!is_rvu_otx2(rvu))236 return cpt_10k_unregister_interrupts(block, offs);237 238 /* Disable all CPT AF interrupts */239 for (i = 0; i < CPT_AF_INT_VEC_RVU; i++)240 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1C(i), ~0ULL);241 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1C, 0x1);242 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1C, 0x1);243 244 for (i = 0; i < CPT_AF_INT_VEC_CNT; i++)245 if (rvu->irq_allocated[offs + i]) {246 free_irq(pci_irq_vector(rvu->pdev, offs + i), block);247 rvu->irq_allocated[offs + i] = false;248 }249}250 251void rvu_cpt_unregister_interrupts(struct rvu *rvu)252{253 cpt_unregister_interrupts(rvu, BLKADDR_CPT0);254 cpt_unregister_interrupts(rvu, BLKADDR_CPT1);255}256 257static int cpt_10k_register_interrupts(struct rvu_block *block, int off)258{259 int rvu_intr_vec, ras_intr_vec;260 struct rvu *rvu = block->rvu;261 int blkaddr = block->addr;262 irq_handler_t flt_fn;263 int i, ret, flt_vecs;264 u16 max_engs;265 u8 nr;266 267 max_engs = cpt_max_engines_get(rvu);268 flt_vecs = cpt_10k_flt_nvecs_get(rvu, max_engs);269 270 for (i = CPT_10K_AF_INT_VEC_FLT0; i < flt_vecs; i++) {271 sprintf(&rvu->irq_name[(off + i) * NAME_SIZE], "CPTAF FLT%d", i);272 273 switch (i) {274 case CPT_10K_AF_INT_VEC_FLT0:275 flt_fn = rvu_cpt_af_flt0_intr_handler;276 break;277 case CPT_10K_AF_INT_VEC_FLT1:278 flt_fn = rvu_cpt_af_flt1_intr_handler;279 break;280 case CPT_10K_AF_INT_VEC_FLT2:281 flt_fn = rvu_cpt_af_flt2_intr_handler;282 break;283 }284 ret = rvu_cpt_do_register_interrupt(block, off + i,285 flt_fn, &rvu->irq_name[(off + i) * NAME_SIZE]);286 if (ret)287 goto err;288 289 nr = (max_engs > 64) ? 64 : max_engs;290 max_engs -= nr;291 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1S(i),292 INTR_MASK(nr));293 }294 295 rvu_intr_vec = flt_vecs;296 ras_intr_vec = rvu_intr_vec + 1;297 298 ret = rvu_cpt_do_register_interrupt(block, off + rvu_intr_vec,299 rvu_cpt_af_rvu_intr_handler,300 "CPTAF RVU");301 if (ret)302 goto err;303 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1S, 0x1);304 305 ret = rvu_cpt_do_register_interrupt(block, off + ras_intr_vec,306 rvu_cpt_af_ras_intr_handler,307 "CPTAF RAS");308 if (ret)309 goto err;310 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1S, 0x1);311 312 return 0;313err:314 rvu_cpt_unregister_interrupts(rvu);315 return ret;316}317 318static int cpt_register_interrupts(struct rvu *rvu, int blkaddr)319{320 struct rvu_hwinfo *hw = rvu->hw;321 struct rvu_block *block;322 irq_handler_t flt_fn;323 int i, offs, ret = 0;324 325 if (!is_block_implemented(rvu->hw, blkaddr))326 return 0;327 328 block = &hw->block[blkaddr];329 offs = rvu_read64(rvu, blkaddr, CPT_PRIV_AF_INT_CFG) & 0x7FF;330 if (!offs) {331 dev_warn(rvu->dev,332 "Failed to get CPT_AF_INT vector offsets\n");333 return 0;334 }335 336 if (!is_rvu_otx2(rvu))337 return cpt_10k_register_interrupts(block, offs);338 339 for (i = CPT_AF_INT_VEC_FLT0; i < CPT_AF_INT_VEC_RVU; i++) {340 sprintf(&rvu->irq_name[(offs + i) * NAME_SIZE], "CPTAF FLT%d", i);341 switch (i) {342 case CPT_AF_INT_VEC_FLT0:343 flt_fn = rvu_cpt_af_flt0_intr_handler;344 break;345 case CPT_AF_INT_VEC_FLT1:346 flt_fn = rvu_cpt_af_flt1_intr_handler;347 break;348 }349 ret = rvu_cpt_do_register_interrupt(block, offs + i,350 flt_fn, &rvu->irq_name[(offs + i) * NAME_SIZE]);351 if (ret)352 goto err;353 rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1S(i), ~0ULL);354 }355 356 ret = rvu_cpt_do_register_interrupt(block, offs + CPT_AF_INT_VEC_RVU,357 rvu_cpt_af_rvu_intr_handler,358 "CPTAF RVU");359 if (ret)360 goto err;361 rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1S, 0x1);362 363 ret = rvu_cpt_do_register_interrupt(block, offs + CPT_AF_INT_VEC_RAS,364 rvu_cpt_af_ras_intr_handler,365 "CPTAF RAS");366 if (ret)367 goto err;368 rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1S, 0x1);369 370 return 0;371err:372 rvu_cpt_unregister_interrupts(rvu);373 return ret;374}375 376int rvu_cpt_register_interrupts(struct rvu *rvu)377{378 int ret;379 380 ret = cpt_register_interrupts(rvu, BLKADDR_CPT0);381 if (ret)382 return ret;383 384 return cpt_register_interrupts(rvu, BLKADDR_CPT1);385}386 387static int get_cpt_pf_num(struct rvu *rvu)388{389 int i, domain_nr, cpt_pf_num = -1;390 struct pci_dev *pdev;391 392 domain_nr = pci_domain_nr(rvu->pdev->bus);393 for (i = 0; i < rvu->hw->total_pfs; i++) {394 pdev = pci_get_domain_bus_and_slot(domain_nr, i + 1, 0);395 if (!pdev)396 continue;397 398 if (pdev->device == PCI_DEVID_OTX2_CPT_PF ||399 pdev->device == PCI_DEVID_OTX2_CPT10K_PF) {400 cpt_pf_num = i;401 put_device(&pdev->dev);402 break;403 }404 put_device(&pdev->dev);405 }406 return cpt_pf_num;407}408 409static bool is_cpt_pf(struct rvu *rvu, u16 pcifunc)410{411 int cpt_pf_num = rvu->cpt_pf_num;412 413 if (rvu_get_pf(pcifunc) != cpt_pf_num)414 return false;415 if (pcifunc & RVU_PFVF_FUNC_MASK)416 return false;417 418 return true;419}420 421static bool is_cpt_vf(struct rvu *rvu, u16 pcifunc)422{423 int cpt_pf_num = rvu->cpt_pf_num;424 425 if (rvu_get_pf(pcifunc) != cpt_pf_num)426 return false;427 if (!(pcifunc & RVU_PFVF_FUNC_MASK))428 return false;429 430 return true;431}432 433static int validate_and_get_cpt_blkaddr(int req_blkaddr)434{435 int blkaddr;436 437 blkaddr = req_blkaddr ? req_blkaddr : BLKADDR_CPT0;438 if (blkaddr != BLKADDR_CPT0 && blkaddr != BLKADDR_CPT1)439 return -EINVAL;440 441 return blkaddr;442}443 444int rvu_mbox_handler_cpt_lf_alloc(struct rvu *rvu,445 struct cpt_lf_alloc_req_msg *req,446 struct msg_rsp *rsp)447{448 u16 pcifunc = req->hdr.pcifunc;449 struct rvu_block *block;450 int cptlf, blkaddr;451 int num_lfs, slot;452 u64 val;453 454 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);455 if (blkaddr < 0)456 return blkaddr;457 458 if (req->eng_grpmsk == 0x0)459 return CPT_AF_ERR_GRP_INVALID;460 461 block = &rvu->hw->block[blkaddr];462 num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc),463 block->addr);464 if (!num_lfs)465 return CPT_AF_ERR_LF_INVALID;466 467 /* Check if requested 'CPTLF <=> NIXLF' mapping is valid */468 if (req->nix_pf_func) {469 /* If default, use 'this' CPTLF's PFFUNC */470 if (req->nix_pf_func == RVU_DEFAULT_PF_FUNC)471 req->nix_pf_func = pcifunc;472 if (!is_pffunc_map_valid(rvu, req->nix_pf_func, BLKTYPE_NIX))473 return CPT_AF_ERR_NIX_PF_FUNC_INVALID;474 }475 476 /* Check if requested 'CPTLF <=> SSOLF' mapping is valid */477 if (req->sso_pf_func) {478 /* If default, use 'this' CPTLF's PFFUNC */479 if (req->sso_pf_func == RVU_DEFAULT_PF_FUNC)480 req->sso_pf_func = pcifunc;481 if (!is_pffunc_map_valid(rvu, req->sso_pf_func, BLKTYPE_SSO))482 return CPT_AF_ERR_SSO_PF_FUNC_INVALID;483 }484 485 for (slot = 0; slot < num_lfs; slot++) {486 cptlf = rvu_get_lf(rvu, block, pcifunc, slot);487 if (cptlf < 0)488 return CPT_AF_ERR_LF_INVALID;489 490 /* Set CPT LF group and priority */491 val = (u64)req->eng_grpmsk << 48 | 1;492 if (!is_rvu_otx2(rvu)) {493 if (req->ctx_ilen_valid)494 val |= (req->ctx_ilen << 17);495 else496 val |= (CPT_CTX_ILEN << 17);497 }498 499 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val);500 501 /* Set CPT LF NIX_PF_FUNC and SSO_PF_FUNC. EXE_LDWB is set502 * on reset.503 */504 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf));505 val &= ~(GENMASK_ULL(63, 48) | GENMASK_ULL(47, 32));506 val |= ((u64)req->nix_pf_func << 48 |507 (u64)req->sso_pf_func << 32);508 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), val);509 }510 511 return 0;512}513 514static int cpt_lf_free(struct rvu *rvu, struct msg_req *req, int blkaddr)515{516 u16 pcifunc = req->hdr.pcifunc;517 int num_lfs, cptlf, slot, err;518 struct rvu_block *block;519 520 block = &rvu->hw->block[blkaddr];521 num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc),522 block->addr);523 if (!num_lfs)524 return 0;525 526 for (slot = 0; slot < num_lfs; slot++) {527 cptlf = rvu_get_lf(rvu, block, pcifunc, slot);528 if (cptlf < 0)529 return CPT_AF_ERR_LF_INVALID;530 531 /* Perform teardown */532 rvu_cpt_lf_teardown(rvu, pcifunc, blkaddr, cptlf, slot);533 534 /* Reset LF */535 err = rvu_lf_reset(rvu, block, cptlf);536 if (err) {537 dev_err(rvu->dev, "Failed to reset blkaddr %d LF%d\n",538 block->addr, cptlf);539 }540 }541 542 return 0;543}544 545int rvu_mbox_handler_cpt_lf_free(struct rvu *rvu, struct msg_req *req,546 struct msg_rsp *rsp)547{548 int ret;549 550 ret = cpt_lf_free(rvu, req, BLKADDR_CPT0);551 if (ret)552 return ret;553 554 if (is_block_implemented(rvu->hw, BLKADDR_CPT1))555 ret = cpt_lf_free(rvu, req, BLKADDR_CPT1);556 557 return ret;558}559 560static int cpt_inline_ipsec_cfg_inbound(struct rvu *rvu, int blkaddr, u8 cptlf,561 struct cpt_inline_ipsec_cfg_msg *req)562{563 u16 sso_pf_func = req->sso_pf_func;564 u8 nix_sel;565 u64 val;566 567 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf));568 if (req->enable && (val & BIT_ULL(16))) {569 /* IPSec inline outbound path is already enabled for a given570 * CPT LF, HRM states that inline inbound & outbound paths571 * must not be enabled at the same time for a given CPT LF572 */573 return CPT_AF_ERR_INLINE_IPSEC_INB_ENA;574 }575 /* Check if requested 'CPTLF <=> SSOLF' mapping is valid */576 if (sso_pf_func && !is_pffunc_map_valid(rvu, sso_pf_func, BLKTYPE_SSO))577 return CPT_AF_ERR_SSO_PF_FUNC_INVALID;578 579 nix_sel = (blkaddr == BLKADDR_CPT1) ? 1 : 0;580 /* Enable CPT LF for IPsec inline inbound operations */581 if (req->enable)582 val |= BIT_ULL(9);583 else584 val &= ~BIT_ULL(9);585 586 val |= (u64)nix_sel << 8;587 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val);588 589 if (sso_pf_func) {590 /* Set SSO_PF_FUNC */591 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf));592 val |= (u64)sso_pf_func << 32;593 val |= (u64)req->nix_pf_func << 48;594 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), val);595 }596 if (req->sso_pf_func_ovrd)597 /* Set SSO_PF_FUNC_OVRD for inline IPSec */598 rvu_write64(rvu, blkaddr, CPT_AF_ECO, 0x1);599 600 /* Configure the X2P Link register with the cpt base channel number and601 * range of channels it should propagate to X2P602 */603 if (!is_rvu_otx2(rvu)) {604 val = (ilog2(NIX_CHAN_CPT_X2P_MASK + 1) << 16);605 val |= (u64)rvu->hw->cpt_chan_base;606 607 rvu_write64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(0), val);608 rvu_write64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(1), val);609 }610 611 return 0;612}613 614static int cpt_inline_ipsec_cfg_outbound(struct rvu *rvu, int blkaddr, u8 cptlf,615 struct cpt_inline_ipsec_cfg_msg *req)616{617 u16 nix_pf_func = req->nix_pf_func;618 int nix_blkaddr;619 u8 nix_sel;620 u64 val;621 622 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf));623 if (req->enable && (val & BIT_ULL(9))) {624 /* IPSec inline inbound path is already enabled for a given625 * CPT LF, HRM states that inline inbound & outbound paths626 * must not be enabled at the same time for a given CPT LF627 */628 return CPT_AF_ERR_INLINE_IPSEC_OUT_ENA;629 }630 631 /* Check if requested 'CPTLF <=> NIXLF' mapping is valid */632 if (nix_pf_func && !is_pffunc_map_valid(rvu, nix_pf_func, BLKTYPE_NIX))633 return CPT_AF_ERR_NIX_PF_FUNC_INVALID;634 635 /* Enable CPT LF for IPsec inline outbound operations */636 if (req->enable)637 val |= BIT_ULL(16);638 else639 val &= ~BIT_ULL(16);640 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val);641 642 if (nix_pf_func) {643 /* Set NIX_PF_FUNC */644 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf));645 val |= (u64)nix_pf_func << 48;646 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), val);647 648 nix_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, nix_pf_func);649 nix_sel = (nix_blkaddr == BLKADDR_NIX0) ? 0 : 1;650 651 val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf));652 val |= (u64)nix_sel << 8;653 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val);654 }655 656 return 0;657}658 659int rvu_mbox_handler_cpt_inline_ipsec_cfg(struct rvu *rvu,660 struct cpt_inline_ipsec_cfg_msg *req,661 struct msg_rsp *rsp)662{663 u16 pcifunc = req->hdr.pcifunc;664 struct rvu_block *block;665 int cptlf, blkaddr, ret;666 u16 actual_slot;667 668 blkaddr = rvu_get_blkaddr_from_slot(rvu, BLKTYPE_CPT, pcifunc,669 req->slot, &actual_slot);670 if (blkaddr < 0)671 return CPT_AF_ERR_LF_INVALID;672 673 block = &rvu->hw->block[blkaddr];674 675 cptlf = rvu_get_lf(rvu, block, pcifunc, actual_slot);676 if (cptlf < 0)677 return CPT_AF_ERR_LF_INVALID;678 679 switch (req->dir) {680 case CPT_INLINE_INBOUND:681 ret = cpt_inline_ipsec_cfg_inbound(rvu, blkaddr, cptlf, req);682 break;683 684 case CPT_INLINE_OUTBOUND:685 ret = cpt_inline_ipsec_cfg_outbound(rvu, blkaddr, cptlf, req);686 break;687 688 default:689 return CPT_AF_ERR_PARAM;690 }691 692 return ret;693}694 695static bool validate_and_update_reg_offset(struct rvu *rvu,696 struct cpt_rd_wr_reg_msg *req,697 u64 *reg_offset)698{699 u64 offset = req->reg_offset;700 int blkaddr, num_lfs, lf;701 struct rvu_block *block;702 struct rvu_pfvf *pfvf;703 704 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);705 if (blkaddr < 0)706 return false;707 708 /* Registers that can be accessed from PF/VF */709 if ((offset & 0xFF000) == CPT_AF_LFX_CTL(0) ||710 (offset & 0xFF000) == CPT_AF_LFX_CTL2(0)) {711 if (offset & 7)712 return false;713 714 lf = (offset & 0xFFF) >> 3;715 block = &rvu->hw->block[blkaddr];716 pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);717 num_lfs = rvu_get_rsrc_mapcount(pfvf, block->addr);718 if (lf >= num_lfs)719 /* Slot is not valid for that PF/VF */720 return false;721 722 /* Translate local LF used by VFs to global CPT LF */723 lf = rvu_get_lf(rvu, &rvu->hw->block[blkaddr],724 req->hdr.pcifunc, lf);725 if (lf < 0)726 return false;727 728 /* Translate local LF's offset to global CPT LF's offset to729 * access LFX register.730 */731 *reg_offset = (req->reg_offset & 0xFF000) + (lf << 3);732 733 return true;734 } else if (!(req->hdr.pcifunc & RVU_PFVF_FUNC_MASK)) {735 /* Registers that can be accessed from PF */736 switch (offset) {737 case CPT_AF_DIAG:738 case CPT_AF_CTL:739 case CPT_AF_PF_FUNC:740 case CPT_AF_BLK_RST:741 case CPT_AF_CONSTANTS1:742 case CPT_AF_CTX_FLUSH_TIMER:743 case CPT_AF_RXC_CFG1:744 return true;745 }746 747 switch (offset & 0xFF000) {748 case CPT_AF_EXEX_STS(0):749 case CPT_AF_EXEX_CTL(0):750 case CPT_AF_EXEX_CTL2(0):751 case CPT_AF_EXEX_UCODE_BASE(0):752 if (offset & 7)753 return false;754 break;755 default:756 return false;757 }758 return true;759 }760 return false;761}762 763int rvu_mbox_handler_cpt_rd_wr_register(struct rvu *rvu,764 struct cpt_rd_wr_reg_msg *req,765 struct cpt_rd_wr_reg_msg *rsp)766{767 u64 offset = req->reg_offset;768 int blkaddr;769 770 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);771 if (blkaddr < 0)772 return blkaddr;773 774 /* This message is accepted only if sent from CPT PF/VF */775 if (!is_cpt_pf(rvu, req->hdr.pcifunc) &&776 !is_cpt_vf(rvu, req->hdr.pcifunc))777 return CPT_AF_ERR_ACCESS_DENIED;778 779 if (!validate_and_update_reg_offset(rvu, req, &offset))780 return CPT_AF_ERR_ACCESS_DENIED;781 782 rsp->reg_offset = req->reg_offset;783 rsp->ret_val = req->ret_val;784 rsp->is_write = req->is_write;785 786 if (req->is_write)787 rvu_write64(rvu, blkaddr, offset, req->val);788 else789 rsp->val = rvu_read64(rvu, blkaddr, offset);790 791 return 0;792}793 794static void get_ctx_pc(struct rvu *rvu, struct cpt_sts_rsp *rsp, int blkaddr)795{796 struct rvu_hwinfo *hw = rvu->hw;797 798 if (is_rvu_otx2(rvu))799 return;800 801 rsp->ctx_mis_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_MIS_PC);802 rsp->ctx_hit_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_HIT_PC);803 rsp->ctx_aop_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_AOP_PC);804 rsp->ctx_aop_lat_pc = rvu_read64(rvu, blkaddr,805 CPT_AF_CTX_AOP_LATENCY_PC);806 rsp->ctx_ifetch_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_IFETCH_PC);807 rsp->ctx_ifetch_lat_pc = rvu_read64(rvu, blkaddr,808 CPT_AF_CTX_IFETCH_LATENCY_PC);809 rsp->ctx_ffetch_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FFETCH_PC);810 rsp->ctx_ffetch_lat_pc = rvu_read64(rvu, blkaddr,811 CPT_AF_CTX_FFETCH_LATENCY_PC);812 rsp->ctx_wback_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FFETCH_PC);813 rsp->ctx_wback_lat_pc = rvu_read64(rvu, blkaddr,814 CPT_AF_CTX_FFETCH_LATENCY_PC);815 rsp->ctx_psh_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FFETCH_PC);816 rsp->ctx_psh_lat_pc = rvu_read64(rvu, blkaddr,817 CPT_AF_CTX_FFETCH_LATENCY_PC);818 rsp->ctx_err = rvu_read64(rvu, blkaddr, CPT_AF_CTX_ERR);819 rsp->ctx_enc_id = rvu_read64(rvu, blkaddr, CPT_AF_CTX_ENC_ID);820 rsp->ctx_flush_timer = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FLUSH_TIMER);821 rsp->x2p_link_cfg0 = rvu_read64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(0));822 rsp->x2p_link_cfg1 = rvu_read64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(1));823 824 if (!hw->cap.cpt_rxc)825 return;826 rsp->rxc_time = rvu_read64(rvu, blkaddr, CPT_AF_RXC_TIME);827 rsp->rxc_time_cfg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_TIME_CFG);828 rsp->rxc_active_sts = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ACTIVE_STS);829 rsp->rxc_zombie_sts = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ZOMBIE_STS);830 rsp->rxc_dfrg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_DFRG);831}832 833static void get_eng_sts(struct rvu *rvu, struct cpt_sts_rsp *rsp, int blkaddr)834{835 u16 max_ses, max_ies, max_aes;836 u32 e_min = 0, e_max = 0;837 u64 reg;838 839 reg = rvu_read64(rvu, blkaddr, CPT_AF_CONSTANTS1);840 max_ses = reg & 0xffff;841 max_ies = (reg >> 16) & 0xffff;842 max_aes = (reg >> 32) & 0xffff;843 844 /* Get AE status */845 e_min = max_ses + max_ies;846 e_max = max_ses + max_ies + max_aes;847 cpt_get_eng_sts(e_min, e_max, rsp, ae);848 /* Get SE status */849 e_min = 0;850 e_max = max_ses;851 cpt_get_eng_sts(e_min, e_max, rsp, se);852 /* Get IE status */853 e_min = max_ses;854 e_max = max_ses + max_ies;855 cpt_get_eng_sts(e_min, e_max, rsp, ie);856}857 858int rvu_mbox_handler_cpt_sts(struct rvu *rvu, struct cpt_sts_req *req,859 struct cpt_sts_rsp *rsp)860{861 int blkaddr;862 863 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);864 if (blkaddr < 0)865 return blkaddr;866 867 /* This message is accepted only if sent from CPT PF/VF */868 if (!is_cpt_pf(rvu, req->hdr.pcifunc) &&869 !is_cpt_vf(rvu, req->hdr.pcifunc))870 return CPT_AF_ERR_ACCESS_DENIED;871 872 get_ctx_pc(rvu, rsp, blkaddr);873 874 /* Get CPT engines status */875 get_eng_sts(rvu, rsp, blkaddr);876 877 /* Read CPT instruction PC registers */878 rsp->inst_req_pc = rvu_read64(rvu, blkaddr, CPT_AF_INST_REQ_PC);879 rsp->inst_lat_pc = rvu_read64(rvu, blkaddr, CPT_AF_INST_LATENCY_PC);880 rsp->rd_req_pc = rvu_read64(rvu, blkaddr, CPT_AF_RD_REQ_PC);881 rsp->rd_lat_pc = rvu_read64(rvu, blkaddr, CPT_AF_RD_LATENCY_PC);882 rsp->rd_uc_pc = rvu_read64(rvu, blkaddr, CPT_AF_RD_UC_PC);883 rsp->active_cycles_pc = rvu_read64(rvu, blkaddr,884 CPT_AF_ACTIVE_CYCLES_PC);885 rsp->exe_err_info = rvu_read64(rvu, blkaddr, CPT_AF_EXE_ERR_INFO);886 rsp->cptclk_cnt = rvu_read64(rvu, blkaddr, CPT_AF_CPTCLK_CNT);887 rsp->diag = rvu_read64(rvu, blkaddr, CPT_AF_DIAG);888 889 return 0;890}891 892#define RXC_ZOMBIE_THRES GENMASK_ULL(59, 48)893#define RXC_ZOMBIE_LIMIT GENMASK_ULL(43, 32)894#define RXC_ACTIVE_THRES GENMASK_ULL(27, 16)895#define RXC_ACTIVE_LIMIT GENMASK_ULL(11, 0)896#define RXC_ACTIVE_COUNT GENMASK_ULL(60, 48)897#define RXC_ZOMBIE_COUNT GENMASK_ULL(60, 48)898 899static void cpt_rxc_time_cfg(struct rvu *rvu, struct cpt_rxc_time_cfg_req *req,900 int blkaddr, struct cpt_rxc_time_cfg_req *save)901{902 u64 dfrg_reg;903 904 if (save) {905 /* Save older config */906 dfrg_reg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_DFRG);907 save->zombie_thres = FIELD_GET(RXC_ZOMBIE_THRES, dfrg_reg);908 save->zombie_limit = FIELD_GET(RXC_ZOMBIE_LIMIT, dfrg_reg);909 save->active_thres = FIELD_GET(RXC_ACTIVE_THRES, dfrg_reg);910 save->active_limit = FIELD_GET(RXC_ACTIVE_LIMIT, dfrg_reg);911 912 save->step = rvu_read64(rvu, blkaddr, CPT_AF_RXC_TIME_CFG);913 }914 915 dfrg_reg = FIELD_PREP(RXC_ZOMBIE_THRES, req->zombie_thres);916 dfrg_reg |= FIELD_PREP(RXC_ZOMBIE_LIMIT, req->zombie_limit);917 dfrg_reg |= FIELD_PREP(RXC_ACTIVE_THRES, req->active_thres);918 dfrg_reg |= FIELD_PREP(RXC_ACTIVE_LIMIT, req->active_limit);919 920 rvu_write64(rvu, blkaddr, CPT_AF_RXC_TIME_CFG, req->step);921 rvu_write64(rvu, blkaddr, CPT_AF_RXC_DFRG, dfrg_reg);922}923 924int rvu_mbox_handler_cpt_rxc_time_cfg(struct rvu *rvu,925 struct cpt_rxc_time_cfg_req *req,926 struct msg_rsp *rsp)927{928 int blkaddr;929 930 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);931 if (blkaddr < 0)932 return blkaddr;933 934 /* This message is accepted only if sent from CPT PF/VF */935 if (!is_cpt_pf(rvu, req->hdr.pcifunc) &&936 !is_cpt_vf(rvu, req->hdr.pcifunc))937 return CPT_AF_ERR_ACCESS_DENIED;938 939 cpt_rxc_time_cfg(rvu, req, blkaddr, NULL);940 941 return 0;942}943 944int rvu_mbox_handler_cpt_ctx_cache_sync(struct rvu *rvu, struct msg_req *req,945 struct msg_rsp *rsp)946{947 return rvu_cpt_ctx_flush(rvu, req->hdr.pcifunc);948}949 950int rvu_mbox_handler_cpt_lf_reset(struct rvu *rvu, struct cpt_lf_rst_req *req,951 struct msg_rsp *rsp)952{953 u16 pcifunc = req->hdr.pcifunc;954 struct rvu_block *block;955 int cptlf, blkaddr, ret;956 u16 actual_slot;957 u64 ctl, ctl2;958 959 blkaddr = rvu_get_blkaddr_from_slot(rvu, BLKTYPE_CPT, pcifunc,960 req->slot, &actual_slot);961 if (blkaddr < 0)962 return CPT_AF_ERR_LF_INVALID;963 964 block = &rvu->hw->block[blkaddr];965 966 cptlf = rvu_get_lf(rvu, block, pcifunc, actual_slot);967 if (cptlf < 0)968 return CPT_AF_ERR_LF_INVALID;969 ctl = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf));970 ctl2 = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf));971 972 ret = rvu_lf_reset(rvu, block, cptlf);973 if (ret)974 dev_err(rvu->dev, "Failed to reset blkaddr %d LF%d\n",975 block->addr, cptlf);976 977 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), ctl);978 rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), ctl2);979 980 return 0;981}982 983int rvu_mbox_handler_cpt_flt_eng_info(struct rvu *rvu, struct cpt_flt_eng_info_req *req,984 struct cpt_flt_eng_info_rsp *rsp)985{986 struct rvu_block *block;987 unsigned long flags;988 int blkaddr, vec;989 int flt_vecs;990 u16 max_engs;991 992 blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);993 if (blkaddr < 0)994 return blkaddr;995 996 block = &rvu->hw->block[blkaddr];997 max_engs = cpt_max_engines_get(rvu);998 flt_vecs = cpt_10k_flt_nvecs_get(rvu, max_engs);999 for (vec = 0; vec < flt_vecs; vec++) {1000 spin_lock_irqsave(&rvu->cpt_intr_lock, flags);1001 rsp->flt_eng_map[vec] = block->cpt_flt_eng_map[vec];1002 rsp->rcvrd_eng_map[vec] = block->cpt_rcvrd_eng_map[vec];1003 if (req->reset) {1004 block->cpt_flt_eng_map[vec] = 0x0;1005 block->cpt_rcvrd_eng_map[vec] = 0x0;1006 }1007 spin_unlock_irqrestore(&rvu->cpt_intr_lock, flags);1008 }1009 return 0;1010}1011 1012static void cpt_rxc_teardown(struct rvu *rvu, int blkaddr)1013{1014 struct cpt_rxc_time_cfg_req req, prev;1015 struct rvu_hwinfo *hw = rvu->hw;1016 int timeout = 2000;1017 u64 reg;1018 1019 if (!hw->cap.cpt_rxc)1020 return;1021 1022 /* Set time limit to minimum values, so that rxc entries will be1023 * flushed out quickly.1024 */1025 req.step = 1;1026 req.zombie_thres = 1;1027 req.zombie_limit = 1;1028 req.active_thres = 1;1029 req.active_limit = 1;1030 1031 cpt_rxc_time_cfg(rvu, &req, blkaddr, &prev);1032 1033 do {1034 reg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ACTIVE_STS);1035 udelay(1);1036 if (FIELD_GET(RXC_ACTIVE_COUNT, reg))1037 timeout--;1038 else1039 break;1040 } while (timeout);1041 1042 if (timeout == 0)1043 dev_warn(rvu->dev, "Poll for RXC active count hits hard loop counter\n");1044 1045 timeout = 2000;1046 do {1047 reg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ZOMBIE_STS);1048 udelay(1);1049 if (FIELD_GET(RXC_ZOMBIE_COUNT, reg))1050 timeout--;1051 else1052 break;1053 } while (timeout);1054 1055 if (timeout == 0)1056 dev_warn(rvu->dev, "Poll for RXC zombie count hits hard loop counter\n");1057 1058 /* Restore config */1059 cpt_rxc_time_cfg(rvu, &prev, blkaddr, NULL);1060}1061 1062#define INFLIGHT GENMASK_ULL(8, 0)1063#define GRB_CNT GENMASK_ULL(39, 32)1064#define GWB_CNT GENMASK_ULL(47, 40)1065#define XQ_XOR GENMASK_ULL(63, 63)1066#define DQPTR GENMASK_ULL(19, 0)1067#define NQPTR GENMASK_ULL(51, 32)1068 1069static void cpt_lf_disable_iqueue(struct rvu *rvu, int blkaddr, int slot)1070{1071 int timeout = 1000000;1072 u64 inprog, inst_ptr;1073 u64 qsize, pending;1074 int i = 0;1075 1076 /* Disable instructions enqueuing */1077 rvu_write64(rvu, blkaddr, CPT_AF_BAR2_ALIASX(slot, CPT_LF_CTL), 0x0);1078 1079 inprog = rvu_read64(rvu, blkaddr,1080 CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG));1081 inprog |= BIT_ULL(16);1082 rvu_write64(rvu, blkaddr,1083 CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG), inprog);1084 1085 qsize = rvu_read64(rvu, blkaddr,1086 CPT_AF_BAR2_ALIASX(slot, CPT_LF_Q_SIZE)) & 0x7FFF;1087 do {1088 inst_ptr = rvu_read64(rvu, blkaddr,1089 CPT_AF_BAR2_ALIASX(slot, CPT_LF_Q_INST_PTR));1090 pending = (FIELD_GET(XQ_XOR, inst_ptr) * qsize * 40) +1091 FIELD_GET(NQPTR, inst_ptr) -1092 FIELD_GET(DQPTR, inst_ptr);1093 udelay(1);1094 timeout--;1095 } while ((pending != 0) && (timeout != 0));1096 1097 if (timeout == 0)1098 dev_warn(rvu->dev, "TIMEOUT: CPT poll on pending instructions\n");1099 1100 timeout = 1000000;1101 /* Wait for CPT queue to become execution-quiescent */1102 do {1103 inprog = rvu_read64(rvu, blkaddr,1104 CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG));1105 1106 if ((FIELD_GET(INFLIGHT, inprog) == 0) &&1107 (FIELD_GET(GRB_CNT, inprog) == 0)) {1108 i++;1109 } else {1110 i = 0;1111 timeout--;1112 }1113 } while ((timeout != 0) && (i < 10));1114 1115 if (timeout == 0)1116 dev_warn(rvu->dev, "TIMEOUT: CPT poll on inflight count\n");1117 /* Wait for 2 us to flush all queue writes to memory */1118 udelay(2);1119}1120 1121int rvu_cpt_lf_teardown(struct rvu *rvu, u16 pcifunc, int blkaddr, int lf, int slot)1122{1123 u64 reg;1124 1125 if (is_cpt_pf(rvu, pcifunc) || is_cpt_vf(rvu, pcifunc))1126 cpt_rxc_teardown(rvu, blkaddr);1127 1128 mutex_lock(&rvu->alias_lock);1129 /* Enable BAR2 ALIAS for this pcifunc. */1130 reg = BIT_ULL(16) | pcifunc;1131 rvu_bar2_sel_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, reg);1132 1133 cpt_lf_disable_iqueue(rvu, blkaddr, slot);1134 1135 rvu_bar2_sel_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, 0);1136 mutex_unlock(&rvu->alias_lock);1137 1138 return 0;1139}1140 1141#define CPT_RES_LEN 161142#define CPT_SE_IE_EGRP 1ULL1143 1144static int cpt_inline_inb_lf_cmd_send(struct rvu *rvu, int blkaddr,1145 int nix_blkaddr)1146{1147 int cpt_pf_num = rvu->cpt_pf_num;1148 struct cpt_inst_lmtst_req *req;1149 dma_addr_t res_daddr;1150 int timeout = 3000;1151 u8 cpt_idx;1152 u64 *inst;1153 u16 *res;1154 int rc;1155 1156 res = kzalloc(CPT_RES_LEN, GFP_KERNEL);1157 if (!res)1158 return -ENOMEM;1159 1160 res_daddr = dma_map_single(rvu->dev, res, CPT_RES_LEN,1161 DMA_BIDIRECTIONAL);1162 if (dma_mapping_error(rvu->dev, res_daddr)) {1163 dev_err(rvu->dev, "DMA mapping failed for CPT result\n");1164 rc = -EFAULT;1165 goto res_free;1166 }1167 *res = 0xFFFF;1168 1169 /* Send mbox message to CPT PF */1170 req = (struct cpt_inst_lmtst_req *)1171 otx2_mbox_alloc_msg_rsp(&rvu->afpf_wq_info.mbox_up,1172 cpt_pf_num, sizeof(*req),1173 sizeof(struct msg_rsp));1174 if (!req) {1175 rc = -ENOMEM;1176 goto res_daddr_unmap;1177 }1178 req->hdr.sig = OTX2_MBOX_REQ_SIG;1179 req->hdr.id = MBOX_MSG_CPT_INST_LMTST;1180 1181 inst = req->inst;1182 /* Prepare CPT_INST_S */1183 inst[0] = 0;1184 inst[1] = res_daddr;1185 /* AF PF FUNC */1186 inst[2] = 0;1187 /* Set QORD */1188 inst[3] = 1;1189 inst[4] = 0;1190 inst[5] = 0;1191 inst[6] = 0;1192 /* Set EGRP */1193 inst[7] = CPT_SE_IE_EGRP << 61;1194 1195 /* Subtract 1 from the NIX-CPT credit count to preserve1196 * credit counts.1197 */1198 cpt_idx = (blkaddr == BLKADDR_CPT0) ? 0 : 1;1199 rvu_write64(rvu, nix_blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx),1200 BIT_ULL(22) - 1);1201 1202 otx2_mbox_msg_send(&rvu->afpf_wq_info.mbox_up, cpt_pf_num);1203 rc = otx2_mbox_wait_for_rsp(&rvu->afpf_wq_info.mbox_up, cpt_pf_num);1204 if (rc)1205 dev_warn(rvu->dev, "notification to pf %d failed\n",1206 cpt_pf_num);1207 /* Wait for CPT instruction to be completed */1208 do {1209 mdelay(1);1210 if (*res == 0xFFFF)1211 timeout--;1212 else1213 break;1214 } while (timeout);1215 1216 if (timeout == 0)1217 dev_warn(rvu->dev, "Poll for result hits hard loop counter\n");1218 1219res_daddr_unmap:1220 dma_unmap_single(rvu->dev, res_daddr, CPT_RES_LEN, DMA_BIDIRECTIONAL);1221res_free:1222 kfree(res);1223 1224 return 0;1225}1226 1227#define CTX_CAM_PF_FUNC GENMASK_ULL(61, 46)1228#define CTX_CAM_CPTR GENMASK_ULL(45, 0)1229 1230int rvu_cpt_ctx_flush(struct rvu *rvu, u16 pcifunc)1231{1232 int nix_blkaddr, blkaddr;1233 u16 max_ctx_entries, i;1234 int slot = 0, num_lfs;1235 u64 reg, cam_data;1236 int rc;1237 1238 nix_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);1239 if (nix_blkaddr < 0)1240 return -EINVAL;1241 1242 if (is_rvu_otx2(rvu))1243 return 0;1244 1245 blkaddr = (nix_blkaddr == BLKADDR_NIX1) ? BLKADDR_CPT1 : BLKADDR_CPT0;1246 1247 /* Submit CPT_INST_S to track when all packets have been1248 * flushed through for the NIX PF FUNC in inline inbound case.1249 */1250 rc = cpt_inline_inb_lf_cmd_send(rvu, blkaddr, nix_blkaddr);1251 if (rc)1252 return rc;1253 1254 /* Wait for rxc entries to be flushed out */1255 cpt_rxc_teardown(rvu, blkaddr);1256 1257 reg = rvu_read64(rvu, blkaddr, CPT_AF_CONSTANTS0);1258 max_ctx_entries = (reg >> 48) & 0xFFF;1259 1260 mutex_lock(&rvu->rsrc_lock);1261 1262 num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc),1263 blkaddr);1264 if (num_lfs == 0) {1265 dev_warn(rvu->dev, "CPT LF is not configured\n");1266 goto unlock;1267 }1268 1269 /* Enable BAR2 ALIAS for this pcifunc. */1270 reg = BIT_ULL(16) | pcifunc;1271 rvu_bar2_sel_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, reg);1272 1273 for (i = 0; i < max_ctx_entries; i++) {1274 cam_data = rvu_read64(rvu, blkaddr, CPT_AF_CTX_CAM_DATA(i));1275 1276 if ((FIELD_GET(CTX_CAM_PF_FUNC, cam_data) == pcifunc) &&1277 FIELD_GET(CTX_CAM_CPTR, cam_data)) {1278 reg = BIT_ULL(46) | FIELD_GET(CTX_CAM_CPTR, cam_data);1279 rvu_write64(rvu, blkaddr,1280 CPT_AF_BAR2_ALIASX(slot, CPT_LF_CTX_FLUSH),1281 reg);1282 }1283 }1284 rvu_bar2_sel_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, 0);1285 1286unlock:1287 mutex_unlock(&rvu->rsrc_lock);1288 1289 return 0;1290}1291 1292#define MAX_RXC_ICB_CNT GENMASK_ULL(40, 32)1293 1294int rvu_cpt_init(struct rvu *rvu)1295{1296 struct rvu_hwinfo *hw = rvu->hw;1297 u64 reg_val;1298 1299 /* Retrieve CPT PF number */1300 rvu->cpt_pf_num = get_cpt_pf_num(rvu);1301 if (is_block_implemented(rvu->hw, BLKADDR_CPT0) && !is_rvu_otx2(rvu) &&1302 !is_cn10kb(rvu))1303 hw->cap.cpt_rxc = true;1304 1305 if (hw->cap.cpt_rxc && !is_cn10ka_a0(rvu) && !is_cn10ka_a1(rvu)) {1306 /* Set CPT_AF_RXC_CFG1:max_rxc_icb_cnt to 0xc0 to not effect1307 * inline inbound peak performance1308 */1309 reg_val = rvu_read64(rvu, BLKADDR_CPT0, CPT_AF_RXC_CFG1);1310 reg_val &= ~MAX_RXC_ICB_CNT;1311 reg_val |= FIELD_PREP(MAX_RXC_ICB_CNT,1312 CPT_DFLT_MAX_RXC_ICB_CNT);1313 rvu_write64(rvu, BLKADDR_CPT0, CPT_AF_RXC_CFG1, reg_val);1314 }1315 1316 spin_lock_init(&rvu->cpt_intr_lock);1317 1318 return 0;1319}1320