3481 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Marvell RVU Admin Function driver3 *4 * Copyright (C) 2018 Marvell.5 *6 */7 8#include <linux/bitfield.h>9#include <linux/module.h>10#include <linux/pci.h>11 12#include "rvu_struct.h"13#include "rvu_reg.h"14#include "rvu.h"15#include "npc.h"16#include "cgx.h"17#include "npc_profile.h"18#include "rvu_npc_hash.h"19 20#define RSVD_MCAM_ENTRIES_PER_PF 3 /* Broadcast, Promisc and AllMulticast */21#define RSVD_MCAM_ENTRIES_PER_NIXLF 1 /* Ucast for LFs */22 23#define NPC_PARSE_RESULT_DMAC_OFFSET 824#define NPC_HW_TSTAMP_OFFSET 8ULL25#define NPC_KEX_CHAN_MASK 0xFFFULL26#define NPC_KEX_PF_FUNC_MASK 0xFFFFULL27 28#define ALIGN_8B_CEIL(__a) (((__a) + 7) & (-8))29 30static const char def_pfl_name[] = "default";31 32static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam,33 int blkaddr, u16 pcifunc);34static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam,35 u16 pcifunc);36 37bool is_npc_intf_tx(u8 intf)38{39 return !!(intf & 0x1);40}41 42bool is_npc_intf_rx(u8 intf)43{44 return !(intf & 0x1);45}46 47bool is_npc_interface_valid(struct rvu *rvu, u8 intf)48{49 struct rvu_hwinfo *hw = rvu->hw;50 51 return intf < hw->npc_intfs;52}53 54int rvu_npc_get_tx_nibble_cfg(struct rvu *rvu, u64 nibble_ena)55{56 /* Due to a HW issue in these silicon versions, parse nibble enable57 * configuration has to be identical for both Rx and Tx interfaces.58 */59 if (is_rvu_96xx_B0(rvu))60 return nibble_ena;61 return 0;62}63 64void rvu_npc_set_pkind(struct rvu *rvu, int pkind, struct rvu_pfvf *pfvf)65{66 int blkaddr;67 u64 val = 0;68 69 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);70 if (blkaddr < 0)71 return;72 73 /* Config CPI base for the PKIND */74 val = pkind | 1ULL << 62;75 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_CPI_DEFX(pkind, 0), val);76}77 78int rvu_npc_get_pkind(struct rvu *rvu, u16 pf)79{80 struct npc_pkind *pkind = &rvu->hw->pkind;81 u32 map;82 int i;83 84 for (i = 0; i < pkind->rsrc.max; i++) {85 map = pkind->pfchan_map[i];86 if (((map >> 16) & 0x3F) == pf)87 return i;88 }89 return -1;90}91 92#define NPC_AF_ACTION0_PTR_ADVANCE GENMASK_ULL(27, 20)93 94int npc_config_ts_kpuaction(struct rvu *rvu, int pf, u16 pcifunc, bool enable)95{96 int pkind, blkaddr;97 u64 val;98 99 pkind = rvu_npc_get_pkind(rvu, pf);100 if (pkind < 0) {101 dev_err(rvu->dev, "%s: pkind not mapped\n", __func__);102 return -EINVAL;103 }104 105 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc);106 if (blkaddr < 0) {107 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__);108 return -EINVAL;109 }110 111 val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind));112 val &= ~NPC_AF_ACTION0_PTR_ADVANCE;113 /* If timestamp is enabled then configure NPC to shift 8 bytes */114 if (enable)115 val |= FIELD_PREP(NPC_AF_ACTION0_PTR_ADVANCE,116 NPC_HW_TSTAMP_OFFSET);117 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val);118 119 return 0;120}121 122static int npc_get_ucast_mcam_index(struct npc_mcam *mcam, u16 pcifunc,123 int nixlf)124{125 struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam);126 struct rvu *rvu = hw->rvu;127 int blkaddr = 0, max = 0;128 struct rvu_block *block;129 struct rvu_pfvf *pfvf;130 131 pfvf = rvu_get_pfvf(rvu, pcifunc);132 /* Given a PF/VF and NIX LF number calculate the unicast mcam133 * entry index based on the NIX block assigned to the PF/VF.134 */135 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr);136 while (blkaddr) {137 if (pfvf->nix_blkaddr == blkaddr)138 break;139 block = &rvu->hw->block[blkaddr];140 max += block->lf.max;141 blkaddr = rvu_get_next_nix_blkaddr(rvu, blkaddr);142 }143 144 return mcam->nixlf_offset + (max + nixlf) * RSVD_MCAM_ENTRIES_PER_NIXLF;145}146 147int npc_get_nixlf_mcam_index(struct npc_mcam *mcam,148 u16 pcifunc, int nixlf, int type)149{150 int pf = rvu_get_pf(pcifunc);151 int index;152 153 /* Check if this is for a PF */154 if (pf && !(pcifunc & RVU_PFVF_FUNC_MASK)) {155 /* Reserved entries exclude PF0 */156 pf--;157 index = mcam->pf_offset + (pf * RSVD_MCAM_ENTRIES_PER_PF);158 /* Broadcast address matching entry should be first so159 * that the packet can be replicated to all VFs.160 */161 if (type == NIXLF_BCAST_ENTRY)162 return index;163 else if (type == NIXLF_ALLMULTI_ENTRY)164 return index + 1;165 else if (type == NIXLF_PROMISC_ENTRY)166 return index + 2;167 }168 169 return npc_get_ucast_mcam_index(mcam, pcifunc, nixlf);170}171 172int npc_get_bank(struct npc_mcam *mcam, int index)173{174 int bank = index / mcam->banksize;175 176 /* 0,1 & 2,3 banks are combined for this keysize */177 if (mcam->keysize == NPC_MCAM_KEY_X2)178 return bank ? 2 : 0;179 180 return bank;181}182 183bool is_mcam_entry_enabled(struct rvu *rvu, struct npc_mcam *mcam,184 int blkaddr, int index)185{186 int bank = npc_get_bank(mcam, index);187 u64 cfg;188 189 index &= (mcam->banksize - 1);190 cfg = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_CFG(index, bank));191 return (cfg & 1);192}193 194void npc_enable_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam,195 int blkaddr, int index, bool enable)196{197 int bank = npc_get_bank(mcam, index);198 int actbank = bank;199 200 index &= (mcam->banksize - 1);201 for (; bank < (actbank + mcam->banks_per_entry); bank++) {202 rvu_write64(rvu, blkaddr,203 NPC_AF_MCAMEX_BANKX_CFG(index, bank),204 enable ? 1 : 0);205 }206}207 208static void npc_clear_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam,209 int blkaddr, int index)210{211 int bank = npc_get_bank(mcam, index);212 int actbank = bank;213 214 index &= (mcam->banksize - 1);215 for (; bank < (actbank + mcam->banks_per_entry); bank++) {216 rvu_write64(rvu, blkaddr,217 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1), 0);218 rvu_write64(rvu, blkaddr,219 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0), 0);220 221 rvu_write64(rvu, blkaddr,222 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), 0);223 rvu_write64(rvu, blkaddr,224 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), 0);225 226 rvu_write64(rvu, blkaddr,227 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), 0);228 rvu_write64(rvu, blkaddr,229 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), 0);230 }231}232 233static void npc_get_keyword(struct mcam_entry *entry, int idx,234 u64 *cam0, u64 *cam1)235{236 u64 kw_mask = 0x00;237 238#define CAM_MASK(n) (BIT_ULL(n) - 1)239 240 /* 0, 2, 4, 6 indices refer to BANKX_CAMX_W0 and241 * 1, 3, 5, 7 indices refer to BANKX_CAMX_W1.242 *243 * Also, only 48 bits of BANKX_CAMX_W1 are valid.244 */245 switch (idx) {246 case 0:247 /* BANK(X)_CAM_W0<63:0> = MCAM_KEY[KW0]<63:0> */248 *cam1 = entry->kw[0];249 kw_mask = entry->kw_mask[0];250 break;251 case 1:252 /* BANK(X)_CAM_W1<47:0> = MCAM_KEY[KW1]<47:0> */253 *cam1 = entry->kw[1] & CAM_MASK(48);254 kw_mask = entry->kw_mask[1] & CAM_MASK(48);255 break;256 case 2:257 /* BANK(X + 1)_CAM_W0<15:0> = MCAM_KEY[KW1]<63:48>258 * BANK(X + 1)_CAM_W0<63:16> = MCAM_KEY[KW2]<47:0>259 */260 *cam1 = (entry->kw[1] >> 48) & CAM_MASK(16);261 *cam1 |= ((entry->kw[2] & CAM_MASK(48)) << 16);262 kw_mask = (entry->kw_mask[1] >> 48) & CAM_MASK(16);263 kw_mask |= ((entry->kw_mask[2] & CAM_MASK(48)) << 16);264 break;265 case 3:266 /* BANK(X + 1)_CAM_W1<15:0> = MCAM_KEY[KW2]<63:48>267 * BANK(X + 1)_CAM_W1<47:16> = MCAM_KEY[KW3]<31:0>268 */269 *cam1 = (entry->kw[2] >> 48) & CAM_MASK(16);270 *cam1 |= ((entry->kw[3] & CAM_MASK(32)) << 16);271 kw_mask = (entry->kw_mask[2] >> 48) & CAM_MASK(16);272 kw_mask |= ((entry->kw_mask[3] & CAM_MASK(32)) << 16);273 break;274 case 4:275 /* BANK(X + 2)_CAM_W0<31:0> = MCAM_KEY[KW3]<63:32>276 * BANK(X + 2)_CAM_W0<63:32> = MCAM_KEY[KW4]<31:0>277 */278 *cam1 = (entry->kw[3] >> 32) & CAM_MASK(32);279 *cam1 |= ((entry->kw[4] & CAM_MASK(32)) << 32);280 kw_mask = (entry->kw_mask[3] >> 32) & CAM_MASK(32);281 kw_mask |= ((entry->kw_mask[4] & CAM_MASK(32)) << 32);282 break;283 case 5:284 /* BANK(X + 2)_CAM_W1<31:0> = MCAM_KEY[KW4]<63:32>285 * BANK(X + 2)_CAM_W1<47:32> = MCAM_KEY[KW5]<15:0>286 */287 *cam1 = (entry->kw[4] >> 32) & CAM_MASK(32);288 *cam1 |= ((entry->kw[5] & CAM_MASK(16)) << 32);289 kw_mask = (entry->kw_mask[4] >> 32) & CAM_MASK(32);290 kw_mask |= ((entry->kw_mask[5] & CAM_MASK(16)) << 32);291 break;292 case 6:293 /* BANK(X + 3)_CAM_W0<47:0> = MCAM_KEY[KW5]<63:16>294 * BANK(X + 3)_CAM_W0<63:48> = MCAM_KEY[KW6]<15:0>295 */296 *cam1 = (entry->kw[5] >> 16) & CAM_MASK(48);297 *cam1 |= ((entry->kw[6] & CAM_MASK(16)) << 48);298 kw_mask = (entry->kw_mask[5] >> 16) & CAM_MASK(48);299 kw_mask |= ((entry->kw_mask[6] & CAM_MASK(16)) << 48);300 break;301 case 7:302 /* BANK(X + 3)_CAM_W1<47:0> = MCAM_KEY[KW6]<63:16> */303 *cam1 = (entry->kw[6] >> 16) & CAM_MASK(48);304 kw_mask = (entry->kw_mask[6] >> 16) & CAM_MASK(48);305 break;306 }307 308 *cam1 &= kw_mask;309 *cam0 = ~*cam1 & kw_mask;310}311 312static void npc_fill_entryword(struct mcam_entry *entry, int idx,313 u64 cam0, u64 cam1)314{315 /* Similar to npc_get_keyword, but fills mcam_entry structure from316 * CAM registers.317 */318 switch (idx) {319 case 0:320 entry->kw[0] = cam1;321 entry->kw_mask[0] = cam1 ^ cam0;322 break;323 case 1:324 entry->kw[1] = cam1;325 entry->kw_mask[1] = cam1 ^ cam0;326 break;327 case 2:328 entry->kw[1] |= (cam1 & CAM_MASK(16)) << 48;329 entry->kw[2] = (cam1 >> 16) & CAM_MASK(48);330 entry->kw_mask[1] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48;331 entry->kw_mask[2] = ((cam1 ^ cam0) >> 16) & CAM_MASK(48);332 break;333 case 3:334 entry->kw[2] |= (cam1 & CAM_MASK(16)) << 48;335 entry->kw[3] = (cam1 >> 16) & CAM_MASK(32);336 entry->kw_mask[2] |= ((cam1 ^ cam0) & CAM_MASK(16)) << 48;337 entry->kw_mask[3] = ((cam1 ^ cam0) >> 16) & CAM_MASK(32);338 break;339 case 4:340 entry->kw[3] |= (cam1 & CAM_MASK(32)) << 32;341 entry->kw[4] = (cam1 >> 32) & CAM_MASK(32);342 entry->kw_mask[3] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32;343 entry->kw_mask[4] = ((cam1 ^ cam0) >> 32) & CAM_MASK(32);344 break;345 case 5:346 entry->kw[4] |= (cam1 & CAM_MASK(32)) << 32;347 entry->kw[5] = (cam1 >> 32) & CAM_MASK(16);348 entry->kw_mask[4] |= ((cam1 ^ cam0) & CAM_MASK(32)) << 32;349 entry->kw_mask[5] = ((cam1 ^ cam0) >> 32) & CAM_MASK(16);350 break;351 case 6:352 entry->kw[5] |= (cam1 & CAM_MASK(48)) << 16;353 entry->kw[6] = (cam1 >> 48) & CAM_MASK(16);354 entry->kw_mask[5] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16;355 entry->kw_mask[6] = ((cam1 ^ cam0) >> 48) & CAM_MASK(16);356 break;357 case 7:358 entry->kw[6] |= (cam1 & CAM_MASK(48)) << 16;359 entry->kw_mask[6] |= ((cam1 ^ cam0) & CAM_MASK(48)) << 16;360 break;361 }362}363 364static u64 npc_get_default_entry_action(struct rvu *rvu, struct npc_mcam *mcam,365 int blkaddr, u16 pf_func)366{367 int bank, nixlf, index;368 369 /* get ucast entry rule entry index */370 if (nix_get_nixlf(rvu, pf_func, &nixlf, NULL)) {371 dev_err(rvu->dev, "%s: nixlf not attached to pcifunc:0x%x\n",372 __func__, pf_func);373 /* Action 0 is drop */374 return 0;375 }376 377 index = npc_get_nixlf_mcam_index(mcam, pf_func, nixlf,378 NIXLF_UCAST_ENTRY);379 bank = npc_get_bank(mcam, index);380 index &= (mcam->banksize - 1);381 382 return rvu_read64(rvu, blkaddr,383 NPC_AF_MCAMEX_BANKX_ACTION(index, bank));384}385 386static void npc_fixup_vf_rule(struct rvu *rvu, struct npc_mcam *mcam,387 int blkaddr, int index, struct mcam_entry *entry,388 bool *enable)389{390 struct rvu_npc_mcam_rule *rule;391 u16 owner, target_func;392 struct rvu_pfvf *pfvf;393 u64 rx_action;394 395 owner = mcam->entry2pfvf_map[index];396 target_func = (entry->action >> 4) & 0xffff;397 /* do nothing when target is LBK/PF or owner is not PF */398 if (is_pffunc_af(owner) || is_lbk_vf(rvu, target_func) ||399 (owner & RVU_PFVF_FUNC_MASK) ||400 !(target_func & RVU_PFVF_FUNC_MASK))401 return;402 403 /* save entry2target_pffunc */404 pfvf = rvu_get_pfvf(rvu, target_func);405 mcam->entry2target_pffunc[index] = target_func;406 407 /* don't enable rule when nixlf not attached or initialized */408 if (!(is_nixlf_attached(rvu, target_func) &&409 test_bit(NIXLF_INITIALIZED, &pfvf->flags)))410 *enable = false;411 412 /* fix up not needed for the rules added by user(ntuple filters) */413 list_for_each_entry(rule, &mcam->mcam_rules, list) {414 if (rule->entry == index)415 return;416 }417 418 /* AF modifies given action iff PF/VF has requested for it */419 if ((entry->action & 0xFULL) != NIX_RX_ACTION_DEFAULT)420 return;421 422 /* copy VF default entry action to the VF mcam entry */423 rx_action = npc_get_default_entry_action(rvu, mcam, blkaddr,424 target_func);425 if (rx_action)426 entry->action = rx_action;427}428 429static void npc_config_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam,430 int blkaddr, int index, u8 intf,431 struct mcam_entry *entry, bool enable)432{433 int bank = npc_get_bank(mcam, index);434 int kw = 0, actbank, actindex;435 u8 tx_intf_mask = ~intf & 0x3;436 u8 tx_intf = intf;437 u64 cam0, cam1;438 439 actbank = bank; /* Save bank id, to set action later on */440 actindex = index;441 index &= (mcam->banksize - 1);442 443 /* Disable before mcam entry update */444 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, false);445 446 /* Clear mcam entry to avoid writes being suppressed by NPC */447 npc_clear_mcam_entry(rvu, mcam, blkaddr, actindex);448 449 /* CAM1 takes the comparison value and450 * CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'.451 * CAM1<n> = 0 & CAM0<n> = 1 => match if key<n> = 0452 * CAM1<n> = 1 & CAM0<n> = 0 => match if key<n> = 1453 * CAM1<n> = 0 & CAM0<n> = 0 => always match i.e dontcare.454 */455 for (; bank < (actbank + mcam->banks_per_entry); bank++, kw = kw + 2) {456 /* Interface should be set in all banks */457 if (is_npc_intf_tx(intf)) {458 /* Last bit must be set and rest don't care459 * for TX interfaces460 */461 tx_intf_mask = 0x1;462 tx_intf = intf & tx_intf_mask;463 tx_intf_mask = ~tx_intf & tx_intf_mask;464 }465 466 rvu_write64(rvu, blkaddr,467 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 1),468 tx_intf);469 rvu_write64(rvu, blkaddr,470 NPC_AF_MCAMEX_BANKX_CAMX_INTF(index, bank, 0),471 tx_intf_mask);472 473 /* Set the match key */474 npc_get_keyword(entry, kw, &cam0, &cam1);475 rvu_write64(rvu, blkaddr,476 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 1), cam1);477 rvu_write64(rvu, blkaddr,478 NPC_AF_MCAMEX_BANKX_CAMX_W0(index, bank, 0), cam0);479 480 npc_get_keyword(entry, kw + 1, &cam0, &cam1);481 rvu_write64(rvu, blkaddr,482 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 1), cam1);483 rvu_write64(rvu, blkaddr,484 NPC_AF_MCAMEX_BANKX_CAMX_W1(index, bank, 0), cam0);485 }486 487 /* PF installing VF rule */488 if (is_npc_intf_rx(intf) && actindex < mcam->bmap_entries)489 npc_fixup_vf_rule(rvu, mcam, blkaddr, actindex, entry, &enable);490 491 /* Set 'action' */492 rvu_write64(rvu, blkaddr,493 NPC_AF_MCAMEX_BANKX_ACTION(index, actbank), entry->action);494 495 /* Set TAG 'action' */496 rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_TAG_ACT(index, actbank),497 entry->vtag_action);498 499 /* Enable the entry */500 if (enable)501 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, true);502}503 504void npc_read_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam,505 int blkaddr, u16 src,506 struct mcam_entry *entry, u8 *intf, u8 *ena)507{508 int sbank = npc_get_bank(mcam, src);509 int bank, kw = 0;510 u64 cam0, cam1;511 512 src &= (mcam->banksize - 1);513 bank = sbank;514 515 for (; bank < (sbank + mcam->banks_per_entry); bank++, kw = kw + 2) {516 cam1 = rvu_read64(rvu, blkaddr,517 NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 1));518 cam0 = rvu_read64(rvu, blkaddr,519 NPC_AF_MCAMEX_BANKX_CAMX_W0(src, bank, 0));520 npc_fill_entryword(entry, kw, cam0, cam1);521 522 cam1 = rvu_read64(rvu, blkaddr,523 NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 1));524 cam0 = rvu_read64(rvu, blkaddr,525 NPC_AF_MCAMEX_BANKX_CAMX_W1(src, bank, 0));526 npc_fill_entryword(entry, kw + 1, cam0, cam1);527 }528 529 entry->action = rvu_read64(rvu, blkaddr,530 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank));531 entry->vtag_action =532 rvu_read64(rvu, blkaddr,533 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank));534 *intf = rvu_read64(rvu, blkaddr,535 NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank, 1)) & 3;536 *ena = rvu_read64(rvu, blkaddr,537 NPC_AF_MCAMEX_BANKX_CFG(src, sbank)) & 1;538}539 540static void npc_copy_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam,541 int blkaddr, u16 src, u16 dest)542{543 int dbank = npc_get_bank(mcam, dest);544 int sbank = npc_get_bank(mcam, src);545 u64 cfg, sreg, dreg;546 int bank, i;547 548 src &= (mcam->banksize - 1);549 dest &= (mcam->banksize - 1);550 551 /* Copy INTF's, W0's, W1's CAM0 and CAM1 configuration */552 for (bank = 0; bank < mcam->banks_per_entry; bank++) {553 sreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(src, sbank + bank, 0);554 dreg = NPC_AF_MCAMEX_BANKX_CAMX_INTF(dest, dbank + bank, 0);555 for (i = 0; i < 6; i++) {556 cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8));557 rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg);558 }559 }560 561 /* Copy action */562 cfg = rvu_read64(rvu, blkaddr,563 NPC_AF_MCAMEX_BANKX_ACTION(src, sbank));564 rvu_write64(rvu, blkaddr,565 NPC_AF_MCAMEX_BANKX_ACTION(dest, dbank), cfg);566 567 /* Copy TAG action */568 cfg = rvu_read64(rvu, blkaddr,569 NPC_AF_MCAMEX_BANKX_TAG_ACT(src, sbank));570 rvu_write64(rvu, blkaddr,571 NPC_AF_MCAMEX_BANKX_TAG_ACT(dest, dbank), cfg);572 573 /* Enable or disable */574 cfg = rvu_read64(rvu, blkaddr,575 NPC_AF_MCAMEX_BANKX_CFG(src, sbank));576 rvu_write64(rvu, blkaddr,577 NPC_AF_MCAMEX_BANKX_CFG(dest, dbank), cfg);578}579 580u64 npc_get_mcam_action(struct rvu *rvu, struct npc_mcam *mcam,581 int blkaddr, int index)582{583 int bank = npc_get_bank(mcam, index);584 585 index &= (mcam->banksize - 1);586 return rvu_read64(rvu, blkaddr,587 NPC_AF_MCAMEX_BANKX_ACTION(index, bank));588}589 590void npc_set_mcam_action(struct rvu *rvu, struct npc_mcam *mcam,591 int blkaddr, int index, u64 cfg)592{593 int bank = npc_get_bank(mcam, index);594 595 index &= (mcam->banksize - 1);596 return rvu_write64(rvu, blkaddr,597 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), cfg);598}599 600void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc,601 int nixlf, u64 chan, u8 *mac_addr)602{603 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);604 struct npc_install_flow_req req = { 0 };605 struct npc_install_flow_rsp rsp = { 0 };606 struct npc_mcam *mcam = &rvu->hw->mcam;607 struct nix_rx_action action = { 0 };608 int blkaddr, index;609 610 /* AF's and SDP VFs work in promiscuous mode */611 if (is_lbk_vf(rvu, pcifunc) || is_sdp_vf(rvu, pcifunc))612 return;613 614 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);615 if (blkaddr < 0)616 return;617 618 /* Ucast rule should not be installed if DMAC619 * extraction is not supported by the profile.620 */621 if (!npc_is_feature_supported(rvu, BIT_ULL(NPC_DMAC), pfvf->nix_rx_intf))622 return;623 624 index = npc_get_nixlf_mcam_index(mcam, pcifunc,625 nixlf, NIXLF_UCAST_ENTRY);626 627 /* Don't change the action if entry is already enabled628 * Otherwise RSS action may get overwritten.629 */630 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, index)) {631 *(u64 *)&action = npc_get_mcam_action(rvu, mcam,632 blkaddr, index);633 } else {634 action.op = NIX_RX_ACTIONOP_UCAST;635 action.pf_func = pcifunc;636 }637 638 req.default_rule = 1;639 ether_addr_copy(req.packet.dmac, mac_addr);640 eth_broadcast_addr((u8 *)&req.mask.dmac);641 req.features = BIT_ULL(NPC_DMAC);642 req.channel = chan;643 req.chan_mask = 0xFFFU;644 req.intf = pfvf->nix_rx_intf;645 req.op = action.op;646 req.hdr.pcifunc = 0; /* AF is requester */647 req.vf = action.pf_func;648 req.index = action.index;649 req.match_id = action.match_id;650 req.flow_key_alg = action.flow_key_alg;651 652 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp);653}654 655void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc,656 int nixlf, u64 chan, u8 chan_cnt)657{658 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);659 struct npc_install_flow_req req = { 0 };660 struct npc_install_flow_rsp rsp = { 0 };661 struct npc_mcam *mcam = &rvu->hw->mcam;662 struct rvu_hwinfo *hw = rvu->hw;663 int blkaddr, ucast_idx, index;664 struct nix_rx_action action = { 0 };665 u64 relaxed_mask;666 u8 flow_key_alg;667 668 if (!hw->cap.nix_rx_multicast && is_cgx_vf(rvu, pcifunc))669 return;670 671 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);672 if (blkaddr < 0)673 return;674 675 index = npc_get_nixlf_mcam_index(mcam, pcifunc,676 nixlf, NIXLF_PROMISC_ENTRY);677 678 if (is_cgx_vf(rvu, pcifunc))679 index = npc_get_nixlf_mcam_index(mcam,680 pcifunc & ~RVU_PFVF_FUNC_MASK,681 nixlf, NIXLF_PROMISC_ENTRY);682 683 /* If the corresponding PF's ucast action is RSS,684 * use the same action for promisc also685 */686 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc,687 nixlf, NIXLF_UCAST_ENTRY);688 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx))689 *(u64 *)&action = npc_get_mcam_action(rvu, mcam,690 blkaddr, ucast_idx);691 692 if (action.op != NIX_RX_ACTIONOP_RSS) {693 *(u64 *)&action = 0;694 action.op = NIX_RX_ACTIONOP_UCAST;695 }696 697 flow_key_alg = action.flow_key_alg;698 699 /* RX_ACTION set to MCAST for CGX PF's */700 if (hw->cap.nix_rx_multicast && pfvf->use_mce_list &&701 is_pf_cgxmapped(rvu, rvu_get_pf(pcifunc))) {702 *(u64 *)&action = 0;703 action.op = NIX_RX_ACTIONOP_MCAST;704 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK);705 action.index = pfvf->promisc_mce_idx;706 }707 708 /* For cn10k the upper two bits of the channel number are709 * cpt channel number. with masking out these bits in the710 * mcam entry, same entry used for NIX will allow packets711 * received from cpt for parsing.712 */713 if (!is_rvu_otx2(rvu)) {714 req.chan_mask = NIX_CHAN_CPT_X2P_MASK;715 } else {716 req.chan_mask = 0xFFFU;717 }718 719 if (chan_cnt > 1) {720 if (!is_power_of_2(chan_cnt)) {721 dev_err(rvu->dev,722 "%s: channel count more than 1, must be power of 2\n", __func__);723 return;724 }725 relaxed_mask = GENMASK_ULL(BITS_PER_LONG_LONG - 1,726 ilog2(chan_cnt));727 req.chan_mask &= relaxed_mask;728 }729 730 req.channel = chan;731 req.intf = pfvf->nix_rx_intf;732 req.entry = index;733 req.op = action.op;734 req.hdr.pcifunc = 0; /* AF is requester */735 req.vf = pcifunc;736 req.index = action.index;737 req.match_id = action.match_id;738 req.flow_key_alg = flow_key_alg;739 740 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp);741}742 743void rvu_npc_enable_promisc_entry(struct rvu *rvu, u16 pcifunc,744 int nixlf, bool enable)745{746 struct npc_mcam *mcam = &rvu->hw->mcam;747 int blkaddr, index;748 749 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);750 if (blkaddr < 0)751 return;752 753 /* Get 'pcifunc' of PF device */754 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK;755 756 index = npc_get_nixlf_mcam_index(mcam, pcifunc,757 nixlf, NIXLF_PROMISC_ENTRY);758 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable);759}760 761void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc,762 int nixlf, u64 chan)763{764 struct rvu_pfvf *pfvf;765 struct npc_install_flow_req req = { 0 };766 struct npc_install_flow_rsp rsp = { 0 };767 struct npc_mcam *mcam = &rvu->hw->mcam;768 struct rvu_hwinfo *hw = rvu->hw;769 int blkaddr, index;770 771 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);772 if (blkaddr < 0)773 return;774 775 /* Skip LBK VFs */776 if (is_lbk_vf(rvu, pcifunc))777 return;778 779 /* If pkt replication is not supported,780 * then only PF is allowed to add a bcast match entry.781 */782 if (!hw->cap.nix_rx_multicast && is_vf(pcifunc))783 return;784 785 /* Get 'pcifunc' of PF device */786 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK;787 pfvf = rvu_get_pfvf(rvu, pcifunc);788 789 /* Bcast rule should not be installed if both DMAC790 * and LXMB extraction is not supported by the profile.791 */792 if (!npc_is_feature_supported(rvu, BIT_ULL(NPC_DMAC), pfvf->nix_rx_intf) &&793 !npc_is_feature_supported(rvu, BIT_ULL(NPC_LXMB), pfvf->nix_rx_intf))794 return;795 796 index = npc_get_nixlf_mcam_index(mcam, pcifunc,797 nixlf, NIXLF_BCAST_ENTRY);798 799 if (!hw->cap.nix_rx_multicast) {800 /* Early silicon doesn't support pkt replication,801 * so install entry with UCAST action, so that PF802 * receives all broadcast packets.803 */804 req.op = NIX_RX_ACTIONOP_UCAST;805 } else {806 req.op = NIX_RX_ACTIONOP_MCAST;807 req.index = pfvf->bcast_mce_idx;808 }809 810 eth_broadcast_addr((u8 *)&req.packet.dmac);811 eth_broadcast_addr((u8 *)&req.mask.dmac);812 req.features = BIT_ULL(NPC_DMAC);813 req.channel = chan;814 req.chan_mask = 0xFFFU;815 req.intf = pfvf->nix_rx_intf;816 req.entry = index;817 req.hdr.pcifunc = 0; /* AF is requester */818 req.vf = pcifunc;819 820 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp);821}822 823void rvu_npc_enable_bcast_entry(struct rvu *rvu, u16 pcifunc, int nixlf,824 bool enable)825{826 struct npc_mcam *mcam = &rvu->hw->mcam;827 int blkaddr, index;828 829 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);830 if (blkaddr < 0)831 return;832 833 /* Get 'pcifunc' of PF device */834 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK;835 836 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf,837 NIXLF_BCAST_ENTRY);838 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable);839}840 841void rvu_npc_install_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf,842 u64 chan)843{844 struct npc_install_flow_req req = { 0 };845 struct npc_install_flow_rsp rsp = { 0 };846 struct npc_mcam *mcam = &rvu->hw->mcam;847 struct rvu_hwinfo *hw = rvu->hw;848 int blkaddr, ucast_idx, index;849 u8 mac_addr[ETH_ALEN] = { 0 };850 struct nix_rx_action action = { 0 };851 struct rvu_pfvf *pfvf;852 u8 flow_key_alg;853 u16 vf_func;854 855 /* Only CGX PF/VF can add allmulticast entry */856 if (is_lbk_vf(rvu, pcifunc) && is_sdp_vf(rvu, pcifunc))857 return;858 859 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);860 if (blkaddr < 0)861 return;862 863 /* Get 'pcifunc' of PF device */864 vf_func = pcifunc & RVU_PFVF_FUNC_MASK;865 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK;866 pfvf = rvu_get_pfvf(rvu, pcifunc);867 868 /* Mcast rule should not be installed if both DMAC869 * and LXMB extraction is not supported by the profile.870 */871 if (!npc_is_feature_supported(rvu, BIT_ULL(NPC_DMAC), pfvf->nix_rx_intf) &&872 !npc_is_feature_supported(rvu, BIT_ULL(NPC_LXMB), pfvf->nix_rx_intf))873 return;874 875 index = npc_get_nixlf_mcam_index(mcam, pcifunc,876 nixlf, NIXLF_ALLMULTI_ENTRY);877 878 /* If the corresponding PF's ucast action is RSS,879 * use the same action for multicast entry also880 */881 ucast_idx = npc_get_nixlf_mcam_index(mcam, pcifunc,882 nixlf, NIXLF_UCAST_ENTRY);883 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, ucast_idx))884 *(u64 *)&action = npc_get_mcam_action(rvu, mcam,885 blkaddr, ucast_idx);886 887 flow_key_alg = action.flow_key_alg;888 if (action.op != NIX_RX_ACTIONOP_RSS) {889 *(u64 *)&action = 0;890 action.op = NIX_RX_ACTIONOP_UCAST;891 action.pf_func = pcifunc;892 }893 894 /* RX_ACTION set to MCAST for CGX PF's */895 if (hw->cap.nix_rx_multicast && pfvf->use_mce_list) {896 *(u64 *)&action = 0;897 action.op = NIX_RX_ACTIONOP_MCAST;898 action.index = pfvf->mcast_mce_idx;899 }900 901 mac_addr[0] = 0x01; /* LSB bit of 1st byte in DMAC */902 ether_addr_copy(req.packet.dmac, mac_addr);903 ether_addr_copy(req.mask.dmac, mac_addr);904 req.features = BIT_ULL(NPC_DMAC);905 906 /* For cn10k the upper two bits of the channel number are907 * cpt channel number. with masking out these bits in the908 * mcam entry, same entry used for NIX will allow packets909 * received from cpt for parsing.910 */911 if (!is_rvu_otx2(rvu))912 req.chan_mask = NIX_CHAN_CPT_X2P_MASK;913 else914 req.chan_mask = 0xFFFU;915 916 req.channel = chan;917 req.intf = pfvf->nix_rx_intf;918 req.entry = index;919 req.op = action.op;920 req.hdr.pcifunc = 0; /* AF is requester */921 req.vf = pcifunc | vf_func;922 req.index = action.index;923 req.match_id = action.match_id;924 req.flow_key_alg = flow_key_alg;925 926 rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp);927}928 929void rvu_npc_enable_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf,930 bool enable)931{932 struct npc_mcam *mcam = &rvu->hw->mcam;933 int blkaddr, index;934 935 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);936 if (blkaddr < 0)937 return;938 939 /* Get 'pcifunc' of PF device */940 pcifunc = pcifunc & ~RVU_PFVF_FUNC_MASK;941 942 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf,943 NIXLF_ALLMULTI_ENTRY);944 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable);945}946 947static void npc_update_vf_flow_entry(struct rvu *rvu, struct npc_mcam *mcam,948 int blkaddr, u16 pcifunc, u64 rx_action)949{950 int actindex, index, bank, entry;951 struct rvu_npc_mcam_rule *rule;952 bool enable, update;953 954 if (!(pcifunc & RVU_PFVF_FUNC_MASK))955 return;956 957 mutex_lock(&mcam->lock);958 for (index = 0; index < mcam->bmap_entries; index++) {959 if (mcam->entry2target_pffunc[index] == pcifunc) {960 update = true;961 /* update not needed for the rules added via ntuple filters */962 list_for_each_entry(rule, &mcam->mcam_rules, list) {963 if (rule->entry == index)964 update = false;965 }966 if (!update)967 continue;968 bank = npc_get_bank(mcam, index);969 actindex = index;970 entry = index & (mcam->banksize - 1);971 972 /* read vf flow entry enable status */973 enable = is_mcam_entry_enabled(rvu, mcam, blkaddr,974 actindex);975 /* disable before mcam entry update */976 npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex,977 false);978 /* update 'action' */979 rvu_write64(rvu, blkaddr,980 NPC_AF_MCAMEX_BANKX_ACTION(entry, bank),981 rx_action);982 if (enable)983 npc_enable_mcam_entry(rvu, mcam, blkaddr,984 actindex, true);985 }986 }987 mutex_unlock(&mcam->lock);988}989 990static void npc_update_rx_action_with_alg_idx(struct rvu *rvu, struct nix_rx_action action,991 struct rvu_pfvf *pfvf, int mcam_index, int blkaddr,992 int alg_idx)993 994{995 struct npc_mcam *mcam = &rvu->hw->mcam;996 struct rvu_hwinfo *hw = rvu->hw;997 int bank, op_rss;998 999 if (!is_mcam_entry_enabled(rvu, mcam, blkaddr, mcam_index))1000 return;1001 1002 op_rss = (!hw->cap.nix_rx_multicast || !pfvf->use_mce_list);1003 1004 bank = npc_get_bank(mcam, mcam_index);1005 mcam_index &= (mcam->banksize - 1);1006 1007 /* If Rx action is MCAST update only RSS algorithm index */1008 if (!op_rss) {1009 *(u64 *)&action = rvu_read64(rvu, blkaddr,1010 NPC_AF_MCAMEX_BANKX_ACTION(mcam_index, bank));1011 1012 action.flow_key_alg = alg_idx;1013 }1014 rvu_write64(rvu, blkaddr,1015 NPC_AF_MCAMEX_BANKX_ACTION(mcam_index, bank), *(u64 *)&action);1016}1017 1018void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf,1019 int group, int alg_idx, int mcam_index)1020{1021 struct npc_mcam *mcam = &rvu->hw->mcam;1022 struct nix_rx_action action;1023 int blkaddr, index, bank;1024 struct rvu_pfvf *pfvf;1025 1026 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);1027 if (blkaddr < 0)1028 return;1029 1030 /* Check if this is for reserved default entry */1031 if (mcam_index < 0) {1032 if (group != DEFAULT_RSS_CONTEXT_GROUP)1033 return;1034 index = npc_get_nixlf_mcam_index(mcam, pcifunc,1035 nixlf, NIXLF_UCAST_ENTRY);1036 } else {1037 /* TODO: validate this mcam index */1038 index = mcam_index;1039 }1040 1041 if (index >= mcam->total_entries)1042 return;1043 1044 bank = npc_get_bank(mcam, index);1045 index &= (mcam->banksize - 1);1046 1047 *(u64 *)&action = rvu_read64(rvu, blkaddr,1048 NPC_AF_MCAMEX_BANKX_ACTION(index, bank));1049 /* Ignore if no action was set earlier */1050 if (!*(u64 *)&action)1051 return;1052 1053 action.op = NIX_RX_ACTIONOP_RSS;1054 action.pf_func = pcifunc;1055 action.index = group;1056 action.flow_key_alg = alg_idx;1057 1058 rvu_write64(rvu, blkaddr,1059 NPC_AF_MCAMEX_BANKX_ACTION(index, bank), *(u64 *)&action);1060 1061 /* update the VF flow rule action with the VF default entry action */1062 if (mcam_index < 0)1063 npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc,1064 *(u64 *)&action);1065 1066 /* update the action change in default rule */1067 pfvf = rvu_get_pfvf(rvu, pcifunc);1068 if (pfvf->def_ucast_rule)1069 pfvf->def_ucast_rule->rx_action = action;1070 1071 index = npc_get_nixlf_mcam_index(mcam, pcifunc,1072 nixlf, NIXLF_PROMISC_ENTRY);1073 1074 /* If PF's promiscuous entry is enabled,1075 * Set RSS action for that entry as well1076 */1077 npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, blkaddr,1078 alg_idx);1079 1080 index = npc_get_nixlf_mcam_index(mcam, pcifunc,1081 nixlf, NIXLF_ALLMULTI_ENTRY);1082 /* If PF's allmulti entry is enabled,1083 * Set RSS action for that entry as well1084 */1085 npc_update_rx_action_with_alg_idx(rvu, action, pfvf, index, blkaddr,1086 alg_idx);1087}1088 1089void npc_enadis_default_mce_entry(struct rvu *rvu, u16 pcifunc,1090 int nixlf, int type, bool enable)1091{1092 struct npc_mcam *mcam = &rvu->hw->mcam;1093 struct rvu_hwinfo *hw = rvu->hw;1094 struct nix_mce_list *mce_list;1095 int index, blkaddr, mce_idx;1096 struct rvu_pfvf *pfvf;1097 1098 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);1099 if (blkaddr < 0)1100 return;1101 1102 index = npc_get_nixlf_mcam_index(mcam, pcifunc & ~RVU_PFVF_FUNC_MASK,1103 nixlf, type);1104 1105 /* disable MCAM entry when packet replication is not supported by hw */1106 if (!hw->cap.nix_rx_multicast && !is_vf(pcifunc)) {1107 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable);1108 return;1109 }1110 1111 /* return incase mce list is not enabled */1112 pfvf = rvu_get_pfvf(rvu, pcifunc & ~RVU_PFVF_FUNC_MASK);1113 if (hw->cap.nix_rx_multicast && is_vf(pcifunc) &&1114 type != NIXLF_BCAST_ENTRY && !pfvf->use_mce_list)1115 return;1116 1117 nix_get_mce_list(rvu, pcifunc, type, &mce_list, &mce_idx);1118 1119 nix_update_mce_list(rvu, pcifunc, mce_list,1120 mce_idx, index, enable);1121 if (enable)1122 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable);1123}1124 1125static void npc_enadis_default_entries(struct rvu *rvu, u16 pcifunc,1126 int nixlf, bool enable)1127{1128 struct npc_mcam *mcam = &rvu->hw->mcam;1129 int index, blkaddr;1130 1131 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);1132 if (blkaddr < 0)1133 return;1134 1135 /* Ucast MCAM match entry of this PF/VF */1136 index = npc_get_nixlf_mcam_index(mcam, pcifunc,1137 nixlf, NIXLF_UCAST_ENTRY);1138 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, enable);1139 1140 /* Nothing to do for VFs, on platforms where pkt replication1141 * is not supported1142 */1143 if ((pcifunc & RVU_PFVF_FUNC_MASK) && !rvu->hw->cap.nix_rx_multicast)1144 return;1145 1146 /* add/delete pf_func to broadcast MCE list */1147 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf,1148 NIXLF_BCAST_ENTRY, enable);1149}1150 1151void rvu_npc_disable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf)1152{1153 if (nixlf < 0)1154 return;1155 1156 npc_enadis_default_entries(rvu, pcifunc, nixlf, false);1157 1158 /* Delete multicast and promisc MCAM entries */1159 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf,1160 NIXLF_ALLMULTI_ENTRY, false);1161 npc_enadis_default_mce_entry(rvu, pcifunc, nixlf,1162 NIXLF_PROMISC_ENTRY, false);1163}1164 1165bool rvu_npc_enable_mcam_by_entry_index(struct rvu *rvu, int entry, int intf, bool enable)1166{1167 int blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);1168 struct npc_mcam *mcam = &rvu->hw->mcam;1169 struct rvu_npc_mcam_rule *rule, *tmp;1170 1171 mutex_lock(&mcam->lock);1172 1173 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) {1174 if (rule->intf != intf)1175 continue;1176 1177 if (rule->entry != entry)1178 continue;1179 1180 rule->enable = enable;1181 mutex_unlock(&mcam->lock);1182 1183 npc_enable_mcam_entry(rvu, mcam, blkaddr,1184 entry, enable);1185 1186 return true;1187 }1188 1189 mutex_unlock(&mcam->lock);1190 return false;1191}1192 1193void rvu_npc_enable_default_entries(struct rvu *rvu, u16 pcifunc, int nixlf)1194{1195 if (nixlf < 0)1196 return;1197 1198 /* Enables only broadcast match entry. Promisc/Allmulti are enabled1199 * in set_rx_mode mbox handler.1200 */1201 npc_enadis_default_entries(rvu, pcifunc, nixlf, true);1202}1203 1204void rvu_npc_disable_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf)1205{1206 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);1207 struct npc_mcam *mcam = &rvu->hw->mcam;1208 struct rvu_npc_mcam_rule *rule, *tmp;1209 int blkaddr;1210 1211 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);1212 if (blkaddr < 0)1213 return;1214 1215 mutex_lock(&mcam->lock);1216 1217 /* Disable MCAM entries directing traffic to this 'pcifunc' */1218 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) {1219 if (is_npc_intf_rx(rule->intf) &&1220 rule->rx_action.pf_func == pcifunc &&1221 rule->rx_action.op != NIX_RX_ACTIONOP_MCAST) {1222 npc_enable_mcam_entry(rvu, mcam, blkaddr,1223 rule->entry, false);1224 rule->enable = false;1225 /* Indicate that default rule is disabled */1226 if (rule->default_rule) {1227 pfvf->def_ucast_rule = NULL;1228 list_del(&rule->list);1229 kfree(rule);1230 }1231 }1232 }1233 1234 mutex_unlock(&mcam->lock);1235 1236 npc_mcam_disable_flows(rvu, pcifunc);1237 1238 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf);1239}1240 1241void rvu_npc_free_mcam_entries(struct rvu *rvu, u16 pcifunc, int nixlf)1242{1243 struct npc_mcam *mcam = &rvu->hw->mcam;1244 struct rvu_npc_mcam_rule *rule, *tmp;1245 int blkaddr;1246 1247 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);1248 if (blkaddr < 0)1249 return;1250 1251 mutex_lock(&mcam->lock);1252 1253 /* Free all MCAM entries owned by this 'pcifunc' */1254 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc);1255 1256 /* Free all MCAM counters owned by this 'pcifunc' */1257 npc_mcam_free_all_counters(rvu, mcam, pcifunc);1258 1259 /* Delete MCAM entries owned by this 'pcifunc' */1260 list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) {1261 if (rule->owner == pcifunc && !rule->default_rule) {1262 list_del(&rule->list);1263 kfree(rule);1264 }1265 }1266 1267 mutex_unlock(&mcam->lock);1268 1269 rvu_npc_disable_default_entries(rvu, pcifunc, nixlf);1270}1271 1272static void npc_program_mkex_rx(struct rvu *rvu, int blkaddr,1273 struct npc_mcam_kex *mkex, u8 intf)1274{1275 int lid, lt, ld, fl;1276 1277 if (is_npc_intf_tx(intf))1278 return;1279 1280 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf),1281 mkex->keyx_cfg[NIX_INTF_RX]);1282 1283 /* Program LDATA */1284 for (lid = 0; lid < NPC_MAX_LID; lid++) {1285 for (lt = 0; lt < NPC_MAX_LT; lt++) {1286 for (ld = 0; ld < NPC_MAX_LD; ld++)1287 SET_KEX_LD(intf, lid, lt, ld,1288 mkex->intf_lid_lt_ld[NIX_INTF_RX]1289 [lid][lt][ld]);1290 }1291 }1292 /* Program LFLAGS */1293 for (ld = 0; ld < NPC_MAX_LD; ld++) {1294 for (fl = 0; fl < NPC_MAX_LFL; fl++)1295 SET_KEX_LDFLAGS(intf, ld, fl,1296 mkex->intf_ld_flags[NIX_INTF_RX]1297 [ld][fl]);1298 }1299}1300 1301static void npc_program_mkex_tx(struct rvu *rvu, int blkaddr,1302 struct npc_mcam_kex *mkex, u8 intf)1303{1304 int lid, lt, ld, fl;1305 1306 if (is_npc_intf_rx(intf))1307 return;1308 1309 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf),1310 mkex->keyx_cfg[NIX_INTF_TX]);1311 1312 /* Program LDATA */1313 for (lid = 0; lid < NPC_MAX_LID; lid++) {1314 for (lt = 0; lt < NPC_MAX_LT; lt++) {1315 for (ld = 0; ld < NPC_MAX_LD; ld++)1316 SET_KEX_LD(intf, lid, lt, ld,1317 mkex->intf_lid_lt_ld[NIX_INTF_TX]1318 [lid][lt][ld]);1319 }1320 }1321 /* Program LFLAGS */1322 for (ld = 0; ld < NPC_MAX_LD; ld++) {1323 for (fl = 0; fl < NPC_MAX_LFL; fl++)1324 SET_KEX_LDFLAGS(intf, ld, fl,1325 mkex->intf_ld_flags[NIX_INTF_TX]1326 [ld][fl]);1327 }1328}1329 1330static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr,1331 struct npc_mcam_kex *mkex)1332{1333 struct rvu_hwinfo *hw = rvu->hw;1334 u8 intf;1335 int ld;1336 1337 for (ld = 0; ld < NPC_MAX_LD; ld++)1338 rvu_write64(rvu, blkaddr, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld),1339 mkex->kex_ld_flags[ld]);1340 1341 for (intf = 0; intf < hw->npc_intfs; intf++) {1342 npc_program_mkex_rx(rvu, blkaddr, mkex, intf);1343 npc_program_mkex_tx(rvu, blkaddr, mkex, intf);1344 }1345 1346 /* Programme mkex hash profile */1347 npc_program_mkex_hash(rvu, blkaddr);1348}1349 1350static int npc_fwdb_prfl_img_map(struct rvu *rvu, void __iomem **prfl_img_addr,1351 u64 *size)1352{1353 u64 prfl_addr, prfl_sz;1354 1355 if (!rvu->fwdata)1356 return -EINVAL;1357 1358 prfl_addr = rvu->fwdata->mcam_addr;1359 prfl_sz = rvu->fwdata->mcam_sz;1360 1361 if (!prfl_addr || !prfl_sz)1362 return -EINVAL;1363 1364 *prfl_img_addr = ioremap_wc(prfl_addr, prfl_sz);1365 if (!(*prfl_img_addr))1366 return -ENOMEM;1367 1368 *size = prfl_sz;1369 1370 return 0;1371}1372 1373/* strtoull of "mkexprof" with base:36 */1374#define MKEX_END_SIGN 0xdeadbeef1375 1376static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr,1377 const char *mkex_profile)1378{1379 struct device *dev = &rvu->pdev->dev;1380 struct npc_mcam_kex *mcam_kex;1381 void __iomem *mkex_prfl_addr = NULL;1382 u64 prfl_sz;1383 int ret;1384 1385 /* If user not selected mkex profile */1386 if (rvu->kpu_fwdata_sz ||1387 !strncmp(mkex_profile, def_pfl_name, MKEX_NAME_LEN))1388 goto program_mkex;1389 1390 /* Setting up the mapping for mkex profile image */1391 ret = npc_fwdb_prfl_img_map(rvu, &mkex_prfl_addr, &prfl_sz);1392 if (ret < 0)1393 goto program_mkex;1394 1395 mcam_kex = (struct npc_mcam_kex __force *)mkex_prfl_addr;1396 1397 while (((s64)prfl_sz > 0) && (mcam_kex->mkex_sign != MKEX_END_SIGN)) {1398 /* Compare with mkex mod_param name string */1399 if (mcam_kex->mkex_sign == MKEX_SIGN &&1400 !strncmp(mcam_kex->name, mkex_profile, MKEX_NAME_LEN)) {1401 /* Due to an errata (35786) in A0/B0 pass silicon,1402 * parse nibble enable configuration has to be1403 * identical for both Rx and Tx interfaces.1404 */1405 if (!is_rvu_96xx_B0(rvu) ||1406 mcam_kex->keyx_cfg[NIX_INTF_RX] == mcam_kex->keyx_cfg[NIX_INTF_TX])1407 rvu->kpu.mkex = mcam_kex;1408 goto program_mkex;1409 }1410 1411 mcam_kex++;1412 prfl_sz -= sizeof(struct npc_mcam_kex);1413 }1414 dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile);1415 1416program_mkex:1417 dev_info(rvu->dev, "Using %s mkex profile\n", rvu->kpu.mkex->name);1418 /* Program selected mkex profile */1419 npc_program_mkex_profile(rvu, blkaddr, rvu->kpu.mkex);1420 if (mkex_prfl_addr)1421 iounmap(mkex_prfl_addr);1422}1423 1424static void npc_config_kpuaction(struct rvu *rvu, int blkaddr,1425 const struct npc_kpu_profile_action *kpuaction,1426 int kpu, int entry, bool pkind)1427{1428 struct npc_kpu_action0 action0 = {0};1429 struct npc_kpu_action1 action1 = {0};1430 u64 reg;1431 1432 action1.errlev = kpuaction->errlev;1433 action1.errcode = kpuaction->errcode;1434 action1.dp0_offset = kpuaction->dp0_offset;1435 action1.dp1_offset = kpuaction->dp1_offset;1436 action1.dp2_offset = kpuaction->dp2_offset;1437 1438 if (pkind)1439 reg = NPC_AF_PKINDX_ACTION1(entry);1440 else1441 reg = NPC_AF_KPUX_ENTRYX_ACTION1(kpu, entry);1442 1443 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1);1444 1445 action0.byp_count = kpuaction->bypass_count;1446 action0.capture_ena = kpuaction->cap_ena;1447 action0.parse_done = kpuaction->parse_done;1448 action0.next_state = kpuaction->next_state;1449 action0.capture_lid = kpuaction->lid;1450 action0.capture_ltype = kpuaction->ltype;1451 action0.capture_flags = kpuaction->flags;1452 action0.ptr_advance = kpuaction->ptr_advance;1453 action0.var_len_offset = kpuaction->offset;1454 action0.var_len_mask = kpuaction->mask;1455 action0.var_len_right = kpuaction->right;1456 action0.var_len_shift = kpuaction->shift;1457 1458 if (pkind)1459 reg = NPC_AF_PKINDX_ACTION0(entry);1460 else1461 reg = NPC_AF_KPUX_ENTRYX_ACTION0(kpu, entry);1462 1463 rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0);1464}1465 1466static void npc_config_kpucam(struct rvu *rvu, int blkaddr,1467 const struct npc_kpu_profile_cam *kpucam,1468 int kpu, int entry)1469{1470 struct npc_kpu_cam cam0 = {0};1471 struct npc_kpu_cam cam1 = {0};1472 1473 cam1.state = kpucam->state & kpucam->state_mask;1474 cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask;1475 cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask;1476 cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask;1477 1478 cam0.state = ~kpucam->state & kpucam->state_mask;1479 cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask;1480 cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask;1481 cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask;1482 1483 rvu_write64(rvu, blkaddr,1484 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 0), *(u64 *)&cam0);1485 rvu_write64(rvu, blkaddr,1486 NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 1), *(u64 *)&cam1);1487}1488 1489static inline u64 enable_mask(int count)1490{1491 return (((count) < 64) ? ~(BIT_ULL(count) - 1) : (0x00ULL));1492}1493 1494static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu,1495 const struct npc_kpu_profile *profile)1496{1497 int entry, num_entries, max_entries;1498 u64 entry_mask;1499 1500 if (profile->cam_entries != profile->action_entries) {1501 dev_err(rvu->dev,1502 "KPU%d: CAM and action entries [%d != %d] not equal\n",1503 kpu, profile->cam_entries, profile->action_entries);1504 }1505 1506 max_entries = rvu->hw->npc_kpu_entries;1507 1508 /* Program CAM match entries for previous KPU extracted data */1509 num_entries = min_t(int, profile->cam_entries, max_entries);1510 for (entry = 0; entry < num_entries; entry++)1511 npc_config_kpucam(rvu, blkaddr,1512 &profile->cam[entry], kpu, entry);1513 1514 /* Program this KPU's actions */1515 num_entries = min_t(int, profile->action_entries, max_entries);1516 for (entry = 0; entry < num_entries; entry++)1517 npc_config_kpuaction(rvu, blkaddr, &profile->action[entry],1518 kpu, entry, false);1519 1520 /* Enable all programmed entries */1521 num_entries = min_t(int, profile->action_entries, profile->cam_entries);1522 entry_mask = enable_mask(num_entries);1523 /* Disable first KPU_MAX_CST_ENT entries for built-in profile */1524 if (!rvu->kpu.custom)1525 entry_mask |= GENMASK_ULL(KPU_MAX_CST_ENT - 1, 0);1526 rvu_write64(rvu, blkaddr,1527 NPC_AF_KPUX_ENTRY_DISX(kpu, 0), entry_mask);1528 if (num_entries > 64) {1529 rvu_write64(rvu, blkaddr,1530 NPC_AF_KPUX_ENTRY_DISX(kpu, 1),1531 enable_mask(num_entries - 64));1532 }1533 1534 /* Enable this KPU */1535 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(kpu), 0x01);1536}1537 1538static int npc_prepare_default_kpu(struct npc_kpu_profile_adapter *profile)1539{1540 profile->custom = 0;1541 profile->name = def_pfl_name;1542 profile->version = NPC_KPU_PROFILE_VER;1543 profile->ikpu = ikpu_action_entries;1544 profile->pkinds = ARRAY_SIZE(ikpu_action_entries);1545 profile->kpu = npc_kpu_profiles;1546 profile->kpus = ARRAY_SIZE(npc_kpu_profiles);1547 profile->lt_def = &npc_lt_defaults;1548 profile->mkex = &npc_mkex_default;1549 profile->mkex_hash = &npc_mkex_hash_default;1550 1551 return 0;1552}1553 1554static int npc_apply_custom_kpu(struct rvu *rvu,1555 struct npc_kpu_profile_adapter *profile)1556{1557 size_t hdr_sz = sizeof(struct npc_kpu_profile_fwdata), offset = 0;1558 struct npc_kpu_profile_fwdata *fw = rvu->kpu_fwdata;1559 struct npc_kpu_profile_action *action;1560 struct npc_kpu_profile_cam *cam;1561 struct npc_kpu_fwdata *fw_kpu;1562 int entries;1563 u16 kpu, entry;1564 1565 if (rvu->kpu_fwdata_sz < hdr_sz) {1566 dev_warn(rvu->dev, "Invalid KPU profile size\n");1567 return -EINVAL;1568 }1569 if (le64_to_cpu(fw->signature) != KPU_SIGN) {1570 dev_warn(rvu->dev, "Invalid KPU profile signature %llx\n",1571 fw->signature);1572 return -EINVAL;1573 }1574 /* Verify if the using known profile structure */1575 if (NPC_KPU_VER_MAJ(profile->version) >1576 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)) {1577 dev_warn(rvu->dev, "Not supported Major version: %d > %d\n",1578 NPC_KPU_VER_MAJ(profile->version),1579 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER));1580 return -EINVAL;1581 }1582 /* Verify if profile is aligned with the required kernel changes */1583 if (NPC_KPU_VER_MIN(profile->version) <1584 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER)) {1585 dev_warn(rvu->dev,1586 "Invalid KPU profile version: %d.%d.%d expected version <= %d.%d.%d\n",1587 NPC_KPU_VER_MAJ(profile->version),1588 NPC_KPU_VER_MIN(profile->version),1589 NPC_KPU_VER_PATCH(profile->version),1590 NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER),1591 NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER),1592 NPC_KPU_VER_PATCH(NPC_KPU_PROFILE_VER));1593 return -EINVAL;1594 }1595 /* Verify if profile fits the HW */1596 if (fw->kpus > profile->kpus) {1597 dev_warn(rvu->dev, "Not enough KPUs: %d > %ld\n", fw->kpus,1598 profile->kpus);1599 return -EINVAL;1600 }1601 1602 profile->custom = 1;1603 profile->name = fw->name;1604 profile->version = le64_to_cpu(fw->version);1605 profile->mkex = &fw->mkex;1606 profile->lt_def = &fw->lt_def;1607 1608 for (kpu = 0; kpu < fw->kpus; kpu++) {1609 fw_kpu = (struct npc_kpu_fwdata *)(fw->data + offset);1610 if (fw_kpu->entries > KPU_MAX_CST_ENT)1611 dev_warn(rvu->dev,1612 "Too many custom entries on KPU%d: %d > %d\n",1613 kpu, fw_kpu->entries, KPU_MAX_CST_ENT);1614 entries = min(fw_kpu->entries, KPU_MAX_CST_ENT);1615 cam = (struct npc_kpu_profile_cam *)fw_kpu->data;1616 offset += sizeof(*fw_kpu) + fw_kpu->entries * sizeof(*cam);1617 action = (struct npc_kpu_profile_action *)(fw->data + offset);1618 offset += fw_kpu->entries * sizeof(*action);1619 if (rvu->kpu_fwdata_sz < hdr_sz + offset) {1620 dev_warn(rvu->dev,1621 "Profile size mismatch on KPU%i parsing.\n",1622 kpu + 1);1623 return -EINVAL;1624 }1625 for (entry = 0; entry < entries; entry++) {1626 profile->kpu[kpu].cam[entry] = cam[entry];1627 profile->kpu[kpu].action[entry] = action[entry];1628 }1629 }1630 1631 return 0;1632}1633 1634static int npc_load_kpu_prfl_img(struct rvu *rvu, void __iomem *prfl_addr,1635 u64 prfl_sz, const char *kpu_profile)1636{1637 struct npc_kpu_profile_fwdata *kpu_data = NULL;1638 int rc = -EINVAL;1639 1640 kpu_data = (struct npc_kpu_profile_fwdata __force *)prfl_addr;1641 if (le64_to_cpu(kpu_data->signature) == KPU_SIGN &&1642 !strncmp(kpu_data->name, kpu_profile, KPU_NAME_LEN)) {1643 dev_info(rvu->dev, "Loading KPU profile from firmware db: %s\n",1644 kpu_profile);1645 rvu->kpu_fwdata = kpu_data;1646 rvu->kpu_fwdata_sz = prfl_sz;1647 rvu->kpu_prfl_addr = prfl_addr;1648 rc = 0;1649 }1650 1651 return rc;1652}1653 1654static int npc_fwdb_detect_load_prfl_img(struct rvu *rvu, uint64_t prfl_sz,1655 const char *kpu_profile)1656{1657 struct npc_coalesced_kpu_prfl *img_data = NULL;1658 int i = 0, rc = -EINVAL;1659 void __iomem *kpu_prfl_addr;1660 u32 offset;1661 1662 img_data = (struct npc_coalesced_kpu_prfl __force *)rvu->kpu_prfl_addr;1663 if (le64_to_cpu(img_data->signature) == KPU_SIGN &&1664 !strncmp(img_data->name, kpu_profile, KPU_NAME_LEN)) {1665 /* Loaded profile is a single KPU profile. */1666 rc = npc_load_kpu_prfl_img(rvu, rvu->kpu_prfl_addr,1667 prfl_sz, kpu_profile);1668 goto done;1669 }1670 1671 /* Loaded profile is coalesced image, offset of first KPU profile.*/1672 offset = offsetof(struct npc_coalesced_kpu_prfl, prfl_sz) +1673 (img_data->num_prfl * sizeof(uint16_t));1674 /* Check if mapped image is coalesced image. */1675 while (i < img_data->num_prfl) {1676 /* Profile image offsets are rounded up to next 8 multiple.*/1677 offset = ALIGN_8B_CEIL(offset);1678 kpu_prfl_addr = (void __iomem *)((uintptr_t)rvu->kpu_prfl_addr +1679 offset);1680 rc = npc_load_kpu_prfl_img(rvu, kpu_prfl_addr,1681 img_data->prfl_sz[i], kpu_profile);1682 if (!rc)1683 break;1684 /* Calculating offset of profile image based on profile size.*/1685 offset += img_data->prfl_sz[i];1686 i++;1687 }1688done:1689 return rc;1690}1691 1692static int npc_load_kpu_profile_fwdb(struct rvu *rvu, const char *kpu_profile)1693{1694 int ret = -EINVAL;1695 u64 prfl_sz;1696 1697 /* Setting up the mapping for NPC profile image */1698 ret = npc_fwdb_prfl_img_map(rvu, &rvu->kpu_prfl_addr, &prfl_sz);1699 if (ret < 0)1700 goto done;1701 1702 /* Detect if profile is coalesced or single KPU profile and load */1703 ret = npc_fwdb_detect_load_prfl_img(rvu, prfl_sz, kpu_profile);1704 if (ret == 0)1705 goto done;1706 1707 /* Cleaning up if KPU profile image from fwdata is not valid. */1708 if (rvu->kpu_prfl_addr) {1709 iounmap(rvu->kpu_prfl_addr);1710 rvu->kpu_prfl_addr = NULL;1711 rvu->kpu_fwdata_sz = 0;1712 rvu->kpu_fwdata = NULL;1713 }1714 1715done:1716 return ret;1717}1718 1719static void npc_load_kpu_profile(struct rvu *rvu)1720{1721 struct npc_kpu_profile_adapter *profile = &rvu->kpu;1722 const char *kpu_profile = rvu->kpu_pfl_name;1723 const struct firmware *fw = NULL;1724 bool retry_fwdb = false;1725 1726 /* If user not specified profile customization */1727 if (!strncmp(kpu_profile, def_pfl_name, KPU_NAME_LEN))1728 goto revert_to_default;1729 /* First prepare default KPU, then we'll customize top entries. */1730 npc_prepare_default_kpu(profile);1731 1732 /* Order of preceedence for load loading NPC profile (high to low)1733 * Firmware binary in filesystem.1734 * Firmware database method.1735 * Default KPU profile.1736 */1737 if (!request_firmware_direct(&fw, kpu_profile, rvu->dev)) {1738 dev_info(rvu->dev, "Loading KPU profile from firmware: %s\n",1739 kpu_profile);1740 rvu->kpu_fwdata = kzalloc(fw->size, GFP_KERNEL);1741 if (rvu->kpu_fwdata) {1742 memcpy(rvu->kpu_fwdata, fw->data, fw->size);1743 rvu->kpu_fwdata_sz = fw->size;1744 }1745 release_firmware(fw);1746 retry_fwdb = true;1747 goto program_kpu;1748 }1749 1750load_image_fwdb:1751 /* Loading the KPU profile using firmware database */1752 if (npc_load_kpu_profile_fwdb(rvu, kpu_profile))1753 goto revert_to_default;1754 1755program_kpu:1756 /* Apply profile customization if firmware was loaded. */1757 if (!rvu->kpu_fwdata_sz || npc_apply_custom_kpu(rvu, profile)) {1758 /* If image from firmware filesystem fails to load or invalid1759 * retry with firmware database method.1760 */1761 if (rvu->kpu_fwdata || rvu->kpu_fwdata_sz) {1762 /* Loading image from firmware database failed. */1763 if (rvu->kpu_prfl_addr) {1764 iounmap(rvu->kpu_prfl_addr);1765 rvu->kpu_prfl_addr = NULL;1766 } else {1767 kfree(rvu->kpu_fwdata);1768 }1769 rvu->kpu_fwdata = NULL;1770 rvu->kpu_fwdata_sz = 0;1771 if (retry_fwdb) {1772 retry_fwdb = false;1773 goto load_image_fwdb;1774 }1775 }1776 1777 dev_warn(rvu->dev,1778 "Can't load KPU profile %s. Using default.\n",1779 kpu_profile);1780 kfree(rvu->kpu_fwdata);1781 rvu->kpu_fwdata = NULL;1782 goto revert_to_default;1783 }1784 1785 dev_info(rvu->dev, "Using custom profile '%s', version %d.%d.%d\n",1786 profile->name, NPC_KPU_VER_MAJ(profile->version),1787 NPC_KPU_VER_MIN(profile->version),1788 NPC_KPU_VER_PATCH(profile->version));1789 1790 return;1791 1792revert_to_default:1793 npc_prepare_default_kpu(profile);1794}1795 1796static void npc_parser_profile_init(struct rvu *rvu, int blkaddr)1797{1798 struct rvu_hwinfo *hw = rvu->hw;1799 int num_pkinds, num_kpus, idx;1800 1801 /* Disable all KPUs and their entries */1802 for (idx = 0; idx < hw->npc_kpus; idx++) {1803 rvu_write64(rvu, blkaddr,1804 NPC_AF_KPUX_ENTRY_DISX(idx, 0), ~0ULL);1805 rvu_write64(rvu, blkaddr,1806 NPC_AF_KPUX_ENTRY_DISX(idx, 1), ~0ULL);1807 rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00);1808 }1809 1810 /* Load and customize KPU profile. */1811 npc_load_kpu_profile(rvu);1812 1813 /* First program IKPU profile i.e PKIND configs.1814 * Check HW max count to avoid configuring junk or1815 * writing to unsupported CSR addresses.1816 */1817 num_pkinds = rvu->kpu.pkinds;1818 num_pkinds = min_t(int, hw->npc_pkinds, num_pkinds);1819 1820 for (idx = 0; idx < num_pkinds; idx++)1821 npc_config_kpuaction(rvu, blkaddr, &rvu->kpu.ikpu[idx], 0, idx, true);1822 1823 /* Program KPU CAM and Action profiles */1824 num_kpus = rvu->kpu.kpus;1825 num_kpus = min_t(int, hw->npc_kpus, num_kpus);1826 1827 for (idx = 0; idx < num_kpus; idx++)1828 npc_program_kpu_profile(rvu, blkaddr, idx, &rvu->kpu.kpu[idx]);1829}1830 1831void npc_mcam_rsrcs_deinit(struct rvu *rvu)1832{1833 struct npc_mcam *mcam = &rvu->hw->mcam;1834 1835 bitmap_free(mcam->bmap);1836 bitmap_free(mcam->bmap_reverse);1837 kfree(mcam->entry2pfvf_map);1838 kfree(mcam->cntr2pfvf_map);1839 kfree(mcam->entry2cntr_map);1840 kfree(mcam->cntr_refcnt);1841 kfree(mcam->entry2target_pffunc);1842 kfree(mcam->counters.bmap);1843}1844 1845int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr)1846{1847 int nixlf_count = rvu_get_nixlf_count(rvu);1848 struct npc_mcam *mcam = &rvu->hw->mcam;1849 int rsvd, err;1850 u16 index;1851 int cntr;1852 u64 cfg;1853 1854 /* Actual number of MCAM entries vary by entry size */1855 cfg = (rvu_read64(rvu, blkaddr,1856 NPC_AF_INTFX_KEX_CFG(0)) >> 32) & 0x07;1857 mcam->total_entries = (mcam->banks / BIT_ULL(cfg)) * mcam->banksize;1858 mcam->keysize = cfg;1859 1860 /* Number of banks combined per MCAM entry */1861 if (cfg == NPC_MCAM_KEY_X4)1862 mcam->banks_per_entry = 4;1863 else if (cfg == NPC_MCAM_KEY_X2)1864 mcam->banks_per_entry = 2;1865 else1866 mcam->banks_per_entry = 1;1867 1868 /* Reserve one MCAM entry for each of the NIX LF to1869 * guarantee space to install default matching DMAC rule.1870 * Also reserve 2 MCAM entries for each PF for default1871 * channel based matching or 'bcast & promisc' matching to1872 * support BCAST and PROMISC modes of operation for PFs.1873 * PF0 is excluded.1874 */1875 rsvd = (nixlf_count * RSVD_MCAM_ENTRIES_PER_NIXLF) +1876 ((rvu->hw->total_pfs - 1) * RSVD_MCAM_ENTRIES_PER_PF);1877 if (mcam->total_entries <= rsvd) {1878 dev_warn(rvu->dev,1879 "Insufficient NPC MCAM size %d for pkt I/O, exiting\n",1880 mcam->total_entries);1881 return -ENOMEM;1882 }1883 1884 mcam->bmap_entries = mcam->total_entries - rsvd;1885 mcam->nixlf_offset = mcam->bmap_entries;1886 mcam->pf_offset = mcam->nixlf_offset + nixlf_count;1887 1888 /* Allocate bitmaps for managing MCAM entries */1889 mcam->bmap = bitmap_zalloc(mcam->bmap_entries, GFP_KERNEL);1890 if (!mcam->bmap)1891 return -ENOMEM;1892 1893 mcam->bmap_reverse = bitmap_zalloc(mcam->bmap_entries, GFP_KERNEL);1894 if (!mcam->bmap_reverse)1895 goto free_bmap;1896 1897 mcam->bmap_fcnt = mcam->bmap_entries;1898 1899 /* Alloc memory for saving entry to RVU PFFUNC allocation mapping */1900 mcam->entry2pfvf_map = kcalloc(mcam->bmap_entries, sizeof(u16),1901 GFP_KERNEL);1902 1903 if (!mcam->entry2pfvf_map)1904 goto free_bmap_reverse;1905 1906 /* Reserve 1/8th of MCAM entries at the bottom for low priority1907 * allocations and another 1/8th at the top for high priority1908 * allocations.1909 */1910 mcam->lprio_count = mcam->bmap_entries / 8;1911 if (mcam->lprio_count > BITS_PER_LONG)1912 mcam->lprio_count = round_down(mcam->lprio_count,1913 BITS_PER_LONG);1914 mcam->lprio_start = mcam->bmap_entries - mcam->lprio_count;1915 mcam->hprio_count = mcam->lprio_count;1916 mcam->hprio_end = mcam->hprio_count;1917 1918 /* Allocate bitmap for managing MCAM counters and memory1919 * for saving counter to RVU PFFUNC allocation mapping.1920 */1921 err = rvu_alloc_bitmap(&mcam->counters);1922 if (err)1923 goto free_entry_map;1924 1925 mcam->cntr2pfvf_map = kcalloc(mcam->counters.max, sizeof(u16),1926 GFP_KERNEL);1927 if (!mcam->cntr2pfvf_map)1928 goto free_cntr_bmap;1929 1930 /* Alloc memory for MCAM entry to counter mapping and for tracking1931 * counter's reference count.1932 */1933 mcam->entry2cntr_map = kcalloc(mcam->bmap_entries, sizeof(u16),1934 GFP_KERNEL);1935 if (!mcam->entry2cntr_map)1936 goto free_cntr_map;1937 1938 mcam->cntr_refcnt = kcalloc(mcam->counters.max, sizeof(u16),1939 GFP_KERNEL);1940 if (!mcam->cntr_refcnt)1941 goto free_entry_cntr_map;1942 1943 /* Alloc memory for saving target device of mcam rule */1944 mcam->entry2target_pffunc = kmalloc_array(mcam->total_entries,1945 sizeof(u16), GFP_KERNEL);1946 if (!mcam->entry2target_pffunc)1947 goto free_cntr_refcnt;1948 1949 for (index = 0; index < mcam->bmap_entries; index++) {1950 mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP;1951 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP;1952 }1953 1954 for (cntr = 0; cntr < mcam->counters.max; cntr++)1955 mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP;1956 1957 mutex_init(&mcam->lock);1958 1959 return 0;1960 1961free_cntr_refcnt:1962 kfree(mcam->cntr_refcnt);1963free_entry_cntr_map:1964 kfree(mcam->entry2cntr_map);1965free_cntr_map:1966 kfree(mcam->cntr2pfvf_map);1967free_cntr_bmap:1968 kfree(mcam->counters.bmap);1969free_entry_map:1970 kfree(mcam->entry2pfvf_map);1971free_bmap_reverse:1972 bitmap_free(mcam->bmap_reverse);1973free_bmap:1974 bitmap_free(mcam->bmap);1975 1976 return -ENOMEM;1977}1978 1979static void rvu_npc_hw_init(struct rvu *rvu, int blkaddr)1980{1981 struct npc_pkind *pkind = &rvu->hw->pkind;1982 struct npc_mcam *mcam = &rvu->hw->mcam;1983 struct rvu_hwinfo *hw = rvu->hw;1984 u64 npc_const, npc_const1;1985 u64 npc_const2 = 0;1986 1987 npc_const = rvu_read64(rvu, blkaddr, NPC_AF_CONST);1988 npc_const1 = rvu_read64(rvu, blkaddr, NPC_AF_CONST1);1989 if (npc_const1 & BIT_ULL(63))1990 npc_const2 = rvu_read64(rvu, blkaddr, NPC_AF_CONST2);1991 1992 pkind->rsrc.max = NPC_UNRESERVED_PKIND_COUNT;1993 hw->npc_pkinds = (npc_const1 >> 12) & 0xFFULL;1994 hw->npc_kpu_entries = npc_const1 & 0xFFFULL;1995 hw->npc_kpus = (npc_const >> 8) & 0x1FULL;1996 hw->npc_intfs = npc_const & 0xFULL;1997 hw->npc_counters = (npc_const >> 48) & 0xFFFFULL;1998 1999 mcam->banks = (npc_const >> 44) & 0xFULL;2000 mcam->banksize = (npc_const >> 28) & 0xFFFFULL;2001 hw->npc_stat_ena = BIT_ULL(9);2002 /* Extended set */2003 if (npc_const2) {2004 hw->npc_ext_set = true;2005 /* 96xx supports only match_stats and npc_counters2006 * reflected in NPC_AF_CONST reg.2007 * STAT_SEL and ENA are at [0:8] and 9 bit positions.2008 * 98xx has both match_stat and ext and npc_counter2009 * reflected in NPC_AF_CONST22010 * STAT_SEL_EXT added at [12:14] bit position.2011 * cn10k supports only ext and hence npc_counters in2012 * NPC_AF_CONST is 0 and npc_counters reflected in NPC_AF_CONST2.2013 * STAT_SEL bitpos incremented from [0:8] to [0:11] and ENA bit moved to 632014 */2015 if (!hw->npc_counters)2016 hw->npc_stat_ena = BIT_ULL(63);2017 hw->npc_counters = (npc_const2 >> 16) & 0xFFFFULL;2018 mcam->banksize = npc_const2 & 0xFFFFULL;2019 }2020 2021 mcam->counters.max = hw->npc_counters;2022}2023 2024static void rvu_npc_setup_interfaces(struct rvu *rvu, int blkaddr)2025{2026 struct npc_mcam_kex *mkex = rvu->kpu.mkex;2027 struct npc_mcam *mcam = &rvu->hw->mcam;2028 struct rvu_hwinfo *hw = rvu->hw;2029 u64 nibble_ena, rx_kex, tx_kex;2030 u8 intf;2031 2032 /* Reserve last counter for MCAM RX miss action which is set to2033 * drop packet. This way we will know how many pkts didn't match2034 * any MCAM entry.2035 */2036 mcam->counters.max--;2037 mcam->rx_miss_act_cntr = mcam->counters.max;2038 2039 rx_kex = mkex->keyx_cfg[NIX_INTF_RX];2040 tx_kex = mkex->keyx_cfg[NIX_INTF_TX];2041 nibble_ena = FIELD_GET(NPC_PARSE_NIBBLE, rx_kex);2042 2043 nibble_ena = rvu_npc_get_tx_nibble_cfg(rvu, nibble_ena);2044 if (nibble_ena) {2045 tx_kex &= ~NPC_PARSE_NIBBLE;2046 tx_kex |= FIELD_PREP(NPC_PARSE_NIBBLE, nibble_ena);2047 mkex->keyx_cfg[NIX_INTF_TX] = tx_kex;2048 }2049 2050 /* Configure RX interfaces */2051 for (intf = 0; intf < hw->npc_intfs; intf++) {2052 if (is_npc_intf_tx(intf))2053 continue;2054 2055 /* Set RX MCAM search key size. LA..LE (ltype only) + Channel */2056 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf),2057 rx_kex);2058 2059 /* If MCAM lookup doesn't result in a match, drop the received2060 * packet. And map this action to a counter to count dropped2061 * packets.2062 */2063 rvu_write64(rvu, blkaddr,2064 NPC_AF_INTFX_MISS_ACT(intf), NIX_RX_ACTIONOP_DROP);2065 2066 /* NPC_AF_INTFX_MISS_STAT_ACT[14:12] - counter[11:9]2067 * NPC_AF_INTFX_MISS_STAT_ACT[8:0] - counter[8:0]2068 */2069 rvu_write64(rvu, blkaddr,2070 NPC_AF_INTFX_MISS_STAT_ACT(intf),2071 ((mcam->rx_miss_act_cntr >> 9) << 12) |2072 hw->npc_stat_ena | mcam->rx_miss_act_cntr);2073 }2074 2075 /* Configure TX interfaces */2076 for (intf = 0; intf < hw->npc_intfs; intf++) {2077 if (is_npc_intf_rx(intf))2078 continue;2079 2080 /* Extract Ltypes LID_LA to LID_LE */2081 rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf),2082 tx_kex);2083 2084 /* Set TX miss action to UCAST_DEFAULT i.e2085 * transmit the packet on NIX LF SQ's default channel.2086 */2087 rvu_write64(rvu, blkaddr,2088 NPC_AF_INTFX_MISS_ACT(intf),2089 NIX_TX_ACTIONOP_UCAST_DEFAULT);2090 }2091}2092 2093int rvu_npc_init(struct rvu *rvu)2094{2095 struct npc_kpu_profile_adapter *kpu = &rvu->kpu;2096 struct npc_pkind *pkind = &rvu->hw->pkind;2097 struct npc_mcam *mcam = &rvu->hw->mcam;2098 int blkaddr, entry, bank, err;2099 2100 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);2101 if (blkaddr < 0) {2102 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__);2103 return -ENODEV;2104 }2105 2106 rvu_npc_hw_init(rvu, blkaddr);2107 2108 /* First disable all MCAM entries, to stop traffic towards NIXLFs */2109 for (bank = 0; bank < mcam->banks; bank++) {2110 for (entry = 0; entry < mcam->banksize; entry++)2111 rvu_write64(rvu, blkaddr,2112 NPC_AF_MCAMEX_BANKX_CFG(entry, bank), 0);2113 }2114 2115 err = rvu_alloc_bitmap(&pkind->rsrc);2116 if (err)2117 return err;2118 /* Reserve PKIND#0 for LBKs. Power reset value of LBK_CH_PKIND is '0',2119 * no need to configure PKIND for all LBKs separately.2120 */2121 rvu_alloc_rsrc(&pkind->rsrc);2122 2123 /* Allocate mem for pkind to PF and channel mapping info */2124 pkind->pfchan_map = devm_kcalloc(rvu->dev, pkind->rsrc.max,2125 sizeof(u32), GFP_KERNEL);2126 if (!pkind->pfchan_map)2127 return -ENOMEM;2128 2129 /* Configure KPU profile */2130 npc_parser_profile_init(rvu, blkaddr);2131 2132 /* Config Outer L2, IPv4's NPC layer info */2133 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OL2,2134 (kpu->lt_def->pck_ol2.lid << 8) | (kpu->lt_def->pck_ol2.ltype_match << 4) |2135 kpu->lt_def->pck_ol2.ltype_mask);2136 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OIP4,2137 (kpu->lt_def->pck_oip4.lid << 8) | (kpu->lt_def->pck_oip4.ltype_match << 4) |2138 kpu->lt_def->pck_oip4.ltype_mask);2139 2140 /* Config Inner IPV4 NPC layer info */2141 rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_IIP4,2142 (kpu->lt_def->pck_iip4.lid << 8) | (kpu->lt_def->pck_iip4.ltype_match << 4) |2143 kpu->lt_def->pck_iip4.ltype_mask);2144 2145 /* Enable below for Rx pkts.2146 * - Outer IPv4 header checksum validation.2147 * - Detect outer L2 broadcast address and set NPC_RESULT_S[L2B].2148 * - Detect outer L2 multicast address and set NPC_RESULT_S[L2M].2149 * - Inner IPv4 header checksum validation.2150 * - Set non zero checksum error code value2151 */2152 rvu_write64(rvu, blkaddr, NPC_AF_PCK_CFG,2153 rvu_read64(rvu, blkaddr, NPC_AF_PCK_CFG) |2154 ((u64)NPC_EC_OIP4_CSUM << 32) | (NPC_EC_IIP4_CSUM << 24) |2155 BIT_ULL(7) | BIT_ULL(6) | BIT_ULL(2) | BIT_ULL(1));2156 2157 rvu_npc_setup_interfaces(rvu, blkaddr);2158 2159 npc_config_secret_key(rvu, blkaddr);2160 /* Configure MKEX profile */2161 npc_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name);2162 2163 err = npc_mcam_rsrcs_init(rvu, blkaddr);2164 if (err)2165 return err;2166 2167 err = npc_flow_steering_init(rvu, blkaddr);2168 if (err) {2169 dev_err(rvu->dev,2170 "Incorrect mkex profile loaded using default mkex\n");2171 npc_load_mkex_profile(rvu, blkaddr, def_pfl_name);2172 }2173 2174 return 0;2175}2176 2177void rvu_npc_freemem(struct rvu *rvu)2178{2179 struct npc_pkind *pkind = &rvu->hw->pkind;2180 struct npc_mcam *mcam = &rvu->hw->mcam;2181 2182 kfree(pkind->rsrc.bmap);2183 npc_mcam_rsrcs_deinit(rvu);2184 if (rvu->kpu_prfl_addr)2185 iounmap(rvu->kpu_prfl_addr);2186 else2187 kfree(rvu->kpu_fwdata);2188 mutex_destroy(&mcam->lock);2189}2190 2191void rvu_npc_get_mcam_entry_alloc_info(struct rvu *rvu, u16 pcifunc,2192 int blkaddr, int *alloc_cnt,2193 int *enable_cnt)2194{2195 struct npc_mcam *mcam = &rvu->hw->mcam;2196 int entry;2197 2198 *alloc_cnt = 0;2199 *enable_cnt = 0;2200 2201 for (entry = 0; entry < mcam->bmap_entries; entry++) {2202 if (mcam->entry2pfvf_map[entry] == pcifunc) {2203 (*alloc_cnt)++;2204 if (is_mcam_entry_enabled(rvu, mcam, blkaddr, entry))2205 (*enable_cnt)++;2206 }2207 }2208}2209 2210void rvu_npc_get_mcam_counter_alloc_info(struct rvu *rvu, u16 pcifunc,2211 int blkaddr, int *alloc_cnt,2212 int *enable_cnt)2213{2214 struct npc_mcam *mcam = &rvu->hw->mcam;2215 int cntr;2216 2217 *alloc_cnt = 0;2218 *enable_cnt = 0;2219 2220 for (cntr = 0; cntr < mcam->counters.max; cntr++) {2221 if (mcam->cntr2pfvf_map[cntr] == pcifunc) {2222 (*alloc_cnt)++;2223 if (mcam->cntr_refcnt[cntr])2224 (*enable_cnt)++;2225 }2226 }2227}2228 2229static int npc_mcam_verify_entry(struct npc_mcam *mcam,2230 u16 pcifunc, int entry)2231{2232 /* verify AF installed entries */2233 if (is_pffunc_af(pcifunc))2234 return 0;2235 /* Verify if entry is valid and if it is indeed2236 * allocated to the requesting PFFUNC.2237 */2238 if (entry >= mcam->bmap_entries)2239 return NPC_MCAM_INVALID_REQ;2240 2241 if (pcifunc != mcam->entry2pfvf_map[entry])2242 return NPC_MCAM_PERM_DENIED;2243 2244 return 0;2245}2246 2247static int npc_mcam_verify_counter(struct npc_mcam *mcam,2248 u16 pcifunc, int cntr)2249{2250 /* Verify if counter is valid and if it is indeed2251 * allocated to the requesting PFFUNC.2252 */2253 if (cntr >= mcam->counters.max)2254 return NPC_MCAM_INVALID_REQ;2255 2256 if (pcifunc != mcam->cntr2pfvf_map[cntr])2257 return NPC_MCAM_PERM_DENIED;2258 2259 return 0;2260}2261 2262static void npc_map_mcam_entry_and_cntr(struct rvu *rvu, struct npc_mcam *mcam,2263 int blkaddr, u16 entry, u16 cntr)2264{2265 u16 index = entry & (mcam->banksize - 1);2266 u32 bank = npc_get_bank(mcam, entry);2267 struct rvu_hwinfo *hw = rvu->hw;2268 2269 /* Set mapping and increment counter's refcnt */2270 mcam->entry2cntr_map[entry] = cntr;2271 mcam->cntr_refcnt[cntr]++;2272 /* Enable stats */2273 rvu_write64(rvu, blkaddr,2274 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank),2275 ((cntr >> 9) << 12) | hw->npc_stat_ena | cntr);2276}2277 2278static void npc_unmap_mcam_entry_and_cntr(struct rvu *rvu,2279 struct npc_mcam *mcam,2280 int blkaddr, u16 entry, u16 cntr)2281{2282 u16 index = entry & (mcam->banksize - 1);2283 u32 bank = npc_get_bank(mcam, entry);2284 2285 /* Remove mapping and reduce counter's refcnt */2286 mcam->entry2cntr_map[entry] = NPC_MCAM_INVALID_MAP;2287 mcam->cntr_refcnt[cntr]--;2288 /* Disable stats */2289 rvu_write64(rvu, blkaddr,2290 NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 0x00);2291}2292 2293/* Sets MCAM entry in bitmap as used. Update2294 * reverse bitmap too. Should be called with2295 * 'mcam->lock' held.2296 */2297static void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index)2298{2299 u16 entry, rentry;2300 2301 entry = index;2302 rentry = mcam->bmap_entries - index - 1;2303 2304 __set_bit(entry, mcam->bmap);2305 __set_bit(rentry, mcam->bmap_reverse);2306 mcam->bmap_fcnt--;2307}2308 2309/* Sets MCAM entry in bitmap as free. Update2310 * reverse bitmap too. Should be called with2311 * 'mcam->lock' held.2312 */2313static void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index)2314{2315 u16 entry, rentry;2316 2317 entry = index;2318 rentry = mcam->bmap_entries - index - 1;2319 2320 __clear_bit(entry, mcam->bmap);2321 __clear_bit(rentry, mcam->bmap_reverse);2322 mcam->bmap_fcnt++;2323}2324 2325static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam,2326 int blkaddr, u16 pcifunc)2327{2328 u16 index, cntr;2329 2330 /* Scan all MCAM entries and free the ones mapped to 'pcifunc' */2331 for (index = 0; index < mcam->bmap_entries; index++) {2332 if (mcam->entry2pfvf_map[index] == pcifunc) {2333 mcam->entry2pfvf_map[index] = NPC_MCAM_INVALID_MAP;2334 /* Free the entry in bitmap */2335 npc_mcam_clear_bit(mcam, index);2336 /* Disable the entry */2337 npc_enable_mcam_entry(rvu, mcam, blkaddr, index, false);2338 2339 /* Update entry2counter mapping */2340 cntr = mcam->entry2cntr_map[index];2341 if (cntr != NPC_MCAM_INVALID_MAP)2342 npc_unmap_mcam_entry_and_cntr(rvu, mcam,2343 blkaddr, index,2344 cntr);2345 mcam->entry2target_pffunc[index] = 0x0;2346 }2347 }2348}2349 2350static void npc_mcam_free_all_counters(struct rvu *rvu, struct npc_mcam *mcam,2351 u16 pcifunc)2352{2353 u16 cntr;2354 2355 /* Scan all MCAM counters and free the ones mapped to 'pcifunc' */2356 for (cntr = 0; cntr < mcam->counters.max; cntr++) {2357 if (mcam->cntr2pfvf_map[cntr] == pcifunc) {2358 mcam->cntr2pfvf_map[cntr] = NPC_MCAM_INVALID_MAP;2359 mcam->cntr_refcnt[cntr] = 0;2360 rvu_free_rsrc(&mcam->counters, cntr);2361 /* This API is expected to be called after freeing2362 * MCAM entries, which inturn will remove2363 * 'entry to counter' mapping.2364 * No need to do it again.2365 */2366 }2367 }2368}2369 2370/* Find area of contiguous free entries of size 'nr'.2371 * If not found return max contiguous free entries available.2372 */2373static u16 npc_mcam_find_zero_area(unsigned long *map, u16 size, u16 start,2374 u16 nr, u16 *max_area)2375{2376 u16 max_area_start = 0;2377 u16 index, next, end;2378 2379 *max_area = 0;2380 2381again:2382 index = find_next_zero_bit(map, size, start);2383 if (index >= size)2384 return max_area_start;2385 2386 end = ((index + nr) >= size) ? size : index + nr;2387 next = find_next_bit(map, end, index);2388 if (*max_area < (next - index)) {2389 *max_area = next - index;2390 max_area_start = index;2391 }2392 2393 if (next < end) {2394 start = next + 1;2395 goto again;2396 }2397 2398 return max_area_start;2399}2400 2401/* Find number of free MCAM entries available2402 * within range i.e in between 'start' and 'end'.2403 */2404static u16 npc_mcam_get_free_count(unsigned long *map, u16 start, u16 end)2405{2406 u16 index, next;2407 u16 fcnt = 0;2408 2409again:2410 if (start >= end)2411 return fcnt;2412 2413 index = find_next_zero_bit(map, end, start);2414 if (index >= end)2415 return fcnt;2416 2417 next = find_next_bit(map, end, index);2418 if (next <= end) {2419 fcnt += next - index;2420 start = next + 1;2421 goto again;2422 }2423 2424 fcnt += end - index;2425 return fcnt;2426}2427 2428static void2429npc_get_mcam_search_range_priority(struct npc_mcam *mcam,2430 struct npc_mcam_alloc_entry_req *req,2431 u16 *start, u16 *end, bool *reverse)2432{2433 u16 fcnt;2434 2435 if (req->priority == NPC_MCAM_HIGHER_PRIO)2436 goto hprio;2437 2438 /* For a low priority entry allocation2439 * - If reference entry is not in hprio zone then2440 * search range: ref_entry to end.2441 * - If reference entry is in hprio zone and if2442 * request can be accomodated in non-hprio zone then2443 * search range: 'start of middle zone' to 'end'2444 * - else search in reverse, so that less number of hprio2445 * zone entries are allocated.2446 */2447 2448 *reverse = false;2449 *start = req->ref_entry + 1;2450 *end = mcam->bmap_entries;2451 2452 if (req->ref_entry >= mcam->hprio_end)2453 return;2454 2455 fcnt = npc_mcam_get_free_count(mcam->bmap,2456 mcam->hprio_end, mcam->bmap_entries);2457 if (fcnt > req->count)2458 *start = mcam->hprio_end;2459 else2460 *reverse = true;2461 return;2462 2463hprio:2464 /* For a high priority entry allocation, search is always2465 * in reverse to preserve hprio zone entries.2466 * - If reference entry is not in lprio zone then2467 * search range: 0 to ref_entry.2468 * - If reference entry is in lprio zone and if2469 * request can be accomodated in middle zone then2470 * search range: 'hprio_end' to 'lprio_start'2471 */2472 2473 *reverse = true;2474 *start = 0;2475 *end = req->ref_entry;2476 2477 if (req->ref_entry <= mcam->lprio_start)2478 return;2479 2480 fcnt = npc_mcam_get_free_count(mcam->bmap,2481 mcam->hprio_end, mcam->lprio_start);2482 if (fcnt < req->count)2483 return;2484 *start = mcam->hprio_end;2485 *end = mcam->lprio_start;2486}2487 2488static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc,2489 struct npc_mcam_alloc_entry_req *req,2490 struct npc_mcam_alloc_entry_rsp *rsp)2491{2492 u16 entry_list[NPC_MAX_NONCONTIG_ENTRIES];2493 u16 fcnt, hp_fcnt, lp_fcnt;2494 u16 start, end, index;2495 int entry, next_start;2496 bool reverse = false;2497 unsigned long *bmap;2498 u16 max_contig;2499 2500 mutex_lock(&mcam->lock);2501 2502 /* Check if there are any free entries */2503 if (!mcam->bmap_fcnt) {2504 mutex_unlock(&mcam->lock);2505 return NPC_MCAM_ALLOC_FAILED;2506 }2507 2508 /* MCAM entries are divided into high priority, middle and2509 * low priority zones. Idea is to not allocate top and lower2510 * most entries as much as possible, this is to increase2511 * probability of honouring priority allocation requests.2512 *2513 * Two bitmaps are used for mcam entry management,2514 * mcam->bmap for forward search i.e '0 to mcam->bmap_entries'.2515 * mcam->bmap_reverse for reverse search i.e 'mcam->bmap_entries to 0'.2516 *2517 * Reverse bitmap is used to allocate entries2518 * - when a higher priority entry is requested2519 * - when available free entries are less.2520 * Lower priority ones out of avaialble free entries are always2521 * chosen when 'high vs low' question arises.2522 *2523 * For a VF base MCAM match rule is set by its PF. And all the2524 * further MCAM rules installed by VF on its own are2525 * concatenated with the base rule set by its PF. Hence PF entries2526 * should be at lower priority compared to VF entries. Otherwise2527 * base rule is hit always and rules installed by VF will be of2528 * no use. Hence if the request is from PF then allocate low2529 * priority entries.2530 */2531 if (!(pcifunc & RVU_PFVF_FUNC_MASK))2532 goto lprio_alloc;2533 2534 /* Get the search range for priority allocation request */2535 if (req->priority) {2536 npc_get_mcam_search_range_priority(mcam, req,2537 &start, &end, &reverse);2538 goto alloc;2539 }2540 2541 /* Find out the search range for non-priority allocation request2542 *2543 * Get MCAM free entry count in middle zone.2544 */2545 lp_fcnt = npc_mcam_get_free_count(mcam->bmap,2546 mcam->lprio_start,2547 mcam->bmap_entries);2548 hp_fcnt = npc_mcam_get_free_count(mcam->bmap, 0, mcam->hprio_end);2549 fcnt = mcam->bmap_fcnt - lp_fcnt - hp_fcnt;2550 2551 /* Check if request can be accomodated in the middle zone */2552 if (fcnt > req->count) {2553 start = mcam->hprio_end;2554 end = mcam->lprio_start;2555 } else if ((fcnt + (hp_fcnt / 2) + (lp_fcnt / 2)) > req->count) {2556 /* Expand search zone from half of hprio zone to2557 * half of lprio zone.2558 */2559 start = mcam->hprio_end / 2;2560 end = mcam->bmap_entries - (mcam->lprio_count / 2);2561 reverse = true;2562 } else {2563 /* Not enough free entries, search all entries in reverse,2564 * so that low priority ones will get used up.2565 */2566lprio_alloc:2567 reverse = true;2568 start = 0;2569 end = mcam->bmap_entries;2570 /* Ensure PF requests are always at bottom and if PF requests2571 * for higher/lower priority entry wrt reference entry then2572 * honour that criteria and start search for entries from bottom2573 * and not in mid zone.2574 */2575 if (!(pcifunc & RVU_PFVF_FUNC_MASK) &&2576 req->priority == NPC_MCAM_HIGHER_PRIO)2577 end = req->ref_entry;2578 2579 if (!(pcifunc & RVU_PFVF_FUNC_MASK) &&2580 req->priority == NPC_MCAM_LOWER_PRIO)2581 start = req->ref_entry;2582 }2583 2584alloc:2585 if (reverse) {2586 bmap = mcam->bmap_reverse;2587 start = mcam->bmap_entries - start;2588 end = mcam->bmap_entries - end;2589 swap(start, end);2590 } else {2591 bmap = mcam->bmap;2592 }2593 2594 if (req->contig) {2595 /* Allocate requested number of contiguous entries, if2596 * unsuccessful find max contiguous entries available.2597 */2598 index = npc_mcam_find_zero_area(bmap, end, start,2599 req->count, &max_contig);2600 rsp->count = max_contig;2601 if (reverse)2602 rsp->entry = mcam->bmap_entries - index - max_contig;2603 else2604 rsp->entry = index;2605 } else {2606 /* Allocate requested number of non-contiguous entries,2607 * if unsuccessful allocate as many as possible.2608 */2609 rsp->count = 0;2610 next_start = start;2611 for (entry = 0; entry < req->count; entry++) {2612 index = find_next_zero_bit(bmap, end, next_start);2613 if (index >= end)2614 break;2615 2616 next_start = start + (index - start) + 1;2617 2618 /* Save the entry's index */2619 if (reverse)2620 index = mcam->bmap_entries - index - 1;2621 entry_list[entry] = index;2622 rsp->count++;2623 }2624 }2625 2626 /* If allocating requested no of entries is unsucessful,2627 * expand the search range to full bitmap length and retry.2628 */2629 if (!req->priority && (rsp->count < req->count) &&2630 ((end - start) != mcam->bmap_entries)) {2631 reverse = true;2632 start = 0;2633 end = mcam->bmap_entries;2634 goto alloc;2635 }2636 2637 /* For priority entry allocation requests, if allocation is2638 * failed then expand search to max possible range and retry.2639 */2640 if (req->priority && rsp->count < req->count) {2641 if (req->priority == NPC_MCAM_LOWER_PRIO &&2642 (start != (req->ref_entry + 1))) {2643 start = req->ref_entry + 1;2644 end = mcam->bmap_entries;2645 reverse = false;2646 goto alloc;2647 } else if ((req->priority == NPC_MCAM_HIGHER_PRIO) &&2648 ((end - start) != req->ref_entry)) {2649 start = 0;2650 end = req->ref_entry;2651 reverse = true;2652 goto alloc;2653 }2654 }2655 2656 /* Copy MCAM entry indices into mbox response entry_list.2657 * Requester always expects indices in ascending order, so2658 * reverse the list if reverse bitmap is used for allocation.2659 */2660 if (!req->contig && rsp->count) {2661 index = 0;2662 for (entry = rsp->count - 1; entry >= 0; entry--) {2663 if (reverse)2664 rsp->entry_list[index++] = entry_list[entry];2665 else2666 rsp->entry_list[entry] = entry_list[entry];2667 }2668 }2669 2670 /* Mark the allocated entries as used and set nixlf mapping */2671 for (entry = 0; entry < rsp->count; entry++) {2672 index = req->contig ?2673 (rsp->entry + entry) : rsp->entry_list[entry];2674 npc_mcam_set_bit(mcam, index);2675 mcam->entry2pfvf_map[index] = pcifunc;2676 mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP;2677 }2678 2679 /* Update available free count in mbox response */2680 rsp->free_count = mcam->bmap_fcnt;2681 2682 mutex_unlock(&mcam->lock);2683 return 0;2684}2685 2686/* Marks bitmaps to reserved the mcam slot */2687void npc_mcam_rsrcs_reserve(struct rvu *rvu, int blkaddr, int entry_idx)2688{2689 struct npc_mcam *mcam = &rvu->hw->mcam;2690 2691 npc_mcam_set_bit(mcam, entry_idx);2692}2693 2694int rvu_mbox_handler_npc_mcam_alloc_entry(struct rvu *rvu,2695 struct npc_mcam_alloc_entry_req *req,2696 struct npc_mcam_alloc_entry_rsp *rsp)2697{2698 struct npc_mcam *mcam = &rvu->hw->mcam;2699 u16 pcifunc = req->hdr.pcifunc;2700 int blkaddr;2701 2702 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);2703 if (blkaddr < 0)2704 return NPC_MCAM_INVALID_REQ;2705 2706 rsp->entry = NPC_MCAM_ENTRY_INVALID;2707 rsp->free_count = 0;2708 2709 /* Check if ref_entry is greater that the range2710 * then set it to max value.2711 */2712 if (req->ref_entry > mcam->bmap_entries)2713 req->ref_entry = mcam->bmap_entries;2714 2715 /* ref_entry can't be '0' if requested priority is high.2716 * Can't be last entry if requested priority is low.2717 */2718 if ((!req->ref_entry && req->priority == NPC_MCAM_HIGHER_PRIO) ||2719 ((req->ref_entry == mcam->bmap_entries) &&2720 req->priority == NPC_MCAM_LOWER_PRIO))2721 return NPC_MCAM_INVALID_REQ;2722 2723 /* Since list of allocated indices needs to be sent to requester,2724 * max number of non-contiguous entries per mbox msg is limited.2725 */2726 if (!req->contig && req->count > NPC_MAX_NONCONTIG_ENTRIES) {2727 dev_err(rvu->dev,2728 "%s: %d Non-contiguous MCAM entries requested is more than max (%d) allowed\n",2729 __func__, req->count, NPC_MAX_NONCONTIG_ENTRIES);2730 return NPC_MCAM_INVALID_REQ;2731 }2732 2733 /* Alloc request from PFFUNC with no NIXLF attached should be denied */2734 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc))2735 return NPC_MCAM_ALLOC_DENIED;2736 2737 return npc_mcam_alloc_entries(mcam, pcifunc, req, rsp);2738}2739 2740int rvu_mbox_handler_npc_mcam_free_entry(struct rvu *rvu,2741 struct npc_mcam_free_entry_req *req,2742 struct msg_rsp *rsp)2743{2744 struct npc_mcam *mcam = &rvu->hw->mcam;2745 u16 pcifunc = req->hdr.pcifunc;2746 int blkaddr, rc = 0;2747 u16 cntr;2748 2749 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);2750 if (blkaddr < 0)2751 return NPC_MCAM_INVALID_REQ;2752 2753 /* Free request from PFFUNC with no NIXLF attached, ignore */2754 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc))2755 return NPC_MCAM_INVALID_REQ;2756 2757 mutex_lock(&mcam->lock);2758 2759 if (req->all)2760 goto free_all;2761 2762 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry);2763 if (rc)2764 goto exit;2765 2766 mcam->entry2pfvf_map[req->entry] = NPC_MCAM_INVALID_MAP;2767 mcam->entry2target_pffunc[req->entry] = 0x0;2768 npc_mcam_clear_bit(mcam, req->entry);2769 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false);2770 2771 /* Update entry2counter mapping */2772 cntr = mcam->entry2cntr_map[req->entry];2773 if (cntr != NPC_MCAM_INVALID_MAP)2774 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr,2775 req->entry, cntr);2776 2777 goto exit;2778 2779free_all:2780 /* Free up all entries allocated to requesting PFFUNC */2781 npc_mcam_free_all_entries(rvu, mcam, blkaddr, pcifunc);2782exit:2783 mutex_unlock(&mcam->lock);2784 return rc;2785}2786 2787int rvu_mbox_handler_npc_mcam_read_entry(struct rvu *rvu,2788 struct npc_mcam_read_entry_req *req,2789 struct npc_mcam_read_entry_rsp *rsp)2790{2791 struct npc_mcam *mcam = &rvu->hw->mcam;2792 u16 pcifunc = req->hdr.pcifunc;2793 int blkaddr, rc;2794 2795 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);2796 if (blkaddr < 0)2797 return NPC_MCAM_INVALID_REQ;2798 2799 mutex_lock(&mcam->lock);2800 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry);2801 if (!rc) {2802 npc_read_mcam_entry(rvu, mcam, blkaddr, req->entry,2803 &rsp->entry_data,2804 &rsp->intf, &rsp->enable);2805 }2806 2807 mutex_unlock(&mcam->lock);2808 return rc;2809}2810 2811int rvu_mbox_handler_npc_mcam_write_entry(struct rvu *rvu,2812 struct npc_mcam_write_entry_req *req,2813 struct msg_rsp *rsp)2814{2815 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);2816 struct npc_mcam *mcam = &rvu->hw->mcam;2817 u16 pcifunc = req->hdr.pcifunc;2818 int blkaddr, rc;2819 u8 nix_intf;2820 2821 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);2822 if (blkaddr < 0)2823 return NPC_MCAM_INVALID_REQ;2824 2825 mutex_lock(&mcam->lock);2826 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry);2827 if (rc)2828 goto exit;2829 2830 if (req->set_cntr &&2831 npc_mcam_verify_counter(mcam, pcifunc, req->cntr)) {2832 rc = NPC_MCAM_INVALID_REQ;2833 goto exit;2834 }2835 2836 if (!is_npc_interface_valid(rvu, req->intf)) {2837 rc = NPC_MCAM_INVALID_REQ;2838 goto exit;2839 }2840 2841 if (is_npc_intf_tx(req->intf))2842 nix_intf = pfvf->nix_tx_intf;2843 else2844 nix_intf = pfvf->nix_rx_intf;2845 2846 /* For AF installed rules, the nix_intf should be set to target NIX */2847 if (is_pffunc_af(req->hdr.pcifunc))2848 nix_intf = req->intf;2849 2850 npc_config_mcam_entry(rvu, mcam, blkaddr, req->entry, nix_intf,2851 &req->entry_data, req->enable_entry);2852 2853 if (req->set_cntr)2854 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr,2855 req->entry, req->cntr);2856 2857 rc = 0;2858exit:2859 mutex_unlock(&mcam->lock);2860 return rc;2861}2862 2863int rvu_mbox_handler_npc_mcam_ena_entry(struct rvu *rvu,2864 struct npc_mcam_ena_dis_entry_req *req,2865 struct msg_rsp *rsp)2866{2867 struct npc_mcam *mcam = &rvu->hw->mcam;2868 u16 pcifunc = req->hdr.pcifunc;2869 int blkaddr, rc;2870 2871 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);2872 if (blkaddr < 0)2873 return NPC_MCAM_INVALID_REQ;2874 2875 mutex_lock(&mcam->lock);2876 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry);2877 mutex_unlock(&mcam->lock);2878 if (rc)2879 return rc;2880 2881 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, true);2882 2883 return 0;2884}2885 2886int rvu_mbox_handler_npc_mcam_dis_entry(struct rvu *rvu,2887 struct npc_mcam_ena_dis_entry_req *req,2888 struct msg_rsp *rsp)2889{2890 struct npc_mcam *mcam = &rvu->hw->mcam;2891 u16 pcifunc = req->hdr.pcifunc;2892 int blkaddr, rc;2893 2894 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);2895 if (blkaddr < 0)2896 return NPC_MCAM_INVALID_REQ;2897 2898 mutex_lock(&mcam->lock);2899 rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry);2900 mutex_unlock(&mcam->lock);2901 if (rc)2902 return rc;2903 2904 npc_enable_mcam_entry(rvu, mcam, blkaddr, req->entry, false);2905 2906 return 0;2907}2908 2909int rvu_mbox_handler_npc_mcam_shift_entry(struct rvu *rvu,2910 struct npc_mcam_shift_entry_req *req,2911 struct npc_mcam_shift_entry_rsp *rsp)2912{2913 struct npc_mcam *mcam = &rvu->hw->mcam;2914 u16 pcifunc = req->hdr.pcifunc;2915 u16 old_entry, new_entry;2916 int blkaddr, rc = 0;2917 u16 index, cntr;2918 2919 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);2920 if (blkaddr < 0)2921 return NPC_MCAM_INVALID_REQ;2922 2923 if (req->shift_count > NPC_MCAM_MAX_SHIFTS)2924 return NPC_MCAM_INVALID_REQ;2925 2926 mutex_lock(&mcam->lock);2927 for (index = 0; index < req->shift_count; index++) {2928 old_entry = req->curr_entry[index];2929 new_entry = req->new_entry[index];2930 2931 /* Check if both old and new entries are valid and2932 * does belong to this PFFUNC or not.2933 */2934 rc = npc_mcam_verify_entry(mcam, pcifunc, old_entry);2935 if (rc)2936 break;2937 2938 rc = npc_mcam_verify_entry(mcam, pcifunc, new_entry);2939 if (rc)2940 break;2941 2942 /* new_entry should not have a counter mapped */2943 if (mcam->entry2cntr_map[new_entry] != NPC_MCAM_INVALID_MAP) {2944 rc = NPC_MCAM_PERM_DENIED;2945 break;2946 }2947 2948 /* Disable the new_entry */2949 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, false);2950 2951 /* Copy rule from old entry to new entry */2952 npc_copy_mcam_entry(rvu, mcam, blkaddr, old_entry, new_entry);2953 2954 /* Copy counter mapping, if any */2955 cntr = mcam->entry2cntr_map[old_entry];2956 if (cntr != NPC_MCAM_INVALID_MAP) {2957 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr,2958 old_entry, cntr);2959 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr,2960 new_entry, cntr);2961 }2962 2963 /* Enable new_entry and disable old_entry */2964 npc_enable_mcam_entry(rvu, mcam, blkaddr, new_entry, true);2965 npc_enable_mcam_entry(rvu, mcam, blkaddr, old_entry, false);2966 }2967 2968 /* If shift has failed then report the failed index */2969 if (index != req->shift_count) {2970 rc = NPC_MCAM_PERM_DENIED;2971 rsp->failed_entry_idx = index;2972 }2973 2974 mutex_unlock(&mcam->lock);2975 return rc;2976}2977 2978int rvu_mbox_handler_npc_mcam_alloc_counter(struct rvu *rvu,2979 struct npc_mcam_alloc_counter_req *req,2980 struct npc_mcam_alloc_counter_rsp *rsp)2981{2982 struct npc_mcam *mcam = &rvu->hw->mcam;2983 u16 pcifunc = req->hdr.pcifunc;2984 u16 max_contig, cntr;2985 int blkaddr, index;2986 2987 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);2988 if (blkaddr < 0)2989 return NPC_MCAM_INVALID_REQ;2990 2991 /* If the request is from a PFFUNC with no NIXLF attached, ignore */2992 if (!is_pffunc_af(pcifunc) && !is_nixlf_attached(rvu, pcifunc))2993 return NPC_MCAM_INVALID_REQ;2994 2995 /* Since list of allocated counter IDs needs to be sent to requester,2996 * max number of non-contiguous counters per mbox msg is limited.2997 */2998 if (!req->contig && req->count > NPC_MAX_NONCONTIG_COUNTERS)2999 return NPC_MCAM_INVALID_REQ;3000 3001 mutex_lock(&mcam->lock);3002 3003 /* Check if unused counters are available or not */3004 if (!rvu_rsrc_free_count(&mcam->counters)) {3005 mutex_unlock(&mcam->lock);3006 return NPC_MCAM_ALLOC_FAILED;3007 }3008 3009 rsp->count = 0;3010 3011 if (req->contig) {3012 /* Allocate requested number of contiguous counters, if3013 * unsuccessful find max contiguous entries available.3014 */3015 index = npc_mcam_find_zero_area(mcam->counters.bmap,3016 mcam->counters.max, 0,3017 req->count, &max_contig);3018 rsp->count = max_contig;3019 rsp->cntr = index;3020 for (cntr = index; cntr < (index + max_contig); cntr++) {3021 __set_bit(cntr, mcam->counters.bmap);3022 mcam->cntr2pfvf_map[cntr] = pcifunc;3023 }3024 } else {3025 /* Allocate requested number of non-contiguous counters,3026 * if unsuccessful allocate as many as possible.3027 */3028 for (cntr = 0; cntr < req->count; cntr++) {3029 index = rvu_alloc_rsrc(&mcam->counters);3030 if (index < 0)3031 break;3032 rsp->cntr_list[cntr] = index;3033 rsp->count++;3034 mcam->cntr2pfvf_map[index] = pcifunc;3035 }3036 }3037 3038 mutex_unlock(&mcam->lock);3039 return 0;3040}3041 3042int rvu_mbox_handler_npc_mcam_free_counter(struct rvu *rvu,3043 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp)3044{3045 struct npc_mcam *mcam = &rvu->hw->mcam;3046 u16 index, entry = 0;3047 int blkaddr, err;3048 3049 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);3050 if (blkaddr < 0)3051 return NPC_MCAM_INVALID_REQ;3052 3053 mutex_lock(&mcam->lock);3054 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr);3055 if (err) {3056 mutex_unlock(&mcam->lock);3057 return err;3058 }3059 3060 /* Mark counter as free/unused */3061 mcam->cntr2pfvf_map[req->cntr] = NPC_MCAM_INVALID_MAP;3062 rvu_free_rsrc(&mcam->counters, req->cntr);3063 3064 /* Disable all MCAM entry's stats which are using this counter */3065 while (entry < mcam->bmap_entries) {3066 if (!mcam->cntr_refcnt[req->cntr])3067 break;3068 3069 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry);3070 if (index >= mcam->bmap_entries)3071 break;3072 entry = index + 1;3073 if (mcam->entry2cntr_map[index] != req->cntr)3074 continue;3075 3076 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr,3077 index, req->cntr);3078 }3079 3080 mutex_unlock(&mcam->lock);3081 return 0;3082}3083 3084int rvu_mbox_handler_npc_mcam_unmap_counter(struct rvu *rvu,3085 struct npc_mcam_unmap_counter_req *req, struct msg_rsp *rsp)3086{3087 struct npc_mcam *mcam = &rvu->hw->mcam;3088 u16 index, entry = 0;3089 int blkaddr, rc;3090 3091 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);3092 if (blkaddr < 0)3093 return NPC_MCAM_INVALID_REQ;3094 3095 mutex_lock(&mcam->lock);3096 rc = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr);3097 if (rc)3098 goto exit;3099 3100 /* Unmap the MCAM entry and counter */3101 if (!req->all) {3102 rc = npc_mcam_verify_entry(mcam, req->hdr.pcifunc, req->entry);3103 if (rc)3104 goto exit;3105 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr,3106 req->entry, req->cntr);3107 goto exit;3108 }3109 3110 /* Disable all MCAM entry's stats which are using this counter */3111 while (entry < mcam->bmap_entries) {3112 if (!mcam->cntr_refcnt[req->cntr])3113 break;3114 3115 index = find_next_bit(mcam->bmap, mcam->bmap_entries, entry);3116 if (index >= mcam->bmap_entries)3117 break;3118 entry = index + 1;3119 3120 if (mcam->entry2cntr_map[index] != req->cntr)3121 continue;3122 3123 npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr,3124 index, req->cntr);3125 }3126exit:3127 mutex_unlock(&mcam->lock);3128 return rc;3129}3130 3131int rvu_mbox_handler_npc_mcam_clear_counter(struct rvu *rvu,3132 struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp)3133{3134 struct npc_mcam *mcam = &rvu->hw->mcam;3135 int blkaddr, err;3136 3137 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);3138 if (blkaddr < 0)3139 return NPC_MCAM_INVALID_REQ;3140 3141 mutex_lock(&mcam->lock);3142 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr);3143 mutex_unlock(&mcam->lock);3144 if (err)3145 return err;3146 3147 rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr), 0x00);3148 3149 return 0;3150}3151 3152int rvu_mbox_handler_npc_mcam_counter_stats(struct rvu *rvu,3153 struct npc_mcam_oper_counter_req *req,3154 struct npc_mcam_oper_counter_rsp *rsp)3155{3156 struct npc_mcam *mcam = &rvu->hw->mcam;3157 int blkaddr, err;3158 3159 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);3160 if (blkaddr < 0)3161 return NPC_MCAM_INVALID_REQ;3162 3163 mutex_lock(&mcam->lock);3164 err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr);3165 mutex_unlock(&mcam->lock);3166 if (err)3167 return err;3168 3169 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(req->cntr));3170 rsp->stat &= BIT_ULL(48) - 1;3171 3172 return 0;3173}3174 3175int rvu_mbox_handler_npc_mcam_alloc_and_write_entry(struct rvu *rvu,3176 struct npc_mcam_alloc_and_write_entry_req *req,3177 struct npc_mcam_alloc_and_write_entry_rsp *rsp)3178{3179 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);3180 struct npc_mcam_alloc_counter_req cntr_req;3181 struct npc_mcam_alloc_counter_rsp cntr_rsp;3182 struct npc_mcam_alloc_entry_req entry_req;3183 struct npc_mcam_alloc_entry_rsp entry_rsp;3184 struct npc_mcam *mcam = &rvu->hw->mcam;3185 u16 entry = NPC_MCAM_ENTRY_INVALID;3186 u16 cntr = NPC_MCAM_ENTRY_INVALID;3187 int blkaddr, rc;3188 u8 nix_intf;3189 3190 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);3191 if (blkaddr < 0)3192 return NPC_MCAM_INVALID_REQ;3193 3194 if (!is_npc_interface_valid(rvu, req->intf))3195 return NPC_MCAM_INVALID_REQ;3196 3197 /* Try to allocate a MCAM entry */3198 entry_req.hdr.pcifunc = req->hdr.pcifunc;3199 entry_req.contig = true;3200 entry_req.priority = req->priority;3201 entry_req.ref_entry = req->ref_entry;3202 entry_req.count = 1;3203 3204 rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu,3205 &entry_req, &entry_rsp);3206 if (rc)3207 return rc;3208 3209 if (!entry_rsp.count)3210 return NPC_MCAM_ALLOC_FAILED;3211 3212 entry = entry_rsp.entry;3213 3214 if (!req->alloc_cntr)3215 goto write_entry;3216 3217 /* Now allocate counter */3218 cntr_req.hdr.pcifunc = req->hdr.pcifunc;3219 cntr_req.contig = true;3220 cntr_req.count = 1;3221 3222 rc = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp);3223 if (rc) {3224 /* Free allocated MCAM entry */3225 mutex_lock(&mcam->lock);3226 mcam->entry2pfvf_map[entry] = NPC_MCAM_INVALID_MAP;3227 npc_mcam_clear_bit(mcam, entry);3228 mutex_unlock(&mcam->lock);3229 return rc;3230 }3231 3232 cntr = cntr_rsp.cntr;3233 3234write_entry:3235 mutex_lock(&mcam->lock);3236 3237 if (is_npc_intf_tx(req->intf))3238 nix_intf = pfvf->nix_tx_intf;3239 else3240 nix_intf = pfvf->nix_rx_intf;3241 3242 npc_config_mcam_entry(rvu, mcam, blkaddr, entry, nix_intf,3243 &req->entry_data, req->enable_entry);3244 3245 if (req->alloc_cntr)3246 npc_map_mcam_entry_and_cntr(rvu, mcam, blkaddr, entry, cntr);3247 mutex_unlock(&mcam->lock);3248 3249 rsp->entry = entry;3250 rsp->cntr = cntr;3251 3252 return 0;3253}3254 3255#define GET_KEX_CFG(intf) \3256 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_INTFX_KEX_CFG(intf))3257 3258#define GET_KEX_FLAGS(ld) \3259 rvu_read64(rvu, BLKADDR_NPC, NPC_AF_KEX_LDATAX_FLAGS_CFG(ld))3260 3261#define GET_KEX_LD(intf, lid, lt, ld) \3262 rvu_read64(rvu, BLKADDR_NPC, \3263 NPC_AF_INTFX_LIDX_LTX_LDX_CFG(intf, lid, lt, ld))3264 3265#define GET_KEX_LDFLAGS(intf, ld, fl) \3266 rvu_read64(rvu, BLKADDR_NPC, \3267 NPC_AF_INTFX_LDATAX_FLAGSX_CFG(intf, ld, fl))3268 3269int rvu_mbox_handler_npc_get_kex_cfg(struct rvu *rvu, struct msg_req *req,3270 struct npc_get_kex_cfg_rsp *rsp)3271{3272 int lid, lt, ld, fl;3273 3274 rsp->rx_keyx_cfg = GET_KEX_CFG(NIX_INTF_RX);3275 rsp->tx_keyx_cfg = GET_KEX_CFG(NIX_INTF_TX);3276 for (lid = 0; lid < NPC_MAX_LID; lid++) {3277 for (lt = 0; lt < NPC_MAX_LT; lt++) {3278 for (ld = 0; ld < NPC_MAX_LD; ld++) {3279 rsp->intf_lid_lt_ld[NIX_INTF_RX][lid][lt][ld] =3280 GET_KEX_LD(NIX_INTF_RX, lid, lt, ld);3281 rsp->intf_lid_lt_ld[NIX_INTF_TX][lid][lt][ld] =3282 GET_KEX_LD(NIX_INTF_TX, lid, lt, ld);3283 }3284 }3285 }3286 for (ld = 0; ld < NPC_MAX_LD; ld++)3287 rsp->kex_ld_flags[ld] = GET_KEX_FLAGS(ld);3288 3289 for (ld = 0; ld < NPC_MAX_LD; ld++) {3290 for (fl = 0; fl < NPC_MAX_LFL; fl++) {3291 rsp->intf_ld_flags[NIX_INTF_RX][ld][fl] =3292 GET_KEX_LDFLAGS(NIX_INTF_RX, ld, fl);3293 rsp->intf_ld_flags[NIX_INTF_TX][ld][fl] =3294 GET_KEX_LDFLAGS(NIX_INTF_TX, ld, fl);3295 }3296 }3297 memcpy(rsp->mkex_pfl_name, rvu->mkex_pfl_name, MKEX_NAME_LEN);3298 return 0;3299}3300 3301static int3302npc_set_var_len_offset_pkind(struct rvu *rvu, u16 pcifunc, u64 pkind,3303 u8 var_len_off, u8 var_len_off_mask, u8 shift_dir)3304{3305 struct npc_kpu_action0 *act0;3306 u8 shift_count = 0;3307 int blkaddr;3308 u64 val;3309 3310 if (!var_len_off_mask)3311 return -EINVAL;3312 3313 if (var_len_off_mask != 0xff) {3314 if (shift_dir)3315 shift_count = __ffs(var_len_off_mask);3316 else3317 shift_count = (8 - __fls(var_len_off_mask));3318 }3319 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, pcifunc);3320 if (blkaddr < 0) {3321 dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__);3322 return -EINVAL;3323 }3324 val = rvu_read64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind));3325 act0 = (struct npc_kpu_action0 *)&val;3326 act0->var_len_shift = shift_count;3327 act0->var_len_right = shift_dir;3328 act0->var_len_mask = var_len_off_mask;3329 act0->var_len_offset = var_len_off;3330 rvu_write64(rvu, blkaddr, NPC_AF_PKINDX_ACTION0(pkind), val);3331 return 0;3332}3333 3334int rvu_npc_set_parse_mode(struct rvu *rvu, u16 pcifunc, u64 mode, u8 dir,3335 u64 pkind, u8 var_len_off, u8 var_len_off_mask,3336 u8 shift_dir)3337 3338{3339 struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);3340 int blkaddr, nixlf, rc, intf_mode;3341 int pf = rvu_get_pf(pcifunc);3342 u64 rxpkind, txpkind;3343 u8 cgx_id, lmac_id;3344 3345 /* use default pkind to disable edsa/higig */3346 rxpkind = rvu_npc_get_pkind(rvu, pf);3347 txpkind = NPC_TX_DEF_PKIND;3348 intf_mode = NPC_INTF_MODE_DEF;3349 3350 if (mode & OTX2_PRIV_FLAGS_CUSTOM) {3351 if (pkind == NPC_RX_CUSTOM_PRE_L2_PKIND) {3352 rc = npc_set_var_len_offset_pkind(rvu, pcifunc, pkind,3353 var_len_off,3354 var_len_off_mask,3355 shift_dir);3356 if (rc)3357 return rc;3358 }3359 rxpkind = pkind;3360 txpkind = pkind;3361 }3362 3363 if (dir & PKIND_RX) {3364 /* rx pkind set req valid only for cgx mapped PFs */3365 if (!is_cgx_config_permitted(rvu, pcifunc))3366 return 0;3367 rvu_get_cgx_lmac_id(rvu->pf2cgxlmac_map[pf], &cgx_id, &lmac_id);3368 3369 rc = cgx_set_pkind(rvu_cgx_pdata(cgx_id, rvu), lmac_id,3370 rxpkind);3371 if (rc)3372 return rc;3373 }3374 3375 if (dir & PKIND_TX) {3376 /* Tx pkind set request valid if PCIFUNC has NIXLF attached */3377 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, &blkaddr);3378 if (rc)3379 return rc;3380 3381 rvu_write64(rvu, blkaddr, NIX_AF_LFX_TX_PARSE_CFG(nixlf),3382 txpkind);3383 }3384 3385 pfvf->intf_mode = intf_mode;3386 return 0;3387}3388 3389int rvu_mbox_handler_npc_set_pkind(struct rvu *rvu, struct npc_set_pkind *req,3390 struct msg_rsp *rsp)3391{3392 return rvu_npc_set_parse_mode(rvu, req->hdr.pcifunc, req->mode,3393 req->dir, req->pkind, req->var_len_off,3394 req->var_len_off_mask, req->shift_dir);3395}3396 3397int rvu_mbox_handler_npc_read_base_steer_rule(struct rvu *rvu,3398 struct msg_req *req,3399 struct npc_mcam_read_base_rule_rsp *rsp)3400{3401 struct npc_mcam *mcam = &rvu->hw->mcam;3402 int index, blkaddr, nixlf, rc = 0;3403 u16 pcifunc = req->hdr.pcifunc;3404 struct rvu_pfvf *pfvf;3405 u8 intf, enable;3406 3407 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);3408 if (blkaddr < 0)3409 return NPC_MCAM_INVALID_REQ;3410 3411 /* Return the channel number in case of PF */3412 if (!(pcifunc & RVU_PFVF_FUNC_MASK)) {3413 pfvf = rvu_get_pfvf(rvu, pcifunc);3414 rsp->entry.kw[0] = pfvf->rx_chan_base;3415 rsp->entry.kw_mask[0] = 0xFFFULL;3416 goto out;3417 }3418 3419 /* Find the pkt steering rule installed by PF to this VF */3420 mutex_lock(&mcam->lock);3421 for (index = 0; index < mcam->bmap_entries; index++) {3422 if (mcam->entry2target_pffunc[index] == pcifunc)3423 goto read_entry;3424 }3425 3426 rc = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL);3427 if (rc < 0) {3428 mutex_unlock(&mcam->lock);3429 goto out;3430 }3431 /* Read the default ucast entry if there is no pkt steering rule */3432 index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf,3433 NIXLF_UCAST_ENTRY);3434read_entry:3435 /* Read the mcam entry */3436 npc_read_mcam_entry(rvu, mcam, blkaddr, index, &rsp->entry, &intf,3437 &enable);3438 mutex_unlock(&mcam->lock);3439out:3440 return rc;3441}3442 3443int rvu_mbox_handler_npc_mcam_entry_stats(struct rvu *rvu,3444 struct npc_mcam_get_stats_req *req,3445 struct npc_mcam_get_stats_rsp *rsp)3446{3447 struct npc_mcam *mcam = &rvu->hw->mcam;3448 u16 index, cntr;3449 int blkaddr;3450 u64 regval;3451 u32 bank;3452 3453 blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);3454 if (blkaddr < 0)3455 return NPC_MCAM_INVALID_REQ;3456 3457 mutex_lock(&mcam->lock);3458 3459 index = req->entry & (mcam->banksize - 1);3460 bank = npc_get_bank(mcam, req->entry);3461 3462 /* read MCAM entry STAT_ACT register */3463 regval = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank));3464 3465 if (!(regval & rvu->hw->npc_stat_ena)) {3466 rsp->stat_ena = 0;3467 mutex_unlock(&mcam->lock);3468 return 0;3469 }3470 3471 cntr = regval & 0x1FF;3472 3473 rsp->stat_ena = 1;3474 rsp->stat = rvu_read64(rvu, blkaddr, NPC_AF_MATCH_STATX(cntr));3475 rsp->stat &= BIT_ULL(48) - 1;3476 3477 mutex_unlock(&mcam->lock);3478 3479 return 0;3480}3481