488 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Marvell RVU Ethernet driver3 *4 * Copyright (C) 2021 Marvell.5 *6 */7 8#include "cn10k.h"9#include "otx2_reg.h"10#include "otx2_struct.h"11 12static struct dev_hw_ops otx2_hw_ops = {13 .sq_aq_init = otx2_sq_aq_init,14 .sqe_flush = otx2_sqe_flush,15 .aura_freeptr = otx2_aura_freeptr,16 .refill_pool_ptrs = otx2_refill_pool_ptrs,17};18 19static struct dev_hw_ops cn10k_hw_ops = {20 .sq_aq_init = cn10k_sq_aq_init,21 .sqe_flush = cn10k_sqe_flush,22 .aura_freeptr = cn10k_aura_freeptr,23 .refill_pool_ptrs = cn10k_refill_pool_ptrs,24};25 26int cn10k_lmtst_init(struct otx2_nic *pfvf)27{28 29 struct lmtst_tbl_setup_req *req;30 struct otx2_lmt_info *lmt_info;31 int err, cpu;32 33 if (!test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) {34 pfvf->hw_ops = &otx2_hw_ops;35 return 0;36 }37 38 pfvf->hw_ops = &cn10k_hw_ops;39 /* Total LMTLINES = num_online_cpus() * 32 (For Burst flush).*/40 pfvf->tot_lmt_lines = (num_online_cpus() * LMT_BURST_SIZE);41 pfvf->hw.lmt_info = alloc_percpu(struct otx2_lmt_info);42 43 mutex_lock(&pfvf->mbox.lock);44 req = otx2_mbox_alloc_msg_lmtst_tbl_setup(&pfvf->mbox);45 if (!req) {46 mutex_unlock(&pfvf->mbox.lock);47 return -ENOMEM;48 }49 50 req->use_local_lmt_region = true;51 52 err = qmem_alloc(pfvf->dev, &pfvf->dync_lmt, pfvf->tot_lmt_lines,53 LMT_LINE_SIZE);54 if (err) {55 mutex_unlock(&pfvf->mbox.lock);56 return err;57 }58 pfvf->hw.lmt_base = (u64 *)pfvf->dync_lmt->base;59 req->lmt_iova = (u64)pfvf->dync_lmt->iova;60 61 err = otx2_sync_mbox_msg(&pfvf->mbox);62 mutex_unlock(&pfvf->mbox.lock);63 64 for_each_possible_cpu(cpu) {65 lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, cpu);66 lmt_info->lmt_addr = ((u64)pfvf->hw.lmt_base +67 (cpu * LMT_BURST_SIZE * LMT_LINE_SIZE));68 lmt_info->lmt_id = cpu * LMT_BURST_SIZE;69 }70 71 return 0;72}73EXPORT_SYMBOL(cn10k_lmtst_init);74 75int cn10k_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura)76{77 struct nix_cn10k_aq_enq_req *aq;78 struct otx2_nic *pfvf = dev;79 80 /* Get memory to put this msg */81 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox);82 if (!aq)83 return -ENOMEM;84 85 aq->sq.cq = pfvf->hw.rx_queues + qidx;86 aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */87 aq->sq.cq_ena = 1;88 aq->sq.ena = 1;89 aq->sq.smq = otx2_get_smq_idx(pfvf, qidx);90 aq->sq.smq_rr_weight = mtu_to_dwrr_weight(pfvf, pfvf->tx_max_pktlen);91 aq->sq.default_chan = pfvf->hw.tx_chan_base;92 aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */93 aq->sq.sqb_aura = sqb_aura;94 aq->sq.sq_int_ena = NIX_SQINT_BITS;95 aq->sq.qint_idx = 0;96 /* Due pipelining impact minimum 2000 unused SQ CQE's97 * need to maintain to avoid CQ overflow.98 */99 aq->sq.cq_limit = ((SEND_CQ_SKID * 256) / (pfvf->qset.sqe_cnt));100 101 /* Fill AQ info */102 aq->qidx = qidx;103 aq->ctype = NIX_AQ_CTYPE_SQ;104 aq->op = NIX_AQ_INSTOP_INIT;105 106 return otx2_sync_mbox_msg(&pfvf->mbox);107}108 109#define NPA_MAX_BURST 16110int cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq)111{112 struct otx2_nic *pfvf = dev;113 int cnt = cq->pool_ptrs;114 u64 ptrs[NPA_MAX_BURST];115 dma_addr_t bufptr;116 int num_ptrs = 1;117 118 /* Refill pool with new buffers */119 while (cq->pool_ptrs) {120 if (otx2_alloc_buffer(pfvf, cq, &bufptr)) {121 if (num_ptrs--)122 __cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs,123 num_ptrs);124 break;125 }126 cq->pool_ptrs--;127 ptrs[num_ptrs] = (u64)bufptr + OTX2_HEAD_ROOM;128 num_ptrs++;129 if (num_ptrs == NPA_MAX_BURST || cq->pool_ptrs == 0) {130 __cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs,131 num_ptrs);132 num_ptrs = 1;133 }134 }135 return cnt - cq->pool_ptrs;136}137 138void cn10k_sqe_flush(void *dev, struct otx2_snd_queue *sq, int size, int qidx)139{140 struct otx2_lmt_info *lmt_info;141 struct otx2_nic *pfvf = dev;142 u64 val = 0, tar_addr = 0;143 144 lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, smp_processor_id());145 /* FIXME: val[0:10] LMT_ID.146 * [12:15] no of LMTST - 1 in the burst.147 * [19:63] data size of each LMTST in the burst except first.148 */149 val = (lmt_info->lmt_id & 0x7FF);150 /* Target address for LMTST flush tells HW how many 128bit151 * words are present.152 * tar_addr[6:4] size of first LMTST - 1 in units of 128b.153 */154 tar_addr |= sq->io_addr | (((size / 16) - 1) & 0x7) << 4;155 dma_wmb();156 memcpy((u64 *)lmt_info->lmt_addr, sq->sqe_base, size);157 cn10k_lmt_flush(val, tar_addr);158 159 sq->head++;160 sq->head &= (sq->sqe_cnt - 1);161}162 163int cn10k_free_all_ipolicers(struct otx2_nic *pfvf)164{165 struct nix_bandprof_free_req *req;166 int rc;167 168 if (is_dev_otx2(pfvf->pdev))169 return 0;170 171 mutex_lock(&pfvf->mbox.lock);172 173 req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox);174 if (!req) {175 rc = -ENOMEM;176 goto out;177 }178 179 /* Free all bandwidth profiles allocated */180 req->free_all = true;181 182 rc = otx2_sync_mbox_msg(&pfvf->mbox);183out:184 mutex_unlock(&pfvf->mbox.lock);185 return rc;186}187 188int cn10k_alloc_leaf_profile(struct otx2_nic *pfvf, u16 *leaf)189{190 struct nix_bandprof_alloc_req *req;191 struct nix_bandprof_alloc_rsp *rsp;192 int rc;193 194 req = otx2_mbox_alloc_msg_nix_bandprof_alloc(&pfvf->mbox);195 if (!req)196 return -ENOMEM;197 198 req->prof_count[BAND_PROF_LEAF_LAYER] = 1;199 200 rc = otx2_sync_mbox_msg(&pfvf->mbox);201 if (rc)202 goto out;203 204 rsp = (struct nix_bandprof_alloc_rsp *)205 otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr);206 if (!rsp->prof_count[BAND_PROF_LEAF_LAYER]) {207 rc = -EIO;208 goto out;209 }210 211 *leaf = rsp->prof_idx[BAND_PROF_LEAF_LAYER][0];212out:213 if (rc) {214 dev_warn(pfvf->dev,215 "Failed to allocate ingress bandwidth policer\n");216 }217 218 return rc;219}220 221int cn10k_alloc_matchall_ipolicer(struct otx2_nic *pfvf)222{223 struct otx2_hw *hw = &pfvf->hw;224 int ret;225 226 mutex_lock(&pfvf->mbox.lock);227 228 ret = cn10k_alloc_leaf_profile(pfvf, &hw->matchall_ipolicer);229 230 mutex_unlock(&pfvf->mbox.lock);231 232 return ret;233}234 235#define POLICER_TIMESTAMP 1 /* 1 second */236#define MAX_RATE_EXP 22 /* Valid rate exponent range: 0 - 22 */237 238static void cn10k_get_ingress_burst_cfg(u32 burst, u32 *burst_exp,239 u32 *burst_mantissa)240{241 int tmp;242 243 /* Burst is calculated as244 * (1+[BURST_MANTISSA]/256)*2^[BURST_EXPONENT]245 * This is the upper limit on number tokens (bytes) that246 * can be accumulated in the bucket.247 */248 *burst_exp = ilog2(burst);249 if (burst < 256) {250 /* No float: can't express mantissa in this case */251 *burst_mantissa = 0;252 return;253 }254 255 if (*burst_exp > MAX_RATE_EXP)256 *burst_exp = MAX_RATE_EXP;257 258 /* Calculate mantissa259 * Find remaining bytes 'burst - 2^burst_exp'260 * mantissa = (remaining bytes) / 2^ (burst_exp - 8)261 */262 tmp = burst - rounddown_pow_of_two(burst);263 *burst_mantissa = tmp / (1UL << (*burst_exp - 8));264}265 266static void cn10k_get_ingress_rate_cfg(u64 rate, u32 *rate_exp,267 u32 *rate_mantissa, u32 *rdiv)268{269 u32 div = 0;270 u32 exp = 0;271 u64 tmp;272 273 /* Figure out mantissa, exponent and divider from given max pkt rate274 *275 * To achieve desired rate HW adds276 * (1+[RATE_MANTISSA]/256)*2^[RATE_EXPONENT] tokens (bytes) at every277 * policer timeunit * 2^rdiv ie 2 * 2^rdiv usecs, to the token bucket.278 * Here policer timeunit is 2 usecs and rate is in bits per sec.279 * Since floating point cannot be used below algorithm uses 1000000280 * scale factor to support rates upto 100Gbps.281 */282 tmp = rate * 32 * 2;283 if (tmp < 256000000) {284 while (tmp < 256000000) {285 tmp = tmp * 2;286 div++;287 }288 } else {289 for (exp = 0; tmp >= 512000000 && exp <= MAX_RATE_EXP; exp++)290 tmp = tmp / 2;291 292 if (exp > MAX_RATE_EXP)293 exp = MAX_RATE_EXP;294 }295 296 *rate_mantissa = (tmp - 256000000) / 1000000;297 *rate_exp = exp;298 *rdiv = div;299}300 301int cn10k_map_unmap_rq_policer(struct otx2_nic *pfvf, int rq_idx,302 u16 policer, bool map)303{304 struct nix_cn10k_aq_enq_req *aq;305 306 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox);307 if (!aq)308 return -ENOMEM;309 310 /* Enable policing and set the bandwidth profile (policer) index */311 if (map)312 aq->rq.policer_ena = 1;313 else314 aq->rq.policer_ena = 0;315 aq->rq_mask.policer_ena = 1;316 317 aq->rq.band_prof_id = policer;318 aq->rq_mask.band_prof_id = GENMASK(9, 0);319 320 /* Fill AQ info */321 aq->qidx = rq_idx;322 aq->ctype = NIX_AQ_CTYPE_RQ;323 aq->op = NIX_AQ_INSTOP_WRITE;324 325 return otx2_sync_mbox_msg(&pfvf->mbox);326}327 328int cn10k_free_leaf_profile(struct otx2_nic *pfvf, u16 leaf)329{330 struct nix_bandprof_free_req *req;331 332 req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox);333 if (!req)334 return -ENOMEM;335 336 req->prof_count[BAND_PROF_LEAF_LAYER] = 1;337 req->prof_idx[BAND_PROF_LEAF_LAYER][0] = leaf;338 339 return otx2_sync_mbox_msg(&pfvf->mbox);340}341 342int cn10k_free_matchall_ipolicer(struct otx2_nic *pfvf)343{344 struct otx2_hw *hw = &pfvf->hw;345 int qidx, rc;346 347 mutex_lock(&pfvf->mbox.lock);348 349 /* Remove RQ's policer mapping */350 for (qidx = 0; qidx < hw->rx_queues; qidx++)351 cn10k_map_unmap_rq_policer(pfvf, qidx,352 hw->matchall_ipolicer, false);353 354 rc = cn10k_free_leaf_profile(pfvf, hw->matchall_ipolicer);355 356 mutex_unlock(&pfvf->mbox.lock);357 return rc;358}359 360int cn10k_set_ipolicer_rate(struct otx2_nic *pfvf, u16 profile,361 u32 burst, u64 rate, bool pps)362{363 struct nix_cn10k_aq_enq_req *aq;364 u32 burst_exp, burst_mantissa;365 u32 rate_exp, rate_mantissa;366 u32 rdiv;367 368 /* Get exponent and mantissa values for the desired rate */369 cn10k_get_ingress_burst_cfg(burst, &burst_exp, &burst_mantissa);370 cn10k_get_ingress_rate_cfg(rate, &rate_exp, &rate_mantissa, &rdiv);371 372 /* Init bandwidth profile */373 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox);374 if (!aq)375 return -ENOMEM;376 377 /* Set initial color mode to blind */378 aq->prof.icolor = 0x03;379 aq->prof_mask.icolor = 0x03;380 381 /* Set rate and burst values */382 aq->prof.cir_exponent = rate_exp;383 aq->prof_mask.cir_exponent = 0x1F;384 385 aq->prof.cir_mantissa = rate_mantissa;386 aq->prof_mask.cir_mantissa = 0xFF;387 388 aq->prof.cbs_exponent = burst_exp;389 aq->prof_mask.cbs_exponent = 0x1F;390 391 aq->prof.cbs_mantissa = burst_mantissa;392 aq->prof_mask.cbs_mantissa = 0xFF;393 394 aq->prof.rdiv = rdiv;395 aq->prof_mask.rdiv = 0xF;396 397 if (pps) {398 /* The amount of decremented tokens is calculated according to399 * the following equation:400 * max([ LMODE ? 0 : (packet_length - LXPTR)] +401 * ([ADJUST_MANTISSA]/256 - 1) * 2^[ADJUST_EXPONENT],402 * 1/256)403 * if LMODE is 1 then rate limiting will be based on404 * PPS otherwise bps.405 * The aim of the ADJUST value is to specify a token cost per406 * packet in contrary to the packet length that specifies a407 * cost per byte. To rate limit based on PPS adjust mantissa408 * is set as 384 and exponent as 1 so that number of tokens409 * decremented becomes 1 i.e, 1 token per packeet.410 */411 aq->prof.adjust_exponent = 1;412 aq->prof_mask.adjust_exponent = 0x1F;413 414 aq->prof.adjust_mantissa = 384;415 aq->prof_mask.adjust_mantissa = 0x1FF;416 417 aq->prof.lmode = 0x1;418 aq->prof_mask.lmode = 0x1;419 }420 421 /* Two rate three color marker422 * With PEIR/EIR set to zero, color will be either green or red423 */424 aq->prof.meter_algo = 2;425 aq->prof_mask.meter_algo = 0x3;426 427 aq->prof.rc_action = NIX_RX_BAND_PROF_ACTIONRESULT_DROP;428 aq->prof_mask.rc_action = 0x3;429 430 aq->prof.yc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS;431 aq->prof_mask.yc_action = 0x3;432 433 aq->prof.gc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS;434 aq->prof_mask.gc_action = 0x3;435 436 /* Setting exponent value as 24 and mantissa as 0 configures437 * the bucket with zero values making bucket unused. Peak438 * information rate and Excess information rate buckets are439 * unused here.440 */441 aq->prof.peir_exponent = 24;442 aq->prof_mask.peir_exponent = 0x1F;443 444 aq->prof.peir_mantissa = 0;445 aq->prof_mask.peir_mantissa = 0xFF;446 447 aq->prof.pebs_exponent = 24;448 aq->prof_mask.pebs_exponent = 0x1F;449 450 aq->prof.pebs_mantissa = 0;451 aq->prof_mask.pebs_mantissa = 0xFF;452 453 aq->prof.hl_en = 0;454 aq->prof_mask.hl_en = 1;455 456 /* Fill AQ info */457 aq->qidx = profile;458 aq->ctype = NIX_AQ_CTYPE_BANDPROF;459 aq->op = NIX_AQ_INSTOP_WRITE;460 461 return otx2_sync_mbox_msg(&pfvf->mbox);462}463 464int cn10k_set_matchall_ipolicer_rate(struct otx2_nic *pfvf,465 u32 burst, u64 rate)466{467 struct otx2_hw *hw = &pfvf->hw;468 int qidx, rc;469 470 mutex_lock(&pfvf->mbox.lock);471 472 rc = cn10k_set_ipolicer_rate(pfvf, hw->matchall_ipolicer, burst,473 rate, false);474 if (rc)475 goto out;476 477 for (qidx = 0; qidx < hw->rx_queues; qidx++) {478 rc = cn10k_map_unmap_rq_policer(pfvf, qidx,479 hw->matchall_ipolicer, true);480 if (rc)481 break;482 }483 484out:485 mutex_unlock(&pfvf->mbox.lock);486 return rc;487}488