1188 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/* Copyright(c) 2023 Intel Corporation */3 4#define dev_fmt(fmt) "RateLimiting: " fmt5 6#include <asm/errno.h>7#include <asm/div64.h>8 9#include <linux/dev_printk.h>10#include <linux/kernel.h>11#include <linux/pci.h>12#include <linux/slab.h>13#include <linux/units.h>14 15#include "adf_accel_devices.h"16#include "adf_common_drv.h"17#include "adf_rl_admin.h"18#include "adf_rl.h"19#include "adf_sysfs_rl.h"20 21#define RL_TOKEN_GRANULARITY_PCIEIN_BUCKET 0U22#define RL_TOKEN_GRANULARITY_PCIEOUT_BUCKET 0U23#define RL_TOKEN_PCIE_SIZE 6424#define RL_TOKEN_ASYM_SIZE 102425#define RL_CSR_SIZE 4U26#define RL_CAPABILITY_MASK GENMASK(6, 4)27#define RL_CAPABILITY_VALUE 0x7028#define RL_VALIDATE_NON_ZERO(input) ((input) == 0)29#define ROOT_MASK GENMASK(1, 0)30#define CLUSTER_MASK GENMASK(3, 0)31#define LEAF_MASK GENMASK(5, 0)32 33static int validate_user_input(struct adf_accel_dev *accel_dev,34 struct adf_rl_sla_input_data *sla_in,35 bool is_update)36{37 const unsigned long rp_mask = sla_in->rp_mask;38 size_t rp_mask_size;39 int i, cnt;40 41 if (sla_in->pir < sla_in->cir) {42 dev_notice(&GET_DEV(accel_dev),43 "PIR must be >= CIR, setting PIR to CIR\n");44 sla_in->pir = sla_in->cir;45 }46 47 if (!is_update) {48 cnt = 0;49 rp_mask_size = sizeof(sla_in->rp_mask) * BITS_PER_BYTE;50 for_each_set_bit(i, &rp_mask, rp_mask_size) {51 if (++cnt > RL_RP_CNT_PER_LEAF_MAX) {52 dev_notice(&GET_DEV(accel_dev),53 "Too many ring pairs selected for this SLA\n");54 return -EINVAL;55 }56 }57 58 if (sla_in->srv >= ADF_SVC_NONE) {59 dev_notice(&GET_DEV(accel_dev),60 "Wrong service type\n");61 return -EINVAL;62 }63 64 if (sla_in->type > RL_LEAF) {65 dev_notice(&GET_DEV(accel_dev),66 "Wrong node type\n");67 return -EINVAL;68 }69 70 if (sla_in->parent_id < RL_PARENT_DEFAULT_ID ||71 sla_in->parent_id >= RL_NODES_CNT_MAX) {72 dev_notice(&GET_DEV(accel_dev),73 "Wrong parent ID\n");74 return -EINVAL;75 }76 }77 78 return 0;79}80 81static int validate_sla_id(struct adf_accel_dev *accel_dev, int sla_id)82{83 struct rl_sla *sla;84 85 if (sla_id <= RL_SLA_EMPTY_ID || sla_id >= RL_NODES_CNT_MAX) {86 dev_notice(&GET_DEV(accel_dev), "Provided ID is out of bounds\n");87 return -EINVAL;88 }89 90 sla = accel_dev->rate_limiting->sla[sla_id];91 92 if (!sla) {93 dev_notice(&GET_DEV(accel_dev), "SLA with provided ID does not exist\n");94 return -EINVAL;95 }96 97 if (sla->type != RL_LEAF) {98 dev_notice(&GET_DEV(accel_dev), "This ID is reserved for internal use\n");99 return -EINVAL;100 }101 102 return 0;103}104 105/**106 * find_parent() - Find the parent for a new SLA107 * @rl_data: pointer to ratelimiting data108 * @sla_in: pointer to user input data for a new SLA109 *110 * Function returns a pointer to the parent SLA. If the parent ID is provided111 * as input in the user data, then such ID is validated and the parent SLA112 * is returned.113 * Otherwise, it returns the default parent SLA (root or cluster) for114 * the new object.115 *116 * Return:117 * * Pointer to the parent SLA object118 * * NULL - when parent cannot be found119 */120static struct rl_sla *find_parent(struct adf_rl *rl_data,121 struct adf_rl_sla_input_data *sla_in)122{123 int input_parent_id = sla_in->parent_id;124 struct rl_sla *root = NULL;125 struct rl_sla *parent_sla;126 int i;127 128 if (sla_in->type == RL_ROOT)129 return NULL;130 131 if (input_parent_id > RL_PARENT_DEFAULT_ID) {132 parent_sla = rl_data->sla[input_parent_id];133 /*134 * SLA can be a parent if it has the same service as the child135 * and its type is higher in the hierarchy,136 * for example the parent type of a LEAF must be a CLUSTER.137 */138 if (parent_sla && parent_sla->srv == sla_in->srv &&139 parent_sla->type == sla_in->type - 1)140 return parent_sla;141 142 return NULL;143 }144 145 /* If input_parent_id is not valid, get root for this service type. */146 for (i = 0; i < RL_ROOT_MAX; i++) {147 if (rl_data->root[i] && rl_data->root[i]->srv == sla_in->srv) {148 root = rl_data->root[i];149 break;150 }151 }152 153 if (!root)154 return NULL;155 156 /*157 * If the type of this SLA is cluster, then return the root.158 * Otherwise, find the default (i.e. first) cluster for this service.159 */160 if (sla_in->type == RL_CLUSTER)161 return root;162 163 for (i = 0; i < RL_CLUSTER_MAX; i++) {164 if (rl_data->cluster[i] && rl_data->cluster[i]->parent == root)165 return rl_data->cluster[i];166 }167 168 return NULL;169}170 171static enum adf_cfg_service_type srv_to_cfg_svc_type(enum adf_base_services rl_srv)172{173 switch (rl_srv) {174 case ADF_SVC_ASYM:175 return ASYM;176 case ADF_SVC_SYM:177 return SYM;178 case ADF_SVC_DC:179 return COMP;180 default:181 return UNUSED;182 }183}184 185/**186 * adf_rl_get_sla_arr_of_type() - Returns a pointer to SLA type specific array187 * @rl_data: pointer to ratelimiting data188 * @type: SLA type189 * @sla_arr: pointer to variable where requested pointer will be stored190 *191 * Return: Max number of elements allowed for the returned array192 */193u32 adf_rl_get_sla_arr_of_type(struct adf_rl *rl_data, enum rl_node_type type,194 struct rl_sla ***sla_arr)195{196 switch (type) {197 case RL_LEAF:198 *sla_arr = rl_data->leaf;199 return RL_LEAF_MAX;200 case RL_CLUSTER:201 *sla_arr = rl_data->cluster;202 return RL_CLUSTER_MAX;203 case RL_ROOT:204 *sla_arr = rl_data->root;205 return RL_ROOT_MAX;206 default:207 *sla_arr = NULL;208 return 0;209 }210}211 212static bool is_service_enabled(struct adf_accel_dev *accel_dev,213 enum adf_base_services rl_srv)214{215 enum adf_cfg_service_type arb_srv = srv_to_cfg_svc_type(rl_srv);216 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);217 u8 rps_per_bundle = hw_data->num_banks_per_vf;218 int i;219 220 for (i = 0; i < rps_per_bundle; i++) {221 if (GET_SRV_TYPE(accel_dev, i) == arb_srv)222 return true;223 }224 225 return false;226}227 228/**229 * prepare_rp_ids() - Creates an array of ring pair IDs from bitmask230 * @accel_dev: pointer to acceleration device structure231 * @sla: SLA object data where result will be written232 * @rp_mask: bitmask of ring pair IDs233 *234 * Function tries to convert provided bitmap to an array of IDs. It checks if235 * RPs aren't in use, are assigned to SLA service or if a number of provided236 * IDs is not too big. If successful, writes the result into the field237 * sla->ring_pairs_cnt.238 *239 * Return:240 * * 0 - ok241 * * -EINVAL - ring pairs array cannot be created from provided mask242 */243static int prepare_rp_ids(struct adf_accel_dev *accel_dev, struct rl_sla *sla,244 const unsigned long rp_mask)245{246 enum adf_cfg_service_type arb_srv = srv_to_cfg_svc_type(sla->srv);247 u16 rps_per_bundle = GET_HW_DATA(accel_dev)->num_banks_per_vf;248 bool *rp_in_use = accel_dev->rate_limiting->rp_in_use;249 size_t rp_cnt_max = ARRAY_SIZE(sla->ring_pairs_ids);250 u16 rp_id_max = GET_HW_DATA(accel_dev)->num_banks;251 u16 cnt = 0;252 u16 rp_id;253 254 for_each_set_bit(rp_id, &rp_mask, rp_id_max) {255 if (cnt >= rp_cnt_max) {256 dev_notice(&GET_DEV(accel_dev),257 "Assigned more ring pairs than supported");258 return -EINVAL;259 }260 261 if (rp_in_use[rp_id]) {262 dev_notice(&GET_DEV(accel_dev),263 "RP %u already assigned to other SLA", rp_id);264 return -EINVAL;265 }266 267 if (GET_SRV_TYPE(accel_dev, rp_id % rps_per_bundle) != arb_srv) {268 dev_notice(&GET_DEV(accel_dev),269 "RP %u does not support SLA service", rp_id);270 return -EINVAL;271 }272 273 sla->ring_pairs_ids[cnt++] = rp_id;274 }275 276 sla->ring_pairs_cnt = cnt;277 278 return 0;279}280 281static void mark_rps_usage(struct rl_sla *sla, bool *rp_in_use, bool used)282{283 u16 rp_id;284 int i;285 286 for (i = 0; i < sla->ring_pairs_cnt; i++) {287 rp_id = sla->ring_pairs_ids[i];288 rp_in_use[rp_id] = used;289 }290}291 292static void assign_rps_to_leaf(struct adf_accel_dev *accel_dev,293 struct rl_sla *sla, bool clear)294{295 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);296 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);297 u32 base_offset = hw_data->rl_data.r2l_offset;298 u32 node_id = clear ? 0U : (sla->node_id & LEAF_MASK);299 u32 offset;300 int i;301 302 for (i = 0; i < sla->ring_pairs_cnt; i++) {303 offset = base_offset + (RL_CSR_SIZE * sla->ring_pairs_ids[i]);304 ADF_CSR_WR(pmisc_addr, offset, node_id);305 }306}307 308static void assign_leaf_to_cluster(struct adf_accel_dev *accel_dev,309 struct rl_sla *sla, bool clear)310{311 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);312 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);313 u32 base_offset = hw_data->rl_data.l2c_offset;314 u32 node_id = sla->node_id & LEAF_MASK;315 u32 parent_id = clear ? 0U : (sla->parent->node_id & CLUSTER_MASK);316 u32 offset;317 318 offset = base_offset + (RL_CSR_SIZE * node_id);319 ADF_CSR_WR(pmisc_addr, offset, parent_id);320}321 322static void assign_cluster_to_root(struct adf_accel_dev *accel_dev,323 struct rl_sla *sla, bool clear)324{325 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);326 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);327 u32 base_offset = hw_data->rl_data.c2s_offset;328 u32 node_id = sla->node_id & CLUSTER_MASK;329 u32 parent_id = clear ? 0U : (sla->parent->node_id & ROOT_MASK);330 u32 offset;331 332 offset = base_offset + (RL_CSR_SIZE * node_id);333 ADF_CSR_WR(pmisc_addr, offset, parent_id);334}335 336static void assign_node_to_parent(struct adf_accel_dev *accel_dev,337 struct rl_sla *sla, bool clear_assignment)338{339 switch (sla->type) {340 case RL_LEAF:341 assign_rps_to_leaf(accel_dev, sla, clear_assignment);342 assign_leaf_to_cluster(accel_dev, sla, clear_assignment);343 break;344 case RL_CLUSTER:345 assign_cluster_to_root(accel_dev, sla, clear_assignment);346 break;347 default:348 break;349 }350}351 352/**353 * can_parent_afford_sla() - Verifies if parent allows to create an SLA354 * @sla_in: pointer to user input data for a new SLA355 * @sla_parent: pointer to parent SLA object356 * @sla_cir: current child CIR value (only for update)357 * @is_update: request is a update358 *359 * Algorithm verifies if parent has enough remaining budget to take assignment360 * of a child with provided parameters. In update case current CIR value must be361 * returned to budget first.362 * PIR value cannot exceed the PIR assigned to parent.363 *364 * Return:365 * * true - SLA can be created366 * * false - SLA cannot be created367 */368static bool can_parent_afford_sla(struct adf_rl_sla_input_data *sla_in,369 struct rl_sla *sla_parent, u32 sla_cir,370 bool is_update)371{372 u32 rem_cir = sla_parent->rem_cir;373 374 if (is_update)375 rem_cir += sla_cir;376 377 if (sla_in->cir > rem_cir || sla_in->pir > sla_parent->pir)378 return false;379 380 return true;381}382 383/**384 * can_node_afford_update() - Verifies if SLA can be updated with input data385 * @sla_in: pointer to user input data for a new SLA386 * @sla: pointer to SLA object selected for update387 *388 * Algorithm verifies if a new CIR value is big enough to satisfy currently389 * assigned child SLAs and if PIR can be updated390 *391 * Return:392 * * true - SLA can be updated393 * * false - SLA cannot be updated394 */395static bool can_node_afford_update(struct adf_rl_sla_input_data *sla_in,396 struct rl_sla *sla)397{398 u32 cir_in_use = sla->cir - sla->rem_cir;399 400 /* new CIR cannot be smaller then currently consumed value */401 if (cir_in_use > sla_in->cir)402 return false;403 404 /* PIR of root/cluster cannot be reduced in node with assigned children */405 if (sla_in->pir < sla->pir && sla->type != RL_LEAF && cir_in_use > 0)406 return false;407 408 return true;409}410 411static bool is_enough_budget(struct adf_rl *rl_data, struct rl_sla *sla,412 struct adf_rl_sla_input_data *sla_in,413 bool is_update)414{415 u32 max_val = rl_data->device_data->scale_ref;416 struct rl_sla *parent = sla->parent;417 bool ret = true;418 419 if (sla_in->cir > max_val || sla_in->pir > max_val)420 ret = false;421 422 switch (sla->type) {423 case RL_LEAF:424 ret &= can_parent_afford_sla(sla_in, parent, sla->cir,425 is_update);426 break;427 case RL_CLUSTER:428 ret &= can_parent_afford_sla(sla_in, parent, sla->cir,429 is_update);430 431 if (is_update)432 ret &= can_node_afford_update(sla_in, sla);433 434 break;435 case RL_ROOT:436 if (is_update)437 ret &= can_node_afford_update(sla_in, sla);438 439 break;440 default:441 ret = false;442 break;443 }444 445 return ret;446}447 448static void update_budget(struct rl_sla *sla, u32 old_cir, bool is_update)449{450 switch (sla->type) {451 case RL_LEAF:452 if (is_update)453 sla->parent->rem_cir += old_cir;454 455 sla->parent->rem_cir -= sla->cir;456 sla->rem_cir = 0;457 break;458 case RL_CLUSTER:459 if (is_update) {460 sla->parent->rem_cir += old_cir;461 sla->rem_cir = sla->cir - (old_cir - sla->rem_cir);462 } else {463 sla->rem_cir = sla->cir;464 }465 466 sla->parent->rem_cir -= sla->cir;467 break;468 case RL_ROOT:469 if (is_update)470 sla->rem_cir = sla->cir - (old_cir - sla->rem_cir);471 else472 sla->rem_cir = sla->cir;473 break;474 default:475 break;476 }477}478 479/**480 * get_next_free_sla_id() - finds next free ID in the SLA array481 * @rl_data: Pointer to ratelimiting data structure482 *483 * Return:484 * * 0 : RL_NODES_CNT_MAX - correct ID485 * * -ENOSPC - all SLA slots are in use486 */487static int get_next_free_sla_id(struct adf_rl *rl_data)488{489 int i = 0;490 491 while (i < RL_NODES_CNT_MAX && rl_data->sla[i++])492 ;493 494 if (i == RL_NODES_CNT_MAX)495 return -ENOSPC;496 497 return i - 1;498}499 500/**501 * get_next_free_node_id() - finds next free ID in the array of that node type502 * @rl_data: Pointer to ratelimiting data structure503 * @sla: Pointer to SLA object for which the ID is searched504 *505 * Return:506 * * 0 : RL_[NODE_TYPE]_MAX - correct ID507 * * -ENOSPC - all slots of that type are in use508 */509static int get_next_free_node_id(struct adf_rl *rl_data, struct rl_sla *sla)510{511 struct adf_hw_device_data *hw_device = GET_HW_DATA(rl_data->accel_dev);512 int max_id, i, step, rp_per_leaf;513 struct rl_sla **sla_list;514 515 rp_per_leaf = hw_device->num_banks / hw_device->num_banks_per_vf;516 517 /*518 * Static nodes mapping:519 * root0 - cluster[0,4,8,12] - leaf[0-15]520 * root1 - cluster[1,5,9,13] - leaf[16-31]521 * root2 - cluster[2,6,10,14] - leaf[32-47]522 */523 switch (sla->type) {524 case RL_LEAF:525 i = sla->srv * rp_per_leaf;526 step = 1;527 max_id = i + rp_per_leaf;528 sla_list = rl_data->leaf;529 break;530 case RL_CLUSTER:531 i = sla->srv;532 step = 4;533 max_id = RL_CLUSTER_MAX;534 sla_list = rl_data->cluster;535 break;536 case RL_ROOT:537 return sla->srv;538 default:539 return -EINVAL;540 }541 542 while (i < max_id && sla_list[i])543 i += step;544 545 if (i >= max_id)546 return -ENOSPC;547 548 return i;549}550 551u32 adf_rl_calculate_slice_tokens(struct adf_accel_dev *accel_dev, u32 sla_val,552 enum adf_base_services svc_type)553{554 struct adf_rl_hw_data *device_data = &accel_dev->hw_device->rl_data;555 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);556 u64 avail_slice_cycles, allocated_tokens;557 558 if (!sla_val)559 return 0;560 561 avail_slice_cycles = hw_data->clock_frequency;562 563 switch (svc_type) {564 case ADF_SVC_ASYM:565 avail_slice_cycles *= device_data->slices.pke_cnt;566 break;567 case ADF_SVC_SYM:568 avail_slice_cycles *= device_data->slices.cph_cnt;569 break;570 case ADF_SVC_DC:571 avail_slice_cycles *= device_data->slices.dcpr_cnt;572 break;573 default:574 break;575 }576 577 do_div(avail_slice_cycles, device_data->scan_interval);578 allocated_tokens = avail_slice_cycles * sla_val;579 do_div(allocated_tokens, device_data->scale_ref);580 581 return allocated_tokens;582}583 584u32 adf_rl_calculate_ae_cycles(struct adf_accel_dev *accel_dev, u32 sla_val,585 enum adf_base_services svc_type)586{587 struct adf_rl_hw_data *device_data = &accel_dev->hw_device->rl_data;588 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);589 u64 allocated_ae_cycles, avail_ae_cycles;590 591 if (!sla_val)592 return 0;593 594 avail_ae_cycles = hw_data->clock_frequency;595 avail_ae_cycles *= hw_data->get_num_aes(hw_data) - 1;596 do_div(avail_ae_cycles, device_data->scan_interval);597 598 sla_val *= device_data->max_tp[svc_type];599 sla_val /= device_data->scale_ref;600 601 allocated_ae_cycles = (sla_val * avail_ae_cycles);602 do_div(allocated_ae_cycles, device_data->max_tp[svc_type]);603 604 return allocated_ae_cycles;605}606 607u32 adf_rl_calculate_pci_bw(struct adf_accel_dev *accel_dev, u32 sla_val,608 enum adf_base_services svc_type, bool is_bw_out)609{610 struct adf_rl_hw_data *device_data = &accel_dev->hw_device->rl_data;611 u64 sla_to_bytes, allocated_bw, sla_scaled;612 613 if (!sla_val)614 return 0;615 616 sla_to_bytes = sla_val;617 sla_to_bytes *= device_data->max_tp[svc_type];618 do_div(sla_to_bytes, device_data->scale_ref);619 620 sla_to_bytes *= (svc_type == ADF_SVC_ASYM) ? RL_TOKEN_ASYM_SIZE :621 BYTES_PER_MBIT;622 if (svc_type == ADF_SVC_DC && is_bw_out)623 sla_to_bytes *= device_data->slices.dcpr_cnt -624 device_data->dcpr_correction;625 626 sla_scaled = sla_to_bytes * device_data->pcie_scale_mul;627 do_div(sla_scaled, device_data->pcie_scale_div);628 allocated_bw = sla_scaled;629 do_div(allocated_bw, RL_TOKEN_PCIE_SIZE);630 do_div(allocated_bw, device_data->scan_interval);631 632 return allocated_bw;633}634 635/**636 * add_new_sla_entry() - creates a new SLA object and fills it with user data637 * @accel_dev: pointer to acceleration device structure638 * @sla_in: pointer to user input data for a new SLA639 * @sla_out: Pointer to variable that will contain the address of a new640 * SLA object if the operation succeeds641 *642 * Return:643 * * 0 - ok644 * * -ENOMEM - memory allocation failed645 * * -EINVAL - invalid user input646 * * -ENOSPC - all available SLAs are in use647 */648static int add_new_sla_entry(struct adf_accel_dev *accel_dev,649 struct adf_rl_sla_input_data *sla_in,650 struct rl_sla **sla_out)651{652 struct adf_rl *rl_data = accel_dev->rate_limiting;653 struct rl_sla *sla;654 int ret = 0;655 656 sla = kzalloc(sizeof(*sla), GFP_KERNEL);657 if (!sla) {658 ret = -ENOMEM;659 goto ret_err;660 }661 *sla_out = sla;662 663 if (!is_service_enabled(accel_dev, sla_in->srv)) {664 dev_notice(&GET_DEV(accel_dev),665 "Provided service is not enabled\n");666 ret = -EINVAL;667 goto ret_err;668 }669 670 sla->srv = sla_in->srv;671 sla->type = sla_in->type;672 ret = get_next_free_node_id(rl_data, sla);673 if (ret < 0) {674 dev_notice(&GET_DEV(accel_dev),675 "Exceeded number of available nodes for that service\n");676 goto ret_err;677 }678 sla->node_id = ret;679 680 ret = get_next_free_sla_id(rl_data);681 if (ret < 0) {682 dev_notice(&GET_DEV(accel_dev),683 "Allocated maximum SLAs number\n");684 goto ret_err;685 }686 sla->sla_id = ret;687 688 sla->parent = find_parent(rl_data, sla_in);689 if (!sla->parent && sla->type != RL_ROOT) {690 if (sla_in->parent_id != RL_PARENT_DEFAULT_ID)691 dev_notice(&GET_DEV(accel_dev),692 "Provided parent ID does not exist or cannot be parent for this SLA.");693 else694 dev_notice(&GET_DEV(accel_dev),695 "Unable to find parent node for this service. Is service enabled?");696 ret = -EINVAL;697 goto ret_err;698 }699 700 if (sla->type == RL_LEAF) {701 ret = prepare_rp_ids(accel_dev, sla, sla_in->rp_mask);702 if (!sla->ring_pairs_cnt || ret) {703 dev_notice(&GET_DEV(accel_dev),704 "Unable to find ring pairs to assign to the leaf");705 if (!ret)706 ret = -EINVAL;707 708 goto ret_err;709 }710 }711 712 return 0;713 714ret_err:715 kfree(sla);716 *sla_out = NULL;717 718 return ret;719}720 721static int initialize_default_nodes(struct adf_accel_dev *accel_dev)722{723 struct adf_rl *rl_data = accel_dev->rate_limiting;724 struct adf_rl_hw_data *device_data = rl_data->device_data;725 struct adf_rl_sla_input_data sla_in = { };726 int ret = 0;727 int i;728 729 /* Init root for each enabled service */730 sla_in.type = RL_ROOT;731 sla_in.parent_id = RL_PARENT_DEFAULT_ID;732 733 for (i = 0; i < ADF_SVC_NONE; i++) {734 if (!is_service_enabled(accel_dev, i))735 continue;736 737 sla_in.cir = device_data->scale_ref;738 sla_in.pir = sla_in.cir;739 sla_in.srv = i;740 741 ret = adf_rl_add_sla(accel_dev, &sla_in);742 if (ret)743 return ret;744 }745 746 /* Init default cluster for each root */747 sla_in.type = RL_CLUSTER;748 for (i = 0; i < ADF_SVC_NONE; i++) {749 if (!rl_data->root[i])750 continue;751 752 sla_in.cir = rl_data->root[i]->cir;753 sla_in.pir = sla_in.cir;754 sla_in.srv = rl_data->root[i]->srv;755 756 ret = adf_rl_add_sla(accel_dev, &sla_in);757 if (ret)758 return ret;759 }760 761 return 0;762}763 764static void clear_sla(struct adf_rl *rl_data, struct rl_sla *sla)765{766 bool *rp_in_use = rl_data->rp_in_use;767 struct rl_sla **sla_type_arr = NULL;768 int i, sla_id, node_id;769 u32 old_cir;770 771 sla_id = sla->sla_id;772 node_id = sla->node_id;773 old_cir = sla->cir;774 sla->cir = 0;775 sla->pir = 0;776 777 for (i = 0; i < sla->ring_pairs_cnt; i++)778 rp_in_use[sla->ring_pairs_ids[i]] = false;779 780 update_budget(sla, old_cir, true);781 adf_rl_get_sla_arr_of_type(rl_data, sla->type, &sla_type_arr);782 assign_node_to_parent(rl_data->accel_dev, sla, true);783 adf_rl_send_admin_delete_msg(rl_data->accel_dev, node_id, sla->type);784 mark_rps_usage(sla, rl_data->rp_in_use, false);785 786 kfree(sla);787 rl_data->sla[sla_id] = NULL;788 sla_type_arr[node_id] = NULL;789}790 791static void free_all_sla(struct adf_accel_dev *accel_dev)792{793 struct adf_rl *rl_data = accel_dev->rate_limiting;794 int sla_id;795 796 mutex_lock(&rl_data->rl_lock);797 798 for (sla_id = 0; sla_id < RL_NODES_CNT_MAX; sla_id++) {799 if (!rl_data->sla[sla_id])800 continue;801 802 kfree(rl_data->sla[sla_id]);803 rl_data->sla[sla_id] = NULL;804 }805 806 mutex_unlock(&rl_data->rl_lock);807}808 809/**810 * add_update_sla() - handles the creation and the update of an SLA811 * @accel_dev: pointer to acceleration device structure812 * @sla_in: pointer to user input data for a new/updated SLA813 * @is_update: flag to indicate if this is an update or an add operation814 *815 * Return:816 * * 0 - ok817 * * -ENOMEM - memory allocation failed818 * * -EINVAL - user input data cannot be used to create SLA819 * * -ENOSPC - all available SLAs are in use820 */821static int add_update_sla(struct adf_accel_dev *accel_dev,822 struct adf_rl_sla_input_data *sla_in, bool is_update)823{824 struct adf_rl *rl_data = accel_dev->rate_limiting;825 struct rl_sla **sla_type_arr = NULL;826 struct rl_sla *sla = NULL;827 u32 old_cir = 0;828 int ret;829 830 if (!sla_in) {831 dev_warn(&GET_DEV(accel_dev),832 "SLA input data pointer is missing\n");833 return -EFAULT;834 }835 836 mutex_lock(&rl_data->rl_lock);837 838 /* Input validation */839 ret = validate_user_input(accel_dev, sla_in, is_update);840 if (ret)841 goto ret_err;842 843 if (is_update) {844 ret = validate_sla_id(accel_dev, sla_in->sla_id);845 if (ret)846 goto ret_err;847 848 sla = rl_data->sla[sla_in->sla_id];849 old_cir = sla->cir;850 } else {851 ret = add_new_sla_entry(accel_dev, sla_in, &sla);852 if (ret)853 goto ret_err;854 }855 856 if (!is_enough_budget(rl_data, sla, sla_in, is_update)) {857 dev_notice(&GET_DEV(accel_dev),858 "Input value exceeds the remaining budget%s\n",859 is_update ? " or more budget is already in use" : "");860 ret = -EINVAL;861 goto ret_err;862 }863 sla->cir = sla_in->cir;864 sla->pir = sla_in->pir;865 866 /* Apply SLA */867 assign_node_to_parent(accel_dev, sla, false);868 ret = adf_rl_send_admin_add_update_msg(accel_dev, sla, is_update);869 if (ret) {870 dev_notice(&GET_DEV(accel_dev),871 "Failed to apply an SLA\n");872 goto ret_err;873 }874 update_budget(sla, old_cir, is_update);875 876 if (!is_update) {877 mark_rps_usage(sla, rl_data->rp_in_use, true);878 adf_rl_get_sla_arr_of_type(rl_data, sla->type, &sla_type_arr);879 sla_type_arr[sla->node_id] = sla;880 rl_data->sla[sla->sla_id] = sla;881 }882 883 sla_in->sla_id = sla->sla_id;884 goto ret_ok;885 886ret_err:887 if (!is_update) {888 sla_in->sla_id = -1;889 kfree(sla);890 }891ret_ok:892 mutex_unlock(&rl_data->rl_lock);893 return ret;894}895 896/**897 * adf_rl_add_sla() - handles the creation of an SLA898 * @accel_dev: pointer to acceleration device structure899 * @sla_in: pointer to user input data required to add an SLA900 *901 * Return:902 * * 0 - ok903 * * -ENOMEM - memory allocation failed904 * * -EINVAL - invalid user input905 * * -ENOSPC - all available SLAs are in use906 */907int adf_rl_add_sla(struct adf_accel_dev *accel_dev,908 struct adf_rl_sla_input_data *sla_in)909{910 return add_update_sla(accel_dev, sla_in, false);911}912 913/**914 * adf_rl_update_sla() - handles the update of an SLA915 * @accel_dev: pointer to acceleration device structure916 * @sla_in: pointer to user input data required to update an SLA917 *918 * Return:919 * * 0 - ok920 * * -EINVAL - user input data cannot be used to update SLA921 */922int adf_rl_update_sla(struct adf_accel_dev *accel_dev,923 struct adf_rl_sla_input_data *sla_in)924{925 return add_update_sla(accel_dev, sla_in, true);926}927 928/**929 * adf_rl_get_sla() - returns an existing SLA data930 * @accel_dev: pointer to acceleration device structure931 * @sla_in: pointer to user data where SLA info will be stored932 *933 * The sla_id for which data are requested should be set in sla_id structure934 *935 * Return:936 * * 0 - ok937 * * -EINVAL - provided sla_id does not exist938 */939int adf_rl_get_sla(struct adf_accel_dev *accel_dev,940 struct adf_rl_sla_input_data *sla_in)941{942 struct rl_sla *sla;943 int ret, i;944 945 ret = validate_sla_id(accel_dev, sla_in->sla_id);946 if (ret)947 return ret;948 949 sla = accel_dev->rate_limiting->sla[sla_in->sla_id];950 sla_in->type = sla->type;951 sla_in->srv = sla->srv;952 sla_in->cir = sla->cir;953 sla_in->pir = sla->pir;954 sla_in->rp_mask = 0U;955 if (sla->parent)956 sla_in->parent_id = sla->parent->sla_id;957 else958 sla_in->parent_id = RL_PARENT_DEFAULT_ID;959 960 for (i = 0; i < sla->ring_pairs_cnt; i++)961 sla_in->rp_mask |= BIT(sla->ring_pairs_ids[i]);962 963 return 0;964}965 966/**967 * adf_rl_get_capability_remaining() - returns the remaining SLA value (CIR) for968 * selected service or provided sla_id969 * @accel_dev: pointer to acceleration device structure970 * @srv: service ID for which capability is requested971 * @sla_id: ID of the cluster or root to which we want assign a new SLA972 *973 * Check if the provided SLA id is valid. If it is and the service matches974 * the requested service and the type is cluster or root, return the remaining975 * capability.976 * If the provided ID does not match the service or type, return the remaining977 * capacity of the default cluster for that service.978 *979 * Return:980 * * Positive value - correct remaining value981 * * -EINVAL - algorithm cannot find a remaining value for provided data982 */983int adf_rl_get_capability_remaining(struct adf_accel_dev *accel_dev,984 enum adf_base_services srv, int sla_id)985{986 struct adf_rl *rl_data = accel_dev->rate_limiting;987 struct rl_sla *sla = NULL;988 int i;989 990 if (srv >= ADF_SVC_NONE)991 return -EINVAL;992 993 if (sla_id > RL_SLA_EMPTY_ID && !validate_sla_id(accel_dev, sla_id)) {994 sla = rl_data->sla[sla_id];995 996 if (sla->srv == srv && sla->type <= RL_CLUSTER)997 goto ret_ok;998 }999 1000 for (i = 0; i < RL_CLUSTER_MAX; i++) {1001 if (!rl_data->cluster[i])1002 continue;1003 1004 if (rl_data->cluster[i]->srv == srv) {1005 sla = rl_data->cluster[i];1006 goto ret_ok;1007 }1008 }1009 1010 return -EINVAL;1011ret_ok:1012 return sla->rem_cir;1013}1014 1015/**1016 * adf_rl_remove_sla() - removes provided sla_id1017 * @accel_dev: pointer to acceleration device structure1018 * @sla_id: ID of the cluster or root to which we want assign an new SLA1019 *1020 * Return:1021 * * 0 - ok1022 * * -EINVAL - wrong sla_id or it still have assigned children1023 */1024int adf_rl_remove_sla(struct adf_accel_dev *accel_dev, u32 sla_id)1025{1026 struct adf_rl *rl_data = accel_dev->rate_limiting;1027 struct rl_sla *sla;1028 int ret = 0;1029 1030 mutex_lock(&rl_data->rl_lock);1031 ret = validate_sla_id(accel_dev, sla_id);1032 if (ret)1033 goto err_ret;1034 1035 sla = rl_data->sla[sla_id];1036 1037 if (sla->type < RL_LEAF && sla->rem_cir != sla->cir) {1038 dev_notice(&GET_DEV(accel_dev),1039 "To remove parent SLA all its children must be removed first");1040 ret = -EINVAL;1041 goto err_ret;1042 }1043 1044 clear_sla(rl_data, sla);1045 1046err_ret:1047 mutex_unlock(&rl_data->rl_lock);1048 return ret;1049}1050 1051/**1052 * adf_rl_remove_sla_all() - removes all SLAs from device1053 * @accel_dev: pointer to acceleration device structure1054 * @incl_default: set to true if default SLAs also should be removed1055 */1056void adf_rl_remove_sla_all(struct adf_accel_dev *accel_dev, bool incl_default)1057{1058 struct adf_rl *rl_data = accel_dev->rate_limiting;1059 int end_type = incl_default ? RL_ROOT : RL_LEAF;1060 struct rl_sla **sla_type_arr = NULL;1061 u32 max_id;1062 int i, j;1063 1064 mutex_lock(&rl_data->rl_lock);1065 1066 /* Unregister and remove all SLAs */1067 for (j = RL_LEAF; j >= end_type; j--) {1068 max_id = adf_rl_get_sla_arr_of_type(rl_data, j, &sla_type_arr);1069 1070 for (i = 0; i < max_id; i++) {1071 if (!sla_type_arr[i])1072 continue;1073 1074 clear_sla(rl_data, sla_type_arr[i]);1075 }1076 }1077 1078 mutex_unlock(&rl_data->rl_lock);1079}1080 1081int adf_rl_init(struct adf_accel_dev *accel_dev)1082{1083 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);1084 struct adf_rl_hw_data *rl_hw_data = &hw_data->rl_data;1085 struct adf_rl *rl;1086 int ret = 0;1087 1088 /* Validate device parameters */1089 if (RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[ADF_SVC_ASYM]) ||1090 RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[ADF_SVC_SYM]) ||1091 RL_VALIDATE_NON_ZERO(rl_hw_data->max_tp[ADF_SVC_DC]) ||1092 RL_VALIDATE_NON_ZERO(rl_hw_data->scan_interval) ||1093 RL_VALIDATE_NON_ZERO(rl_hw_data->pcie_scale_div) ||1094 RL_VALIDATE_NON_ZERO(rl_hw_data->pcie_scale_mul) ||1095 RL_VALIDATE_NON_ZERO(rl_hw_data->scale_ref)) {1096 ret = -EOPNOTSUPP;1097 goto err_ret;1098 }1099 1100 rl = kzalloc(sizeof(*rl), GFP_KERNEL);1101 if (!rl) {1102 ret = -ENOMEM;1103 goto err_ret;1104 }1105 1106 mutex_init(&rl->rl_lock);1107 rl->device_data = &accel_dev->hw_device->rl_data;1108 rl->accel_dev = accel_dev;1109 init_rwsem(&rl->user_input.lock);1110 accel_dev->rate_limiting = rl;1111 1112err_ret:1113 return ret;1114}1115 1116int adf_rl_start(struct adf_accel_dev *accel_dev)1117{1118 struct adf_rl_hw_data *rl_hw_data = &GET_HW_DATA(accel_dev)->rl_data;1119 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);1120 u16 fw_caps = GET_HW_DATA(accel_dev)->fw_capabilities;1121 int ret;1122 1123 if (!accel_dev->rate_limiting) {1124 ret = -EOPNOTSUPP;1125 goto ret_err;1126 }1127 1128 if ((fw_caps & RL_CAPABILITY_MASK) != RL_CAPABILITY_VALUE) {1129 dev_info(&GET_DEV(accel_dev), "feature not supported by FW\n");1130 ret = -EOPNOTSUPP;1131 goto ret_free;1132 }1133 1134 ADF_CSR_WR(pmisc_addr, rl_hw_data->pciin_tb_offset,1135 RL_TOKEN_GRANULARITY_PCIEIN_BUCKET);1136 ADF_CSR_WR(pmisc_addr, rl_hw_data->pciout_tb_offset,1137 RL_TOKEN_GRANULARITY_PCIEOUT_BUCKET);1138 1139 ret = adf_rl_send_admin_init_msg(accel_dev, &rl_hw_data->slices);1140 if (ret) {1141 dev_err(&GET_DEV(accel_dev), "initialization failed\n");1142 goto ret_free;1143 }1144 1145 ret = initialize_default_nodes(accel_dev);1146 if (ret) {1147 dev_err(&GET_DEV(accel_dev),1148 "failed to initialize default SLAs\n");1149 goto ret_sla_rm;1150 }1151 1152 ret = adf_sysfs_rl_add(accel_dev);1153 if (ret) {1154 dev_err(&GET_DEV(accel_dev), "failed to add sysfs interface\n");1155 goto ret_sysfs_rm;1156 }1157 1158 return 0;1159 1160ret_sysfs_rm:1161 adf_sysfs_rl_rm(accel_dev);1162ret_sla_rm:1163 adf_rl_remove_sla_all(accel_dev, true);1164ret_free:1165 kfree(accel_dev->rate_limiting);1166 accel_dev->rate_limiting = NULL;1167ret_err:1168 return ret;1169}1170 1171void adf_rl_stop(struct adf_accel_dev *accel_dev)1172{1173 if (!accel_dev->rate_limiting)1174 return;1175 1176 adf_sysfs_rl_rm(accel_dev);1177 free_all_sla(accel_dev);1178}1179 1180void adf_rl_exit(struct adf_accel_dev *accel_dev)1181{1182 if (!accel_dev->rate_limiting)1183 return;1184 1185 kfree(accel_dev->rate_limiting);1186 accel_dev->rate_limiting = NULL;1187}1188