1589 lines · c
1// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.02/* Copyright (c) 2018 Mellanox Technologies. All rights reserved */3 4#include <linux/bitmap.h>5#include <linux/errno.h>6#include <linux/genalloc.h>7#include <linux/gfp.h>8#include <linux/kernel.h>9#include <linux/list.h>10#include <linux/mutex.h>11#include <linux/objagg.h>12#include <linux/rtnetlink.h>13#include <linux/slab.h>14 15#include "core.h"16#include "reg.h"17#include "spectrum.h"18#include "spectrum_acl_tcam.h"19 20/* gen_pool_alloc() returns 0 when allocation fails, so use an offset */21#define MLXSW_SP_ACL_ERP_GENALLOC_OFFSET 0x10022#define MLXSW_SP_ACL_ERP_MAX_PER_REGION 1623 24struct mlxsw_sp_acl_erp_core {25 unsigned int erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_MAX + 1];26 struct gen_pool *erp_tables;27 struct mlxsw_sp *mlxsw_sp;28 struct mlxsw_sp_acl_bf *bf;29 unsigned int num_erp_banks;30};31 32struct mlxsw_sp_acl_erp_key {33 char mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN];34#define __MASK_LEN 0x3835#define __MASK_IDX(i) (__MASK_LEN - (i) - 1)36 bool ctcam;37};38 39struct mlxsw_sp_acl_erp {40 struct mlxsw_sp_acl_erp_key key;41 u8 id;42 u8 index;43 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);44 struct list_head list;45 struct mlxsw_sp_acl_erp_table *erp_table;46};47 48struct mlxsw_sp_acl_erp_master_mask {49 DECLARE_BITMAP(bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);50 unsigned int count[MLXSW_SP_ACL_TCAM_MASK_LEN];51};52 53struct mlxsw_sp_acl_erp_table {54 struct mlxsw_sp_acl_erp_master_mask master_mask;55 DECLARE_BITMAP(erp_id_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION);56 DECLARE_BITMAP(erp_index_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION);57 struct list_head atcam_erps_list;58 struct mlxsw_sp_acl_erp_core *erp_core;59 struct mlxsw_sp_acl_atcam_region *aregion;60 const struct mlxsw_sp_acl_erp_table_ops *ops;61 unsigned long base_index;62 unsigned int num_atcam_erps;63 unsigned int num_max_atcam_erps;64 unsigned int num_ctcam_erps;65 unsigned int num_deltas;66 struct objagg *objagg;67 struct mutex objagg_lock; /* guards objagg manipulation */68};69 70struct mlxsw_sp_acl_erp_table_ops {71 struct mlxsw_sp_acl_erp *72 (*erp_create)(struct mlxsw_sp_acl_erp_table *erp_table,73 struct mlxsw_sp_acl_erp_key *key);74 void (*erp_destroy)(struct mlxsw_sp_acl_erp_table *erp_table,75 struct mlxsw_sp_acl_erp *erp);76};77 78static struct mlxsw_sp_acl_erp *79mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,80 struct mlxsw_sp_acl_erp_key *key);81static void82mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,83 struct mlxsw_sp_acl_erp *erp);84static struct mlxsw_sp_acl_erp *85mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,86 struct mlxsw_sp_acl_erp_key *key);87static void88mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,89 struct mlxsw_sp_acl_erp *erp);90static struct mlxsw_sp_acl_erp *91mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,92 struct mlxsw_sp_acl_erp_key *key);93static void94mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,95 struct mlxsw_sp_acl_erp *erp);96static void97mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,98 struct mlxsw_sp_acl_erp *erp);99 100static const struct mlxsw_sp_acl_erp_table_ops erp_multiple_masks_ops = {101 .erp_create = mlxsw_sp_acl_erp_mask_create,102 .erp_destroy = mlxsw_sp_acl_erp_mask_destroy,103};104 105static const struct mlxsw_sp_acl_erp_table_ops erp_two_masks_ops = {106 .erp_create = mlxsw_sp_acl_erp_mask_create,107 .erp_destroy = mlxsw_sp_acl_erp_second_mask_destroy,108};109 110static const struct mlxsw_sp_acl_erp_table_ops erp_single_mask_ops = {111 .erp_create = mlxsw_sp_acl_erp_second_mask_create,112 .erp_destroy = mlxsw_sp_acl_erp_first_mask_destroy,113};114 115static const struct mlxsw_sp_acl_erp_table_ops erp_no_mask_ops = {116 .erp_create = mlxsw_sp_acl_erp_first_mask_create,117 .erp_destroy = mlxsw_sp_acl_erp_no_mask_destroy,118};119 120static bool121mlxsw_sp_acl_erp_table_is_used(const struct mlxsw_sp_acl_erp_table *erp_table)122{123 return erp_table->ops != &erp_single_mask_ops &&124 erp_table->ops != &erp_no_mask_ops;125}126 127static unsigned int128mlxsw_sp_acl_erp_bank_get(const struct mlxsw_sp_acl_erp *erp)129{130 return erp->index % erp->erp_table->erp_core->num_erp_banks;131}132 133static unsigned int134mlxsw_sp_acl_erp_table_entry_size(const struct mlxsw_sp_acl_erp_table *erp_table)135{136 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion;137 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;138 139 return erp_core->erpt_entries_size[aregion->type];140}141 142static int mlxsw_sp_acl_erp_id_get(struct mlxsw_sp_acl_erp_table *erp_table,143 u8 *p_id)144{145 u8 id;146 147 id = find_first_zero_bit(erp_table->erp_id_bitmap,148 MLXSW_SP_ACL_ERP_MAX_PER_REGION);149 if (id < MLXSW_SP_ACL_ERP_MAX_PER_REGION) {150 __set_bit(id, erp_table->erp_id_bitmap);151 *p_id = id;152 return 0;153 }154 155 return -ENOBUFS;156}157 158static void mlxsw_sp_acl_erp_id_put(struct mlxsw_sp_acl_erp_table *erp_table,159 u8 id)160{161 __clear_bit(id, erp_table->erp_id_bitmap);162}163 164static void165mlxsw_sp_acl_erp_master_mask_bit_set(unsigned long bit,166 struct mlxsw_sp_acl_erp_master_mask *mask)167{168 if (mask->count[bit]++ == 0)169 __set_bit(bit, mask->bitmap);170}171 172static void173mlxsw_sp_acl_erp_master_mask_bit_clear(unsigned long bit,174 struct mlxsw_sp_acl_erp_master_mask *mask)175{176 if (--mask->count[bit] == 0)177 __clear_bit(bit, mask->bitmap);178}179 180static int181mlxsw_sp_acl_erp_master_mask_update(struct mlxsw_sp_acl_erp_table *erp_table)182{183 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;184 struct mlxsw_sp *mlxsw_sp = region->mlxsw_sp;185 char percr_pl[MLXSW_REG_PERCR_LEN];186 char *master_mask;187 188 mlxsw_reg_percr_pack(percr_pl, region->id);189 master_mask = mlxsw_reg_percr_master_mask_data(percr_pl);190 bitmap_to_arr32((u32 *) master_mask, erp_table->master_mask.bitmap,191 MLXSW_SP_ACL_TCAM_MASK_LEN);192 193 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl);194}195 196static int197mlxsw_sp_acl_erp_master_mask_set(struct mlxsw_sp_acl_erp_table *erp_table,198 struct mlxsw_sp_acl_erp_key *key)199{200 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);201 unsigned long bit;202 int err;203 204 bitmap_from_arr32(mask_bitmap, (u32 *) key->mask,205 MLXSW_SP_ACL_TCAM_MASK_LEN);206 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)207 mlxsw_sp_acl_erp_master_mask_bit_set(bit,208 &erp_table->master_mask);209 210 err = mlxsw_sp_acl_erp_master_mask_update(erp_table);211 if (err)212 goto err_master_mask_update;213 214 return 0;215 216err_master_mask_update:217 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)218 mlxsw_sp_acl_erp_master_mask_bit_clear(bit,219 &erp_table->master_mask);220 return err;221}222 223static int224mlxsw_sp_acl_erp_master_mask_clear(struct mlxsw_sp_acl_erp_table *erp_table,225 struct mlxsw_sp_acl_erp_key *key)226{227 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN);228 unsigned long bit;229 int err;230 231 bitmap_from_arr32(mask_bitmap, (u32 *) key->mask,232 MLXSW_SP_ACL_TCAM_MASK_LEN);233 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)234 mlxsw_sp_acl_erp_master_mask_bit_clear(bit,235 &erp_table->master_mask);236 237 err = mlxsw_sp_acl_erp_master_mask_update(erp_table);238 if (err)239 goto err_master_mask_update;240 241 return 0;242 243err_master_mask_update:244 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN)245 mlxsw_sp_acl_erp_master_mask_bit_set(bit,246 &erp_table->master_mask);247 return err;248}249 250static struct mlxsw_sp_acl_erp *251mlxsw_sp_acl_erp_generic_create(struct mlxsw_sp_acl_erp_table *erp_table,252 struct mlxsw_sp_acl_erp_key *key)253{254 struct mlxsw_sp_acl_erp *erp;255 int err;256 257 erp = kzalloc(sizeof(*erp), GFP_KERNEL);258 if (!erp)259 return ERR_PTR(-ENOMEM);260 261 err = mlxsw_sp_acl_erp_id_get(erp_table, &erp->id);262 if (err)263 goto err_erp_id_get;264 265 memcpy(&erp->key, key, sizeof(*key));266 list_add(&erp->list, &erp_table->atcam_erps_list);267 erp_table->num_atcam_erps++;268 erp->erp_table = erp_table;269 270 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key);271 if (err)272 goto err_master_mask_set;273 274 return erp;275 276err_master_mask_set:277 erp_table->num_atcam_erps--;278 list_del(&erp->list);279 mlxsw_sp_acl_erp_id_put(erp_table, erp->id);280err_erp_id_get:281 kfree(erp);282 return ERR_PTR(err);283}284 285static void286mlxsw_sp_acl_erp_generic_destroy(struct mlxsw_sp_acl_erp *erp)287{288 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;289 290 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key);291 erp_table->num_atcam_erps--;292 list_del(&erp->list);293 mlxsw_sp_acl_erp_id_put(erp_table, erp->id);294 kfree(erp);295}296 297static int298mlxsw_sp_acl_erp_table_alloc(struct mlxsw_sp_acl_erp_core *erp_core,299 unsigned int num_erps,300 enum mlxsw_sp_acl_atcam_region_type region_type,301 unsigned long *p_index)302{303 unsigned int num_rows, entry_size;304 unsigned long index;305 306 /* We only allow allocations of entire rows */307 if (num_erps % erp_core->num_erp_banks != 0)308 return -EINVAL;309 310 entry_size = erp_core->erpt_entries_size[region_type];311 num_rows = num_erps / erp_core->num_erp_banks;312 313 index = gen_pool_alloc(erp_core->erp_tables, num_rows * entry_size);314 if (!index)315 return -ENOBUFS;316 317 *p_index = index - MLXSW_SP_ACL_ERP_GENALLOC_OFFSET;318 319 return 0;320}321 322static void323mlxsw_sp_acl_erp_table_free(struct mlxsw_sp_acl_erp_core *erp_core,324 unsigned int num_erps,325 enum mlxsw_sp_acl_atcam_region_type region_type,326 unsigned long index)327{328 unsigned long base_index;329 unsigned int entry_size;330 size_t size;331 332 entry_size = erp_core->erpt_entries_size[region_type];333 base_index = index + MLXSW_SP_ACL_ERP_GENALLOC_OFFSET;334 size = num_erps / erp_core->num_erp_banks * entry_size;335 gen_pool_free(erp_core->erp_tables, base_index, size);336}337 338static struct mlxsw_sp_acl_erp *339mlxsw_sp_acl_erp_table_master_rp(struct mlxsw_sp_acl_erp_table *erp_table)340{341 if (!list_is_singular(&erp_table->atcam_erps_list))342 return NULL;343 344 return list_first_entry(&erp_table->atcam_erps_list,345 struct mlxsw_sp_acl_erp, list);346}347 348static int mlxsw_sp_acl_erp_index_get(struct mlxsw_sp_acl_erp_table *erp_table,349 u8 *p_index)350{351 u8 index;352 353 index = find_first_zero_bit(erp_table->erp_index_bitmap,354 erp_table->num_max_atcam_erps);355 if (index < erp_table->num_max_atcam_erps) {356 __set_bit(index, erp_table->erp_index_bitmap);357 *p_index = index;358 return 0;359 }360 361 return -ENOBUFS;362}363 364static void mlxsw_sp_acl_erp_index_put(struct mlxsw_sp_acl_erp_table *erp_table,365 u8 index)366{367 __clear_bit(index, erp_table->erp_index_bitmap);368}369 370static void371mlxsw_sp_acl_erp_table_locate(const struct mlxsw_sp_acl_erp_table *erp_table,372 const struct mlxsw_sp_acl_erp *erp,373 u8 *p_erpt_bank, u8 *p_erpt_index)374{375 unsigned int entry_size = mlxsw_sp_acl_erp_table_entry_size(erp_table);376 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;377 unsigned int row;378 379 *p_erpt_bank = erp->index % erp_core->num_erp_banks;380 row = erp->index / erp_core->num_erp_banks;381 *p_erpt_index = erp_table->base_index + row * entry_size;382}383 384static int385mlxsw_sp_acl_erp_table_erp_add(struct mlxsw_sp_acl_erp_table *erp_table,386 struct mlxsw_sp_acl_erp *erp)387{388 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;389 enum mlxsw_reg_perpt_key_size key_size;390 char perpt_pl[MLXSW_REG_PERPT_LEN];391 u8 erpt_bank, erpt_index;392 393 mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index);394 key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type;395 mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id,396 0, erp_table->base_index, erp->index,397 erp->key.mask);398 mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap,399 MLXSW_SP_ACL_ERP_MAX_PER_REGION);400 mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, true);401 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl);402}403 404static void mlxsw_sp_acl_erp_table_erp_del(struct mlxsw_sp_acl_erp *erp)405{406 char empty_mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN] = { 0 };407 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;408 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;409 enum mlxsw_reg_perpt_key_size key_size;410 char perpt_pl[MLXSW_REG_PERPT_LEN];411 u8 erpt_bank, erpt_index;412 413 mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index);414 key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type;415 mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id,416 0, erp_table->base_index, erp->index, empty_mask);417 mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap,418 MLXSW_SP_ACL_ERP_MAX_PER_REGION);419 mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, false);420 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl);421}422 423static int424mlxsw_sp_acl_erp_table_enable(struct mlxsw_sp_acl_erp_table *erp_table,425 bool ctcam_le)426{427 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;428 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;429 char pererp_pl[MLXSW_REG_PERERP_LEN];430 431 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0,432 erp_table->base_index, 0);433 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap,434 MLXSW_SP_ACL_ERP_MAX_PER_REGION);435 436 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);437}438 439static void440mlxsw_sp_acl_erp_table_disable(struct mlxsw_sp_acl_erp_table *erp_table)441{442 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;443 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;444 char pererp_pl[MLXSW_REG_PERERP_LEN];445 struct mlxsw_sp_acl_erp *master_rp;446 447 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table);448 /* It is possible we do not have a master RP when we disable the449 * table when there are no rules in the A-TCAM and the last C-TCAM450 * rule is deleted451 */452 mlxsw_reg_pererp_pack(pererp_pl, region->id, false, false, 0, 0,453 master_rp ? master_rp->id : 0);454 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);455}456 457static int458mlxsw_sp_acl_erp_table_relocate(struct mlxsw_sp_acl_erp_table *erp_table)459{460 struct mlxsw_sp_acl_erp *erp;461 int err;462 463 list_for_each_entry(erp, &erp_table->atcam_erps_list, list) {464 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp);465 if (err)466 goto err_table_erp_add;467 }468 469 return 0;470 471err_table_erp_add:472 list_for_each_entry_continue_reverse(erp, &erp_table->atcam_erps_list,473 list)474 mlxsw_sp_acl_erp_table_erp_del(erp);475 return err;476}477 478static int479mlxsw_sp_acl_erp_table_expand(struct mlxsw_sp_acl_erp_table *erp_table)480{481 unsigned int num_erps, old_num_erps = erp_table->num_max_atcam_erps;482 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;483 unsigned long old_base_index = erp_table->base_index;484 bool ctcam_le = erp_table->num_ctcam_erps > 0;485 int err;486 487 if (erp_table->num_atcam_erps < erp_table->num_max_atcam_erps)488 return 0;489 490 if (erp_table->num_max_atcam_erps == MLXSW_SP_ACL_ERP_MAX_PER_REGION)491 return -ENOBUFS;492 493 num_erps = old_num_erps + erp_core->num_erp_banks;494 err = mlxsw_sp_acl_erp_table_alloc(erp_core, num_erps,495 erp_table->aregion->type,496 &erp_table->base_index);497 if (err)498 return err;499 erp_table->num_max_atcam_erps = num_erps;500 501 err = mlxsw_sp_acl_erp_table_relocate(erp_table);502 if (err)503 goto err_table_relocate;504 505 err = mlxsw_sp_acl_erp_table_enable(erp_table, ctcam_le);506 if (err)507 goto err_table_enable;508 509 mlxsw_sp_acl_erp_table_free(erp_core, old_num_erps,510 erp_table->aregion->type, old_base_index);511 512 return 0;513 514err_table_enable:515err_table_relocate:516 erp_table->num_max_atcam_erps = old_num_erps;517 mlxsw_sp_acl_erp_table_free(erp_core, num_erps,518 erp_table->aregion->type,519 erp_table->base_index);520 erp_table->base_index = old_base_index;521 return err;522}523 524static int525mlxsw_acl_erp_table_bf_add(struct mlxsw_sp_acl_erp_table *erp_table,526 struct mlxsw_sp_acl_erp *erp)527{528 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion;529 unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp);530 struct mlxsw_sp_acl_atcam_entry *aentry;531 int err;532 533 list_for_each_entry(aentry, &aregion->entries_list, list) {534 err = mlxsw_sp_acl_bf_entry_add(aregion->region->mlxsw_sp,535 erp_table->erp_core->bf,536 aregion, erp_bank, aentry);537 if (err)538 goto bf_entry_add_err;539 }540 541 return 0;542 543bf_entry_add_err:544 list_for_each_entry_continue_reverse(aentry, &aregion->entries_list,545 list)546 mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp,547 erp_table->erp_core->bf,548 aregion, erp_bank, aentry);549 return err;550}551 552static void553mlxsw_acl_erp_table_bf_del(struct mlxsw_sp_acl_erp_table *erp_table,554 struct mlxsw_sp_acl_erp *erp)555{556 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion;557 unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp);558 struct mlxsw_sp_acl_atcam_entry *aentry;559 560 list_for_each_entry_reverse(aentry, &aregion->entries_list, list)561 mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp,562 erp_table->erp_core->bf,563 aregion, erp_bank, aentry);564}565 566static int567mlxsw_sp_acl_erp_region_table_trans(struct mlxsw_sp_acl_erp_table *erp_table)568{569 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;570 struct mlxsw_sp_acl_erp *master_rp;571 int err;572 573 /* Initially, allocate a single eRP row. Expand later as needed */574 err = mlxsw_sp_acl_erp_table_alloc(erp_core, erp_core->num_erp_banks,575 erp_table->aregion->type,576 &erp_table->base_index);577 if (err)578 return err;579 erp_table->num_max_atcam_erps = erp_core->num_erp_banks;580 581 /* Transition the sole RP currently configured (the master RP)582 * to the eRP table583 */584 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table);585 if (!master_rp) {586 err = -EINVAL;587 goto err_table_master_rp;588 }589 590 /* Make sure the master RP is using a valid index, as591 * only a single eRP row is currently allocated.592 */593 master_rp->index = 0;594 __set_bit(master_rp->index, erp_table->erp_index_bitmap);595 596 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, master_rp);597 if (err)598 goto err_table_master_rp_add;599 600 /* Update Bloom filter before enabling eRP table, as rules601 * on the master RP were not set to Bloom filter up to this602 * point.603 */604 err = mlxsw_acl_erp_table_bf_add(erp_table, master_rp);605 if (err)606 goto err_table_bf_add;607 608 err = mlxsw_sp_acl_erp_table_enable(erp_table, false);609 if (err)610 goto err_table_enable;611 612 return 0;613 614err_table_enable:615 mlxsw_acl_erp_table_bf_del(erp_table, master_rp);616err_table_bf_add:617 mlxsw_sp_acl_erp_table_erp_del(master_rp);618err_table_master_rp_add:619 __clear_bit(master_rp->index, erp_table->erp_index_bitmap);620err_table_master_rp:621 mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps,622 erp_table->aregion->type,623 erp_table->base_index);624 return err;625}626 627static void628mlxsw_sp_acl_erp_region_master_mask_trans(struct mlxsw_sp_acl_erp_table *erp_table)629{630 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core;631 struct mlxsw_sp_acl_erp *master_rp;632 633 mlxsw_sp_acl_erp_table_disable(erp_table);634 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table);635 if (!master_rp)636 return;637 mlxsw_acl_erp_table_bf_del(erp_table, master_rp);638 mlxsw_sp_acl_erp_table_erp_del(master_rp);639 __clear_bit(master_rp->index, erp_table->erp_index_bitmap);640 mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps,641 erp_table->aregion->type,642 erp_table->base_index);643}644 645static int646mlxsw_sp_acl_erp_region_erp_add(struct mlxsw_sp_acl_erp_table *erp_table,647 struct mlxsw_sp_acl_erp *erp)648{649 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;650 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;651 bool ctcam_le = erp_table->num_ctcam_erps > 0;652 char pererp_pl[MLXSW_REG_PERERP_LEN];653 654 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0,655 erp_table->base_index, 0);656 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap,657 MLXSW_SP_ACL_ERP_MAX_PER_REGION);658 mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, true);659 660 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);661}662 663static void mlxsw_sp_acl_erp_region_erp_del(struct mlxsw_sp_acl_erp *erp)664{665 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;666 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region;667 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp;668 bool ctcam_le = erp_table->num_ctcam_erps > 0;669 char pererp_pl[MLXSW_REG_PERERP_LEN];670 671 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0,672 erp_table->base_index, 0);673 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap,674 MLXSW_SP_ACL_ERP_MAX_PER_REGION);675 mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, false);676 677 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);678}679 680static int681mlxsw_sp_acl_erp_region_ctcam_enable(struct mlxsw_sp_acl_erp_table *erp_table)682{683 /* No need to re-enable lookup in the C-TCAM */684 if (erp_table->num_ctcam_erps > 1)685 return 0;686 687 return mlxsw_sp_acl_erp_table_enable(erp_table, true);688}689 690static void691mlxsw_sp_acl_erp_region_ctcam_disable(struct mlxsw_sp_acl_erp_table *erp_table)692{693 /* Only disable C-TCAM lookup when last C-TCAM eRP is deleted */694 if (erp_table->num_ctcam_erps > 1)695 return;696 697 mlxsw_sp_acl_erp_table_enable(erp_table, false);698}699 700static int701__mlxsw_sp_acl_erp_table_other_inc(struct mlxsw_sp_acl_erp_table *erp_table,702 unsigned int *inc_num)703{704 int err;705 706 /* If there are C-TCAM eRP or deltas in use we need to transition707 * the region to use eRP table, if it is not already done708 */709 if (!mlxsw_sp_acl_erp_table_is_used(erp_table)) {710 err = mlxsw_sp_acl_erp_region_table_trans(erp_table);711 if (err)712 return err;713 }714 715 /* When C-TCAM or deltas are used, the eRP table must be used */716 if (erp_table->ops != &erp_multiple_masks_ops)717 erp_table->ops = &erp_multiple_masks_ops;718 719 (*inc_num)++;720 721 return 0;722}723 724static int mlxsw_sp_acl_erp_ctcam_inc(struct mlxsw_sp_acl_erp_table *erp_table)725{726 return __mlxsw_sp_acl_erp_table_other_inc(erp_table,727 &erp_table->num_ctcam_erps);728}729 730static int mlxsw_sp_acl_erp_delta_inc(struct mlxsw_sp_acl_erp_table *erp_table)731{732 return __mlxsw_sp_acl_erp_table_other_inc(erp_table,733 &erp_table->num_deltas);734}735 736static void737__mlxsw_sp_acl_erp_table_other_dec(struct mlxsw_sp_acl_erp_table *erp_table,738 unsigned int *dec_num)739{740 (*dec_num)--;741 742 /* If there are no C-TCAM eRP or deltas in use, the state we743 * transition to depends on the number of A-TCAM eRPs currently744 * in use.745 */746 if (erp_table->num_ctcam_erps > 0 || erp_table->num_deltas > 0)747 return;748 749 switch (erp_table->num_atcam_erps) {750 case 2:751 /* Keep using the eRP table, but correctly set the752 * operations pointer so that when an A-TCAM eRP is753 * deleted we will transition to use the master mask754 */755 erp_table->ops = &erp_two_masks_ops;756 break;757 case 1:758 /* We only kept the eRP table because we had C-TCAM759 * eRPs in use. Now that the last C-TCAM eRP is gone we760 * can stop using the table and transition to use the761 * master mask762 */763 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table);764 erp_table->ops = &erp_single_mask_ops;765 break;766 case 0:767 /* There are no more eRPs of any kind used by the region768 * so free its eRP table and transition to initial state769 */770 mlxsw_sp_acl_erp_table_disable(erp_table);771 mlxsw_sp_acl_erp_table_free(erp_table->erp_core,772 erp_table->num_max_atcam_erps,773 erp_table->aregion->type,774 erp_table->base_index);775 erp_table->ops = &erp_no_mask_ops;776 break;777 default:778 break;779 }780}781 782static void mlxsw_sp_acl_erp_ctcam_dec(struct mlxsw_sp_acl_erp_table *erp_table)783{784 __mlxsw_sp_acl_erp_table_other_dec(erp_table,785 &erp_table->num_ctcam_erps);786}787 788static void mlxsw_sp_acl_erp_delta_dec(struct mlxsw_sp_acl_erp_table *erp_table)789{790 __mlxsw_sp_acl_erp_table_other_dec(erp_table,791 &erp_table->num_deltas);792}793 794static struct mlxsw_sp_acl_erp *795mlxsw_sp_acl_erp_ctcam_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,796 struct mlxsw_sp_acl_erp_key *key)797{798 struct mlxsw_sp_acl_erp *erp;799 int err;800 801 erp = kzalloc(sizeof(*erp), GFP_KERNEL);802 if (!erp)803 return ERR_PTR(-ENOMEM);804 805 memcpy(&erp->key, key, sizeof(*key));806 bitmap_from_arr32(erp->mask_bitmap, (u32 *) key->mask,807 MLXSW_SP_ACL_TCAM_MASK_LEN);808 809 err = mlxsw_sp_acl_erp_ctcam_inc(erp_table);810 if (err)811 goto err_erp_ctcam_inc;812 813 erp->erp_table = erp_table;814 815 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key);816 if (err)817 goto err_master_mask_set;818 819 err = mlxsw_sp_acl_erp_region_ctcam_enable(erp_table);820 if (err)821 goto err_erp_region_ctcam_enable;822 823 return erp;824 825err_erp_region_ctcam_enable:826 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key);827err_master_mask_set:828 mlxsw_sp_acl_erp_ctcam_dec(erp_table);829err_erp_ctcam_inc:830 kfree(erp);831 return ERR_PTR(err);832}833 834static void835mlxsw_sp_acl_erp_ctcam_mask_destroy(struct mlxsw_sp_acl_erp *erp)836{837 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table;838 839 mlxsw_sp_acl_erp_region_ctcam_disable(erp_table);840 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key);841 mlxsw_sp_acl_erp_ctcam_dec(erp_table);842 kfree(erp);843}844 845static struct mlxsw_sp_acl_erp *846mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,847 struct mlxsw_sp_acl_erp_key *key)848{849 struct mlxsw_sp_acl_erp *erp;850 int err;851 852 if (key->ctcam)853 return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key);854 855 /* Expand the eRP table for the new eRP, if needed */856 err = mlxsw_sp_acl_erp_table_expand(erp_table);857 if (err)858 return ERR_PTR(err);859 860 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key);861 if (IS_ERR(erp))862 return erp;863 864 err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index);865 if (err)866 goto err_erp_index_get;867 868 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp);869 if (err)870 goto err_table_erp_add;871 872 err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp);873 if (err)874 goto err_region_erp_add;875 876 erp_table->ops = &erp_multiple_masks_ops;877 878 return erp;879 880err_region_erp_add:881 mlxsw_sp_acl_erp_table_erp_del(erp);882err_table_erp_add:883 mlxsw_sp_acl_erp_index_put(erp_table, erp->index);884err_erp_index_get:885 mlxsw_sp_acl_erp_generic_destroy(erp);886 return ERR_PTR(err);887}888 889static void890mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,891 struct mlxsw_sp_acl_erp *erp)892{893 if (erp->key.ctcam)894 return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp);895 896 mlxsw_sp_acl_erp_region_erp_del(erp);897 mlxsw_sp_acl_erp_table_erp_del(erp);898 mlxsw_sp_acl_erp_index_put(erp_table, erp->index);899 mlxsw_sp_acl_erp_generic_destroy(erp);900 901 if (erp_table->num_atcam_erps == 2 && erp_table->num_ctcam_erps == 0 &&902 erp_table->num_deltas == 0)903 erp_table->ops = &erp_two_masks_ops;904}905 906static struct mlxsw_sp_acl_erp *907mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,908 struct mlxsw_sp_acl_erp_key *key)909{910 struct mlxsw_sp_acl_erp *erp;911 int err;912 913 if (key->ctcam)914 return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key);915 916 /* Transition to use eRP table instead of master mask */917 err = mlxsw_sp_acl_erp_region_table_trans(erp_table);918 if (err)919 return ERR_PTR(err);920 921 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key);922 if (IS_ERR(erp)) {923 err = PTR_ERR(erp);924 goto err_erp_create;925 }926 927 err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index);928 if (err)929 goto err_erp_index_get;930 931 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp);932 if (err)933 goto err_table_erp_add;934 935 err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp);936 if (err)937 goto err_region_erp_add;938 939 erp_table->ops = &erp_two_masks_ops;940 941 return erp;942 943err_region_erp_add:944 mlxsw_sp_acl_erp_table_erp_del(erp);945err_table_erp_add:946 mlxsw_sp_acl_erp_index_put(erp_table, erp->index);947err_erp_index_get:948 mlxsw_sp_acl_erp_generic_destroy(erp);949err_erp_create:950 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table);951 return ERR_PTR(err);952}953 954static void955mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,956 struct mlxsw_sp_acl_erp *erp)957{958 if (erp->key.ctcam)959 return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp);960 961 mlxsw_sp_acl_erp_region_erp_del(erp);962 mlxsw_sp_acl_erp_table_erp_del(erp);963 mlxsw_sp_acl_erp_index_put(erp_table, erp->index);964 mlxsw_sp_acl_erp_generic_destroy(erp);965 /* Transition to use master mask instead of eRP table */966 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table);967 968 erp_table->ops = &erp_single_mask_ops;969}970 971static struct mlxsw_sp_acl_erp *972mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table,973 struct mlxsw_sp_acl_erp_key *key)974{975 struct mlxsw_sp_acl_erp *erp;976 977 if (key->ctcam)978 return ERR_PTR(-EINVAL);979 980 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key);981 if (IS_ERR(erp))982 return erp;983 984 erp_table->ops = &erp_single_mask_ops;985 986 return erp;987}988 989static void990mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,991 struct mlxsw_sp_acl_erp *erp)992{993 mlxsw_sp_acl_erp_generic_destroy(erp);994 erp_table->ops = &erp_no_mask_ops;995}996 997static void998mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table,999 struct mlxsw_sp_acl_erp *erp)1000{1001 WARN_ON(1);1002}1003 1004struct mlxsw_sp_acl_erp_mask *1005mlxsw_sp_acl_erp_mask_get(struct mlxsw_sp_acl_atcam_region *aregion,1006 const char *mask, bool ctcam)1007{1008 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;1009 struct mlxsw_sp_acl_erp_key key;1010 struct objagg_obj *objagg_obj;1011 1012 memcpy(key.mask, mask, MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN);1013 key.ctcam = ctcam;1014 mutex_lock(&erp_table->objagg_lock);1015 objagg_obj = objagg_obj_get(erp_table->objagg, &key);1016 mutex_unlock(&erp_table->objagg_lock);1017 if (IS_ERR(objagg_obj))1018 return ERR_CAST(objagg_obj);1019 return (struct mlxsw_sp_acl_erp_mask *) objagg_obj;1020}1021 1022void mlxsw_sp_acl_erp_mask_put(struct mlxsw_sp_acl_atcam_region *aregion,1023 struct mlxsw_sp_acl_erp_mask *erp_mask)1024{1025 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;1026 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;1027 1028 mutex_lock(&erp_table->objagg_lock);1029 objagg_obj_put(erp_table->objagg, objagg_obj);1030 mutex_unlock(&erp_table->objagg_lock);1031}1032 1033int mlxsw_sp_acl_erp_bf_insert(struct mlxsw_sp *mlxsw_sp,1034 struct mlxsw_sp_acl_atcam_region *aregion,1035 struct mlxsw_sp_acl_erp_mask *erp_mask,1036 struct mlxsw_sp_acl_atcam_entry *aentry)1037{1038 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;1039 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj);1040 unsigned int erp_bank;1041 1042 if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table))1043 return 0;1044 1045 erp_bank = mlxsw_sp_acl_erp_bank_get(erp);1046 return mlxsw_sp_acl_bf_entry_add(mlxsw_sp,1047 erp->erp_table->erp_core->bf,1048 aregion, erp_bank, aentry);1049}1050 1051void mlxsw_sp_acl_erp_bf_remove(struct mlxsw_sp *mlxsw_sp,1052 struct mlxsw_sp_acl_atcam_region *aregion,1053 struct mlxsw_sp_acl_erp_mask *erp_mask,1054 struct mlxsw_sp_acl_atcam_entry *aentry)1055{1056 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;1057 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj);1058 unsigned int erp_bank;1059 1060 if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table))1061 return;1062 1063 erp_bank = mlxsw_sp_acl_erp_bank_get(erp);1064 mlxsw_sp_acl_bf_entry_del(mlxsw_sp,1065 erp->erp_table->erp_core->bf,1066 aregion, erp_bank, aentry);1067}1068 1069bool1070mlxsw_sp_acl_erp_mask_is_ctcam(const struct mlxsw_sp_acl_erp_mask *erp_mask)1071{1072 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;1073 const struct mlxsw_sp_acl_erp_key *key = objagg_obj_raw(objagg_obj);1074 1075 return key->ctcam;1076}1077 1078u8 mlxsw_sp_acl_erp_mask_erp_id(const struct mlxsw_sp_acl_erp_mask *erp_mask)1079{1080 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;1081 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj);1082 1083 return erp->id;1084}1085 1086struct mlxsw_sp_acl_erp_delta {1087 struct mlxsw_sp_acl_erp_key key;1088 u16 start;1089 u8 mask;1090};1091 1092u16 mlxsw_sp_acl_erp_delta_start(const struct mlxsw_sp_acl_erp_delta *delta)1093{1094 return delta->start;1095}1096 1097u8 mlxsw_sp_acl_erp_delta_mask(const struct mlxsw_sp_acl_erp_delta *delta)1098{1099 return delta->mask;1100}1101 1102u8 mlxsw_sp_acl_erp_delta_value(const struct mlxsw_sp_acl_erp_delta *delta,1103 const char *enc_key)1104{1105 u16 start = delta->start;1106 u8 mask = delta->mask;1107 u16 tmp;1108 1109 if (!mask)1110 return 0;1111 1112 tmp = (unsigned char) enc_key[__MASK_IDX(start / 8)];1113 if (start / 8 + 1 < __MASK_LEN)1114 tmp |= (unsigned char) enc_key[__MASK_IDX(start / 8 + 1)] << 8;1115 tmp >>= start % 8;1116 tmp &= mask;1117 return tmp;1118}1119 1120void mlxsw_sp_acl_erp_delta_clear(const struct mlxsw_sp_acl_erp_delta *delta,1121 const char *enc_key)1122{1123 u16 start = delta->start;1124 u8 mask = delta->mask;1125 unsigned char *byte;1126 u16 tmp;1127 1128 tmp = mask;1129 tmp <<= start % 8;1130 tmp = ~tmp;1131 1132 byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8)];1133 *byte &= tmp & 0xff;1134 if (start / 8 + 1 < __MASK_LEN) {1135 byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8 + 1)];1136 *byte &= (tmp >> 8) & 0xff;1137 }1138}1139 1140static const struct mlxsw_sp_acl_erp_delta1141mlxsw_sp_acl_erp_delta_default = {};1142 1143const struct mlxsw_sp_acl_erp_delta *1144mlxsw_sp_acl_erp_delta(const struct mlxsw_sp_acl_erp_mask *erp_mask)1145{1146 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask;1147 const struct mlxsw_sp_acl_erp_delta *delta;1148 1149 delta = objagg_obj_delta_priv(objagg_obj);1150 if (!delta)1151 delta = &mlxsw_sp_acl_erp_delta_default;1152 return delta;1153}1154 1155static int1156mlxsw_sp_acl_erp_delta_fill(const struct mlxsw_sp_acl_erp_key *parent_key,1157 const struct mlxsw_sp_acl_erp_key *key,1158 u16 *delta_start, u8 *delta_mask)1159{1160 int offset = 0;1161 int si = -1;1162 u16 pmask;1163 u16 mask;1164 int i;1165 1166 /* The difference between 2 masks can be up to 8 consecutive bits. */1167 for (i = 0; i < __MASK_LEN; i++) {1168 if (parent_key->mask[__MASK_IDX(i)] == key->mask[__MASK_IDX(i)])1169 continue;1170 if (si == -1)1171 si = i;1172 else if (si != i - 1)1173 return -EINVAL;1174 }1175 if (si == -1) {1176 /* The masks are the same, this can happen in case eRPs with1177 * the same mask were created in both A-TCAM and C-TCAM.1178 * The only possible condition under which this can happen1179 * is identical rule insertion. Delta is not possible here.1180 */1181 return -EINVAL;1182 }1183 pmask = (unsigned char) parent_key->mask[__MASK_IDX(si)];1184 mask = (unsigned char) key->mask[__MASK_IDX(si)];1185 if (si + 1 < __MASK_LEN) {1186 pmask |= (unsigned char) parent_key->mask[__MASK_IDX(si + 1)] << 8;1187 mask |= (unsigned char) key->mask[__MASK_IDX(si + 1)] << 8;1188 }1189 1190 if ((pmask ^ mask) & pmask)1191 return -EINVAL;1192 mask &= ~pmask;1193 while (!(mask & (1 << offset)))1194 offset++;1195 while (!(mask & 1))1196 mask >>= 1;1197 if (mask & 0xff00)1198 return -EINVAL;1199 1200 *delta_start = si * 8 + offset;1201 *delta_mask = mask;1202 1203 return 0;1204}1205 1206static bool mlxsw_sp_acl_erp_delta_check(void *priv, const void *parent_obj,1207 const void *obj)1208{1209 const struct mlxsw_sp_acl_erp_key *parent_key = parent_obj;1210 const struct mlxsw_sp_acl_erp_key *key = obj;1211 u16 delta_start;1212 u8 delta_mask;1213 int err;1214 1215 err = mlxsw_sp_acl_erp_delta_fill(parent_key, key,1216 &delta_start, &delta_mask);1217 return err ? false : true;1218}1219 1220static void *mlxsw_sp_acl_erp_delta_create(void *priv, void *parent_obj,1221 void *obj)1222{1223 struct mlxsw_sp_acl_erp_key *parent_key = parent_obj;1224 struct mlxsw_sp_acl_atcam_region *aregion = priv;1225 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;1226 struct mlxsw_sp_acl_erp_key *key = obj;1227 struct mlxsw_sp_acl_erp_delta *delta;1228 u16 delta_start;1229 u8 delta_mask;1230 int err;1231 1232 if (parent_key->ctcam || key->ctcam)1233 return ERR_PTR(-EINVAL);1234 err = mlxsw_sp_acl_erp_delta_fill(parent_key, key,1235 &delta_start, &delta_mask);1236 if (err)1237 return ERR_PTR(-EINVAL);1238 1239 delta = kzalloc(sizeof(*delta), GFP_KERNEL);1240 if (!delta)1241 return ERR_PTR(-ENOMEM);1242 delta->start = delta_start;1243 delta->mask = delta_mask;1244 1245 err = mlxsw_sp_acl_erp_delta_inc(erp_table);1246 if (err)1247 goto err_erp_delta_inc;1248 1249 memcpy(&delta->key, key, sizeof(*key));1250 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &delta->key);1251 if (err)1252 goto err_master_mask_set;1253 1254 return delta;1255 1256err_master_mask_set:1257 mlxsw_sp_acl_erp_delta_dec(erp_table);1258err_erp_delta_inc:1259 kfree(delta);1260 return ERR_PTR(err);1261}1262 1263static void mlxsw_sp_acl_erp_delta_destroy(void *priv, void *delta_priv)1264{1265 struct mlxsw_sp_acl_erp_delta *delta = delta_priv;1266 struct mlxsw_sp_acl_atcam_region *aregion = priv;1267 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;1268 1269 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &delta->key);1270 mlxsw_sp_acl_erp_delta_dec(erp_table);1271 kfree(delta);1272}1273 1274static void *mlxsw_sp_acl_erp_root_create(void *priv, void *obj,1275 unsigned int root_id)1276{1277 struct mlxsw_sp_acl_atcam_region *aregion = priv;1278 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;1279 struct mlxsw_sp_acl_erp_key *key = obj;1280 1281 if (!key->ctcam &&1282 root_id != OBJAGG_OBJ_ROOT_ID_INVALID &&1283 root_id >= MLXSW_SP_ACL_ERP_MAX_PER_REGION)1284 return ERR_PTR(-ENOBUFS);1285 return erp_table->ops->erp_create(erp_table, key);1286}1287 1288static void mlxsw_sp_acl_erp_root_destroy(void *priv, void *root_priv)1289{1290 struct mlxsw_sp_acl_atcam_region *aregion = priv;1291 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;1292 1293 erp_table->ops->erp_destroy(erp_table, root_priv);1294}1295 1296static const struct objagg_ops mlxsw_sp_acl_erp_objagg_ops = {1297 .obj_size = sizeof(struct mlxsw_sp_acl_erp_key),1298 .delta_check = mlxsw_sp_acl_erp_delta_check,1299 .delta_create = mlxsw_sp_acl_erp_delta_create,1300 .delta_destroy = mlxsw_sp_acl_erp_delta_destroy,1301 .root_create = mlxsw_sp_acl_erp_root_create,1302 .root_destroy = mlxsw_sp_acl_erp_root_destroy,1303};1304 1305static struct mlxsw_sp_acl_erp_table *1306mlxsw_sp_acl_erp_table_create(struct mlxsw_sp_acl_atcam_region *aregion,1307 struct objagg_hints *hints)1308{1309 struct mlxsw_sp_acl_erp_table *erp_table;1310 int err;1311 1312 erp_table = kzalloc(sizeof(*erp_table), GFP_KERNEL);1313 if (!erp_table)1314 return ERR_PTR(-ENOMEM);1315 1316 erp_table->objagg = objagg_create(&mlxsw_sp_acl_erp_objagg_ops,1317 hints, aregion);1318 if (IS_ERR(erp_table->objagg)) {1319 err = PTR_ERR(erp_table->objagg);1320 goto err_objagg_create;1321 }1322 1323 erp_table->erp_core = aregion->atcam->erp_core;1324 erp_table->ops = &erp_no_mask_ops;1325 INIT_LIST_HEAD(&erp_table->atcam_erps_list);1326 erp_table->aregion = aregion;1327 mutex_init(&erp_table->objagg_lock);1328 1329 return erp_table;1330 1331err_objagg_create:1332 kfree(erp_table);1333 return ERR_PTR(err);1334}1335 1336static void1337mlxsw_sp_acl_erp_table_destroy(struct mlxsw_sp_acl_erp_table *erp_table)1338{1339 WARN_ON(!list_empty(&erp_table->atcam_erps_list));1340 mutex_destroy(&erp_table->objagg_lock);1341 objagg_destroy(erp_table->objagg);1342 kfree(erp_table);1343}1344 1345static int1346mlxsw_sp_acl_erp_master_mask_init(struct mlxsw_sp_acl_atcam_region *aregion)1347{1348 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;1349 char percr_pl[MLXSW_REG_PERCR_LEN];1350 1351 mlxsw_reg_percr_pack(percr_pl, aregion->region->id);1352 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl);1353}1354 1355static int1356mlxsw_sp_acl_erp_region_param_init(struct mlxsw_sp_acl_atcam_region *aregion)1357{1358 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;1359 char pererp_pl[MLXSW_REG_PERERP_LEN];1360 1361 mlxsw_reg_pererp_pack(pererp_pl, aregion->region->id, false, false, 0,1362 0, 0);1363 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl);1364}1365 1366static int1367mlxsw_sp_acl_erp_hints_check(struct mlxsw_sp *mlxsw_sp,1368 struct mlxsw_sp_acl_atcam_region *aregion,1369 struct objagg_hints *hints, bool *p_rehash_needed)1370{1371 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;1372 const struct objagg_stats *ostats;1373 const struct objagg_stats *hstats;1374 int err;1375 1376 *p_rehash_needed = false;1377 1378 mutex_lock(&erp_table->objagg_lock);1379 ostats = objagg_stats_get(erp_table->objagg);1380 mutex_unlock(&erp_table->objagg_lock);1381 if (IS_ERR(ostats)) {1382 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to get ERP stats\n");1383 return PTR_ERR(ostats);1384 }1385 1386 hstats = objagg_hints_stats_get(hints);1387 if (IS_ERR(hstats)) {1388 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to get ERP hints stats\n");1389 err = PTR_ERR(hstats);1390 goto err_hints_stats_get;1391 }1392 1393 /* Very basic criterion for now. */1394 if (hstats->root_count < ostats->root_count)1395 *p_rehash_needed = true;1396 1397 err = 0;1398 1399 objagg_stats_put(hstats);1400err_hints_stats_get:1401 objagg_stats_put(ostats);1402 return err;1403}1404 1405void *1406mlxsw_sp_acl_erp_rehash_hints_get(struct mlxsw_sp_acl_atcam_region *aregion)1407{1408 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table;1409 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;1410 struct objagg_hints *hints;1411 bool rehash_needed;1412 int err;1413 1414 mutex_lock(&erp_table->objagg_lock);1415 hints = objagg_hints_get(erp_table->objagg,1416 OBJAGG_OPT_ALGO_SIMPLE_GREEDY);1417 mutex_unlock(&erp_table->objagg_lock);1418 if (IS_ERR(hints)) {1419 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to create ERP hints\n");1420 return ERR_CAST(hints);1421 }1422 err = mlxsw_sp_acl_erp_hints_check(mlxsw_sp, aregion, hints,1423 &rehash_needed);1424 if (err)1425 goto errout;1426 1427 if (!rehash_needed) {1428 err = -EAGAIN;1429 goto errout;1430 }1431 return hints;1432 1433errout:1434 objagg_hints_put(hints);1435 return ERR_PTR(err);1436}1437 1438void mlxsw_sp_acl_erp_rehash_hints_put(void *hints_priv)1439{1440 struct objagg_hints *hints = hints_priv;1441 1442 objagg_hints_put(hints);1443}1444 1445int mlxsw_sp_acl_erp_region_init(struct mlxsw_sp_acl_atcam_region *aregion,1446 void *hints_priv)1447{1448 struct mlxsw_sp_acl_erp_table *erp_table;1449 struct objagg_hints *hints = hints_priv;1450 int err;1451 1452 erp_table = mlxsw_sp_acl_erp_table_create(aregion, hints);1453 if (IS_ERR(erp_table))1454 return PTR_ERR(erp_table);1455 aregion->erp_table = erp_table;1456 1457 /* Initialize the region's master mask to all zeroes */1458 err = mlxsw_sp_acl_erp_master_mask_init(aregion);1459 if (err)1460 goto err_erp_master_mask_init;1461 1462 /* Initialize the region to not use the eRP table */1463 err = mlxsw_sp_acl_erp_region_param_init(aregion);1464 if (err)1465 goto err_erp_region_param_init;1466 1467 return 0;1468 1469err_erp_region_param_init:1470err_erp_master_mask_init:1471 mlxsw_sp_acl_erp_table_destroy(erp_table);1472 return err;1473}1474 1475void mlxsw_sp_acl_erp_region_fini(struct mlxsw_sp_acl_atcam_region *aregion)1476{1477 mlxsw_sp_acl_erp_table_destroy(aregion->erp_table);1478}1479 1480static int1481mlxsw_sp_acl_erp_tables_sizes_query(struct mlxsw_sp *mlxsw_sp,1482 struct mlxsw_sp_acl_erp_core *erp_core)1483{1484 unsigned int size;1485 1486 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB) ||1487 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB) ||1488 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB) ||1489 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB))1490 return -EIO;1491 1492 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB);1493 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB] = size;1494 1495 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB);1496 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB] = size;1497 1498 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB);1499 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB] = size;1500 1501 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB);1502 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB] = size;1503 1504 return 0;1505}1506 1507static int mlxsw_sp_acl_erp_tables_init(struct mlxsw_sp *mlxsw_sp,1508 struct mlxsw_sp_acl_erp_core *erp_core)1509{1510 unsigned int erpt_bank_size;1511 int err;1512 1513 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANK_SIZE) ||1514 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANKS))1515 return -EIO;1516 erpt_bank_size = MLXSW_CORE_RES_GET(mlxsw_sp->core,1517 ACL_MAX_ERPT_BANK_SIZE);1518 erp_core->num_erp_banks = MLXSW_CORE_RES_GET(mlxsw_sp->core,1519 ACL_MAX_ERPT_BANKS);1520 1521 erp_core->erp_tables = gen_pool_create(0, -1);1522 if (!erp_core->erp_tables)1523 return -ENOMEM;1524 gen_pool_set_algo(erp_core->erp_tables, gen_pool_best_fit, NULL);1525 1526 err = gen_pool_add(erp_core->erp_tables,1527 MLXSW_SP_ACL_ERP_GENALLOC_OFFSET, erpt_bank_size,1528 -1);1529 if (err)1530 goto err_gen_pool_add;1531 1532 erp_core->bf = mlxsw_sp_acl_bf_init(mlxsw_sp, erp_core->num_erp_banks);1533 if (IS_ERR(erp_core->bf)) {1534 err = PTR_ERR(erp_core->bf);1535 goto err_bf_init;1536 }1537 1538 /* Different regions require masks of different sizes */1539 err = mlxsw_sp_acl_erp_tables_sizes_query(mlxsw_sp, erp_core);1540 if (err)1541 goto err_erp_tables_sizes_query;1542 1543 return 0;1544 1545err_erp_tables_sizes_query:1546 mlxsw_sp_acl_bf_fini(erp_core->bf);1547err_bf_init:1548err_gen_pool_add:1549 gen_pool_destroy(erp_core->erp_tables);1550 return err;1551}1552 1553static void mlxsw_sp_acl_erp_tables_fini(struct mlxsw_sp *mlxsw_sp,1554 struct mlxsw_sp_acl_erp_core *erp_core)1555{1556 mlxsw_sp_acl_bf_fini(erp_core->bf);1557 gen_pool_destroy(erp_core->erp_tables);1558}1559 1560int mlxsw_sp_acl_erps_init(struct mlxsw_sp *mlxsw_sp,1561 struct mlxsw_sp_acl_atcam *atcam)1562{1563 struct mlxsw_sp_acl_erp_core *erp_core;1564 int err;1565 1566 erp_core = kzalloc(sizeof(*erp_core), GFP_KERNEL);1567 if (!erp_core)1568 return -ENOMEM;1569 erp_core->mlxsw_sp = mlxsw_sp;1570 atcam->erp_core = erp_core;1571 1572 err = mlxsw_sp_acl_erp_tables_init(mlxsw_sp, erp_core);1573 if (err)1574 goto err_erp_tables_init;1575 1576 return 0;1577 1578err_erp_tables_init:1579 kfree(erp_core);1580 return err;1581}1582 1583void mlxsw_sp_acl_erps_fini(struct mlxsw_sp *mlxsw_sp,1584 struct mlxsw_sp_acl_atcam *atcam)1585{1586 mlxsw_sp_acl_erp_tables_fini(mlxsw_sp, atcam->erp_core);1587 kfree(atcam->erp_core);1588}1589