558 lines · c
1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)2/* Copyright (C) 2018 Netronome Systems, Inc. */3 4#include <linux/bitfield.h>5#include <linux/bitmap.h>6#include <linux/etherdevice.h>7#include <linux/lockdep.h>8#include <linux/netdevice.h>9#include <linux/rcupdate.h>10#include <linux/rtnetlink.h>11#include <linux/slab.h>12 13#include "../nfpcore/nfp.h"14#include "../nfpcore/nfp_cpp.h"15#include "../nfpcore/nfp_nsp.h"16#include "../nfp_app.h"17#include "../nfp_main.h"18#include "../nfp_net.h"19#include "../nfp_net_repr.h"20#include "../nfp_port.h"21#include "main.h"22 23static u32 nfp_abm_portid(enum nfp_repr_type rtype, unsigned int id)24{25 return FIELD_PREP(NFP_ABM_PORTID_TYPE, rtype) |26 FIELD_PREP(NFP_ABM_PORTID_ID, id);27}28 29static int30nfp_abm_setup_tc(struct nfp_app *app, struct net_device *netdev,31 enum tc_setup_type type, void *type_data)32{33 struct nfp_repr *repr = netdev_priv(netdev);34 struct nfp_port *port;35 36 port = nfp_port_from_netdev(netdev);37 if (!port || port->type != NFP_PORT_PF_PORT)38 return -EOPNOTSUPP;39 40 switch (type) {41 case TC_SETUP_ROOT_QDISC:42 return nfp_abm_setup_root(netdev, repr->app_priv, type_data);43 case TC_SETUP_QDISC_MQ:44 return nfp_abm_setup_tc_mq(netdev, repr->app_priv, type_data);45 case TC_SETUP_QDISC_RED:46 return nfp_abm_setup_tc_red(netdev, repr->app_priv, type_data);47 case TC_SETUP_QDISC_GRED:48 return nfp_abm_setup_tc_gred(netdev, repr->app_priv, type_data);49 case TC_SETUP_BLOCK:50 return nfp_abm_setup_cls_block(netdev, repr, type_data);51 default:52 return -EOPNOTSUPP;53 }54}55 56static struct net_device *57nfp_abm_repr_get(struct nfp_app *app, u32 port_id, bool *redir_egress)58{59 enum nfp_repr_type rtype;60 struct nfp_reprs *reprs;61 u8 port;62 63 rtype = FIELD_GET(NFP_ABM_PORTID_TYPE, port_id);64 port = FIELD_GET(NFP_ABM_PORTID_ID, port_id);65 66 reprs = rcu_dereference(app->reprs[rtype]);67 if (!reprs)68 return NULL;69 70 if (port >= reprs->num_reprs)71 return NULL;72 73 return rcu_dereference(reprs->reprs[port]);74}75 76static int77nfp_abm_spawn_repr(struct nfp_app *app, struct nfp_abm_link *alink,78 enum nfp_port_type ptype)79{80 struct net_device *netdev;81 enum nfp_repr_type rtype;82 struct nfp_reprs *reprs;83 struct nfp_repr *repr;84 struct nfp_port *port;85 unsigned int txqs;86 int err;87 88 if (ptype == NFP_PORT_PHYS_PORT) {89 rtype = NFP_REPR_TYPE_PHYS_PORT;90 txqs = 1;91 } else {92 rtype = NFP_REPR_TYPE_PF;93 txqs = alink->vnic->max_rx_rings;94 }95 96 netdev = nfp_repr_alloc_mqs(app, txqs, 1);97 if (!netdev)98 return -ENOMEM;99 repr = netdev_priv(netdev);100 repr->app_priv = alink;101 102 port = nfp_port_alloc(app, ptype, netdev);103 if (IS_ERR(port)) {104 err = PTR_ERR(port);105 goto err_free_repr;106 }107 108 if (ptype == NFP_PORT_PHYS_PORT) {109 port->eth_forced = true;110 err = nfp_port_init_phy_port(app->pf, app, port, alink->id);111 if (err)112 goto err_free_port;113 } else {114 port->pf_id = alink->abm->pf_id;115 port->pf_split = app->pf->max_data_vnics > 1;116 port->pf_split_id = alink->id;117 port->vnic = alink->vnic->dp.ctrl_bar;118 }119 120 SET_NETDEV_DEV(netdev, &alink->vnic->pdev->dev);121 eth_hw_addr_random(netdev);122 123 err = nfp_repr_init(app, netdev, nfp_abm_portid(rtype, alink->id),124 port, alink->vnic->dp.netdev);125 if (err)126 goto err_free_port;127 128 reprs = nfp_reprs_get_locked(app, rtype);129 WARN(nfp_repr_get_locked(app, reprs, alink->id), "duplicate repr");130 rtnl_lock();131 rcu_assign_pointer(reprs->reprs[alink->id], netdev);132 rtnl_unlock();133 134 nfp_info(app->cpp, "%s Port %d Representor(%s) created\n",135 ptype == NFP_PORT_PF_PORT ? "PCIe" : "Phys",136 alink->id, netdev->name);137 138 return 0;139 140err_free_port:141 nfp_port_free(port);142err_free_repr:143 nfp_repr_free(netdev);144 return err;145}146 147static void148nfp_abm_kill_repr(struct nfp_app *app, struct nfp_abm_link *alink,149 enum nfp_repr_type rtype)150{151 struct net_device *netdev;152 struct nfp_reprs *reprs;153 154 reprs = nfp_reprs_get_locked(app, rtype);155 netdev = nfp_repr_get_locked(app, reprs, alink->id);156 if (!netdev)157 return;158 rtnl_lock();159 rcu_assign_pointer(reprs->reprs[alink->id], NULL);160 rtnl_unlock();161 synchronize_rcu();162 /* Cast to make sure nfp_repr_clean_and_free() takes a nfp_repr */163 nfp_repr_clean_and_free((struct nfp_repr *)netdev_priv(netdev));164}165 166static void167nfp_abm_kill_reprs(struct nfp_abm *abm, struct nfp_abm_link *alink)168{169 nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PF);170 nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PHYS_PORT);171}172 173static void nfp_abm_kill_reprs_all(struct nfp_abm *abm)174{175 struct nfp_pf *pf = abm->app->pf;176 struct nfp_net *nn;177 178 list_for_each_entry(nn, &pf->vnics, vnic_list)179 nfp_abm_kill_reprs(abm, (struct nfp_abm_link *)nn->app_priv);180}181 182static enum devlink_eswitch_mode nfp_abm_eswitch_mode_get(struct nfp_app *app)183{184 struct nfp_abm *abm = app->priv;185 186 return abm->eswitch_mode;187}188 189static int nfp_abm_eswitch_set_legacy(struct nfp_abm *abm)190{191 nfp_abm_kill_reprs_all(abm);192 nfp_abm_ctrl_qm_disable(abm);193 194 abm->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY;195 return 0;196}197 198static void nfp_abm_eswitch_clean_up(struct nfp_abm *abm)199{200 if (abm->eswitch_mode != DEVLINK_ESWITCH_MODE_LEGACY)201 WARN_ON(nfp_abm_eswitch_set_legacy(abm));202}203 204static int nfp_abm_eswitch_set_switchdev(struct nfp_abm *abm)205{206 struct nfp_app *app = abm->app;207 struct nfp_pf *pf = app->pf;208 struct nfp_net *nn;209 int err;210 211 if (!abm->red_support)212 return -EOPNOTSUPP;213 214 err = nfp_abm_ctrl_qm_enable(abm);215 if (err)216 return err;217 218 list_for_each_entry(nn, &pf->vnics, vnic_list) {219 struct nfp_abm_link *alink = nn->app_priv;220 221 err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PHYS_PORT);222 if (err)223 goto err_kill_all_reprs;224 225 err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PF_PORT);226 if (err)227 goto err_kill_all_reprs;228 }229 230 abm->eswitch_mode = DEVLINK_ESWITCH_MODE_SWITCHDEV;231 return 0;232 233err_kill_all_reprs:234 nfp_abm_kill_reprs_all(abm);235 nfp_abm_ctrl_qm_disable(abm);236 return err;237}238 239static int nfp_abm_eswitch_mode_set(struct nfp_app *app, u16 mode)240{241 struct nfp_abm *abm = app->priv;242 243 if (abm->eswitch_mode == mode)244 return 0;245 246 switch (mode) {247 case DEVLINK_ESWITCH_MODE_LEGACY:248 return nfp_abm_eswitch_set_legacy(abm);249 case DEVLINK_ESWITCH_MODE_SWITCHDEV:250 return nfp_abm_eswitch_set_switchdev(abm);251 default:252 return -EINVAL;253 }254}255 256static void257nfp_abm_vnic_set_mac(struct nfp_pf *pf, struct nfp_abm *abm, struct nfp_net *nn,258 unsigned int id)259{260 struct nfp_eth_table_port *eth_port = &pf->eth_tbl->ports[id];261 u8 mac_addr[ETH_ALEN];262 struct nfp_nsp *nsp;263 char hwinfo[32];264 int err;265 266 if (id > pf->eth_tbl->count) {267 nfp_warn(pf->cpp, "No entry for persistent MAC address\n");268 eth_hw_addr_random(nn->dp.netdev);269 return;270 }271 272 snprintf(hwinfo, sizeof(hwinfo), "eth%u.mac.pf%u",273 eth_port->eth_index, abm->pf_id);274 275 nsp = nfp_nsp_open(pf->cpp);276 if (IS_ERR(nsp)) {277 nfp_warn(pf->cpp, "Failed to access the NSP for persistent MAC address: %ld\n",278 PTR_ERR(nsp));279 eth_hw_addr_random(nn->dp.netdev);280 return;281 }282 283 if (!nfp_nsp_has_hwinfo_lookup(nsp)) {284 nfp_warn(pf->cpp, "NSP doesn't support PF MAC generation\n");285 eth_hw_addr_random(nn->dp.netdev);286 nfp_nsp_close(nsp);287 return;288 }289 290 err = nfp_nsp_hwinfo_lookup(nsp, hwinfo, sizeof(hwinfo));291 nfp_nsp_close(nsp);292 if (err) {293 nfp_warn(pf->cpp, "Reading persistent MAC address failed: %d\n",294 err);295 eth_hw_addr_random(nn->dp.netdev);296 return;297 }298 299 if (sscanf(hwinfo, "%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx",300 &mac_addr[0], &mac_addr[1], &mac_addr[2],301 &mac_addr[3], &mac_addr[4], &mac_addr[5]) != 6) {302 nfp_warn(pf->cpp, "Can't parse persistent MAC address (%s)\n",303 hwinfo);304 eth_hw_addr_random(nn->dp.netdev);305 return;306 }307 308 eth_hw_addr_set(nn->dp.netdev, mac_addr);309 ether_addr_copy(nn->dp.netdev->perm_addr, mac_addr);310}311 312static int313nfp_abm_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id)314{315 struct nfp_eth_table_port *eth_port = &app->pf->eth_tbl->ports[id];316 struct nfp_abm *abm = app->priv;317 struct nfp_abm_link *alink;318 int err;319 320 alink = kzalloc(sizeof(*alink), GFP_KERNEL);321 if (!alink)322 return -ENOMEM;323 nn->app_priv = alink;324 alink->abm = abm;325 alink->vnic = nn;326 alink->id = id;327 alink->total_queues = alink->vnic->max_rx_rings;328 329 INIT_LIST_HEAD(&alink->dscp_map);330 331 err = nfp_abm_ctrl_read_params(alink);332 if (err)333 goto err_free_alink;334 335 alink->prio_map = kzalloc(abm->prio_map_len, GFP_KERNEL);336 if (!alink->prio_map) {337 err = -ENOMEM;338 goto err_free_alink;339 }340 341 /* This is a multi-host app, make sure MAC/PHY is up, but don't342 * make the MAC/PHY state follow the state of any of the ports.343 */344 err = nfp_eth_set_configured(app->cpp, eth_port->index, true);345 if (err < 0)346 goto err_free_priomap;347 348 netif_keep_dst(nn->dp.netdev);349 350 nfp_abm_vnic_set_mac(app->pf, abm, nn, id);351 INIT_RADIX_TREE(&alink->qdiscs, GFP_KERNEL);352 353 return 0;354 355err_free_priomap:356 kfree(alink->prio_map);357err_free_alink:358 kfree(alink);359 return err;360}361 362static void nfp_abm_vnic_free(struct nfp_app *app, struct nfp_net *nn)363{364 struct nfp_abm_link *alink = nn->app_priv;365 366 nfp_abm_kill_reprs(alink->abm, alink);367 WARN(!radix_tree_empty(&alink->qdiscs), "left over qdiscs\n");368 kfree(alink->prio_map);369 kfree(alink);370}371 372static int nfp_abm_vnic_init(struct nfp_app *app, struct nfp_net *nn)373{374 struct nfp_abm_link *alink = nn->app_priv;375 376 if (nfp_abm_has_prio(alink->abm))377 return nfp_abm_ctrl_prio_map_update(alink, alink->prio_map);378 return 0;379}380 381static u64 *382nfp_abm_port_get_stats(struct nfp_app *app, struct nfp_port *port, u64 *data)383{384 struct nfp_repr *repr = netdev_priv(port->netdev);385 struct nfp_abm_link *alink;386 unsigned int i;387 388 if (port->type != NFP_PORT_PF_PORT)389 return data;390 alink = repr->app_priv;391 for (i = 0; i < alink->vnic->dp.num_r_vecs; i++) {392 *data++ = nfp_abm_ctrl_stat_non_sto(alink, i);393 *data++ = nfp_abm_ctrl_stat_sto(alink, i);394 }395 return data;396}397 398static int399nfp_abm_port_get_stats_count(struct nfp_app *app, struct nfp_port *port)400{401 struct nfp_repr *repr = netdev_priv(port->netdev);402 struct nfp_abm_link *alink;403 404 if (port->type != NFP_PORT_PF_PORT)405 return 0;406 alink = repr->app_priv;407 return alink->vnic->dp.num_r_vecs * 2;408}409 410static u8 *411nfp_abm_port_get_stats_strings(struct nfp_app *app, struct nfp_port *port,412 u8 *data)413{414 struct nfp_repr *repr = netdev_priv(port->netdev);415 struct nfp_abm_link *alink;416 unsigned int i;417 418 if (port->type != NFP_PORT_PF_PORT)419 return data;420 alink = repr->app_priv;421 for (i = 0; i < alink->vnic->dp.num_r_vecs; i++) {422 ethtool_sprintf(&data, "q%u_no_wait", i);423 ethtool_sprintf(&data, "q%u_delayed", i);424 }425 return data;426}427 428static int nfp_abm_fw_init_reset(struct nfp_abm *abm)429{430 unsigned int i;431 432 if (!abm->red_support)433 return 0;434 435 for (i = 0; i < abm->num_bands * NFP_NET_MAX_RX_RINGS; i++) {436 __nfp_abm_ctrl_set_q_lvl(abm, i, NFP_ABM_LVL_INFINITY);437 __nfp_abm_ctrl_set_q_act(abm, i, NFP_ABM_ACT_DROP);438 }439 440 return nfp_abm_ctrl_qm_disable(abm);441}442 443static int nfp_abm_init(struct nfp_app *app)444{445 struct nfp_pf *pf = app->pf;446 struct nfp_reprs *reprs;447 struct nfp_abm *abm;448 int err;449 450 if (!pf->eth_tbl) {451 nfp_err(pf->cpp, "ABM NIC requires ETH table\n");452 return -EINVAL;453 }454 if (pf->max_data_vnics != pf->eth_tbl->count) {455 nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n",456 pf->max_data_vnics, pf->eth_tbl->count);457 return -EINVAL;458 }459 if (!pf->mac_stats_bar) {460 nfp_warn(app->cpp, "ABM NIC requires mac_stats symbol\n");461 return -EINVAL;462 }463 464 abm = kzalloc(sizeof(*abm), GFP_KERNEL);465 if (!abm)466 return -ENOMEM;467 app->priv = abm;468 abm->app = app;469 470 err = nfp_abm_ctrl_find_addrs(abm);471 if (err)472 goto err_free_abm;473 474 err = -ENOMEM;475 abm->num_thresholds = array_size(abm->num_bands, NFP_NET_MAX_RX_RINGS);476 abm->threshold_undef = bitmap_zalloc(abm->num_thresholds, GFP_KERNEL);477 if (!abm->threshold_undef)478 goto err_free_abm;479 480 abm->thresholds = kvcalloc(abm->num_thresholds,481 sizeof(*abm->thresholds), GFP_KERNEL);482 if (!abm->thresholds)483 goto err_free_thresh_umap;484 485 abm->actions = kvcalloc(abm->num_thresholds, sizeof(*abm->actions),486 GFP_KERNEL);487 if (!abm->actions)488 goto err_free_thresh;489 490 /* We start in legacy mode, make sure advanced queuing is disabled */491 err = nfp_abm_fw_init_reset(abm);492 if (err)493 goto err_free_act;494 495 err = -ENOMEM;496 reprs = nfp_reprs_alloc(pf->max_data_vnics);497 if (!reprs)498 goto err_free_act;499 RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PHYS_PORT], reprs);500 501 reprs = nfp_reprs_alloc(pf->max_data_vnics);502 if (!reprs)503 goto err_free_phys;504 RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PF], reprs);505 506 return 0;507 508err_free_phys:509 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT);510err_free_act:511 kvfree(abm->actions);512err_free_thresh:513 kvfree(abm->thresholds);514err_free_thresh_umap:515 bitmap_free(abm->threshold_undef);516err_free_abm:517 kfree(abm);518 app->priv = NULL;519 return err;520}521 522static void nfp_abm_clean(struct nfp_app *app)523{524 struct nfp_abm *abm = app->priv;525 526 nfp_abm_eswitch_clean_up(abm);527 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PF);528 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT);529 bitmap_free(abm->threshold_undef);530 kvfree(abm->actions);531 kvfree(abm->thresholds);532 kfree(abm);533 app->priv = NULL;534}535 536const struct nfp_app_type app_abm = {537 .id = NFP_APP_ACTIVE_BUFFER_MGMT_NIC,538 .name = "abm",539 540 .init = nfp_abm_init,541 .clean = nfp_abm_clean,542 543 .vnic_alloc = nfp_abm_vnic_alloc,544 .vnic_free = nfp_abm_vnic_free,545 .vnic_init = nfp_abm_vnic_init,546 547 .port_get_stats = nfp_abm_port_get_stats,548 .port_get_stats_count = nfp_abm_port_get_stats_count,549 .port_get_stats_strings = nfp_abm_port_get_stats_strings,550 551 .setup_tc = nfp_abm_setup_tc,552 553 .eswitch_mode_get = nfp_abm_eswitch_mode_get,554 .eswitch_mode_set = nfp_abm_eswitch_mode_set,555 556 .dev_get = nfp_abm_repr_get,557};558