505 lines · c
1/* Broadcom NetXtreme-C/E network driver.2 *3 * Copyright (c) 2016-2018 Broadcom Limited4 *5 * This program is free software; you can redistribute it and/or modify6 * it under the terms of the GNU General Public License as published by7 * the Free Software Foundation.8 */9 10#include <linux/module.h>11 12#include <linux/kernel.h>13#include <linux/errno.h>14#include <linux/interrupt.h>15#include <linux/pci.h>16#include <linux/netdevice.h>17#include <linux/rtnetlink.h>18#include <linux/bitops.h>19#include <linux/irq.h>20#include <asm/byteorder.h>21#include <linux/bitmap.h>22#include <linux/auxiliary_bus.h>23 24#include "bnxt_hsi.h"25#include "bnxt.h"26#include "bnxt_hwrm.h"27#include "bnxt_ulp.h"28 29static DEFINE_IDA(bnxt_aux_dev_ids);30 31static void bnxt_fill_msix_vecs(struct bnxt *bp, struct bnxt_msix_entry *ent)32{33 struct bnxt_en_dev *edev = bp->edev;34 int num_msix, i;35 36 if (!edev->ulp_tbl->msix_requested) {37 netdev_warn(bp->dev, "Requested MSI-X vectors insufficient\n");38 return;39 }40 num_msix = edev->ulp_tbl->msix_requested;41 for (i = 0; i < num_msix; i++) {42 ent[i].vector = bp->irq_tbl[i].vector;43 ent[i].ring_idx = i;44 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS)45 ent[i].db_offset = bp->db_offset;46 else47 ent[i].db_offset = i * 0x80;48 }49}50 51int bnxt_get_ulp_msix_num(struct bnxt *bp)52{53 if (bp->edev)54 return bp->edev->ulp_num_msix_vec;55 return 0;56}57 58void bnxt_set_ulp_msix_num(struct bnxt *bp, int num)59{60 if (bp->edev)61 bp->edev->ulp_num_msix_vec = num;62}63 64int bnxt_get_ulp_msix_num_in_use(struct bnxt *bp)65{66 if (bnxt_ulp_registered(bp->edev))67 return bp->edev->ulp_num_msix_vec;68 return 0;69}70 71int bnxt_get_ulp_stat_ctxs(struct bnxt *bp)72{73 if (bp->edev)74 return bp->edev->ulp_num_ctxs;75 return 0;76}77 78void bnxt_set_ulp_stat_ctxs(struct bnxt *bp, int num_ulp_ctx)79{80 if (bp->edev)81 bp->edev->ulp_num_ctxs = num_ulp_ctx;82}83 84int bnxt_get_ulp_stat_ctxs_in_use(struct bnxt *bp)85{86 if (bnxt_ulp_registered(bp->edev))87 return bp->edev->ulp_num_ctxs;88 return 0;89}90 91void bnxt_set_dflt_ulp_stat_ctxs(struct bnxt *bp)92{93 if (bp->edev) {94 bp->edev->ulp_num_ctxs = BNXT_MIN_ROCE_STAT_CTXS;95 /* Reserve one additional stat_ctx for PF0 (except96 * on 1-port NICs) as it also creates one stat_ctx97 * for PF1 in case of RoCE bonding.98 */99 if (BNXT_PF(bp) && !bp->pf.port_id &&100 bp->port_count > 1)101 bp->edev->ulp_num_ctxs++;102 }103}104 105int bnxt_register_dev(struct bnxt_en_dev *edev,106 struct bnxt_ulp_ops *ulp_ops,107 void *handle)108{109 struct net_device *dev = edev->net;110 struct bnxt *bp = netdev_priv(dev);111 unsigned int max_stat_ctxs;112 struct bnxt_ulp *ulp;113 int rc = 0;114 115 rtnl_lock();116 mutex_lock(&edev->en_dev_lock);117 if (!bp->irq_tbl) {118 rc = -ENODEV;119 goto exit;120 }121 max_stat_ctxs = bnxt_get_max_func_stat_ctxs(bp);122 if (max_stat_ctxs <= BNXT_MIN_ROCE_STAT_CTXS ||123 bp->cp_nr_rings == max_stat_ctxs) {124 rc = -ENOMEM;125 goto exit;126 }127 128 ulp = edev->ulp_tbl;129 ulp->handle = handle;130 rcu_assign_pointer(ulp->ulp_ops, ulp_ops);131 132 if (test_bit(BNXT_STATE_OPEN, &bp->state))133 bnxt_hwrm_vnic_cfg(bp, &bp->vnic_info[BNXT_VNIC_DEFAULT]);134 135 edev->ulp_tbl->msix_requested = bnxt_get_ulp_msix_num(bp);136 137 bnxt_fill_msix_vecs(bp, bp->edev->msix_entries);138 edev->flags |= BNXT_EN_FLAG_MSIX_REQUESTED;139exit:140 mutex_unlock(&edev->en_dev_lock);141 rtnl_unlock();142 return rc;143}144EXPORT_SYMBOL(bnxt_register_dev);145 146void bnxt_unregister_dev(struct bnxt_en_dev *edev)147{148 struct net_device *dev = edev->net;149 struct bnxt *bp = netdev_priv(dev);150 struct bnxt_ulp *ulp;151 int i = 0;152 153 ulp = edev->ulp_tbl;154 rtnl_lock();155 mutex_lock(&edev->en_dev_lock);156 if (ulp->msix_requested)157 edev->flags &= ~BNXT_EN_FLAG_MSIX_REQUESTED;158 edev->ulp_tbl->msix_requested = 0;159 160 if (ulp->max_async_event_id)161 bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, true);162 163 RCU_INIT_POINTER(ulp->ulp_ops, NULL);164 synchronize_rcu();165 ulp->max_async_event_id = 0;166 ulp->async_events_bmap = NULL;167 while (atomic_read(&ulp->ref_count) != 0 && i < 10) {168 msleep(100);169 i++;170 }171 mutex_unlock(&edev->en_dev_lock);172 rtnl_unlock();173 return;174}175EXPORT_SYMBOL(bnxt_unregister_dev);176 177static int bnxt_set_dflt_ulp_msix(struct bnxt *bp)178{179 int roce_msix = BNXT_MAX_ROCE_MSIX;180 181 if (BNXT_VF(bp))182 roce_msix = BNXT_MAX_ROCE_MSIX_VF;183 else if (bp->port_partition_type)184 roce_msix = BNXT_MAX_ROCE_MSIX_NPAR_PF;185 186 /* NQ MSIX vectors should match the number of CPUs plus 1 more for187 * the CREQ MSIX, up to the default.188 */189 return min_t(int, roce_msix, num_online_cpus() + 1);190}191 192int bnxt_send_msg(struct bnxt_en_dev *edev,193 struct bnxt_fw_msg *fw_msg)194{195 struct net_device *dev = edev->net;196 struct bnxt *bp = netdev_priv(dev);197 struct output *resp;198 struct input *req;199 u32 resp_len;200 int rc;201 202 if (bp->fw_reset_state)203 return -EBUSY;204 205 rc = hwrm_req_init(bp, req, 0 /* don't care */);206 if (rc)207 return rc;208 209 rc = hwrm_req_replace(bp, req, fw_msg->msg, fw_msg->msg_len);210 if (rc)211 return rc;212 213 hwrm_req_timeout(bp, req, fw_msg->timeout);214 resp = hwrm_req_hold(bp, req);215 rc = hwrm_req_send(bp, req);216 resp_len = le16_to_cpu(resp->resp_len);217 if (resp_len) {218 if (fw_msg->resp_max_len < resp_len)219 resp_len = fw_msg->resp_max_len;220 221 memcpy(fw_msg->resp, resp, resp_len);222 }223 hwrm_req_drop(bp, req);224 return rc;225}226EXPORT_SYMBOL(bnxt_send_msg);227 228void bnxt_ulp_stop(struct bnxt *bp)229{230 struct bnxt_aux_priv *aux_priv = bp->aux_priv;231 struct bnxt_en_dev *edev = bp->edev;232 233 if (!edev)234 return;235 236 mutex_lock(&edev->en_dev_lock);237 if (!bnxt_ulp_registered(edev)) {238 mutex_unlock(&edev->en_dev_lock);239 return;240 }241 242 edev->flags |= BNXT_EN_FLAG_ULP_STOPPED;243 if (aux_priv) {244 struct auxiliary_device *adev;245 246 adev = &aux_priv->aux_dev;247 if (adev->dev.driver) {248 const struct auxiliary_driver *adrv;249 pm_message_t pm = {};250 251 adrv = to_auxiliary_drv(adev->dev.driver);252 edev->en_state = bp->state;253 adrv->suspend(adev, pm);254 }255 }256 mutex_unlock(&edev->en_dev_lock);257}258 259void bnxt_ulp_start(struct bnxt *bp, int err)260{261 struct bnxt_aux_priv *aux_priv = bp->aux_priv;262 struct bnxt_en_dev *edev = bp->edev;263 264 if (!edev)265 return;266 267 edev->flags &= ~BNXT_EN_FLAG_ULP_STOPPED;268 269 if (err)270 return;271 272 mutex_lock(&edev->en_dev_lock);273 if (!bnxt_ulp_registered(edev)) {274 mutex_unlock(&edev->en_dev_lock);275 return;276 }277 278 if (edev->ulp_tbl->msix_requested)279 bnxt_fill_msix_vecs(bp, edev->msix_entries);280 281 if (aux_priv) {282 struct auxiliary_device *adev;283 284 adev = &aux_priv->aux_dev;285 if (adev->dev.driver) {286 const struct auxiliary_driver *adrv;287 288 adrv = to_auxiliary_drv(adev->dev.driver);289 edev->en_state = bp->state;290 adrv->resume(adev);291 }292 }293 mutex_unlock(&edev->en_dev_lock);294}295 296void bnxt_ulp_irq_stop(struct bnxt *bp)297{298 struct bnxt_en_dev *edev = bp->edev;299 struct bnxt_ulp_ops *ops;300 301 if (!edev || !(edev->flags & BNXT_EN_FLAG_MSIX_REQUESTED))302 return;303 304 if (bnxt_ulp_registered(bp->edev)) {305 struct bnxt_ulp *ulp = edev->ulp_tbl;306 307 if (!ulp->msix_requested)308 return;309 310 ops = rtnl_dereference(ulp->ulp_ops);311 if (!ops || !ops->ulp_irq_stop)312 return;313 ops->ulp_irq_stop(ulp->handle);314 }315}316 317void bnxt_ulp_irq_restart(struct bnxt *bp, int err)318{319 struct bnxt_en_dev *edev = bp->edev;320 struct bnxt_ulp_ops *ops;321 322 if (!edev || !(edev->flags & BNXT_EN_FLAG_MSIX_REQUESTED))323 return;324 325 if (bnxt_ulp_registered(bp->edev)) {326 struct bnxt_ulp *ulp = edev->ulp_tbl;327 struct bnxt_msix_entry *ent = NULL;328 329 if (!ulp->msix_requested)330 return;331 332 ops = rtnl_dereference(ulp->ulp_ops);333 if (!ops || !ops->ulp_irq_restart)334 return;335 336 if (!err) {337 ent = kcalloc(ulp->msix_requested, sizeof(*ent),338 GFP_KERNEL);339 if (!ent)340 return;341 bnxt_fill_msix_vecs(bp, ent);342 }343 ops->ulp_irq_restart(ulp->handle, ent);344 kfree(ent);345 }346}347 348int bnxt_register_async_events(struct bnxt_en_dev *edev,349 unsigned long *events_bmap,350 u16 max_id)351{352 struct net_device *dev = edev->net;353 struct bnxt *bp = netdev_priv(dev);354 struct bnxt_ulp *ulp;355 356 ulp = edev->ulp_tbl;357 ulp->async_events_bmap = events_bmap;358 /* Make sure bnxt_ulp_async_events() sees this order */359 smp_wmb();360 ulp->max_async_event_id = max_id;361 bnxt_hwrm_func_drv_rgtr(bp, events_bmap, max_id + 1, true);362 return 0;363}364EXPORT_SYMBOL(bnxt_register_async_events);365 366void bnxt_rdma_aux_device_uninit(struct bnxt *bp)367{368 struct bnxt_aux_priv *aux_priv;369 struct auxiliary_device *adev;370 371 /* Skip if no auxiliary device init was done. */372 if (!bp->aux_priv)373 return;374 375 aux_priv = bp->aux_priv;376 adev = &aux_priv->aux_dev;377 auxiliary_device_uninit(adev);378}379 380static void bnxt_aux_dev_release(struct device *dev)381{382 struct bnxt_aux_priv *aux_priv =383 container_of(dev, struct bnxt_aux_priv, aux_dev.dev);384 struct bnxt *bp = netdev_priv(aux_priv->edev->net);385 386 ida_free(&bnxt_aux_dev_ids, aux_priv->id);387 kfree(aux_priv->edev->ulp_tbl);388 bp->edev = NULL;389 kfree(aux_priv->edev);390 kfree(aux_priv);391 bp->aux_priv = NULL;392}393 394void bnxt_rdma_aux_device_del(struct bnxt *bp)395{396 if (!bp->edev)397 return;398 399 auxiliary_device_delete(&bp->aux_priv->aux_dev);400}401 402static void bnxt_set_edev_info(struct bnxt_en_dev *edev, struct bnxt *bp)403{404 edev->net = bp->dev;405 edev->pdev = bp->pdev;406 edev->l2_db_size = bp->db_size;407 edev->l2_db_size_nc = bp->db_size;408 edev->l2_db_offset = bp->db_offset;409 mutex_init(&edev->en_dev_lock);410 411 if (bp->flags & BNXT_FLAG_ROCEV1_CAP)412 edev->flags |= BNXT_EN_FLAG_ROCEV1_CAP;413 if (bp->flags & BNXT_FLAG_ROCEV2_CAP)414 edev->flags |= BNXT_EN_FLAG_ROCEV2_CAP;415 if (bp->flags & BNXT_FLAG_VF)416 edev->flags |= BNXT_EN_FLAG_VF;417 418 edev->chip_num = bp->chip_num;419 edev->hw_ring_stats_size = bp->hw_ring_stats_size;420 edev->pf_port_id = bp->pf.port_id;421 edev->en_state = bp->state;422 edev->bar0 = bp->bar0;423}424 425void bnxt_rdma_aux_device_add(struct bnxt *bp)426{427 struct auxiliary_device *aux_dev;428 int rc;429 430 if (!bp->edev)431 return;432 433 aux_dev = &bp->aux_priv->aux_dev;434 rc = auxiliary_device_add(aux_dev);435 if (rc) {436 netdev_warn(bp->dev, "Failed to add auxiliary device for ROCE\n");437 auxiliary_device_uninit(aux_dev);438 bp->flags &= ~BNXT_FLAG_ROCE_CAP;439 }440}441 442void bnxt_rdma_aux_device_init(struct bnxt *bp)443{444 struct auxiliary_device *aux_dev;445 struct bnxt_aux_priv *aux_priv;446 struct bnxt_en_dev *edev;447 struct bnxt_ulp *ulp;448 int rc;449 450 if (!(bp->flags & BNXT_FLAG_ROCE_CAP))451 return;452 453 aux_priv = kzalloc(sizeof(*bp->aux_priv), GFP_KERNEL);454 if (!aux_priv)455 goto exit;456 457 aux_priv->id = ida_alloc(&bnxt_aux_dev_ids, GFP_KERNEL);458 if (aux_priv->id < 0) {459 netdev_warn(bp->dev,460 "ida alloc failed for ROCE auxiliary device\n");461 kfree(aux_priv);462 goto exit;463 }464 465 aux_dev = &aux_priv->aux_dev;466 aux_dev->id = aux_priv->id;467 aux_dev->name = "rdma";468 aux_dev->dev.parent = &bp->pdev->dev;469 aux_dev->dev.release = bnxt_aux_dev_release;470 471 rc = auxiliary_device_init(aux_dev);472 if (rc) {473 ida_free(&bnxt_aux_dev_ids, aux_priv->id);474 kfree(aux_priv);475 goto exit;476 }477 bp->aux_priv = aux_priv;478 479 /* From this point, all cleanup will happen via the .release callback &480 * any error unwinding will need to include a call to481 * auxiliary_device_uninit.482 */483 edev = kzalloc(sizeof(*edev), GFP_KERNEL);484 if (!edev)485 goto aux_dev_uninit;486 487 aux_priv->edev = edev;488 489 ulp = kzalloc(sizeof(*ulp), GFP_KERNEL);490 if (!ulp)491 goto aux_dev_uninit;492 493 edev->ulp_tbl = ulp;494 bp->edev = edev;495 bnxt_set_edev_info(edev, bp);496 bp->ulp_num_msix_want = bnxt_set_dflt_ulp_msix(bp);497 498 return;499 500aux_dev_uninit:501 auxiliary_device_uninit(aux_dev);502exit:503 bp->flags &= ~BNXT_FLAG_ROCE_CAP;504}505