488 lines · c
1// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)2/*3 * Copyright(c) 2020 Intel Corporation.4 *5 */6 7/*8 * This file contains HFI1 support for netdev RX functionality9 */10 11#include "sdma.h"12#include "verbs.h"13#include "netdev.h"14#include "hfi.h"15 16#include <linux/netdevice.h>17#include <linux/etherdevice.h>18#include <rdma/ib_verbs.h>19 20static int hfi1_netdev_setup_ctxt(struct hfi1_netdev_rx *rx,21 struct hfi1_ctxtdata *uctxt)22{23 unsigned int rcvctrl_ops;24 struct hfi1_devdata *dd = rx->dd;25 int ret;26 27 uctxt->rhf_rcv_function_map = netdev_rhf_rcv_functions;28 uctxt->do_interrupt = &handle_receive_interrupt_napi_sp;29 30 /* Now allocate the RcvHdr queue and eager buffers. */31 ret = hfi1_create_rcvhdrq(dd, uctxt);32 if (ret)33 goto done;34 35 ret = hfi1_setup_eagerbufs(uctxt);36 if (ret)37 goto done;38 39 clear_rcvhdrtail(uctxt);40 41 rcvctrl_ops = HFI1_RCVCTRL_CTXT_DIS;42 rcvctrl_ops |= HFI1_RCVCTRL_INTRAVAIL_DIS;43 44 if (!HFI1_CAP_KGET_MASK(uctxt->flags, MULTI_PKT_EGR))45 rcvctrl_ops |= HFI1_RCVCTRL_ONE_PKT_EGR_ENB;46 if (HFI1_CAP_KGET_MASK(uctxt->flags, NODROP_EGR_FULL))47 rcvctrl_ops |= HFI1_RCVCTRL_NO_EGR_DROP_ENB;48 if (HFI1_CAP_KGET_MASK(uctxt->flags, NODROP_RHQ_FULL))49 rcvctrl_ops |= HFI1_RCVCTRL_NO_RHQ_DROP_ENB;50 if (HFI1_CAP_KGET_MASK(uctxt->flags, DMA_RTAIL))51 rcvctrl_ops |= HFI1_RCVCTRL_TAILUPD_ENB;52 53 hfi1_rcvctrl(uctxt->dd, rcvctrl_ops, uctxt);54done:55 return ret;56}57 58static int hfi1_netdev_allocate_ctxt(struct hfi1_devdata *dd,59 struct hfi1_ctxtdata **ctxt)60{61 struct hfi1_ctxtdata *uctxt;62 int ret;63 64 if (dd->flags & HFI1_FROZEN)65 return -EIO;66 67 ret = hfi1_create_ctxtdata(dd->pport, dd->node, &uctxt);68 if (ret < 0) {69 dd_dev_err(dd, "Unable to create ctxtdata, failing open\n");70 return -ENOMEM;71 }72 73 uctxt->flags = HFI1_CAP_KGET(MULTI_PKT_EGR) |74 HFI1_CAP_KGET(NODROP_RHQ_FULL) |75 HFI1_CAP_KGET(NODROP_EGR_FULL) |76 HFI1_CAP_KGET(DMA_RTAIL);77 /* Netdev contexts are always NO_RDMA_RTAIL */78 uctxt->fast_handler = handle_receive_interrupt_napi_fp;79 uctxt->slow_handler = handle_receive_interrupt_napi_sp;80 hfi1_set_seq_cnt(uctxt, 1);81 uctxt->is_vnic = true;82 83 hfi1_stats.sps_ctxts++;84 85 dd_dev_info(dd, "created netdev context %d\n", uctxt->ctxt);86 *ctxt = uctxt;87 88 return 0;89}90 91static void hfi1_netdev_deallocate_ctxt(struct hfi1_devdata *dd,92 struct hfi1_ctxtdata *uctxt)93{94 flush_wc();95 96 /*97 * Disable receive context and interrupt available, reset all98 * RcvCtxtCtrl bits to default values.99 */100 hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS |101 HFI1_RCVCTRL_TIDFLOW_DIS |102 HFI1_RCVCTRL_INTRAVAIL_DIS |103 HFI1_RCVCTRL_ONE_PKT_EGR_DIS |104 HFI1_RCVCTRL_NO_RHQ_DROP_DIS |105 HFI1_RCVCTRL_NO_EGR_DROP_DIS, uctxt);106 107 if (uctxt->msix_intr != CCE_NUM_MSIX_VECTORS)108 msix_free_irq(dd, uctxt->msix_intr);109 110 uctxt->msix_intr = CCE_NUM_MSIX_VECTORS;111 uctxt->event_flags = 0;112 113 hfi1_clear_tids(uctxt);114 hfi1_clear_ctxt_pkey(dd, uctxt);115 116 hfi1_stats.sps_ctxts--;117 118 hfi1_free_ctxt(uctxt);119}120 121static int hfi1_netdev_allot_ctxt(struct hfi1_netdev_rx *rx,122 struct hfi1_ctxtdata **ctxt)123{124 int rc;125 struct hfi1_devdata *dd = rx->dd;126 127 rc = hfi1_netdev_allocate_ctxt(dd, ctxt);128 if (rc) {129 dd_dev_err(dd, "netdev ctxt alloc failed %d\n", rc);130 return rc;131 }132 133 rc = hfi1_netdev_setup_ctxt(rx, *ctxt);134 if (rc) {135 dd_dev_err(dd, "netdev ctxt setup failed %d\n", rc);136 hfi1_netdev_deallocate_ctxt(dd, *ctxt);137 *ctxt = NULL;138 }139 140 return rc;141}142 143/**144 * hfi1_num_netdev_contexts - Count of netdev recv contexts to use.145 * @dd: device on which to allocate netdev contexts146 * @available_contexts: count of available receive contexts147 * @cpu_mask: mask of possible cpus to include for contexts148 *149 * Return: count of physical cores on a node or the remaining available recv150 * contexts for netdev recv context usage up to the maximum of151 * HFI1_MAX_NETDEV_CTXTS.152 * A value of 0 can be returned when acceleration is explicitly turned off,153 * a memory allocation error occurs or when there are no available contexts.154 *155 */156u32 hfi1_num_netdev_contexts(struct hfi1_devdata *dd, u32 available_contexts,157 struct cpumask *cpu_mask)158{159 cpumask_var_t node_cpu_mask;160 unsigned int available_cpus;161 162 if (!HFI1_CAP_IS_KSET(AIP))163 return 0;164 165 /* Always give user contexts priority over netdev contexts */166 if (available_contexts == 0) {167 dd_dev_info(dd, "No receive contexts available for netdevs.\n");168 return 0;169 }170 171 if (!zalloc_cpumask_var(&node_cpu_mask, GFP_KERNEL)) {172 dd_dev_err(dd, "Unable to allocate cpu_mask for netdevs.\n");173 return 0;174 }175 176 cpumask_and(node_cpu_mask, cpu_mask, cpumask_of_node(dd->node));177 178 available_cpus = cpumask_weight(node_cpu_mask);179 180 free_cpumask_var(node_cpu_mask);181 182 return min3(available_cpus, available_contexts,183 (u32)HFI1_MAX_NETDEV_CTXTS);184}185 186static int hfi1_netdev_rxq_init(struct hfi1_netdev_rx *rx)187{188 int i;189 int rc;190 struct hfi1_devdata *dd = rx->dd;191 struct net_device *dev = rx->rx_napi;192 193 rx->num_rx_q = dd->num_netdev_contexts;194 rx->rxq = kcalloc_node(rx->num_rx_q, sizeof(*rx->rxq),195 GFP_KERNEL, dd->node);196 197 if (!rx->rxq) {198 dd_dev_err(dd, "Unable to allocate netdev queue data\n");199 return (-ENOMEM);200 }201 202 for (i = 0; i < rx->num_rx_q; i++) {203 struct hfi1_netdev_rxq *rxq = &rx->rxq[i];204 205 rc = hfi1_netdev_allot_ctxt(rx, &rxq->rcd);206 if (rc)207 goto bail_context_irq_failure;208 209 hfi1_rcd_get(rxq->rcd);210 rxq->rx = rx;211 rxq->rcd->napi = &rxq->napi;212 dd_dev_info(dd, "Setting rcv queue %d napi to context %d\n",213 i, rxq->rcd->ctxt);214 /*215 * Disable BUSY_POLL on this NAPI as this is not supported216 * right now.217 */218 set_bit(NAPI_STATE_NO_BUSY_POLL, &rxq->napi.state);219 netif_napi_add(dev, &rxq->napi, hfi1_netdev_rx_napi);220 rc = msix_netdev_request_rcd_irq(rxq->rcd);221 if (rc)222 goto bail_context_irq_failure;223 }224 225 return 0;226 227bail_context_irq_failure:228 dd_dev_err(dd, "Unable to allot receive context\n");229 for (; i >= 0; i--) {230 struct hfi1_netdev_rxq *rxq = &rx->rxq[i];231 232 if (rxq->rcd) {233 hfi1_netdev_deallocate_ctxt(dd, rxq->rcd);234 hfi1_rcd_put(rxq->rcd);235 rxq->rcd = NULL;236 }237 }238 kfree(rx->rxq);239 rx->rxq = NULL;240 241 return rc;242}243 244static void hfi1_netdev_rxq_deinit(struct hfi1_netdev_rx *rx)245{246 int i;247 struct hfi1_devdata *dd = rx->dd;248 249 for (i = 0; i < rx->num_rx_q; i++) {250 struct hfi1_netdev_rxq *rxq = &rx->rxq[i];251 252 netif_napi_del(&rxq->napi);253 hfi1_netdev_deallocate_ctxt(dd, rxq->rcd);254 hfi1_rcd_put(rxq->rcd);255 rxq->rcd = NULL;256 }257 258 kfree(rx->rxq);259 rx->rxq = NULL;260 rx->num_rx_q = 0;261}262 263static void enable_queues(struct hfi1_netdev_rx *rx)264{265 int i;266 267 for (i = 0; i < rx->num_rx_q; i++) {268 struct hfi1_netdev_rxq *rxq = &rx->rxq[i];269 270 dd_dev_info(rx->dd, "enabling queue %d on context %d\n", i,271 rxq->rcd->ctxt);272 napi_enable(&rxq->napi);273 hfi1_rcvctrl(rx->dd,274 HFI1_RCVCTRL_CTXT_ENB | HFI1_RCVCTRL_INTRAVAIL_ENB,275 rxq->rcd);276 }277}278 279static void disable_queues(struct hfi1_netdev_rx *rx)280{281 int i;282 283 msix_netdev_synchronize_irq(rx->dd);284 285 for (i = 0; i < rx->num_rx_q; i++) {286 struct hfi1_netdev_rxq *rxq = &rx->rxq[i];287 288 dd_dev_info(rx->dd, "disabling queue %d on context %d\n", i,289 rxq->rcd->ctxt);290 291 /* wait for napi if it was scheduled */292 hfi1_rcvctrl(rx->dd,293 HFI1_RCVCTRL_CTXT_DIS | HFI1_RCVCTRL_INTRAVAIL_DIS,294 rxq->rcd);295 napi_synchronize(&rxq->napi);296 napi_disable(&rxq->napi);297 }298}299 300/**301 * hfi1_netdev_rx_init - Incrememnts netdevs counter. When called first time,302 * it allocates receive queue data and calls netif_napi_add303 * for each queue.304 *305 * @dd: hfi1 dev data306 */307int hfi1_netdev_rx_init(struct hfi1_devdata *dd)308{309 struct hfi1_netdev_rx *rx = dd->netdev_rx;310 int res;311 312 if (atomic_fetch_inc(&rx->netdevs))313 return 0;314 315 mutex_lock(&hfi1_mutex);316 res = hfi1_netdev_rxq_init(rx);317 mutex_unlock(&hfi1_mutex);318 return res;319}320 321/**322 * hfi1_netdev_rx_destroy - Decrements netdevs counter, when it reaches 0323 * napi is deleted and receive queses memory is freed.324 *325 * @dd: hfi1 dev data326 */327int hfi1_netdev_rx_destroy(struct hfi1_devdata *dd)328{329 struct hfi1_netdev_rx *rx = dd->netdev_rx;330 331 /* destroy the RX queues only if it is the last netdev going away */332 if (atomic_fetch_add_unless(&rx->netdevs, -1, 0) == 1) {333 mutex_lock(&hfi1_mutex);334 hfi1_netdev_rxq_deinit(rx);335 mutex_unlock(&hfi1_mutex);336 }337 338 return 0;339}340 341/**342 * hfi1_alloc_rx - Allocates the rx support structure343 * @dd: hfi1 dev data344 *345 * Allocate the rx structure to support gathering the receive346 * resources and the dummy netdev.347 *348 * Updates dd struct pointer upon success.349 *350 * Return: 0 (success) -error on failure351 *352 */353int hfi1_alloc_rx(struct hfi1_devdata *dd)354{355 struct hfi1_netdev_rx *rx;356 357 dd_dev_info(dd, "allocating rx size %ld\n", sizeof(*rx));358 rx = kzalloc_node(sizeof(*rx), GFP_KERNEL, dd->node);359 360 if (!rx)361 return -ENOMEM;362 rx->dd = dd;363 rx->rx_napi = alloc_netdev_dummy(0);364 if (!rx->rx_napi) {365 kfree(rx);366 return -ENOMEM;367 }368 369 xa_init(&rx->dev_tbl);370 atomic_set(&rx->enabled, 0);371 atomic_set(&rx->netdevs, 0);372 dd->netdev_rx = rx;373 374 return 0;375}376 377void hfi1_free_rx(struct hfi1_devdata *dd)378{379 if (dd->netdev_rx) {380 dd_dev_info(dd, "hfi1 rx freed\n");381 free_netdev(dd->netdev_rx->rx_napi);382 kfree(dd->netdev_rx);383 dd->netdev_rx = NULL;384 }385}386 387/**388 * hfi1_netdev_enable_queues - This is napi enable function.389 * It enables napi objects associated with queues.390 * When at least one device has called it it increments atomic counter.391 * Disable function decrements counter and when it is 0,392 * calls napi_disable for every queue.393 *394 * @dd: hfi1 dev data395 */396void hfi1_netdev_enable_queues(struct hfi1_devdata *dd)397{398 struct hfi1_netdev_rx *rx;399 400 if (!dd->netdev_rx)401 return;402 403 rx = dd->netdev_rx;404 if (atomic_fetch_inc(&rx->enabled))405 return;406 407 mutex_lock(&hfi1_mutex);408 enable_queues(rx);409 mutex_unlock(&hfi1_mutex);410}411 412void hfi1_netdev_disable_queues(struct hfi1_devdata *dd)413{414 struct hfi1_netdev_rx *rx;415 416 if (!dd->netdev_rx)417 return;418 419 rx = dd->netdev_rx;420 if (atomic_dec_if_positive(&rx->enabled))421 return;422 423 mutex_lock(&hfi1_mutex);424 disable_queues(rx);425 mutex_unlock(&hfi1_mutex);426}427 428/**429 * hfi1_netdev_add_data - Registers data with unique identifier430 * to be requested later this is needed for VNIC and IPoIB VLANs431 * implementations.432 * This call is protected by mutex idr_lock.433 *434 * @dd: hfi1 dev data435 * @id: requested integer id up to INT_MAX436 * @data: data to be associated with index437 */438int hfi1_netdev_add_data(struct hfi1_devdata *dd, int id, void *data)439{440 struct hfi1_netdev_rx *rx = dd->netdev_rx;441 442 return xa_insert(&rx->dev_tbl, id, data, GFP_NOWAIT);443}444 445/**446 * hfi1_netdev_remove_data - Removes data with previously given id.447 * Returns the reference to removed entry.448 *449 * @dd: hfi1 dev data450 * @id: requested integer id up to INT_MAX451 */452void *hfi1_netdev_remove_data(struct hfi1_devdata *dd, int id)453{454 struct hfi1_netdev_rx *rx = dd->netdev_rx;455 456 return xa_erase(&rx->dev_tbl, id);457}458 459/**460 * hfi1_netdev_get_data - Gets data with given id461 *462 * @dd: hfi1 dev data463 * @id: requested integer id up to INT_MAX464 */465void *hfi1_netdev_get_data(struct hfi1_devdata *dd, int id)466{467 struct hfi1_netdev_rx *rx = dd->netdev_rx;468 469 return xa_load(&rx->dev_tbl, id);470}471 472/**473 * hfi1_netdev_get_first_data - Gets first entry with greater or equal id.474 *475 * @dd: hfi1 dev data476 * @start_id: requested integer id up to INT_MAX477 */478void *hfi1_netdev_get_first_data(struct hfi1_devdata *dd, int *start_id)479{480 struct hfi1_netdev_rx *rx = dd->netdev_rx;481 unsigned long index = *start_id;482 void *ret;483 484 ret = xa_find(&rx->dev_tbl, &index, UINT_MAX, XA_PRESENT);485 *start_id = (int)index;486 return ret;487}488