349 lines · c
1// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)2/*3 * Copyright(c) 2018 - 2020 Intel Corporation.4 */5 6#include "hfi.h"7#include "affinity.h"8#include "sdma.h"9#include "netdev.h"10 11/**12 * msix_initialize() - Calculate, request and configure MSIx IRQs13 * @dd: valid hfi1 devdata14 *15 */16int msix_initialize(struct hfi1_devdata *dd)17{18 u32 total;19 int ret;20 struct hfi1_msix_entry *entries;21 22 /*23 * MSIx interrupt count:24 * one for the general, "slow path" interrupt25 * one per used SDMA engine26 * one per kernel receive context27 * one for each VNIC context28 * ...any new IRQs should be added here.29 */30 total = 1 + dd->num_sdma + dd->n_krcv_queues + dd->num_netdev_contexts;31 32 if (total >= CCE_NUM_MSIX_VECTORS)33 return -EINVAL;34 35 ret = pci_alloc_irq_vectors(dd->pcidev, total, total, PCI_IRQ_MSIX);36 if (ret < 0) {37 dd_dev_err(dd, "pci_alloc_irq_vectors() failed: %d\n", ret);38 return ret;39 }40 41 entries = kcalloc(total, sizeof(*dd->msix_info.msix_entries),42 GFP_KERNEL);43 if (!entries) {44 pci_free_irq_vectors(dd->pcidev);45 return -ENOMEM;46 }47 48 dd->msix_info.msix_entries = entries;49 spin_lock_init(&dd->msix_info.msix_lock);50 bitmap_zero(dd->msix_info.in_use_msix, total);51 dd->msix_info.max_requested = total;52 dd_dev_info(dd, "%u MSI-X interrupts allocated\n", total);53 54 return 0;55}56 57/**58 * msix_request_irq() - Allocate a free MSIx IRQ59 * @dd: valid devdata60 * @arg: context information for the IRQ61 * @handler: IRQ handler62 * @thread: IRQ thread handler (could be NULL)63 * @type: affinty IRQ type64 * @name: IRQ name65 *66 * Allocated an MSIx vector if available, and then create the appropriate67 * meta data needed to keep track of the pci IRQ request.68 *69 * Return:70 * < 0 Error71 * >= 0 MSIx vector72 *73 */74static int msix_request_irq(struct hfi1_devdata *dd, void *arg,75 irq_handler_t handler, irq_handler_t thread,76 enum irq_type type, const char *name)77{78 unsigned long nr;79 int irq;80 int ret;81 struct hfi1_msix_entry *me;82 83 /* Allocate an MSIx vector */84 spin_lock(&dd->msix_info.msix_lock);85 nr = find_first_zero_bit(dd->msix_info.in_use_msix,86 dd->msix_info.max_requested);87 if (nr < dd->msix_info.max_requested)88 __set_bit(nr, dd->msix_info.in_use_msix);89 spin_unlock(&dd->msix_info.msix_lock);90 91 if (nr == dd->msix_info.max_requested)92 return -ENOSPC;93 94 if (type < IRQ_SDMA || type >= IRQ_OTHER)95 return -EINVAL;96 97 irq = pci_irq_vector(dd->pcidev, nr);98 ret = pci_request_irq(dd->pcidev, nr, handler, thread, arg, name);99 if (ret) {100 dd_dev_err(dd,101 "%s: request for IRQ %d failed, MSIx %lx, err %d\n",102 name, irq, nr, ret);103 spin_lock(&dd->msix_info.msix_lock);104 __clear_bit(nr, dd->msix_info.in_use_msix);105 spin_unlock(&dd->msix_info.msix_lock);106 return ret;107 }108 109 /*110 * assign arg after pci_request_irq call, so it will be111 * cleaned up112 */113 me = &dd->msix_info.msix_entries[nr];114 me->irq = irq;115 me->arg = arg;116 me->type = type;117 118 /* This is a request, so a failure is not fatal */119 ret = hfi1_get_irq_affinity(dd, me);120 if (ret)121 dd_dev_err(dd, "%s: unable to pin IRQ %d\n", name, ret);122 123 return nr;124}125 126static int msix_request_rcd_irq_common(struct hfi1_ctxtdata *rcd,127 irq_handler_t handler,128 irq_handler_t thread,129 const char *name)130{131 int nr = msix_request_irq(rcd->dd, rcd, handler, thread,132 rcd->is_vnic ? IRQ_NETDEVCTXT : IRQ_RCVCTXT,133 name);134 if (nr < 0)135 return nr;136 137 /*138 * Set the interrupt register and mask for this139 * context's interrupt.140 */141 rcd->ireg = (IS_RCVAVAIL_START + rcd->ctxt) / 64;142 rcd->imask = ((u64)1) << ((IS_RCVAVAIL_START + rcd->ctxt) % 64);143 rcd->msix_intr = nr;144 remap_intr(rcd->dd, IS_RCVAVAIL_START + rcd->ctxt, nr);145 146 return 0;147}148 149/**150 * msix_request_rcd_irq() - Helper function for RCVAVAIL IRQs151 * @rcd: valid rcd context152 *153 */154int msix_request_rcd_irq(struct hfi1_ctxtdata *rcd)155{156 char name[MAX_NAME_SIZE];157 158 snprintf(name, sizeof(name), DRIVER_NAME "_%d kctxt%d",159 rcd->dd->unit, rcd->ctxt);160 161 return msix_request_rcd_irq_common(rcd, receive_context_interrupt,162 receive_context_thread, name);163}164 165/**166 * msix_netdev_request_rcd_irq - Helper function for RCVAVAIL IRQs167 * for netdev context168 * @rcd: valid netdev contexti169 */170int msix_netdev_request_rcd_irq(struct hfi1_ctxtdata *rcd)171{172 char name[MAX_NAME_SIZE];173 174 snprintf(name, sizeof(name), DRIVER_NAME "_%d nd kctxt%d",175 rcd->dd->unit, rcd->ctxt);176 return msix_request_rcd_irq_common(rcd, receive_context_interrupt_napi,177 NULL, name);178}179 180/**181 * msix_request_sdma_irq - Helper for getting SDMA IRQ resources182 * @sde: valid sdma engine183 *184 */185int msix_request_sdma_irq(struct sdma_engine *sde)186{187 int nr;188 char name[MAX_NAME_SIZE];189 190 snprintf(name, sizeof(name), DRIVER_NAME "_%d sdma%d",191 sde->dd->unit, sde->this_idx);192 nr = msix_request_irq(sde->dd, sde, sdma_interrupt, NULL,193 IRQ_SDMA, name);194 if (nr < 0)195 return nr;196 sde->msix_intr = nr;197 remap_sdma_interrupts(sde->dd, sde->this_idx, nr);198 199 return 0;200}201 202/**203 * msix_request_general_irq - Helper for getting general IRQ204 * resources205 * @dd: valid device data206 */207int msix_request_general_irq(struct hfi1_devdata *dd)208{209 int nr;210 char name[MAX_NAME_SIZE];211 212 snprintf(name, sizeof(name), DRIVER_NAME "_%d", dd->unit);213 nr = msix_request_irq(dd, dd, general_interrupt, NULL, IRQ_GENERAL,214 name);215 if (nr < 0)216 return nr;217 218 /* general interrupt must be MSIx vector 0 */219 if (nr) {220 msix_free_irq(dd, (u8)nr);221 dd_dev_err(dd, "Invalid index %d for GENERAL IRQ\n", nr);222 return -EINVAL;223 }224 225 return 0;226}227 228/**229 * enable_sdma_srcs - Helper to enable SDMA IRQ srcs230 * @dd: valid devdata structure231 * @i: index of SDMA engine232 */233static void enable_sdma_srcs(struct hfi1_devdata *dd, int i)234{235 set_intr_bits(dd, IS_SDMA_START + i, IS_SDMA_START + i, true);236 set_intr_bits(dd, IS_SDMA_PROGRESS_START + i,237 IS_SDMA_PROGRESS_START + i, true);238 set_intr_bits(dd, IS_SDMA_IDLE_START + i, IS_SDMA_IDLE_START + i, true);239 set_intr_bits(dd, IS_SDMAENG_ERR_START + i, IS_SDMAENG_ERR_START + i,240 true);241}242 243/**244 * msix_request_irqs() - Allocate all MSIx IRQs245 * @dd: valid devdata structure246 *247 * Helper function to request the used MSIx IRQs.248 *249 */250int msix_request_irqs(struct hfi1_devdata *dd)251{252 int i;253 int ret = msix_request_general_irq(dd);254 255 if (ret)256 return ret;257 258 for (i = 0; i < dd->num_sdma; i++) {259 struct sdma_engine *sde = &dd->per_sdma[i];260 261 ret = msix_request_sdma_irq(sde);262 if (ret)263 return ret;264 enable_sdma_srcs(sde->dd, i);265 }266 267 for (i = 0; i < dd->n_krcv_queues; i++) {268 struct hfi1_ctxtdata *rcd = hfi1_rcd_get_by_index_safe(dd, i);269 270 if (rcd)271 ret = msix_request_rcd_irq(rcd);272 hfi1_rcd_put(rcd);273 if (ret)274 return ret;275 }276 277 return 0;278}279 280/**281 * msix_free_irq() - Free the specified MSIx resources and IRQ282 * @dd: valid devdata283 * @msix_intr: MSIx vector to free.284 *285 */286void msix_free_irq(struct hfi1_devdata *dd, u8 msix_intr)287{288 struct hfi1_msix_entry *me;289 290 if (msix_intr >= dd->msix_info.max_requested)291 return;292 293 me = &dd->msix_info.msix_entries[msix_intr];294 295 if (!me->arg) /* => no irq, no affinity */296 return;297 298 hfi1_put_irq_affinity(dd, me);299 pci_free_irq(dd->pcidev, msix_intr, me->arg);300 301 me->arg = NULL;302 303 spin_lock(&dd->msix_info.msix_lock);304 __clear_bit(msix_intr, dd->msix_info.in_use_msix);305 spin_unlock(&dd->msix_info.msix_lock);306}307 308/**309 * msix_clean_up_interrupts - Free all MSIx IRQ resources310 * @dd: valid device data data structure311 *312 * Free the MSIx and associated PCI resources, if they have been allocated.313 */314void msix_clean_up_interrupts(struct hfi1_devdata *dd)315{316 int i;317 struct hfi1_msix_entry *me = dd->msix_info.msix_entries;318 319 /* remove irqs - must happen before disabling/turning off */320 for (i = 0; i < dd->msix_info.max_requested; i++, me++)321 msix_free_irq(dd, i);322 323 /* clean structures */324 kfree(dd->msix_info.msix_entries);325 dd->msix_info.msix_entries = NULL;326 dd->msix_info.max_requested = 0;327 328 pci_free_irq_vectors(dd->pcidev);329}330 331/**332 * msix_netdev_synchronize_irq - netdev IRQ synchronize333 * @dd: valid devdata334 */335void msix_netdev_synchronize_irq(struct hfi1_devdata *dd)336{337 int i;338 int ctxt_count = hfi1_netdev_ctxt_count(dd);339 340 for (i = 0; i < ctxt_count; i++) {341 struct hfi1_ctxtdata *rcd = hfi1_netdev_get_ctxt(dd, i);342 struct hfi1_msix_entry *me;343 344 me = &dd->msix_info.msix_entries[rcd->msix_intr];345 346 synchronize_irq(me->irq);347 }348}349