409 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * PCI MSI/MSI-X — Exported APIs for device drivers4 *5 * Copyright (C) 2003-2004 Intel6 * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)7 * Copyright (C) 2016 Christoph Hellwig.8 * Copyright (C) 2022 Linutronix GmbH9 */10 11#include <linux/export.h>12#include <linux/irq.h>13 14#include "msi.h"15 16/**17 * pci_enable_msi() - Enable MSI interrupt mode on device18 * @dev: the PCI device to operate on19 *20 * Legacy device driver API to enable MSI interrupts mode on device and21 * allocate a single interrupt vector. On success, the allocated vector22 * Linux IRQ will be saved at @dev->irq. The driver must invoke23 * pci_disable_msi() on cleanup.24 *25 * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API26 * pair should, in general, be used instead.27 *28 * Return: 0 on success, errno otherwise29 */30int pci_enable_msi(struct pci_dev *dev)31{32 int rc = __pci_enable_msi_range(dev, 1, 1, NULL);33 if (rc < 0)34 return rc;35 return 0;36}37EXPORT_SYMBOL(pci_enable_msi);38 39/**40 * pci_disable_msi() - Disable MSI interrupt mode on device41 * @dev: the PCI device to operate on42 *43 * Legacy device driver API to disable MSI interrupt mode on device,44 * free earlier allocated interrupt vectors, and restore INTx emulation.45 * The PCI device Linux IRQ (@dev->irq) is restored to its default46 * pin-assertion IRQ. This is the cleanup pair of pci_enable_msi().47 *48 * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API49 * pair should, in general, be used instead.50 */51void pci_disable_msi(struct pci_dev *dev)52{53 if (!pci_msi_enabled() || !dev || !dev->msi_enabled)54 return;55 56 msi_lock_descs(&dev->dev);57 pci_msi_shutdown(dev);58 pci_free_msi_irqs(dev);59 msi_unlock_descs(&dev->dev);60}61EXPORT_SYMBOL(pci_disable_msi);62 63/**64 * pci_msix_vec_count() - Get number of MSI-X interrupt vectors on device65 * @dev: the PCI device to operate on66 *67 * Return: number of MSI-X interrupt vectors available on this device68 * (i.e., the device's MSI-X capability structure "table size"), -EINVAL69 * if the device is not MSI-X capable, other errnos otherwise.70 */71int pci_msix_vec_count(struct pci_dev *dev)72{73 u16 control;74 75 if (!dev->msix_cap)76 return -EINVAL;77 78 pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control);79 return msix_table_size(control);80}81EXPORT_SYMBOL(pci_msix_vec_count);82 83/**84 * pci_enable_msix_range() - Enable MSI-X interrupt mode on device85 * @dev: the PCI device to operate on86 * @entries: input/output parameter, array of MSI-X configuration entries87 * @minvec: minimum required number of MSI-X vectors88 * @maxvec: maximum desired number of MSI-X vectors89 *90 * Legacy device driver API to enable MSI-X interrupt mode on device and91 * configure its MSI-X capability structure as appropriate. The passed92 * @entries array must have each of its members "entry" field set to a93 * desired (valid) MSI-X vector number, where the range of valid MSI-X94 * vector numbers can be queried through pci_msix_vec_count(). If95 * successful, the driver must invoke pci_disable_msix() on cleanup.96 *97 * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API98 * pair should, in general, be used instead.99 *100 * Return: number of MSI-X vectors allocated (which might be smaller101 * than @maxvecs), where Linux IRQ numbers for such allocated vectors102 * are saved back in the @entries array elements' "vector" field. Return103 * -ENOSPC if less than @minvecs interrupt vectors are available.104 * Return -EINVAL if one of the passed @entries members "entry" field105 * was invalid or a duplicate, or if plain MSI interrupts mode was106 * earlier enabled on device. Return other errnos otherwise.107 */108int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries,109 int minvec, int maxvec)110{111 return __pci_enable_msix_range(dev, entries, minvec, maxvec, NULL, 0);112}113EXPORT_SYMBOL(pci_enable_msix_range);114 115/**116 * pci_msix_can_alloc_dyn - Query whether dynamic allocation after enabling117 * MSI-X is supported118 *119 * @dev: PCI device to operate on120 *121 * Return: True if supported, false otherwise122 */123bool pci_msix_can_alloc_dyn(struct pci_dev *dev)124{125 if (!dev->msix_cap)126 return false;127 128 return pci_msi_domain_supports(dev, MSI_FLAG_PCI_MSIX_ALLOC_DYN, DENY_LEGACY);129}130EXPORT_SYMBOL_GPL(pci_msix_can_alloc_dyn);131 132/**133 * pci_msix_alloc_irq_at - Allocate an MSI-X interrupt after enabling MSI-X134 * at a given MSI-X vector index or any free vector index135 *136 * @dev: PCI device to operate on137 * @index: Index to allocate. If @index == MSI_ANY_INDEX this allocates138 * the next free index in the MSI-X table139 * @affdesc: Optional pointer to an affinity descriptor structure. NULL otherwise140 *141 * Return: A struct msi_map142 *143 * On success msi_map::index contains the allocated index (>= 0) and144 * msi_map::virq contains the allocated Linux interrupt number (> 0).145 *146 * On fail msi_map::index contains the error code and msi_map::virq147 * is set to 0.148 */149struct msi_map pci_msix_alloc_irq_at(struct pci_dev *dev, unsigned int index,150 const struct irq_affinity_desc *affdesc)151{152 struct msi_map map = { .index = -ENOTSUPP };153 154 if (!dev->msix_enabled)155 return map;156 157 if (!pci_msix_can_alloc_dyn(dev))158 return map;159 160 return msi_domain_alloc_irq_at(&dev->dev, MSI_DEFAULT_DOMAIN, index, affdesc, NULL);161}162EXPORT_SYMBOL_GPL(pci_msix_alloc_irq_at);163 164/**165 * pci_msix_free_irq - Free an interrupt on a PCI/MSIX interrupt domain166 *167 * @dev: The PCI device to operate on168 * @map: A struct msi_map describing the interrupt to free169 *170 * Undo an interrupt vector allocation. Does not disable MSI-X.171 */172void pci_msix_free_irq(struct pci_dev *dev, struct msi_map map)173{174 if (WARN_ON_ONCE(map.index < 0 || map.virq <= 0))175 return;176 if (WARN_ON_ONCE(!pci_msix_can_alloc_dyn(dev)))177 return;178 msi_domain_free_irqs_range(&dev->dev, MSI_DEFAULT_DOMAIN, map.index, map.index);179}180EXPORT_SYMBOL_GPL(pci_msix_free_irq);181 182/**183 * pci_disable_msix() - Disable MSI-X interrupt mode on device184 * @dev: the PCI device to operate on185 *186 * Legacy device driver API to disable MSI-X interrupt mode on device,187 * free earlier-allocated interrupt vectors, and restore INTx.188 * The PCI device Linux IRQ (@dev->irq) is restored to its default pin189 * assertion IRQ. This is the cleanup pair of pci_enable_msix_range().190 *191 * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API192 * pair should, in general, be used instead.193 */194void pci_disable_msix(struct pci_dev *dev)195{196 if (!pci_msi_enabled() || !dev || !dev->msix_enabled)197 return;198 199 msi_lock_descs(&dev->dev);200 pci_msix_shutdown(dev);201 pci_free_msi_irqs(dev);202 msi_unlock_descs(&dev->dev);203}204EXPORT_SYMBOL(pci_disable_msix);205 206/**207 * pci_alloc_irq_vectors() - Allocate multiple device interrupt vectors208 * @dev: the PCI device to operate on209 * @min_vecs: minimum required number of vectors (must be >= 1)210 * @max_vecs: maximum desired number of vectors211 * @flags: One or more of:212 *213 * * %PCI_IRQ_MSIX Allow trying MSI-X vector allocations214 * * %PCI_IRQ_MSI Allow trying MSI vector allocations215 *216 * * %PCI_IRQ_INTX Allow trying INTx interrupts, if and217 * only if @min_vecs == 1218 *219 * * %PCI_IRQ_AFFINITY Auto-manage IRQs affinity by spreading220 * the vectors around available CPUs221 *222 * Allocate up to @max_vecs interrupt vectors on device. MSI-X irq223 * vector allocation has a higher precedence over plain MSI, which has a224 * higher precedence over legacy INTx emulation.225 *226 * Upon a successful allocation, the caller should use pci_irq_vector()227 * to get the Linux IRQ number to be passed to request_threaded_irq().228 * The driver must call pci_free_irq_vectors() on cleanup.229 *230 * Return: number of allocated vectors (which might be smaller than231 * @max_vecs), -ENOSPC if less than @min_vecs interrupt vectors are232 * available, other errnos otherwise.233 */234int pci_alloc_irq_vectors(struct pci_dev *dev, unsigned int min_vecs,235 unsigned int max_vecs, unsigned int flags)236{237 return pci_alloc_irq_vectors_affinity(dev, min_vecs, max_vecs,238 flags, NULL);239}240EXPORT_SYMBOL(pci_alloc_irq_vectors);241 242/**243 * pci_alloc_irq_vectors_affinity() - Allocate multiple device interrupt244 * vectors with affinity requirements245 * @dev: the PCI device to operate on246 * @min_vecs: minimum required number of vectors (must be >= 1)247 * @max_vecs: maximum desired number of vectors248 * @flags: allocation flags, as in pci_alloc_irq_vectors()249 * @affd: affinity requirements (can be %NULL).250 *251 * Same as pci_alloc_irq_vectors(), but with the extra @affd parameter.252 * Check that function docs, and &struct irq_affinity, for more details.253 */254int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,255 unsigned int max_vecs, unsigned int flags,256 struct irq_affinity *affd)257{258 struct irq_affinity msi_default_affd = {0};259 int nvecs = -ENOSPC;260 261 if (flags & PCI_IRQ_AFFINITY) {262 if (!affd)263 affd = &msi_default_affd;264 } else {265 if (WARN_ON(affd))266 affd = NULL;267 }268 269 if (flags & PCI_IRQ_MSIX) {270 nvecs = __pci_enable_msix_range(dev, NULL, min_vecs, max_vecs,271 affd, flags);272 if (nvecs > 0)273 return nvecs;274 }275 276 if (flags & PCI_IRQ_MSI) {277 nvecs = __pci_enable_msi_range(dev, min_vecs, max_vecs, affd);278 if (nvecs > 0)279 return nvecs;280 }281 282 /* use INTx IRQ if allowed */283 if (flags & PCI_IRQ_INTX) {284 if (min_vecs == 1 && dev->irq) {285 /*286 * Invoke the affinity spreading logic to ensure that287 * the device driver can adjust queue configuration288 * for the single interrupt case.289 */290 if (affd)291 irq_create_affinity_masks(1, affd);292 pci_intx(dev, 1);293 return 1;294 }295 }296 297 return nvecs;298}299EXPORT_SYMBOL(pci_alloc_irq_vectors_affinity);300 301/**302 * pci_irq_vector() - Get Linux IRQ number of a device interrupt vector303 * @dev: the PCI device to operate on304 * @nr: device-relative interrupt vector index (0-based); has different305 * meanings, depending on interrupt mode:306 *307 * * MSI-X the index in the MSI-X vector table308 * * MSI the index of the enabled MSI vectors309 * * INTx must be 0310 *311 * Return: the Linux IRQ number, or -EINVAL if @nr is out of range312 */313int pci_irq_vector(struct pci_dev *dev, unsigned int nr)314{315 unsigned int irq;316 317 if (!dev->msi_enabled && !dev->msix_enabled)318 return !nr ? dev->irq : -EINVAL;319 320 irq = msi_get_virq(&dev->dev, nr);321 return irq ? irq : -EINVAL;322}323EXPORT_SYMBOL(pci_irq_vector);324 325/**326 * pci_irq_get_affinity() - Get a device interrupt vector affinity327 * @dev: the PCI device to operate on328 * @nr: device-relative interrupt vector index (0-based); has different329 * meanings, depending on interrupt mode:330 *331 * * MSI-X the index in the MSI-X vector table332 * * MSI the index of the enabled MSI vectors333 * * INTx must be 0334 *335 * Return: MSI/MSI-X vector affinity, NULL if @nr is out of range or if336 * the MSI(-X) vector was allocated without explicit affinity337 * requirements (e.g., by pci_enable_msi(), pci_enable_msix_range(), or338 * pci_alloc_irq_vectors() without the %PCI_IRQ_AFFINITY flag). Return a339 * generic set of CPU IDs representing all possible CPUs available340 * during system boot if the device is in legacy INTx mode.341 */342const struct cpumask *pci_irq_get_affinity(struct pci_dev *dev, int nr)343{344 int idx, irq = pci_irq_vector(dev, nr);345 struct msi_desc *desc;346 347 if (WARN_ON_ONCE(irq <= 0))348 return NULL;349 350 desc = irq_get_msi_desc(irq);351 /* Non-MSI does not have the information handy */352 if (!desc)353 return cpu_possible_mask;354 355 /* MSI[X] interrupts can be allocated without affinity descriptor */356 if (!desc->affinity)357 return NULL;358 359 /*360 * MSI has a mask array in the descriptor.361 * MSI-X has a single mask.362 */363 idx = dev->msi_enabled ? nr : 0;364 return &desc->affinity[idx].mask;365}366EXPORT_SYMBOL(pci_irq_get_affinity);367 368/**369 * pci_free_irq_vectors() - Free previously allocated IRQs for a device370 * @dev: the PCI device to operate on371 *372 * Undo the interrupt vector allocations and possible device MSI/MSI-X373 * enablement earlier done through pci_alloc_irq_vectors_affinity() or374 * pci_alloc_irq_vectors().375 */376void pci_free_irq_vectors(struct pci_dev *dev)377{378 pci_disable_msix(dev);379 pci_disable_msi(dev);380}381EXPORT_SYMBOL(pci_free_irq_vectors);382 383/**384 * pci_restore_msi_state() - Restore cached MSI(-X) state on device385 * @dev: the PCI device to operate on386 *387 * Write the Linux-cached MSI(-X) state back on device. This is388 * typically useful upon system resume, or after an error-recovery PCI389 * adapter reset.390 */391void pci_restore_msi_state(struct pci_dev *dev)392{393 __pci_restore_msi_state(dev);394 __pci_restore_msix_state(dev);395}396EXPORT_SYMBOL_GPL(pci_restore_msi_state);397 398/**399 * pci_msi_enabled() - Are MSI(-X) interrupts enabled system-wide?400 *401 * Return: true if MSI has not been globally disabled through ACPI FADT,402 * PCI bridge quirks, or the "pci=nomsi" kernel command-line option.403 */404int pci_msi_enabled(void)405{406 return pci_msi_enable;407}408EXPORT_SYMBOL(pci_msi_enabled);409