447 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * PCI Message Signaled Interrupt (MSI) - irqdomain support4 */5#include <linux/acpi_iort.h>6#include <linux/irqdomain.h>7#include <linux/of_irq.h>8 9#include "msi.h"10 11int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)12{13 struct irq_domain *domain;14 15 domain = dev_get_msi_domain(&dev->dev);16 if (domain && irq_domain_is_hierarchy(domain))17 return msi_domain_alloc_irqs_all_locked(&dev->dev, MSI_DEFAULT_DOMAIN, nvec);18 19 return pci_msi_legacy_setup_msi_irqs(dev, nvec, type);20}21 22void pci_msi_teardown_msi_irqs(struct pci_dev *dev)23{24 struct irq_domain *domain;25 26 domain = dev_get_msi_domain(&dev->dev);27 if (domain && irq_domain_is_hierarchy(domain)) {28 msi_domain_free_irqs_all_locked(&dev->dev, MSI_DEFAULT_DOMAIN);29 } else {30 pci_msi_legacy_teardown_msi_irqs(dev);31 msi_free_msi_descs(&dev->dev);32 }33}34 35/**36 * pci_msi_domain_write_msg - Helper to write MSI message to PCI config space37 * @irq_data: Pointer to interrupt data of the MSI interrupt38 * @msg: Pointer to the message39 */40static void pci_msi_domain_write_msg(struct irq_data *irq_data, struct msi_msg *msg)41{42 struct msi_desc *desc = irq_data_get_msi_desc(irq_data);43 44 /*45 * For MSI-X desc->irq is always equal to irq_data->irq. For46 * MSI only the first interrupt of MULTI MSI passes the test.47 */48 if (desc->irq == irq_data->irq)49 __pci_write_msi_msg(desc, msg);50}51 52/**53 * pci_msi_domain_calc_hwirq - Generate a unique ID for an MSI source54 * @desc: Pointer to the MSI descriptor55 *56 * The ID number is only used within the irqdomain.57 */58static irq_hw_number_t pci_msi_domain_calc_hwirq(struct msi_desc *desc)59{60 struct pci_dev *dev = msi_desc_to_pci_dev(desc);61 62 return (irq_hw_number_t)desc->msi_index |63 pci_dev_id(dev) << 11 |64 ((irq_hw_number_t)(pci_domain_nr(dev->bus) & 0xFFFFFFFF)) << 27;65}66 67static void pci_msi_domain_set_desc(msi_alloc_info_t *arg,68 struct msi_desc *desc)69{70 arg->desc = desc;71 arg->hwirq = pci_msi_domain_calc_hwirq(desc);72}73 74static struct msi_domain_ops pci_msi_domain_ops_default = {75 .set_desc = pci_msi_domain_set_desc,76};77 78static void pci_msi_domain_update_dom_ops(struct msi_domain_info *info)79{80 struct msi_domain_ops *ops = info->ops;81 82 if (ops == NULL) {83 info->ops = &pci_msi_domain_ops_default;84 } else {85 if (ops->set_desc == NULL)86 ops->set_desc = pci_msi_domain_set_desc;87 }88}89 90static void pci_msi_domain_update_chip_ops(struct msi_domain_info *info)91{92 struct irq_chip *chip = info->chip;93 94 BUG_ON(!chip);95 if (!chip->irq_write_msi_msg)96 chip->irq_write_msi_msg = pci_msi_domain_write_msg;97 if (!chip->irq_mask)98 chip->irq_mask = pci_msi_mask_irq;99 if (!chip->irq_unmask)100 chip->irq_unmask = pci_msi_unmask_irq;101}102 103/**104 * pci_msi_create_irq_domain - Create a MSI interrupt domain105 * @fwnode: Optional fwnode of the interrupt controller106 * @info: MSI domain info107 * @parent: Parent irq domain108 *109 * Updates the domain and chip ops and creates a MSI interrupt domain.110 *111 * Returns:112 * A domain pointer or NULL in case of failure.113 */114struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode,115 struct msi_domain_info *info,116 struct irq_domain *parent)117{118 if (WARN_ON(info->flags & MSI_FLAG_LEVEL_CAPABLE))119 info->flags &= ~MSI_FLAG_LEVEL_CAPABLE;120 121 if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS)122 pci_msi_domain_update_dom_ops(info);123 if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS)124 pci_msi_domain_update_chip_ops(info);125 126 /* Let the core code free MSI descriptors when freeing interrupts */127 info->flags |= MSI_FLAG_FREE_MSI_DESCS;128 129 info->flags |= MSI_FLAG_ACTIVATE_EARLY | MSI_FLAG_DEV_SYSFS;130 if (IS_ENABLED(CONFIG_GENERIC_IRQ_RESERVATION_MODE))131 info->flags |= MSI_FLAG_MUST_REACTIVATE;132 133 /* PCI-MSI is oneshot-safe */134 info->chip->flags |= IRQCHIP_ONESHOT_SAFE;135 /* Let the core update the bus token */136 info->bus_token = DOMAIN_BUS_PCI_MSI;137 138 return msi_create_irq_domain(fwnode, info, parent);139}140EXPORT_SYMBOL_GPL(pci_msi_create_irq_domain);141 142/*143 * Per device MSI[-X] domain functionality144 */145static void pci_device_domain_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc)146{147 arg->desc = desc;148 arg->hwirq = desc->msi_index;149}150 151static __always_inline void cond_mask_parent(struct irq_data *data)152{153 struct msi_domain_info *info = data->domain->host_data;154 155 if (unlikely(info->flags & MSI_FLAG_PCI_MSI_MASK_PARENT))156 irq_chip_mask_parent(data);157}158 159static __always_inline void cond_unmask_parent(struct irq_data *data)160{161 struct msi_domain_info *info = data->domain->host_data;162 163 if (unlikely(info->flags & MSI_FLAG_PCI_MSI_MASK_PARENT))164 irq_chip_unmask_parent(data);165}166 167static void pci_irq_mask_msi(struct irq_data *data)168{169 struct msi_desc *desc = irq_data_get_msi_desc(data);170 171 pci_msi_mask(desc, BIT(data->irq - desc->irq));172 cond_mask_parent(data);173}174 175static void pci_irq_unmask_msi(struct irq_data *data)176{177 struct msi_desc *desc = irq_data_get_msi_desc(data);178 179 cond_unmask_parent(data);180 pci_msi_unmask(desc, BIT(data->irq - desc->irq));181}182 183#ifdef CONFIG_GENERIC_IRQ_RESERVATION_MODE184# define MSI_REACTIVATE MSI_FLAG_MUST_REACTIVATE185#else186# define MSI_REACTIVATE 0187#endif188 189#define MSI_COMMON_FLAGS (MSI_FLAG_FREE_MSI_DESCS | \190 MSI_FLAG_ACTIVATE_EARLY | \191 MSI_FLAG_DEV_SYSFS | \192 MSI_REACTIVATE)193 194static const struct msi_domain_template pci_msi_template = {195 .chip = {196 .name = "PCI-MSI",197 .irq_mask = pci_irq_mask_msi,198 .irq_unmask = pci_irq_unmask_msi,199 .irq_write_msi_msg = pci_msi_domain_write_msg,200 .flags = IRQCHIP_ONESHOT_SAFE,201 },202 203 .ops = {204 .set_desc = pci_device_domain_set_desc,205 },206 207 .info = {208 .flags = MSI_COMMON_FLAGS | MSI_FLAG_MULTI_PCI_MSI,209 .bus_token = DOMAIN_BUS_PCI_DEVICE_MSI,210 },211};212 213static void pci_irq_mask_msix(struct irq_data *data)214{215 pci_msix_mask(irq_data_get_msi_desc(data));216 cond_mask_parent(data);217}218 219static void pci_irq_unmask_msix(struct irq_data *data)220{221 cond_unmask_parent(data);222 pci_msix_unmask(irq_data_get_msi_desc(data));223}224 225static void pci_msix_prepare_desc(struct irq_domain *domain, msi_alloc_info_t *arg,226 struct msi_desc *desc)227{228 /* Don't fiddle with preallocated MSI descriptors */229 if (!desc->pci.mask_base)230 msix_prepare_msi_desc(to_pci_dev(desc->dev), desc);231}232 233static const struct msi_domain_template pci_msix_template = {234 .chip = {235 .name = "PCI-MSIX",236 .irq_mask = pci_irq_mask_msix,237 .irq_unmask = pci_irq_unmask_msix,238 .irq_write_msi_msg = pci_msi_domain_write_msg,239 .flags = IRQCHIP_ONESHOT_SAFE,240 },241 242 .ops = {243 .prepare_desc = pci_msix_prepare_desc,244 .set_desc = pci_device_domain_set_desc,245 },246 247 .info = {248 .flags = MSI_COMMON_FLAGS | MSI_FLAG_PCI_MSIX |249 MSI_FLAG_PCI_MSIX_ALLOC_DYN,250 .bus_token = DOMAIN_BUS_PCI_DEVICE_MSIX,251 },252};253 254static bool pci_match_device_domain(struct pci_dev *pdev, enum irq_domain_bus_token bus_token)255{256 return msi_match_device_irq_domain(&pdev->dev, MSI_DEFAULT_DOMAIN, bus_token);257}258 259static bool pci_create_device_domain(struct pci_dev *pdev, const struct msi_domain_template *tmpl,260 unsigned int hwsize)261{262 struct irq_domain *domain = dev_get_msi_domain(&pdev->dev);263 264 if (!domain || !irq_domain_is_msi_parent(domain))265 return true;266 267 return msi_create_device_irq_domain(&pdev->dev, MSI_DEFAULT_DOMAIN, tmpl,268 hwsize, NULL, NULL);269}270 271/**272 * pci_setup_msi_device_domain - Setup a device MSI interrupt domain273 * @pdev: The PCI device to create the domain on274 *275 * Return:276 * True when:277 * - The device does not have a MSI parent irq domain associated,278 * which keeps the legacy architecture specific and the global279 * PCI/MSI domain models working280 * - The MSI domain exists already281 * - The MSI domain was successfully allocated282 * False when:283 * - MSI-X is enabled284 * - The domain creation fails.285 *286 * The created MSI domain is preserved until:287 * - The device is removed288 * - MSI is disabled and a MSI-X domain is created289 */290bool pci_setup_msi_device_domain(struct pci_dev *pdev)291{292 if (WARN_ON_ONCE(pdev->msix_enabled))293 return false;294 295 if (pci_match_device_domain(pdev, DOMAIN_BUS_PCI_DEVICE_MSI))296 return true;297 if (pci_match_device_domain(pdev, DOMAIN_BUS_PCI_DEVICE_MSIX))298 msi_remove_device_irq_domain(&pdev->dev, MSI_DEFAULT_DOMAIN);299 300 return pci_create_device_domain(pdev, &pci_msi_template, 1);301}302 303/**304 * pci_setup_msix_device_domain - Setup a device MSI-X interrupt domain305 * @pdev: The PCI device to create the domain on306 * @hwsize: The size of the MSI-X vector table307 *308 * Return:309 * True when:310 * - The device does not have a MSI parent irq domain associated,311 * which keeps the legacy architecture specific and the global312 * PCI/MSI domain models working313 * - The MSI-X domain exists already314 * - The MSI-X domain was successfully allocated315 * False when:316 * - MSI is enabled317 * - The domain creation fails.318 *319 * The created MSI-X domain is preserved until:320 * - The device is removed321 * - MSI-X is disabled and a MSI domain is created322 */323bool pci_setup_msix_device_domain(struct pci_dev *pdev, unsigned int hwsize)324{325 if (WARN_ON_ONCE(pdev->msi_enabled))326 return false;327 328 if (pci_match_device_domain(pdev, DOMAIN_BUS_PCI_DEVICE_MSIX))329 return true;330 if (pci_match_device_domain(pdev, DOMAIN_BUS_PCI_DEVICE_MSI))331 msi_remove_device_irq_domain(&pdev->dev, MSI_DEFAULT_DOMAIN);332 333 return pci_create_device_domain(pdev, &pci_msix_template, hwsize);334}335 336/**337 * pci_msi_domain_supports - Check for support of a particular feature flag338 * @pdev: The PCI device to operate on339 * @feature_mask: The feature mask to check for (full match)340 * @mode: If ALLOW_LEGACY this grants the feature when there is no irq domain341 * associated to the device. If DENY_LEGACY the lack of an irq domain342 * makes the feature unsupported343 */344bool pci_msi_domain_supports(struct pci_dev *pdev, unsigned int feature_mask,345 enum support_mode mode)346{347 struct msi_domain_info *info;348 struct irq_domain *domain;349 unsigned int supported;350 351 domain = dev_get_msi_domain(&pdev->dev);352 353 if (!domain || !irq_domain_is_hierarchy(domain))354 return mode == ALLOW_LEGACY;355 356 if (!irq_domain_is_msi_parent(domain)) {357 /*358 * For "global" PCI/MSI interrupt domains the associated359 * msi_domain_info::flags is the authoritative source of360 * information.361 */362 info = domain->host_data;363 supported = info->flags;364 } else {365 /*366 * For MSI parent domains the supported feature set367 * is available in the parent ops. This makes checks368 * possible before actually instantiating the369 * per device domain because the parent is never370 * expanding the PCI/MSI functionality.371 */372 supported = domain->msi_parent_ops->supported_flags;373 }374 375 return (supported & feature_mask) == feature_mask;376}377 378/*379 * Users of the generic MSI infrastructure expect a device to have a single ID,380 * so with DMA aliases we have to pick the least-worst compromise. Devices with381 * DMA phantom functions tend to still emit MSIs from the real function number,382 * so we ignore those and only consider topological aliases where either the383 * alias device or RID appears on a different bus number. We also make the384 * reasonable assumption that bridges are walked in an upstream direction (so385 * the last one seen wins), and the much braver assumption that the most likely386 * case is that of PCI->PCIe so we should always use the alias RID. This echoes387 * the logic from intel_irq_remapping's set_msi_sid(), which presumably works388 * well enough in practice; in the face of the horrible PCIe<->PCI-X conditions389 * for taking ownership all we can really do is close our eyes and hope...390 */391static int get_msi_id_cb(struct pci_dev *pdev, u16 alias, void *data)392{393 u32 *pa = data;394 u8 bus = PCI_BUS_NUM(*pa);395 396 if (pdev->bus->number != bus || PCI_BUS_NUM(alias) != bus)397 *pa = alias;398 399 return 0;400}401 402/**403 * pci_msi_domain_get_msi_rid - Get the MSI requester id (RID)404 * @domain: The interrupt domain405 * @pdev: The PCI device.406 *407 * The RID for a device is formed from the alias, with a firmware408 * supplied mapping applied409 *410 * Returns: The RID.411 */412u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev)413{414 struct device_node *of_node;415 u32 rid = pci_dev_id(pdev);416 417 pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid);418 419 of_node = irq_domain_get_of_node(domain);420 rid = of_node ? of_msi_map_id(&pdev->dev, of_node, rid) :421 iort_msi_map_id(&pdev->dev, rid);422 423 return rid;424}425 426/**427 * pci_msi_get_device_domain - Get the MSI domain for a given PCI device428 * @pdev: The PCI device429 *430 * Use the firmware data to find a device-specific MSI domain431 * (i.e. not one that is set as a default).432 *433 * Returns: The corresponding MSI domain or NULL if none has been found.434 */435struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev)436{437 struct irq_domain *dom;438 u32 rid = pci_dev_id(pdev);439 440 pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid);441 dom = of_msi_map_get_device_domain(&pdev->dev, rid, DOMAIN_BUS_PCI_MSI);442 if (!dom)443 dom = iort_get_device_domain(&pdev->dev, rid,444 DOMAIN_BUS_PCI_MSI);445 return dom;446}447