444 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * PCI Backend Operations - respond to PCI requests from Frontend4 *5 * Author: Ryan Wilson <hap9@epoch.ncsc.mil>6 */7 8#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt9#define dev_fmt pr_fmt10 11#include <linux/moduleparam.h>12#include <linux/wait.h>13#include <linux/bitops.h>14#include <xen/events.h>15#include <linux/sched.h>16#include "pciback.h"17 18static irqreturn_t xen_pcibk_guest_interrupt(int irq, void *dev_id);19 20/* Ensure a device is has the fake IRQ handler "turned on/off" and is21 * ready to be exported. This MUST be run after xen_pcibk_reset_device22 * which does the actual PCI device enable/disable.23 */24static void xen_pcibk_control_isr(struct pci_dev *dev, int reset)25{26 struct xen_pcibk_dev_data *dev_data;27 int rc;28 int enable = 0;29 30 dev_data = pci_get_drvdata(dev);31 if (!dev_data)32 return;33 34 /* We don't deal with bridges */35 if (dev->hdr_type != PCI_HEADER_TYPE_NORMAL)36 return;37 38 if (reset) {39 dev_data->enable_intx = 0;40 dev_data->ack_intr = 0;41 }42 enable = dev_data->enable_intx;43 44 /* Asked to disable, but ISR isn't runnig */45 if (!enable && !dev_data->isr_on)46 return;47 48 /* Squirrel away the IRQs in the dev_data. We need this49 * b/c when device transitions to MSI, the dev->irq is50 * overwritten with the MSI vector.51 */52 if (enable)53 dev_data->irq = dev->irq;54 55 /*56 * SR-IOV devices in all use MSI-X and have no legacy57 * interrupts, so inhibit creating a fake IRQ handler for them.58 */59 if (dev_data->irq == 0)60 goto out;61 62 dev_dbg(&dev->dev, "%s: #%d %s %s%s %s-> %s\n",63 dev_data->irq_name,64 dev_data->irq,65 pci_is_enabled(dev) ? "on" : "off",66 dev->msi_enabled ? "MSI" : "",67 dev->msix_enabled ? "MSI/X" : "",68 dev_data->isr_on ? "enable" : "disable",69 enable ? "enable" : "disable");70 71 if (enable) {72 /*73 * The MSI or MSI-X should not have an IRQ handler. Otherwise74 * if the guest terminates we BUG_ON in free_msi_irqs.75 */76 if (dev->msi_enabled || dev->msix_enabled)77 goto out;78 79 rc = request_irq(dev_data->irq,80 xen_pcibk_guest_interrupt, IRQF_SHARED,81 dev_data->irq_name, dev);82 if (rc) {83 dev_err(&dev->dev, "%s: failed to install fake IRQ " \84 "handler for IRQ %d! (rc:%d)\n",85 dev_data->irq_name, dev_data->irq, rc);86 goto out;87 }88 } else {89 free_irq(dev_data->irq, dev);90 dev_data->irq = 0;91 }92 dev_data->isr_on = enable;93 dev_data->ack_intr = enable;94out:95 dev_dbg(&dev->dev, "%s: #%d %s %s%s %s\n",96 dev_data->irq_name,97 dev_data->irq,98 pci_is_enabled(dev) ? "on" : "off",99 dev->msi_enabled ? "MSI" : "",100 dev->msix_enabled ? "MSI/X" : "",101 enable ? (dev_data->isr_on ? "enabled" : "failed to enable") :102 (dev_data->isr_on ? "failed to disable" : "disabled"));103}104 105/* Ensure a device is "turned off" and ready to be exported.106 * (Also see xen_pcibk_config_reset to ensure virtual configuration space is107 * ready to be re-exported)108 */109void xen_pcibk_reset_device(struct pci_dev *dev)110{111 u16 cmd;112 113 xen_pcibk_control_isr(dev, 1 /* reset device */);114 115 /* Disable devices (but not bridges) */116 if (dev->hdr_type == PCI_HEADER_TYPE_NORMAL) {117#ifdef CONFIG_PCI_MSI118 /* The guest could have been abruptly killed without119 * disabling MSI/MSI-X interrupts.*/120 if (dev->msix_enabled)121 pci_disable_msix(dev);122 if (dev->msi_enabled)123 pci_disable_msi(dev);124#endif125 if (pci_is_enabled(dev))126 pci_disable_device(dev);127 128 dev->is_busmaster = 0;129 } else {130 pci_read_config_word(dev, PCI_COMMAND, &cmd);131 if (cmd & (PCI_COMMAND_INVALIDATE)) {132 cmd &= ~(PCI_COMMAND_INVALIDATE);133 pci_write_config_word(dev, PCI_COMMAND, cmd);134 135 dev->is_busmaster = 0;136 }137 }138}139 140#ifdef CONFIG_PCI_MSI141static142int xen_pcibk_enable_msi(struct xen_pcibk_device *pdev,143 struct pci_dev *dev, struct xen_pci_op *op)144{145 struct xen_pcibk_dev_data *dev_data;146 int status;147 148 if (dev->msi_enabled)149 status = -EALREADY;150 else if (dev->msix_enabled)151 status = -ENXIO;152 else153 status = pci_enable_msi(dev);154 155 if (status) {156 dev_warn_ratelimited(&dev->dev, "error enabling MSI for guest %u: err %d\n",157 pdev->xdev->otherend_id, status);158 op->value = 0;159 return XEN_PCI_ERR_op_failed;160 }161 162 /* The value the guest needs is actually the IDT vector, not163 * the local domain's IRQ number. */164 165 op->value = dev->irq ? xen_pirq_from_irq(dev->irq) : 0;166 167 dev_dbg(&dev->dev, "MSI: %d\n", op->value);168 169 dev_data = pci_get_drvdata(dev);170 if (dev_data)171 dev_data->ack_intr = 0;172 173 return 0;174}175 176static177int xen_pcibk_disable_msi(struct xen_pcibk_device *pdev,178 struct pci_dev *dev, struct xen_pci_op *op)179{180 if (dev->msi_enabled) {181 struct xen_pcibk_dev_data *dev_data;182 183 pci_disable_msi(dev);184 185 dev_data = pci_get_drvdata(dev);186 if (dev_data)187 dev_data->ack_intr = 1;188 }189 op->value = dev->irq ? xen_pirq_from_irq(dev->irq) : 0;190 191 dev_dbg(&dev->dev, "MSI: %d\n", op->value);192 193 return 0;194}195 196static197int xen_pcibk_enable_msix(struct xen_pcibk_device *pdev,198 struct pci_dev *dev, struct xen_pci_op *op)199{200 struct xen_pcibk_dev_data *dev_data;201 int i, result;202 struct msix_entry *entries;203 u16 cmd;204 205 dev_dbg(&dev->dev, "enable MSI-X\n");206 207 if (op->value > SH_INFO_MAX_VEC)208 return -EINVAL;209 210 if (dev->msix_enabled)211 return -EALREADY;212 213 /*214 * PCI_COMMAND_MEMORY must be enabled, otherwise we may not be able215 * to access the BARs where the MSI-X entries reside.216 * But VF devices are unique in which the PF needs to be checked.217 */218 pci_read_config_word(pci_physfn(dev), PCI_COMMAND, &cmd);219 if (dev->msi_enabled || !(cmd & PCI_COMMAND_MEMORY))220 return -ENXIO;221 222 entries = kmalloc_array(op->value, sizeof(*entries), GFP_KERNEL);223 if (entries == NULL)224 return -ENOMEM;225 226 for (i = 0; i < op->value; i++) {227 entries[i].entry = op->msix_entries[i].entry;228 entries[i].vector = op->msix_entries[i].vector;229 }230 231 result = pci_enable_msix_exact(dev, entries, op->value);232 if (result == 0) {233 for (i = 0; i < op->value; i++) {234 op->msix_entries[i].entry = entries[i].entry;235 if (entries[i].vector) {236 op->msix_entries[i].vector =237 xen_pirq_from_irq(entries[i].vector);238 dev_dbg(&dev->dev, "MSI-X[%d]: %d\n", i,239 op->msix_entries[i].vector);240 }241 }242 } else243 dev_warn_ratelimited(&dev->dev, "error enabling MSI-X for guest %u: err %d!\n",244 pdev->xdev->otherend_id, result);245 kfree(entries);246 247 op->value = result;248 dev_data = pci_get_drvdata(dev);249 if (dev_data)250 dev_data->ack_intr = 0;251 252 return result > 0 ? 0 : result;253}254 255static256int xen_pcibk_disable_msix(struct xen_pcibk_device *pdev,257 struct pci_dev *dev, struct xen_pci_op *op)258{259 if (dev->msix_enabled) {260 struct xen_pcibk_dev_data *dev_data;261 262 pci_disable_msix(dev);263 264 dev_data = pci_get_drvdata(dev);265 if (dev_data)266 dev_data->ack_intr = 1;267 }268 /*269 * SR-IOV devices (which don't have any legacy IRQ) have270 * an undefined IRQ value of zero.271 */272 op->value = dev->irq ? xen_pirq_from_irq(dev->irq) : 0;273 274 dev_dbg(&dev->dev, "MSI-X: %d\n", op->value);275 276 return 0;277}278#endif279 280static inline bool xen_pcibk_test_op_pending(struct xen_pcibk_device *pdev)281{282 return test_bit(_XEN_PCIF_active,283 (unsigned long *)&pdev->sh_info->flags) &&284 !test_and_set_bit(_PDEVF_op_active, &pdev->flags);285}286 287/*288* Now the same evtchn is used for both pcifront conf_read_write request289* as well as pcie aer front end ack. We use a new work_queue to schedule290* xen_pcibk conf_read_write service for avoiding confict with aer_core291* do_recovery job which also use the system default work_queue292*/293static void xen_pcibk_test_and_schedule_op(struct xen_pcibk_device *pdev)294{295 bool eoi = true;296 297 /* Check that frontend is requesting an operation and that we are not298 * already processing a request */299 if (xen_pcibk_test_op_pending(pdev)) {300 schedule_work(&pdev->op_work);301 eoi = false;302 }303 /*_XEN_PCIB_active should have been cleared by pcifront. And also make304 sure xen_pcibk is waiting for ack by checking _PCIB_op_pending*/305 if (!test_bit(_XEN_PCIB_active, (unsigned long *)&pdev->sh_info->flags)306 && test_bit(_PCIB_op_pending, &pdev->flags)) {307 wake_up(&xen_pcibk_aer_wait_queue);308 eoi = false;309 }310 311 /* EOI if there was nothing to do. */312 if (eoi)313 xen_pcibk_lateeoi(pdev, XEN_EOI_FLAG_SPURIOUS);314}315 316/* Performing the configuration space reads/writes must not be done in atomic317 * context because some of the pci_* functions can sleep (mostly due to ACPI318 * use of semaphores). This function is intended to be called from a work319 * queue in process context taking a struct xen_pcibk_device as a parameter */320 321static void xen_pcibk_do_one_op(struct xen_pcibk_device *pdev)322{323 struct pci_dev *dev;324 struct xen_pcibk_dev_data *dev_data = NULL;325 struct xen_pci_op *op = &pdev->op;326 int test_intx = 0;327#ifdef CONFIG_PCI_MSI328 unsigned int nr = 0;329#endif330 331 *op = pdev->sh_info->op;332 barrier();333 dev = xen_pcibk_get_pci_dev(pdev, op->domain, op->bus, op->devfn);334 335 if (dev == NULL)336 op->err = XEN_PCI_ERR_dev_not_found;337 else {338 dev_data = pci_get_drvdata(dev);339 if (dev_data)340 test_intx = dev_data->enable_intx;341 switch (op->cmd) {342 case XEN_PCI_OP_conf_read:343 op->err = xen_pcibk_config_read(dev,344 op->offset, op->size, &op->value);345 break;346 case XEN_PCI_OP_conf_write:347 op->err = xen_pcibk_config_write(dev,348 op->offset, op->size, op->value);349 break;350#ifdef CONFIG_PCI_MSI351 case XEN_PCI_OP_enable_msi:352 op->err = xen_pcibk_enable_msi(pdev, dev, op);353 break;354 case XEN_PCI_OP_disable_msi:355 op->err = xen_pcibk_disable_msi(pdev, dev, op);356 break;357 case XEN_PCI_OP_enable_msix:358 nr = op->value;359 op->err = xen_pcibk_enable_msix(pdev, dev, op);360 break;361 case XEN_PCI_OP_disable_msix:362 op->err = xen_pcibk_disable_msix(pdev, dev, op);363 break;364#endif365 default:366 op->err = XEN_PCI_ERR_not_implemented;367 break;368 }369 }370 if (!op->err && dev && dev_data) {371 /* Transition detected */372 if ((dev_data->enable_intx != test_intx))373 xen_pcibk_control_isr(dev, 0 /* no reset */);374 }375 pdev->sh_info->op.err = op->err;376 pdev->sh_info->op.value = op->value;377#ifdef CONFIG_PCI_MSI378 if (op->cmd == XEN_PCI_OP_enable_msix && op->err == 0) {379 unsigned int i;380 381 for (i = 0; i < nr; i++)382 pdev->sh_info->op.msix_entries[i].vector =383 op->msix_entries[i].vector;384 }385#endif386 /* Tell the driver domain that we're done. */387 wmb();388 clear_bit(_XEN_PCIF_active, (unsigned long *)&pdev->sh_info->flags);389 notify_remote_via_irq(pdev->evtchn_irq);390 391 /* Mark that we're done. */392 smp_mb__before_atomic(); /* /after/ clearing PCIF_active */393 clear_bit(_PDEVF_op_active, &pdev->flags);394 smp_mb__after_atomic(); /* /before/ final check for work */395}396 397void xen_pcibk_do_op(struct work_struct *data)398{399 struct xen_pcibk_device *pdev =400 container_of(data, struct xen_pcibk_device, op_work);401 402 do {403 xen_pcibk_do_one_op(pdev);404 } while (xen_pcibk_test_op_pending(pdev));405 406 xen_pcibk_lateeoi(pdev, 0);407}408 409irqreturn_t xen_pcibk_handle_event(int irq, void *dev_id)410{411 struct xen_pcibk_device *pdev = dev_id;412 bool eoi;413 414 /* IRQs might come in before pdev->evtchn_irq is written. */415 if (unlikely(pdev->evtchn_irq != irq))416 pdev->evtchn_irq = irq;417 418 eoi = test_and_set_bit(_EOI_pending, &pdev->flags);419 WARN(eoi, "IRQ while EOI pending\n");420 421 xen_pcibk_test_and_schedule_op(pdev);422 423 return IRQ_HANDLED;424}425static irqreturn_t xen_pcibk_guest_interrupt(int irq, void *dev_id)426{427 struct pci_dev *dev = (struct pci_dev *)dev_id;428 struct xen_pcibk_dev_data *dev_data = pci_get_drvdata(dev);429 430 if (dev_data->isr_on && dev_data->ack_intr) {431 dev_data->handled++;432 if ((dev_data->handled % 1000) == 0) {433 if (xen_test_irq_shared(irq)) {434 dev_info(&dev->dev, "%s IRQ line is not shared "435 "with other domains. Turning ISR off\n",436 dev_data->irq_name);437 dev_data->ack_intr = 0;438 }439 }440 return IRQ_HANDLED;441 }442 return IRQ_NONE;443}444