410 lines · c
1// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)2/* Copyright(c) 2014 - 2020 Intel Corporation */3#include <linux/kernel.h>4#include <linux/init.h>5#include <linux/types.h>6#include <linux/pci.h>7#include <linux/slab.h>8#include <linux/errno.h>9#include <linux/interrupt.h>10#include "adf_accel_devices.h"11#include "adf_common_drv.h"12#include "adf_cfg.h"13#include "adf_cfg_strings.h"14#include "adf_cfg_common.h"15#include "adf_transport_access_macros.h"16#include "adf_transport_internal.h"17 18#define ADF_MAX_NUM_VFS 3219static struct workqueue_struct *adf_misc_wq;20 21static int adf_enable_msix(struct adf_accel_dev *accel_dev)22{23 struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;24 struct adf_hw_device_data *hw_data = accel_dev->hw_device;25 u32 msix_num_entries = hw_data->num_banks + 1;26 int ret;27 28 if (hw_data->set_msix_rttable)29 hw_data->set_msix_rttable(accel_dev);30 31 ret = pci_alloc_irq_vectors(pci_dev_info->pci_dev, msix_num_entries,32 msix_num_entries, PCI_IRQ_MSIX);33 if (unlikely(ret < 0)) {34 dev_err(&GET_DEV(accel_dev),35 "Failed to allocate %d MSI-X vectors\n",36 msix_num_entries);37 return ret;38 }39 return 0;40}41 42static void adf_disable_msix(struct adf_accel_pci *pci_dev_info)43{44 pci_free_irq_vectors(pci_dev_info->pci_dev);45}46 47static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr)48{49 struct adf_etr_bank_data *bank = bank_ptr;50 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(bank->accel_dev);51 52 csr_ops->write_csr_int_flag_and_col(bank->csr_addr, bank->bank_number,53 0);54 tasklet_hi_schedule(&bank->resp_handler);55 return IRQ_HANDLED;56}57 58#ifdef CONFIG_PCI_IOV59void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask)60{61 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);62 unsigned long flags;63 64 spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags);65 GET_PFVF_OPS(accel_dev)->enable_vf2pf_interrupts(pmisc_addr, vf_mask);66 spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags);67}68 69void adf_disable_all_vf2pf_interrupts(struct adf_accel_dev *accel_dev)70{71 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);72 unsigned long flags;73 74 spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags);75 GET_PFVF_OPS(accel_dev)->disable_all_vf2pf_interrupts(pmisc_addr);76 spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags);77}78 79static u32 adf_disable_pending_vf2pf_interrupts(struct adf_accel_dev *accel_dev)80{81 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);82 u32 pending;83 84 spin_lock(&accel_dev->pf.vf2pf_ints_lock);85 pending = GET_PFVF_OPS(accel_dev)->disable_pending_vf2pf_interrupts(pmisc_addr);86 spin_unlock(&accel_dev->pf.vf2pf_ints_lock);87 88 return pending;89}90 91static bool adf_handle_vf2pf_int(struct adf_accel_dev *accel_dev)92{93 bool irq_handled = false;94 unsigned long vf_mask;95 96 /* Get the interrupt sources triggered by VFs, except for those already disabled */97 vf_mask = adf_disable_pending_vf2pf_interrupts(accel_dev);98 if (vf_mask) {99 struct adf_accel_vf_info *vf_info;100 int i;101 102 /*103 * Handle VF2PF interrupt unless the VF is malicious and104 * is attempting to flood the host OS with VF2PF interrupts.105 */106 for_each_set_bit(i, &vf_mask, ADF_MAX_NUM_VFS) {107 vf_info = accel_dev->pf.vf_info + i;108 109 if (!__ratelimit(&vf_info->vf2pf_ratelimit)) {110 dev_info(&GET_DEV(accel_dev),111 "Too many ints from VF%d\n",112 vf_info->vf_nr);113 continue;114 }115 116 adf_schedule_vf2pf_handler(vf_info);117 irq_handled = true;118 }119 }120 return irq_handled;121}122#endif /* CONFIG_PCI_IOV */123 124static bool adf_handle_pm_int(struct adf_accel_dev *accel_dev)125{126 struct adf_hw_device_data *hw_data = accel_dev->hw_device;127 128 if (hw_data->handle_pm_interrupt &&129 hw_data->handle_pm_interrupt(accel_dev))130 return true;131 132 return false;133}134 135static bool adf_handle_ras_int(struct adf_accel_dev *accel_dev)136{137 struct adf_ras_ops *ras_ops = &accel_dev->hw_device->ras_ops;138 bool reset_required;139 140 if (ras_ops->handle_interrupt &&141 ras_ops->handle_interrupt(accel_dev, &reset_required)) {142 if (reset_required) {143 dev_err(&GET_DEV(accel_dev), "Fatal error, reset required\n");144 if (adf_notify_fatal_error(accel_dev))145 dev_err(&GET_DEV(accel_dev),146 "Failed to notify fatal error\n");147 }148 149 return true;150 }151 152 return false;153}154 155static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr)156{157 struct adf_accel_dev *accel_dev = dev_ptr;158 159#ifdef CONFIG_PCI_IOV160 /* If SR-IOV is enabled (vf_info is non-NULL), check for VF->PF ints */161 if (accel_dev->pf.vf_info && adf_handle_vf2pf_int(accel_dev))162 return IRQ_HANDLED;163#endif /* CONFIG_PCI_IOV */164 165 if (adf_handle_pm_int(accel_dev))166 return IRQ_HANDLED;167 168 if (adf_handle_ras_int(accel_dev))169 return IRQ_HANDLED;170 171 dev_dbg(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n",172 accel_dev->accel_id);173 174 return IRQ_NONE;175}176 177static void adf_free_irqs(struct adf_accel_dev *accel_dev)178{179 struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;180 struct adf_hw_device_data *hw_data = accel_dev->hw_device;181 struct adf_irq *irqs = pci_dev_info->msix_entries.irqs;182 struct adf_etr_data *etr_data = accel_dev->transport;183 int clust_irq = hw_data->num_banks;184 int irq, i = 0;185 186 if (pci_dev_info->msix_entries.num_entries > 1) {187 for (i = 0; i < hw_data->num_banks; i++) {188 if (irqs[i].enabled) {189 irq = pci_irq_vector(pci_dev_info->pci_dev, i);190 irq_set_affinity_hint(irq, NULL);191 free_irq(irq, &etr_data->banks[i]);192 }193 }194 }195 196 if (irqs[i].enabled) {197 irq = pci_irq_vector(pci_dev_info->pci_dev, clust_irq);198 free_irq(irq, accel_dev);199 }200}201 202static int adf_request_irqs(struct adf_accel_dev *accel_dev)203{204 struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev;205 struct adf_hw_device_data *hw_data = accel_dev->hw_device;206 struct adf_irq *irqs = pci_dev_info->msix_entries.irqs;207 struct adf_etr_data *etr_data = accel_dev->transport;208 int clust_irq = hw_data->num_banks;209 int ret, irq, i = 0;210 char *name;211 212 /* Request msix irq for all banks unless SR-IOV enabled */213 if (!accel_dev->pf.vf_info) {214 for (i = 0; i < hw_data->num_banks; i++) {215 struct adf_etr_bank_data *bank = &etr_data->banks[i];216 unsigned int cpu, cpus = num_online_cpus();217 218 name = irqs[i].name;219 snprintf(name, ADF_MAX_MSIX_VECTOR_NAME,220 "qat%d-bundle%d", accel_dev->accel_id, i);221 irq = pci_irq_vector(pci_dev_info->pci_dev, i);222 if (unlikely(irq < 0)) {223 dev_err(&GET_DEV(accel_dev),224 "Failed to get IRQ number of device vector %d - %s\n",225 i, name);226 ret = irq;227 goto err;228 }229 ret = request_irq(irq, adf_msix_isr_bundle, 0,230 &name[0], bank);231 if (ret) {232 dev_err(&GET_DEV(accel_dev),233 "Failed to allocate IRQ %d for %s\n",234 irq, name);235 goto err;236 }237 238 cpu = ((accel_dev->accel_id * hw_data->num_banks) +239 i) % cpus;240 irq_set_affinity_hint(irq, get_cpu_mask(cpu));241 irqs[i].enabled = true;242 }243 }244 245 /* Request msix irq for AE */246 name = irqs[i].name;247 snprintf(name, ADF_MAX_MSIX_VECTOR_NAME,248 "qat%d-ae-cluster", accel_dev->accel_id);249 irq = pci_irq_vector(pci_dev_info->pci_dev, clust_irq);250 if (unlikely(irq < 0)) {251 dev_err(&GET_DEV(accel_dev),252 "Failed to get IRQ number of device vector %d - %s\n",253 i, name);254 ret = irq;255 goto err;256 }257 ret = request_irq(irq, adf_msix_isr_ae, 0, &name[0], accel_dev);258 if (ret) {259 dev_err(&GET_DEV(accel_dev),260 "Failed to allocate IRQ %d for %s\n", irq, name);261 goto err;262 }263 irqs[i].enabled = true;264 return ret;265err:266 adf_free_irqs(accel_dev);267 return ret;268}269 270static int adf_isr_alloc_msix_vectors_data(struct adf_accel_dev *accel_dev)271{272 struct adf_hw_device_data *hw_data = accel_dev->hw_device;273 u32 msix_num_entries = 1;274 struct adf_irq *irqs;275 276 /* If SR-IOV is disabled (vf_info is NULL), add entries for each bank */277 if (!accel_dev->pf.vf_info)278 msix_num_entries += hw_data->num_banks;279 280 irqs = kcalloc_node(msix_num_entries, sizeof(*irqs),281 GFP_KERNEL, dev_to_node(&GET_DEV(accel_dev)));282 if (!irqs)283 return -ENOMEM;284 285 accel_dev->accel_pci_dev.msix_entries.num_entries = msix_num_entries;286 accel_dev->accel_pci_dev.msix_entries.irqs = irqs;287 return 0;288}289 290static void adf_isr_free_msix_vectors_data(struct adf_accel_dev *accel_dev)291{292 kfree(accel_dev->accel_pci_dev.msix_entries.irqs);293 accel_dev->accel_pci_dev.msix_entries.irqs = NULL;294}295 296static int adf_setup_bh(struct adf_accel_dev *accel_dev)297{298 struct adf_etr_data *priv_data = accel_dev->transport;299 struct adf_hw_device_data *hw_data = accel_dev->hw_device;300 int i;301 302 for (i = 0; i < hw_data->num_banks; i++)303 tasklet_init(&priv_data->banks[i].resp_handler,304 adf_response_handler,305 (unsigned long)&priv_data->banks[i]);306 return 0;307}308 309static void adf_cleanup_bh(struct adf_accel_dev *accel_dev)310{311 struct adf_etr_data *priv_data = accel_dev->transport;312 struct adf_hw_device_data *hw_data = accel_dev->hw_device;313 int i;314 315 for (i = 0; i < hw_data->num_banks; i++) {316 tasklet_disable(&priv_data->banks[i].resp_handler);317 tasklet_kill(&priv_data->banks[i].resp_handler);318 }319}320 321/**322 * adf_isr_resource_free() - Free IRQ for acceleration device323 * @accel_dev: Pointer to acceleration device.324 *325 * Function frees interrupts for acceleration device.326 */327void adf_isr_resource_free(struct adf_accel_dev *accel_dev)328{329 adf_free_irqs(accel_dev);330 adf_cleanup_bh(accel_dev);331 adf_disable_msix(&accel_dev->accel_pci_dev);332 adf_isr_free_msix_vectors_data(accel_dev);333}334EXPORT_SYMBOL_GPL(adf_isr_resource_free);335 336/**337 * adf_isr_resource_alloc() - Allocate IRQ for acceleration device338 * @accel_dev: Pointer to acceleration device.339 *340 * Function allocates interrupts for acceleration device.341 *342 * Return: 0 on success, error code otherwise.343 */344int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev)345{346 int ret;347 348 ret = adf_isr_alloc_msix_vectors_data(accel_dev);349 if (ret)350 goto err_out;351 352 ret = adf_enable_msix(accel_dev);353 if (ret)354 goto err_free_msix_table;355 356 ret = adf_setup_bh(accel_dev);357 if (ret)358 goto err_disable_msix;359 360 ret = adf_request_irqs(accel_dev);361 if (ret)362 goto err_cleanup_bh;363 364 return 0;365 366err_cleanup_bh:367 adf_cleanup_bh(accel_dev);368 369err_disable_msix:370 adf_disable_msix(&accel_dev->accel_pci_dev);371 372err_free_msix_table:373 adf_isr_free_msix_vectors_data(accel_dev);374 375err_out:376 return ret;377}378EXPORT_SYMBOL_GPL(adf_isr_resource_alloc);379 380/**381 * adf_init_misc_wq() - Init misc workqueue382 *383 * Return: 0 on success, error code otherwise.384 */385int __init adf_init_misc_wq(void)386{387 adf_misc_wq = alloc_workqueue("qat_misc_wq", WQ_MEM_RECLAIM, 0);388 389 return !adf_misc_wq ? -ENOMEM : 0;390}391 392void adf_exit_misc_wq(void)393{394 if (adf_misc_wq)395 destroy_workqueue(adf_misc_wq);396 397 adf_misc_wq = NULL;398}399 400bool adf_misc_wq_queue_work(struct work_struct *work)401{402 return queue_work(adf_misc_wq, work);403}404 405bool adf_misc_wq_queue_delayed_work(struct delayed_work *work,406 unsigned long delay)407{408 return queue_delayed_work(adf_misc_wq, work, delay);409}410