657 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */3#include <linux/init.h>4#include <linux/kernel.h>5#include <linux/module.h>6#include <linux/pci.h>7#include <linux/io-64-nonatomic-lo-hi.h>8#include <linux/dmaengine.h>9#include <linux/delay.h>10#include <linux/iommu.h>11#include <linux/sched/mm.h>12#include <uapi/linux/idxd.h>13#include "../dmaengine.h"14#include "idxd.h"15#include "registers.h"16 17enum irq_work_type {18 IRQ_WORK_NORMAL = 0,19 IRQ_WORK_PROCESS_FAULT,20};21 22struct idxd_resubmit {23 struct work_struct work;24 struct idxd_desc *desc;25};26 27struct idxd_int_handle_revoke {28 struct work_struct work;29 struct idxd_device *idxd;30};31 32static void idxd_device_reinit(struct work_struct *work)33{34 struct idxd_device *idxd = container_of(work, struct idxd_device, work);35 struct device *dev = &idxd->pdev->dev;36 int rc, i;37 38 idxd_device_reset(idxd);39 rc = idxd_device_config(idxd);40 if (rc < 0)41 goto out;42 43 rc = idxd_device_enable(idxd);44 if (rc < 0)45 goto out;46 47 for (i = 0; i < idxd->max_wqs; i++) {48 if (test_bit(i, idxd->wq_enable_map)) {49 struct idxd_wq *wq = idxd->wqs[i];50 51 rc = idxd_wq_enable(wq);52 if (rc < 0) {53 clear_bit(i, idxd->wq_enable_map);54 dev_warn(dev, "Unable to re-enable wq %s\n",55 dev_name(wq_confdev(wq)));56 }57 }58 }59 60 return;61 62 out:63 idxd_device_clear_state(idxd);64}65 66/*67 * The function sends a drain descriptor for the interrupt handle. The drain ensures68 * all descriptors with this interrupt handle is flushed and the interrupt69 * will allow the cleanup of the outstanding descriptors.70 */71static void idxd_int_handle_revoke_drain(struct idxd_irq_entry *ie)72{73 struct idxd_wq *wq = ie_to_wq(ie);74 struct idxd_device *idxd = wq->idxd;75 struct device *dev = &idxd->pdev->dev;76 struct dsa_hw_desc desc = {};77 void __iomem *portal;78 int rc;79 80 /* Issue a simple drain operation with interrupt but no completion record */81 desc.flags = IDXD_OP_FLAG_RCI;82 desc.opcode = DSA_OPCODE_DRAIN;83 desc.priv = 1;84 85 if (ie->pasid != IOMMU_PASID_INVALID)86 desc.pasid = ie->pasid;87 desc.int_handle = ie->int_handle;88 portal = idxd_wq_portal_addr(wq);89 90 /*91 * The wmb() makes sure that the descriptor is all there before we92 * issue.93 */94 wmb();95 if (wq_dedicated(wq)) {96 iosubmit_cmds512(portal, &desc, 1);97 } else {98 rc = idxd_enqcmds(wq, portal, &desc);99 /* This should not fail unless hardware failed. */100 if (rc < 0)101 dev_warn(dev, "Failed to submit drain desc on wq %d\n", wq->id);102 }103}104 105static void idxd_abort_invalid_int_handle_descs(struct idxd_irq_entry *ie)106{107 LIST_HEAD(flist);108 struct idxd_desc *d, *t;109 struct llist_node *head;110 111 spin_lock(&ie->list_lock);112 head = llist_del_all(&ie->pending_llist);113 if (head) {114 llist_for_each_entry_safe(d, t, head, llnode)115 list_add_tail(&d->list, &ie->work_list);116 }117 118 list_for_each_entry_safe(d, t, &ie->work_list, list) {119 if (d->completion->status == DSA_COMP_INT_HANDLE_INVAL)120 list_move_tail(&d->list, &flist);121 }122 spin_unlock(&ie->list_lock);123 124 list_for_each_entry_safe(d, t, &flist, list) {125 list_del(&d->list);126 idxd_desc_complete(d, IDXD_COMPLETE_ABORT, true);127 }128}129 130static void idxd_int_handle_revoke(struct work_struct *work)131{132 struct idxd_int_handle_revoke *revoke =133 container_of(work, struct idxd_int_handle_revoke, work);134 struct idxd_device *idxd = revoke->idxd;135 struct pci_dev *pdev = idxd->pdev;136 struct device *dev = &pdev->dev;137 int i, new_handle, rc;138 139 if (!idxd->request_int_handles) {140 kfree(revoke);141 dev_warn(dev, "Unexpected int handle refresh interrupt.\n");142 return;143 }144 145 /*146 * The loop attempts to acquire new interrupt handle for all interrupt147 * vectors that supports a handle. If a new interrupt handle is acquired and the148 * wq is kernel type, the driver will kill the percpu_ref to pause all149 * ongoing descriptor submissions. The interrupt handle is then changed.150 * After change, the percpu_ref is revived and all the pending submissions151 * are woken to try again. A drain is sent to for the interrupt handle152 * at the end to make sure all invalid int handle descriptors are processed.153 */154 for (i = 1; i < idxd->irq_cnt; i++) {155 struct idxd_irq_entry *ie = idxd_get_ie(idxd, i);156 struct idxd_wq *wq = ie_to_wq(ie);157 158 if (ie->int_handle == INVALID_INT_HANDLE)159 continue;160 161 rc = idxd_device_request_int_handle(idxd, i, &new_handle, IDXD_IRQ_MSIX);162 if (rc < 0) {163 dev_warn(dev, "get int handle %d failed: %d\n", i, rc);164 /*165 * Failed to acquire new interrupt handle. Kill the WQ166 * and release all the pending submitters. The submitters will167 * get error return code and handle appropriately.168 */169 ie->int_handle = INVALID_INT_HANDLE;170 idxd_wq_quiesce(wq);171 idxd_abort_invalid_int_handle_descs(ie);172 continue;173 }174 175 /* No change in interrupt handle, nothing needs to be done */176 if (ie->int_handle == new_handle)177 continue;178 179 if (wq->state != IDXD_WQ_ENABLED || wq->type != IDXD_WQT_KERNEL) {180 /*181 * All the MSIX interrupts are allocated at once during probe.182 * Therefore we need to update all interrupts even if the WQ183 * isn't supporting interrupt operations.184 */185 ie->int_handle = new_handle;186 continue;187 }188 189 mutex_lock(&wq->wq_lock);190 reinit_completion(&wq->wq_resurrect);191 192 /* Kill percpu_ref to pause additional descriptor submissions */193 percpu_ref_kill(&wq->wq_active);194 195 /* Wait for all submitters quiesce before we change interrupt handle */196 wait_for_completion(&wq->wq_dead);197 198 ie->int_handle = new_handle;199 200 /* Revive percpu ref and wake up all the waiting submitters */201 percpu_ref_reinit(&wq->wq_active);202 complete_all(&wq->wq_resurrect);203 mutex_unlock(&wq->wq_lock);204 205 /*206 * The delay here is to wait for all possible MOVDIR64B that207 * are issued before percpu_ref_kill() has happened to have208 * reached the PCIe domain before the drain is issued. The driver209 * needs to ensure that the drain descriptor issued does not pass210 * all the other issued descriptors that contain the invalid211 * interrupt handle in order to ensure that the drain descriptor212 * interrupt will allow the cleanup of all the descriptors with213 * invalid interrupt handle.214 */215 if (wq_dedicated(wq))216 udelay(100);217 idxd_int_handle_revoke_drain(ie);218 }219 kfree(revoke);220}221 222static void idxd_evl_fault_work(struct work_struct *work)223{224 struct idxd_evl_fault *fault = container_of(work, struct idxd_evl_fault, work);225 struct idxd_wq *wq = fault->wq;226 struct idxd_device *idxd = wq->idxd;227 struct device *dev = &idxd->pdev->dev;228 struct idxd_evl *evl = idxd->evl;229 struct __evl_entry *entry_head = fault->entry;230 void *cr = (void *)entry_head + idxd->data->evl_cr_off;231 int cr_size = idxd->data->compl_size;232 u8 *status = (u8 *)cr + idxd->data->cr_status_off;233 u8 *result = (u8 *)cr + idxd->data->cr_result_off;234 int copied, copy_size;235 bool *bf;236 237 switch (fault->status) {238 case DSA_COMP_CRA_XLAT:239 if (entry_head->batch && entry_head->first_err_in_batch)240 evl->batch_fail[entry_head->batch_id] = false;241 242 copy_size = cr_size;243 idxd_user_counter_increment(wq, entry_head->pasid, COUNTER_FAULTS);244 break;245 case DSA_COMP_BATCH_EVL_ERR:246 bf = &evl->batch_fail[entry_head->batch_id];247 248 copy_size = entry_head->rcr || *bf ? cr_size : 0;249 if (*bf) {250 if (*status == DSA_COMP_SUCCESS)251 *status = DSA_COMP_BATCH_FAIL;252 *result = 1;253 *bf = false;254 }255 idxd_user_counter_increment(wq, entry_head->pasid, COUNTER_FAULTS);256 break;257 case DSA_COMP_DRAIN_EVL:258 copy_size = cr_size;259 break;260 default:261 copy_size = 0;262 dev_dbg_ratelimited(dev, "Unrecognized error code: %#x\n", fault->status);263 break;264 }265 266 if (copy_size == 0)267 return;268 269 /*270 * Copy completion record to fault_addr in user address space271 * that is found by wq and PASID.272 */273 copied = idxd_copy_cr(wq, entry_head->pasid, entry_head->fault_addr,274 cr, copy_size);275 /*276 * The task that triggered the page fault is unknown currently277 * because multiple threads may share the user address278 * space or the task exits already before this fault.279 * So if the copy fails, SIGSEGV can not be sent to the task.280 * Just print an error for the failure. The user application281 * waiting for the completion record will time out on this282 * failure.283 */284 switch (fault->status) {285 case DSA_COMP_CRA_XLAT:286 if (copied != copy_size) {287 idxd_user_counter_increment(wq, entry_head->pasid, COUNTER_FAULT_FAILS);288 dev_dbg_ratelimited(dev, "Failed to write to completion record: (%d:%d)\n",289 copy_size, copied);290 if (entry_head->batch)291 evl->batch_fail[entry_head->batch_id] = true;292 }293 break;294 case DSA_COMP_BATCH_EVL_ERR:295 if (copied != copy_size) {296 idxd_user_counter_increment(wq, entry_head->pasid, COUNTER_FAULT_FAILS);297 dev_dbg_ratelimited(dev, "Failed to write to batch completion record: (%d:%d)\n",298 copy_size, copied);299 }300 break;301 case DSA_COMP_DRAIN_EVL:302 if (copied != copy_size)303 dev_dbg_ratelimited(dev, "Failed to write to drain completion record: (%d:%d)\n",304 copy_size, copied);305 break;306 }307 308 kmem_cache_free(idxd->evl_cache, fault);309}310 311static void process_evl_entry(struct idxd_device *idxd,312 struct __evl_entry *entry_head, unsigned int index)313{314 struct device *dev = &idxd->pdev->dev;315 struct idxd_evl *evl = idxd->evl;316 u8 status;317 318 if (test_bit(index, evl->bmap)) {319 clear_bit(index, evl->bmap);320 } else {321 status = DSA_COMP_STATUS(entry_head->error);322 323 if (status == DSA_COMP_CRA_XLAT || status == DSA_COMP_DRAIN_EVL ||324 status == DSA_COMP_BATCH_EVL_ERR) {325 struct idxd_evl_fault *fault;326 int ent_size = evl_ent_size(idxd);327 328 if (entry_head->rci)329 dev_dbg(dev, "Completion Int Req set, ignoring!\n");330 331 if (!entry_head->rcr && status == DSA_COMP_DRAIN_EVL)332 return;333 334 fault = kmem_cache_alloc(idxd->evl_cache, GFP_ATOMIC);335 if (fault) {336 struct idxd_wq *wq = idxd->wqs[entry_head->wq_idx];337 338 fault->wq = wq;339 fault->status = status;340 memcpy(&fault->entry, entry_head, ent_size);341 INIT_WORK(&fault->work, idxd_evl_fault_work);342 queue_work(wq->wq, &fault->work);343 } else {344 dev_warn(dev, "Failed to service fault work.\n");345 }346 } else {347 dev_warn_ratelimited(dev, "Device error %#x operation: %#x fault addr: %#llx\n",348 status, entry_head->operation,349 entry_head->fault_addr);350 }351 }352}353 354static void process_evl_entries(struct idxd_device *idxd)355{356 union evl_status_reg evl_status;357 unsigned int h, t;358 struct idxd_evl *evl = idxd->evl;359 struct __evl_entry *entry_head;360 unsigned int ent_size = evl_ent_size(idxd);361 u32 size;362 363 evl_status.bits = 0;364 evl_status.int_pending = 1;365 366 mutex_lock(&evl->lock);367 /* Clear interrupt pending bit */368 iowrite32(evl_status.bits_upper32,369 idxd->reg_base + IDXD_EVLSTATUS_OFFSET + sizeof(u32));370 evl_status.bits = ioread64(idxd->reg_base + IDXD_EVLSTATUS_OFFSET);371 t = evl_status.tail;372 h = evl_status.head;373 size = idxd->evl->size;374 375 while (h != t) {376 entry_head = (struct __evl_entry *)(evl->log + (h * ent_size));377 process_evl_entry(idxd, entry_head, h);378 h = (h + 1) % size;379 }380 381 evl_status.head = h;382 iowrite32(evl_status.bits_lower32, idxd->reg_base + IDXD_EVLSTATUS_OFFSET);383 mutex_unlock(&evl->lock);384}385 386irqreturn_t idxd_misc_thread(int vec, void *data)387{388 struct idxd_irq_entry *irq_entry = data;389 struct idxd_device *idxd = ie_to_idxd(irq_entry);390 struct device *dev = &idxd->pdev->dev;391 union gensts_reg gensts;392 u32 val = 0;393 int i;394 bool err = false;395 u32 cause;396 397 cause = ioread32(idxd->reg_base + IDXD_INTCAUSE_OFFSET);398 if (!cause)399 return IRQ_NONE;400 401 iowrite32(cause, idxd->reg_base + IDXD_INTCAUSE_OFFSET);402 403 if (cause & IDXD_INTC_HALT_STATE)404 goto halt;405 406 if (cause & IDXD_INTC_ERR) {407 spin_lock(&idxd->dev_lock);408 for (i = 0; i < 4; i++)409 idxd->sw_err.bits[i] = ioread64(idxd->reg_base +410 IDXD_SWERR_OFFSET + i * sizeof(u64));411 412 iowrite64(idxd->sw_err.bits[0] & IDXD_SWERR_ACK,413 idxd->reg_base + IDXD_SWERR_OFFSET);414 415 if (idxd->sw_err.valid && idxd->sw_err.wq_idx_valid) {416 int id = idxd->sw_err.wq_idx;417 struct idxd_wq *wq = idxd->wqs[id];418 419 if (wq->type == IDXD_WQT_USER)420 wake_up_interruptible(&wq->err_queue);421 } else {422 int i;423 424 for (i = 0; i < idxd->max_wqs; i++) {425 struct idxd_wq *wq = idxd->wqs[i];426 427 if (wq->type == IDXD_WQT_USER)428 wake_up_interruptible(&wq->err_queue);429 }430 }431 432 spin_unlock(&idxd->dev_lock);433 val |= IDXD_INTC_ERR;434 435 for (i = 0; i < 4; i++)436 dev_warn_ratelimited(dev, "err[%d]: %#16.16llx\n",437 i, idxd->sw_err.bits[i]);438 err = true;439 }440 441 if (cause & IDXD_INTC_INT_HANDLE_REVOKED) {442 struct idxd_int_handle_revoke *revoke;443 444 val |= IDXD_INTC_INT_HANDLE_REVOKED;445 446 revoke = kzalloc(sizeof(*revoke), GFP_ATOMIC);447 if (revoke) {448 revoke->idxd = idxd;449 INIT_WORK(&revoke->work, idxd_int_handle_revoke);450 queue_work(idxd->wq, &revoke->work);451 452 } else {453 dev_err(dev, "Failed to allocate work for int handle revoke\n");454 idxd_wqs_quiesce(idxd);455 }456 }457 458 if (cause & IDXD_INTC_CMD) {459 val |= IDXD_INTC_CMD;460 complete(idxd->cmd_done);461 }462 463 if (cause & IDXD_INTC_OCCUPY) {464 /* Driver does not utilize occupancy interrupt */465 val |= IDXD_INTC_OCCUPY;466 }467 468 if (cause & IDXD_INTC_PERFMON_OVFL) {469 val |= IDXD_INTC_PERFMON_OVFL;470 perfmon_counter_overflow(idxd);471 }472 473 if (cause & IDXD_INTC_EVL) {474 val |= IDXD_INTC_EVL;475 process_evl_entries(idxd);476 }477 478 val ^= cause;479 if (val)480 dev_warn_once(dev, "Unexpected interrupt cause bits set: %#x\n",481 val);482 483 if (!err)484 goto out;485 486halt:487 gensts.bits = ioread32(idxd->reg_base + IDXD_GENSTATS_OFFSET);488 if (gensts.state == IDXD_DEVICE_STATE_HALT) {489 idxd->state = IDXD_DEV_HALTED;490 if (gensts.reset_type == IDXD_DEVICE_RESET_SOFTWARE) {491 /*492 * If we need a software reset, we will throw the work493 * on a system workqueue in order to allow interrupts494 * for the device command completions.495 */496 INIT_WORK(&idxd->work, idxd_device_reinit);497 queue_work(idxd->wq, &idxd->work);498 } else {499 idxd->state = IDXD_DEV_HALTED;500 idxd_wqs_quiesce(idxd);501 idxd_wqs_unmap_portal(idxd);502 idxd_device_clear_state(idxd);503 dev_err(&idxd->pdev->dev,504 "idxd halted, need %s.\n",505 gensts.reset_type == IDXD_DEVICE_RESET_FLR ?506 "FLR" : "system reset");507 }508 }509 510out:511 return IRQ_HANDLED;512}513 514static void idxd_int_handle_resubmit_work(struct work_struct *work)515{516 struct idxd_resubmit *irw = container_of(work, struct idxd_resubmit, work);517 struct idxd_desc *desc = irw->desc;518 struct idxd_wq *wq = desc->wq;519 int rc;520 521 desc->completion->status = 0;522 rc = idxd_submit_desc(wq, desc);523 if (rc < 0) {524 dev_dbg(&wq->idxd->pdev->dev, "Failed to resubmit desc %d to wq %d.\n",525 desc->id, wq->id);526 /*527 * If the error is not -EAGAIN, it means the submission failed due to wq528 * has been killed instead of ENQCMDS failure. Here the driver needs to529 * notify the submitter of the failure by reporting abort status.530 *531 * -EAGAIN comes from ENQCMDS failure. idxd_submit_desc() will handle the532 * abort.533 */534 if (rc != -EAGAIN) {535 desc->completion->status = IDXD_COMP_DESC_ABORT;536 idxd_desc_complete(desc, IDXD_COMPLETE_ABORT, false);537 }538 idxd_free_desc(wq, desc);539 }540 kfree(irw);541}542 543bool idxd_queue_int_handle_resubmit(struct idxd_desc *desc)544{545 struct idxd_wq *wq = desc->wq;546 struct idxd_device *idxd = wq->idxd;547 struct idxd_resubmit *irw;548 549 irw = kzalloc(sizeof(*irw), GFP_KERNEL);550 if (!irw)551 return false;552 553 irw->desc = desc;554 INIT_WORK(&irw->work, idxd_int_handle_resubmit_work);555 queue_work(idxd->wq, &irw->work);556 return true;557}558 559static void irq_process_pending_llist(struct idxd_irq_entry *irq_entry)560{561 struct idxd_desc *desc, *t;562 struct llist_node *head;563 564 head = llist_del_all(&irq_entry->pending_llist);565 if (!head)566 return;567 568 llist_for_each_entry_safe(desc, t, head, llnode) {569 u8 status = desc->completion->status & DSA_COMP_STATUS_MASK;570 571 if (status) {572 /*573 * Check against the original status as ABORT is software defined574 * and 0xff, which DSA_COMP_STATUS_MASK can mask out.575 */576 if (unlikely(desc->completion->status == IDXD_COMP_DESC_ABORT)) {577 idxd_desc_complete(desc, IDXD_COMPLETE_ABORT, true);578 continue;579 }580 581 idxd_desc_complete(desc, IDXD_COMPLETE_NORMAL, true);582 } else {583 spin_lock(&irq_entry->list_lock);584 list_add_tail(&desc->list,585 &irq_entry->work_list);586 spin_unlock(&irq_entry->list_lock);587 }588 }589}590 591static void irq_process_work_list(struct idxd_irq_entry *irq_entry)592{593 LIST_HEAD(flist);594 struct idxd_desc *desc, *n;595 596 /*597 * This lock protects list corruption from access of list outside of the irq handler598 * thread.599 */600 spin_lock(&irq_entry->list_lock);601 if (list_empty(&irq_entry->work_list)) {602 spin_unlock(&irq_entry->list_lock);603 return;604 }605 606 list_for_each_entry_safe(desc, n, &irq_entry->work_list, list) {607 if (desc->completion->status) {608 list_move_tail(&desc->list, &flist);609 }610 }611 612 spin_unlock(&irq_entry->list_lock);613 614 list_for_each_entry_safe(desc, n, &flist, list) {615 /*616 * Check against the original status as ABORT is software defined617 * and 0xff, which DSA_COMP_STATUS_MASK can mask out.618 */619 list_del(&desc->list);620 621 if (unlikely(desc->completion->status == IDXD_COMP_DESC_ABORT)) {622 idxd_desc_complete(desc, IDXD_COMPLETE_ABORT, true);623 continue;624 }625 626 idxd_desc_complete(desc, IDXD_COMPLETE_NORMAL, true);627 }628}629 630irqreturn_t idxd_wq_thread(int irq, void *data)631{632 struct idxd_irq_entry *irq_entry = data;633 634 /*635 * There are two lists we are processing. The pending_llist is where636 * submmiter adds all the submitted descriptor after sending it to637 * the workqueue. It's a lockless singly linked list. The work_list638 * is the common linux double linked list. We are in a scenario of639 * multiple producers and a single consumer. The producers are all640 * the kernel submitters of descriptors, and the consumer is the641 * kernel irq handler thread for the msix vector when using threaded642 * irq. To work with the restrictions of llist to remain lockless,643 * we are doing the following steps:644 * 1. Iterate through the work_list and process any completed645 * descriptor. Delete the completed entries during iteration.646 * 2. llist_del_all() from the pending list.647 * 3. Iterate through the llist that was deleted from the pending list648 * and process the completed entries.649 * 4. If the entry is still waiting on hardware, list_add_tail() to650 * the work_list.651 */652 irq_process_work_list(irq_entry);653 irq_process_pending_llist(irq_entry);654 655 return IRQ_HANDLED;656}657