776 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */3#include <linux/init.h>4#include <linux/kernel.h>5#include <linux/module.h>6#include <linux/pci.h>7#include <linux/device.h>8#include <linux/sched/task.h>9#include <linux/io-64-nonatomic-lo-hi.h>10#include <linux/cdev.h>11#include <linux/fs.h>12#include <linux/poll.h>13#include <linux/iommu.h>14#include <linux/highmem.h>15#include <uapi/linux/idxd.h>16#include <linux/xarray.h>17#include "registers.h"18#include "idxd.h"19 20struct idxd_cdev_context {21 const char *name;22 dev_t devt;23 struct ida minor_ida;24};25 26/*27 * Since user file names are global in DSA devices, define their ida's as28 * global to avoid conflict file names.29 */30static DEFINE_IDA(file_ida);31static DEFINE_MUTEX(ida_lock);32 33/*34 * ictx is an array based off of accelerator types. enum idxd_type35 * is used as index36 */37static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = {38 { .name = "dsa" },39 { .name = "iax" }40};41 42struct idxd_user_context {43 struct idxd_wq *wq;44 struct task_struct *task;45 unsigned int pasid;46 struct mm_struct *mm;47 unsigned int flags;48 struct iommu_sva *sva;49 struct idxd_dev idxd_dev;50 u64 counters[COUNTER_MAX];51 int id;52 pid_t pid;53};54 55static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid);56static void idxd_xa_pasid_remove(struct idxd_user_context *ctx);57 58static inline struct idxd_user_context *dev_to_uctx(struct device *dev)59{60 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);61 62 return container_of(idxd_dev, struct idxd_user_context, idxd_dev);63}64 65static ssize_t cr_faults_show(struct device *dev, struct device_attribute *attr, char *buf)66{67 struct idxd_user_context *ctx = dev_to_uctx(dev);68 69 return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULTS]);70}71static DEVICE_ATTR_RO(cr_faults);72 73static ssize_t cr_fault_failures_show(struct device *dev,74 struct device_attribute *attr, char *buf)75{76 struct idxd_user_context *ctx = dev_to_uctx(dev);77 78 return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULT_FAILS]);79}80static DEVICE_ATTR_RO(cr_fault_failures);81 82static ssize_t pid_show(struct device *dev, struct device_attribute *attr, char *buf)83{84 struct idxd_user_context *ctx = dev_to_uctx(dev);85 86 return sysfs_emit(buf, "%u\n", ctx->pid);87}88static DEVICE_ATTR_RO(pid);89 90static struct attribute *cdev_file_attributes[] = {91 &dev_attr_cr_faults.attr,92 &dev_attr_cr_fault_failures.attr,93 &dev_attr_pid.attr,94 NULL95};96 97static umode_t cdev_file_attr_visible(struct kobject *kobj, struct attribute *a, int n)98{99 struct device *dev = container_of(kobj, typeof(*dev), kobj);100 struct idxd_user_context *ctx = dev_to_uctx(dev);101 struct idxd_wq *wq = ctx->wq;102 103 if (!wq_pasid_enabled(wq))104 return 0;105 106 return a->mode;107}108 109static const struct attribute_group cdev_file_attribute_group = {110 .attrs = cdev_file_attributes,111 .is_visible = cdev_file_attr_visible,112};113 114static const struct attribute_group *cdev_file_attribute_groups[] = {115 &cdev_file_attribute_group,116 NULL117};118 119static void idxd_file_dev_release(struct device *dev)120{121 struct idxd_user_context *ctx = dev_to_uctx(dev);122 struct idxd_wq *wq = ctx->wq;123 struct idxd_device *idxd = wq->idxd;124 int rc;125 126 mutex_lock(&ida_lock);127 ida_free(&file_ida, ctx->id);128 mutex_unlock(&ida_lock);129 130 /* Wait for in-flight operations to complete. */131 if (wq_shared(wq)) {132 idxd_device_drain_pasid(idxd, ctx->pasid);133 } else {134 if (device_user_pasid_enabled(idxd)) {135 /* The wq disable in the disable pasid function will drain the wq */136 rc = idxd_wq_disable_pasid(wq);137 if (rc < 0)138 dev_err(dev, "wq disable pasid failed.\n");139 } else {140 idxd_wq_drain(wq);141 }142 }143 144 if (ctx->sva) {145 idxd_cdev_evl_drain_pasid(wq, ctx->pasid);146 iommu_sva_unbind_device(ctx->sva);147 idxd_xa_pasid_remove(ctx);148 }149 kfree(ctx);150 mutex_lock(&wq->wq_lock);151 idxd_wq_put(wq);152 mutex_unlock(&wq->wq_lock);153}154 155static const struct device_type idxd_cdev_file_type = {156 .name = "idxd_file",157 .release = idxd_file_dev_release,158 .groups = cdev_file_attribute_groups,159};160 161static void idxd_cdev_dev_release(struct device *dev)162{163 struct idxd_cdev *idxd_cdev = dev_to_cdev(dev);164 struct idxd_cdev_context *cdev_ctx;165 struct idxd_wq *wq = idxd_cdev->wq;166 167 cdev_ctx = &ictx[wq->idxd->data->type];168 ida_free(&cdev_ctx->minor_ida, idxd_cdev->minor);169 kfree(idxd_cdev);170}171 172static const struct device_type idxd_cdev_device_type = {173 .name = "idxd_cdev",174 .release = idxd_cdev_dev_release,175};176 177static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode)178{179 struct cdev *cdev = inode->i_cdev;180 181 return container_of(cdev, struct idxd_cdev, cdev);182}183 184static inline struct idxd_wq *inode_wq(struct inode *inode)185{186 struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode);187 188 return idxd_cdev->wq;189}190 191static void idxd_xa_pasid_remove(struct idxd_user_context *ctx)192{193 struct idxd_wq *wq = ctx->wq;194 void *ptr;195 196 mutex_lock(&wq->uc_lock);197 ptr = xa_cmpxchg(&wq->upasid_xa, ctx->pasid, ctx, NULL, GFP_KERNEL);198 if (ptr != (void *)ctx)199 dev_warn(&wq->idxd->pdev->dev, "xarray cmpxchg failed for pasid %u\n",200 ctx->pasid);201 mutex_unlock(&wq->uc_lock);202}203 204void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index)205{206 struct idxd_user_context *ctx;207 208 if (index >= COUNTER_MAX)209 return;210 211 mutex_lock(&wq->uc_lock);212 ctx = xa_load(&wq->upasid_xa, pasid);213 if (!ctx) {214 mutex_unlock(&wq->uc_lock);215 return;216 }217 ctx->counters[index]++;218 mutex_unlock(&wq->uc_lock);219}220 221static int idxd_cdev_open(struct inode *inode, struct file *filp)222{223 struct idxd_user_context *ctx;224 struct idxd_device *idxd;225 struct idxd_wq *wq;226 struct device *dev, *fdev;227 int rc = 0;228 struct iommu_sva *sva;229 unsigned int pasid;230 struct idxd_cdev *idxd_cdev;231 232 wq = inode_wq(inode);233 idxd = wq->idxd;234 dev = &idxd->pdev->dev;235 236 dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq));237 238 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);239 if (!ctx)240 return -ENOMEM;241 242 mutex_lock(&wq->wq_lock);243 244 if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) {245 rc = -EBUSY;246 goto failed;247 }248 249 ctx->wq = wq;250 filp->private_data = ctx;251 ctx->pid = current->pid;252 253 if (device_user_pasid_enabled(idxd)) {254 sva = iommu_sva_bind_device(dev, current->mm);255 if (IS_ERR(sva)) {256 rc = PTR_ERR(sva);257 dev_err(dev, "pasid allocation failed: %d\n", rc);258 goto failed;259 }260 261 pasid = iommu_sva_get_pasid(sva);262 if (pasid == IOMMU_PASID_INVALID) {263 rc = -EINVAL;264 goto failed_get_pasid;265 }266 267 ctx->sva = sva;268 ctx->pasid = pasid;269 ctx->mm = current->mm;270 271 mutex_lock(&wq->uc_lock);272 rc = xa_insert(&wq->upasid_xa, pasid, ctx, GFP_KERNEL);273 mutex_unlock(&wq->uc_lock);274 if (rc < 0)275 dev_warn(dev, "PASID entry already exist in xarray.\n");276 277 if (wq_dedicated(wq)) {278 rc = idxd_wq_set_pasid(wq, pasid);279 if (rc < 0) {280 dev_err(dev, "wq set pasid failed: %d\n", rc);281 goto failed_set_pasid;282 }283 }284 }285 286 idxd_cdev = wq->idxd_cdev;287 mutex_lock(&ida_lock);288 ctx->id = ida_alloc(&file_ida, GFP_KERNEL);289 mutex_unlock(&ida_lock);290 if (ctx->id < 0) {291 dev_warn(dev, "ida alloc failure\n");292 goto failed_ida;293 }294 ctx->idxd_dev.type = IDXD_DEV_CDEV_FILE;295 fdev = user_ctx_dev(ctx);296 device_initialize(fdev);297 fdev->parent = cdev_dev(idxd_cdev);298 fdev->bus = &dsa_bus_type;299 fdev->type = &idxd_cdev_file_type;300 301 rc = dev_set_name(fdev, "file%d", ctx->id);302 if (rc < 0) {303 dev_warn(dev, "set name failure\n");304 goto failed_dev_name;305 }306 307 rc = device_add(fdev);308 if (rc < 0) {309 dev_warn(dev, "file device add failure\n");310 goto failed_dev_add;311 }312 313 idxd_wq_get(wq);314 mutex_unlock(&wq->wq_lock);315 return 0;316 317failed_dev_add:318failed_dev_name:319 put_device(fdev);320failed_ida:321failed_set_pasid:322 if (device_user_pasid_enabled(idxd))323 idxd_xa_pasid_remove(ctx);324failed_get_pasid:325 if (device_user_pasid_enabled(idxd))326 iommu_sva_unbind_device(sva);327failed:328 mutex_unlock(&wq->wq_lock);329 kfree(ctx);330 return rc;331}332 333static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid)334{335 struct idxd_device *idxd = wq->idxd;336 struct idxd_evl *evl = idxd->evl;337 union evl_status_reg status;338 u16 h, t, size;339 int ent_size = evl_ent_size(idxd);340 struct __evl_entry *entry_head;341 342 if (!evl)343 return;344 345 mutex_lock(&evl->lock);346 status.bits = ioread64(idxd->reg_base + IDXD_EVLSTATUS_OFFSET);347 t = status.tail;348 h = status.head;349 size = evl->size;350 351 while (h != t) {352 entry_head = (struct __evl_entry *)(evl->log + (h * ent_size));353 if (entry_head->pasid == pasid && entry_head->wq_idx == wq->id)354 set_bit(h, evl->bmap);355 h = (h + 1) % size;356 }357 drain_workqueue(wq->wq);358 mutex_unlock(&evl->lock);359}360 361static int idxd_cdev_release(struct inode *node, struct file *filep)362{363 struct idxd_user_context *ctx = filep->private_data;364 struct idxd_wq *wq = ctx->wq;365 struct idxd_device *idxd = wq->idxd;366 struct device *dev = &idxd->pdev->dev;367 368 dev_dbg(dev, "%s called\n", __func__);369 filep->private_data = NULL;370 371 device_unregister(user_ctx_dev(ctx));372 373 return 0;374}375 376static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma,377 const char *func)378{379 struct device *dev = &wq->idxd->pdev->dev;380 381 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {382 dev_info_ratelimited(dev,383 "%s: %s: mapping too large: %lu\n",384 current->comm, func,385 vma->vm_end - vma->vm_start);386 return -EINVAL;387 }388 389 return 0;390}391 392static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma)393{394 struct idxd_user_context *ctx = filp->private_data;395 struct idxd_wq *wq = ctx->wq;396 struct idxd_device *idxd = wq->idxd;397 struct pci_dev *pdev = idxd->pdev;398 phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR);399 unsigned long pfn;400 int rc;401 402 dev_dbg(&pdev->dev, "%s called\n", __func__);403 404 /*405 * Due to an erratum in some of the devices supported by the driver,406 * direct user submission to the device can be unsafe.407 * (See the INTEL-SA-01084 security advisory)408 *409 * For the devices that exhibit this behavior, require that the user410 * has CAP_SYS_RAWIO capabilities.411 */412 if (!idxd->user_submission_safe && !capable(CAP_SYS_RAWIO))413 return -EPERM;414 415 rc = check_vma(wq, vma, __func__);416 if (rc < 0)417 return rc;418 419 vm_flags_set(vma, VM_DONTCOPY);420 pfn = (base + idxd_get_wq_portal_full_offset(wq->id,421 IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT;422 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);423 vma->vm_private_data = ctx;424 425 return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE,426 vma->vm_page_prot);427}428 429static int idxd_submit_user_descriptor(struct idxd_user_context *ctx,430 struct dsa_hw_desc __user *udesc)431{432 struct idxd_wq *wq = ctx->wq;433 struct idxd_dev *idxd_dev = &wq->idxd->idxd_dev;434 const uint64_t comp_addr_align = is_dsa_dev(idxd_dev) ? 0x20 : 0x40;435 void __iomem *portal = idxd_wq_portal_addr(wq);436 struct dsa_hw_desc descriptor __aligned(64);437 int rc;438 439 rc = copy_from_user(&descriptor, udesc, sizeof(descriptor));440 if (rc)441 return -EFAULT;442 443 /*444 * DSA devices are capable of indirect ("batch") command submission.445 * On devices where direct user submissions are not safe, we cannot446 * allow this since there is no good way for us to verify these447 * indirect commands.448 */449 if (is_dsa_dev(idxd_dev) && descriptor.opcode == DSA_OPCODE_BATCH &&450 !wq->idxd->user_submission_safe)451 return -EINVAL;452 /*453 * As per the programming specification, the completion address must be454 * aligned to 32 or 64 bytes. If this is violated the hardware455 * engine can get very confused (security issue).456 */457 if (!IS_ALIGNED(descriptor.completion_addr, comp_addr_align))458 return -EINVAL;459 460 if (wq_dedicated(wq))461 iosubmit_cmds512(portal, &descriptor, 1);462 else {463 descriptor.priv = 0;464 descriptor.pasid = ctx->pasid;465 rc = idxd_enqcmds(wq, portal, &descriptor);466 if (rc < 0)467 return rc;468 }469 470 return 0;471}472 473static ssize_t idxd_cdev_write(struct file *filp, const char __user *buf, size_t len,474 loff_t *unused)475{476 struct dsa_hw_desc __user *udesc = (struct dsa_hw_desc __user *)buf;477 struct idxd_user_context *ctx = filp->private_data;478 ssize_t written = 0;479 int i;480 481 for (i = 0; i < len/sizeof(struct dsa_hw_desc); i++) {482 int rc = idxd_submit_user_descriptor(ctx, udesc + i);483 484 if (rc)485 return written ? written : rc;486 487 written += sizeof(struct dsa_hw_desc);488 }489 490 return written;491}492 493static __poll_t idxd_cdev_poll(struct file *filp,494 struct poll_table_struct *wait)495{496 struct idxd_user_context *ctx = filp->private_data;497 struct idxd_wq *wq = ctx->wq;498 struct idxd_device *idxd = wq->idxd;499 __poll_t out = 0;500 501 poll_wait(filp, &wq->err_queue, wait);502 spin_lock(&idxd->dev_lock);503 if (idxd->sw_err.valid)504 out = EPOLLIN | EPOLLRDNORM;505 spin_unlock(&idxd->dev_lock);506 507 return out;508}509 510static const struct file_operations idxd_cdev_fops = {511 .owner = THIS_MODULE,512 .open = idxd_cdev_open,513 .release = idxd_cdev_release,514 .mmap = idxd_cdev_mmap,515 .write = idxd_cdev_write,516 .poll = idxd_cdev_poll,517};518 519int idxd_cdev_get_major(struct idxd_device *idxd)520{521 return MAJOR(ictx[idxd->data->type].devt);522}523 524int idxd_wq_add_cdev(struct idxd_wq *wq)525{526 struct idxd_device *idxd = wq->idxd;527 struct idxd_cdev *idxd_cdev;528 struct cdev *cdev;529 struct device *dev;530 struct idxd_cdev_context *cdev_ctx;531 int rc, minor;532 533 idxd_cdev = kzalloc(sizeof(*idxd_cdev), GFP_KERNEL);534 if (!idxd_cdev)535 return -ENOMEM;536 537 idxd_cdev->idxd_dev.type = IDXD_DEV_CDEV;538 idxd_cdev->wq = wq;539 cdev = &idxd_cdev->cdev;540 dev = cdev_dev(idxd_cdev);541 cdev_ctx = &ictx[wq->idxd->data->type];542 minor = ida_alloc_max(&cdev_ctx->minor_ida, MINORMASK, GFP_KERNEL);543 if (minor < 0) {544 kfree(idxd_cdev);545 return minor;546 }547 idxd_cdev->minor = minor;548 549 device_initialize(dev);550 dev->parent = wq_confdev(wq);551 dev->bus = &dsa_bus_type;552 dev->type = &idxd_cdev_device_type;553 dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor);554 555 rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id);556 if (rc < 0)557 goto err;558 559 wq->idxd_cdev = idxd_cdev;560 cdev_init(cdev, &idxd_cdev_fops);561 rc = cdev_device_add(cdev, dev);562 if (rc) {563 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc);564 goto err;565 }566 567 return 0;568 569 err:570 put_device(dev);571 wq->idxd_cdev = NULL;572 return rc;573}574 575void idxd_wq_del_cdev(struct idxd_wq *wq)576{577 struct idxd_cdev *idxd_cdev;578 579 idxd_cdev = wq->idxd_cdev;580 wq->idxd_cdev = NULL;581 cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev));582 put_device(cdev_dev(idxd_cdev));583}584 585static int idxd_user_drv_probe(struct idxd_dev *idxd_dev)586{587 struct device *dev = &idxd_dev->conf_dev;588 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);589 struct idxd_device *idxd = wq->idxd;590 int rc;591 592 if (idxd->state != IDXD_DEV_ENABLED)593 return -ENXIO;594 595 mutex_lock(&wq->wq_lock);596 597 if (!idxd_wq_driver_name_match(wq, dev)) {598 idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME;599 rc = -ENODEV;600 goto wq_err;601 }602 603 /*604 * User type WQ is enabled only when SVA is enabled for two reasons:605 * - If no IOMMU or IOMMU Passthrough without SVA, userspace606 * can directly access physical address through the WQ.607 * - The IDXD cdev driver does not provide any ways to pin608 * user pages and translate the address from user VA to IOVA or609 * PA without IOMMU SVA. Therefore the application has no way610 * to instruct the device to perform DMA function. This makes611 * the cdev not usable for normal application usage.612 */613 if (!device_user_pasid_enabled(idxd)) {614 idxd->cmd_status = IDXD_SCMD_WQ_USER_NO_IOMMU;615 dev_dbg(&idxd->pdev->dev,616 "User type WQ cannot be enabled without SVA.\n");617 618 rc = -EOPNOTSUPP;619 goto wq_err;620 }621 622 wq->wq = create_workqueue(dev_name(wq_confdev(wq)));623 if (!wq->wq) {624 rc = -ENOMEM;625 goto wq_err;626 }627 628 wq->type = IDXD_WQT_USER;629 rc = idxd_drv_enable_wq(wq);630 if (rc < 0)631 goto err;632 633 rc = idxd_wq_add_cdev(wq);634 if (rc < 0) {635 idxd->cmd_status = IDXD_SCMD_CDEV_ERR;636 goto err_cdev;637 }638 639 idxd->cmd_status = 0;640 mutex_unlock(&wq->wq_lock);641 return 0;642 643err_cdev:644 idxd_drv_disable_wq(wq);645err:646 destroy_workqueue(wq->wq);647 wq->type = IDXD_WQT_NONE;648wq_err:649 mutex_unlock(&wq->wq_lock);650 return rc;651}652 653static void idxd_user_drv_remove(struct idxd_dev *idxd_dev)654{655 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);656 657 mutex_lock(&wq->wq_lock);658 idxd_wq_del_cdev(wq);659 idxd_drv_disable_wq(wq);660 wq->type = IDXD_WQT_NONE;661 destroy_workqueue(wq->wq);662 wq->wq = NULL;663 mutex_unlock(&wq->wq_lock);664}665 666static enum idxd_dev_type dev_types[] = {667 IDXD_DEV_WQ,668 IDXD_DEV_NONE,669};670 671struct idxd_device_driver idxd_user_drv = {672 .probe = idxd_user_drv_probe,673 .remove = idxd_user_drv_remove,674 .name = "user",675 .type = dev_types,676};677EXPORT_SYMBOL_GPL(idxd_user_drv);678 679int idxd_cdev_register(void)680{681 int rc, i;682 683 for (i = 0; i < IDXD_TYPE_MAX; i++) {684 ida_init(&ictx[i].minor_ida);685 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK,686 ictx[i].name);687 if (rc)688 goto err_free_chrdev_region;689 }690 691 return 0;692 693err_free_chrdev_region:694 for (i--; i >= 0; i--)695 unregister_chrdev_region(ictx[i].devt, MINORMASK);696 697 return rc;698}699 700void idxd_cdev_remove(void)701{702 int i;703 704 for (i = 0; i < IDXD_TYPE_MAX; i++) {705 unregister_chrdev_region(ictx[i].devt, MINORMASK);706 ida_destroy(&ictx[i].minor_ida);707 }708}709 710/**711 * idxd_copy_cr - copy completion record to user address space found by wq and712 * PASID713 * @wq: work queue714 * @pasid: PASID715 * @addr: user fault address to write716 * @cr: completion record717 * @len: number of bytes to copy718 *719 * This is called by a work that handles completion record fault.720 *721 * Return: number of bytes copied.722 */723int idxd_copy_cr(struct idxd_wq *wq, ioasid_t pasid, unsigned long addr,724 void *cr, int len)725{726 struct device *dev = &wq->idxd->pdev->dev;727 int left = len, status_size = 1;728 struct idxd_user_context *ctx;729 struct mm_struct *mm;730 731 mutex_lock(&wq->uc_lock);732 733 ctx = xa_load(&wq->upasid_xa, pasid);734 if (!ctx) {735 dev_warn(dev, "No user context\n");736 goto out;737 }738 739 mm = ctx->mm;740 /*741 * The completion record fault handling work is running in kernel742 * thread context. It temporarily switches to the mm to copy cr743 * to addr in the mm.744 */745 kthread_use_mm(mm);746 left = copy_to_user((void __user *)addr + status_size, cr + status_size,747 len - status_size);748 /*749 * Copy status only after the rest of completion record is copied750 * successfully so that the user gets the complete completion record751 * when a non-zero status is polled.752 */753 if (!left) {754 u8 status;755 756 /*757 * Ensure that the completion record's status field is written758 * after the rest of the completion record has been written.759 * This ensures that the user receives the correct completion760 * record information once polling for a non-zero status.761 */762 wmb();763 status = *(u8 *)cr;764 if (put_user(status, (u8 __user *)addr))765 left += status_size;766 } else {767 left += status_size;768 }769 kthread_unuse_mm(mm);770 771out:772 mutex_unlock(&wq->uc_lock);773 774 return len - left;775}776