533 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * Copyright 2014 IBM Corp.4 */5 6#include <linux/pci.h>7#include <linux/slab.h>8#include <linux/file.h>9#include <misc/cxl.h>10#include <linux/module.h>11#include <linux/mount.h>12#include <linux/pseudo_fs.h>13#include <linux/sched/mm.h>14#include <linux/mmu_context.h>15#include <linux/irqdomain.h>16 17#include "cxl.h"18 19/*20 * Since we want to track memory mappings to be able to force-unmap21 * when the AFU is no longer reachable, we need an inode. For devices22 * opened through the cxl user API, this is not a problem, but a23 * userland process can also get a cxl fd through the cxl_get_fd()24 * API, which is used by the cxlflash driver.25 *26 * Therefore we implement our own simple pseudo-filesystem and inode27 * allocator. We don't use the anonymous inode, as we need the28 * meta-data associated with it (address_space) and it is shared by29 * other drivers/processes, so it could lead to cxl unmapping VMAs30 * from random processes.31 */32 33#define CXL_PSEUDO_FS_MAGIC 0x1697697f34 35static int cxl_fs_cnt;36static struct vfsmount *cxl_vfs_mount;37 38static int cxl_fs_init_fs_context(struct fs_context *fc)39{40 return init_pseudo(fc, CXL_PSEUDO_FS_MAGIC) ? 0 : -ENOMEM;41}42 43static struct file_system_type cxl_fs_type = {44 .name = "cxl",45 .owner = THIS_MODULE,46 .init_fs_context = cxl_fs_init_fs_context,47 .kill_sb = kill_anon_super,48};49 50 51void cxl_release_mapping(struct cxl_context *ctx)52{53 if (ctx->kernelapi && ctx->mapping)54 simple_release_fs(&cxl_vfs_mount, &cxl_fs_cnt);55}56 57static struct file *cxl_getfile(const char *name,58 const struct file_operations *fops,59 void *priv, int flags)60{61 struct file *file;62 struct inode *inode;63 int rc;64 65 /* strongly inspired by anon_inode_getfile() */66 67 if (fops->owner && !try_module_get(fops->owner))68 return ERR_PTR(-ENOENT);69 70 rc = simple_pin_fs(&cxl_fs_type, &cxl_vfs_mount, &cxl_fs_cnt);71 if (rc < 0) {72 pr_err("Cannot mount cxl pseudo filesystem: %d\n", rc);73 file = ERR_PTR(rc);74 goto err_module;75 }76 77 inode = alloc_anon_inode(cxl_vfs_mount->mnt_sb);78 if (IS_ERR(inode)) {79 file = ERR_CAST(inode);80 goto err_fs;81 }82 83 file = alloc_file_pseudo(inode, cxl_vfs_mount, name,84 flags & (O_ACCMODE | O_NONBLOCK), fops);85 if (IS_ERR(file))86 goto err_inode;87 88 file->private_data = priv;89 90 return file;91 92err_inode:93 iput(inode);94err_fs:95 simple_release_fs(&cxl_vfs_mount, &cxl_fs_cnt);96err_module:97 module_put(fops->owner);98 return file;99}100 101struct cxl_context *cxl_dev_context_init(struct pci_dev *dev)102{103 struct cxl_afu *afu;104 struct cxl_context *ctx;105 int rc;106 107 afu = cxl_pci_to_afu(dev);108 if (IS_ERR(afu))109 return ERR_CAST(afu);110 111 ctx = cxl_context_alloc();112 if (!ctx)113 return ERR_PTR(-ENOMEM);114 115 ctx->kernelapi = true;116 117 /* Make it a slave context. We can promote it later? */118 rc = cxl_context_init(ctx, afu, false);119 if (rc)120 goto err_ctx;121 122 return ctx;123 124err_ctx:125 kfree(ctx);126 return ERR_PTR(rc);127}128EXPORT_SYMBOL_GPL(cxl_dev_context_init);129 130struct cxl_context *cxl_get_context(struct pci_dev *dev)131{132 return dev->dev.archdata.cxl_ctx;133}134EXPORT_SYMBOL_GPL(cxl_get_context);135 136int cxl_release_context(struct cxl_context *ctx)137{138 if (ctx->status >= STARTED)139 return -EBUSY;140 141 cxl_context_free(ctx);142 143 return 0;144}145EXPORT_SYMBOL_GPL(cxl_release_context);146 147static irq_hw_number_t cxl_find_afu_irq(struct cxl_context *ctx, int num)148{149 __u16 range;150 int r;151 152 for (r = 0; r < CXL_IRQ_RANGES; r++) {153 range = ctx->irqs.range[r];154 if (num < range) {155 return ctx->irqs.offset[r] + num;156 }157 num -= range;158 }159 return 0;160}161 162 163int cxl_set_priv(struct cxl_context *ctx, void *priv)164{165 if (!ctx)166 return -EINVAL;167 168 ctx->priv = priv;169 170 return 0;171}172EXPORT_SYMBOL_GPL(cxl_set_priv);173 174void *cxl_get_priv(struct cxl_context *ctx)175{176 if (!ctx)177 return ERR_PTR(-EINVAL);178 179 return ctx->priv;180}181EXPORT_SYMBOL_GPL(cxl_get_priv);182 183int cxl_allocate_afu_irqs(struct cxl_context *ctx, int num)184{185 int res;186 irq_hw_number_t hwirq;187 188 if (num == 0)189 num = ctx->afu->pp_irqs;190 res = afu_allocate_irqs(ctx, num);191 if (res)192 return res;193 194 if (!cpu_has_feature(CPU_FTR_HVMODE)) {195 /* In a guest, the PSL interrupt is not multiplexed. It was196 * allocated above, and we need to set its handler197 */198 hwirq = cxl_find_afu_irq(ctx, 0);199 if (hwirq)200 cxl_map_irq(ctx->afu->adapter, hwirq, cxl_ops->psl_interrupt, ctx, "psl");201 }202 203 if (ctx->status == STARTED) {204 if (cxl_ops->update_ivtes)205 cxl_ops->update_ivtes(ctx);206 else WARN(1, "BUG: cxl_allocate_afu_irqs must be called prior to starting the context on this platform\n");207 }208 209 return res;210}211EXPORT_SYMBOL_GPL(cxl_allocate_afu_irqs);212 213void cxl_free_afu_irqs(struct cxl_context *ctx)214{215 irq_hw_number_t hwirq;216 unsigned int virq;217 218 if (!cpu_has_feature(CPU_FTR_HVMODE)) {219 hwirq = cxl_find_afu_irq(ctx, 0);220 if (hwirq) {221 virq = irq_find_mapping(NULL, hwirq);222 if (virq)223 cxl_unmap_irq(virq, ctx);224 }225 }226 afu_irq_name_free(ctx);227 cxl_ops->release_irq_ranges(&ctx->irqs, ctx->afu->adapter);228}229EXPORT_SYMBOL_GPL(cxl_free_afu_irqs);230 231int cxl_map_afu_irq(struct cxl_context *ctx, int num,232 irq_handler_t handler, void *cookie, char *name)233{234 irq_hw_number_t hwirq;235 236 /*237 * Find interrupt we are to register.238 */239 hwirq = cxl_find_afu_irq(ctx, num);240 if (!hwirq)241 return -ENOENT;242 243 return cxl_map_irq(ctx->afu->adapter, hwirq, handler, cookie, name);244}245EXPORT_SYMBOL_GPL(cxl_map_afu_irq);246 247void cxl_unmap_afu_irq(struct cxl_context *ctx, int num, void *cookie)248{249 irq_hw_number_t hwirq;250 unsigned int virq;251 252 hwirq = cxl_find_afu_irq(ctx, num);253 if (!hwirq)254 return;255 256 virq = irq_find_mapping(NULL, hwirq);257 if (virq)258 cxl_unmap_irq(virq, cookie);259}260EXPORT_SYMBOL_GPL(cxl_unmap_afu_irq);261 262/*263 * Start a context264 * Code here similar to afu_ioctl_start_work().265 */266int cxl_start_context(struct cxl_context *ctx, u64 wed,267 struct task_struct *task)268{269 int rc = 0;270 bool kernel = true;271 272 pr_devel("%s: pe: %i\n", __func__, ctx->pe);273 274 mutex_lock(&ctx->status_mutex);275 if (ctx->status == STARTED)276 goto out; /* already started */277 278 /*279 * Increment the mapped context count for adapter. This also checks280 * if adapter_context_lock is taken.281 */282 rc = cxl_adapter_context_get(ctx->afu->adapter);283 if (rc)284 goto out;285 286 if (task) {287 ctx->pid = get_task_pid(task, PIDTYPE_PID);288 kernel = false;289 290 /* acquire a reference to the task's mm */291 ctx->mm = get_task_mm(current);292 293 /* ensure this mm_struct can't be freed */294 cxl_context_mm_count_get(ctx);295 296 if (ctx->mm) {297 /* decrement the use count from above */298 mmput(ctx->mm);299 /* make TLBIs for this context global */300 mm_context_add_copro(ctx->mm);301 }302 }303 304 /*305 * Increment driver use count. Enables global TLBIs for hash306 * and callbacks to handle the segment table307 */308 cxl_ctx_get();309 310 /* See the comment in afu_ioctl_start_work() */311 smp_mb();312 313 if ((rc = cxl_ops->attach_process(ctx, kernel, wed, 0))) {314 put_pid(ctx->pid);315 ctx->pid = NULL;316 cxl_adapter_context_put(ctx->afu->adapter);317 cxl_ctx_put();318 if (task) {319 cxl_context_mm_count_put(ctx);320 if (ctx->mm)321 mm_context_remove_copro(ctx->mm);322 }323 goto out;324 }325 326 ctx->status = STARTED;327out:328 mutex_unlock(&ctx->status_mutex);329 return rc;330}331EXPORT_SYMBOL_GPL(cxl_start_context);332 333int cxl_process_element(struct cxl_context *ctx)334{335 return ctx->external_pe;336}337EXPORT_SYMBOL_GPL(cxl_process_element);338 339/* Stop a context. Returns 0 on success, otherwise -Errno */340int cxl_stop_context(struct cxl_context *ctx)341{342 return __detach_context(ctx);343}344EXPORT_SYMBOL_GPL(cxl_stop_context);345 346void cxl_set_master(struct cxl_context *ctx)347{348 ctx->master = true;349}350EXPORT_SYMBOL_GPL(cxl_set_master);351 352/* wrappers around afu_* file ops which are EXPORTED */353int cxl_fd_open(struct inode *inode, struct file *file)354{355 return afu_open(inode, file);356}357EXPORT_SYMBOL_GPL(cxl_fd_open);358int cxl_fd_release(struct inode *inode, struct file *file)359{360 return afu_release(inode, file);361}362EXPORT_SYMBOL_GPL(cxl_fd_release);363long cxl_fd_ioctl(struct file *file, unsigned int cmd, unsigned long arg)364{365 return afu_ioctl(file, cmd, arg);366}367EXPORT_SYMBOL_GPL(cxl_fd_ioctl);368int cxl_fd_mmap(struct file *file, struct vm_area_struct *vm)369{370 return afu_mmap(file, vm);371}372EXPORT_SYMBOL_GPL(cxl_fd_mmap);373__poll_t cxl_fd_poll(struct file *file, struct poll_table_struct *poll)374{375 return afu_poll(file, poll);376}377EXPORT_SYMBOL_GPL(cxl_fd_poll);378ssize_t cxl_fd_read(struct file *file, char __user *buf, size_t count,379 loff_t *off)380{381 return afu_read(file, buf, count, off);382}383EXPORT_SYMBOL_GPL(cxl_fd_read);384 385#define PATCH_FOPS(NAME) if (!fops->NAME) fops->NAME = afu_fops.NAME386 387/* Get a struct file and fd for a context and attach the ops */388struct file *cxl_get_fd(struct cxl_context *ctx, struct file_operations *fops,389 int *fd)390{391 struct file *file;392 int rc, flags, fdtmp;393 char *name = NULL;394 395 /* only allow one per context */396 if (ctx->mapping)397 return ERR_PTR(-EEXIST);398 399 flags = O_RDWR | O_CLOEXEC;400 401 /* This code is similar to anon_inode_getfd() */402 rc = get_unused_fd_flags(flags);403 if (rc < 0)404 return ERR_PTR(rc);405 fdtmp = rc;406 407 /*408 * Patch the file ops. Needs to be careful that this is rentrant safe.409 */410 if (fops) {411 PATCH_FOPS(open);412 PATCH_FOPS(poll);413 PATCH_FOPS(read);414 PATCH_FOPS(release);415 PATCH_FOPS(unlocked_ioctl);416 PATCH_FOPS(compat_ioctl);417 PATCH_FOPS(mmap);418 } else /* use default ops */419 fops = (struct file_operations *)&afu_fops;420 421 name = kasprintf(GFP_KERNEL, "cxl:%d", ctx->pe);422 file = cxl_getfile(name, fops, ctx, flags);423 kfree(name);424 if (IS_ERR(file))425 goto err_fd;426 427 cxl_context_set_mapping(ctx, file->f_mapping);428 *fd = fdtmp;429 return file;430 431err_fd:432 put_unused_fd(fdtmp);433 return NULL;434}435EXPORT_SYMBOL_GPL(cxl_get_fd);436 437struct cxl_context *cxl_fops_get_context(struct file *file)438{439 return file->private_data;440}441EXPORT_SYMBOL_GPL(cxl_fops_get_context);442 443void cxl_set_driver_ops(struct cxl_context *ctx,444 struct cxl_afu_driver_ops *ops)445{446 WARN_ON(!ops->fetch_event || !ops->event_delivered);447 atomic_set(&ctx->afu_driver_events, 0);448 ctx->afu_driver_ops = ops;449}450EXPORT_SYMBOL_GPL(cxl_set_driver_ops);451 452void cxl_context_events_pending(struct cxl_context *ctx,453 unsigned int new_events)454{455 atomic_add(new_events, &ctx->afu_driver_events);456 wake_up_all(&ctx->wq);457}458EXPORT_SYMBOL_GPL(cxl_context_events_pending);459 460int cxl_start_work(struct cxl_context *ctx,461 struct cxl_ioctl_start_work *work)462{463 int rc;464 465 /* code taken from afu_ioctl_start_work */466 if (!(work->flags & CXL_START_WORK_NUM_IRQS))467 work->num_interrupts = ctx->afu->pp_irqs;468 else if ((work->num_interrupts < ctx->afu->pp_irqs) ||469 (work->num_interrupts > ctx->afu->irqs_max)) {470 return -EINVAL;471 }472 473 rc = afu_register_irqs(ctx, work->num_interrupts);474 if (rc)475 return rc;476 477 rc = cxl_start_context(ctx, work->work_element_descriptor, current);478 if (rc < 0) {479 afu_release_irqs(ctx, ctx);480 return rc;481 }482 483 return 0;484}485EXPORT_SYMBOL_GPL(cxl_start_work);486 487void __iomem *cxl_psa_map(struct cxl_context *ctx)488{489 if (ctx->status != STARTED)490 return NULL;491 492 pr_devel("%s: psn_phys%llx size:%llx\n",493 __func__, ctx->psn_phys, ctx->psn_size);494 return ioremap(ctx->psn_phys, ctx->psn_size);495}496EXPORT_SYMBOL_GPL(cxl_psa_map);497 498void cxl_psa_unmap(void __iomem *addr)499{500 iounmap(addr);501}502EXPORT_SYMBOL_GPL(cxl_psa_unmap);503 504int cxl_afu_reset(struct cxl_context *ctx)505{506 struct cxl_afu *afu = ctx->afu;507 int rc;508 509 rc = cxl_ops->afu_reset(afu);510 if (rc)511 return rc;512 513 return cxl_ops->afu_check_and_enable(afu);514}515EXPORT_SYMBOL_GPL(cxl_afu_reset);516 517void cxl_perst_reloads_same_image(struct cxl_afu *afu,518 bool perst_reloads_same_image)519{520 afu->adapter->perst_same_image = perst_reloads_same_image;521}522EXPORT_SYMBOL_GPL(cxl_perst_reloads_same_image);523 524ssize_t cxl_read_adapter_vpd(struct pci_dev *dev, void *buf, size_t count)525{526 struct cxl_afu *afu = cxl_pci_to_afu(dev);527 if (IS_ERR(afu))528 return -ENODEV;529 530 return cxl_ops->read_adapter_vpd(afu->adapter, buf, count);531}532EXPORT_SYMBOL_GPL(cxl_read_adapter_vpd);533