297 lines · c
1// SPDX-License-Identifier: GPL-2.0+2// Copyright 2017 IBM Corp.3#include <linux/sched/mm.h>4#include "trace.h"5#include "ocxl_internal.h"6 7int ocxl_context_alloc(struct ocxl_context **context, struct ocxl_afu *afu,8 struct address_space *mapping)9{10 int pasid;11 struct ocxl_context *ctx;12 13 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);14 if (!ctx)15 return -ENOMEM;16 17 ctx->afu = afu;18 mutex_lock(&afu->contexts_lock);19 pasid = idr_alloc(&afu->contexts_idr, ctx, afu->pasid_base,20 afu->pasid_base + afu->pasid_max, GFP_KERNEL);21 if (pasid < 0) {22 mutex_unlock(&afu->contexts_lock);23 kfree(ctx);24 return pasid;25 }26 afu->pasid_count++;27 mutex_unlock(&afu->contexts_lock);28 29 ctx->pasid = pasid;30 ctx->status = OPENED;31 mutex_init(&ctx->status_mutex);32 ctx->mapping = mapping;33 mutex_init(&ctx->mapping_lock);34 init_waitqueue_head(&ctx->events_wq);35 mutex_init(&ctx->xsl_error_lock);36 mutex_init(&ctx->irq_lock);37 idr_init(&ctx->irq_idr);38 ctx->tidr = 0;39 40 /*41 * Keep a reference on the AFU to make sure it's valid for the42 * duration of the life of the context43 */44 ocxl_afu_get(afu);45 *context = ctx;46 return 0;47}48EXPORT_SYMBOL_GPL(ocxl_context_alloc);49 50/*51 * Callback for when a translation fault triggers an error52 * data: a pointer to the context which triggered the fault53 * addr: the address that triggered the error54 * dsisr: the value of the PPC64 dsisr register55 */56static void xsl_fault_error(void *data, u64 addr, u64 dsisr)57{58 struct ocxl_context *ctx = data;59 60 mutex_lock(&ctx->xsl_error_lock);61 ctx->xsl_error.addr = addr;62 ctx->xsl_error.dsisr = dsisr;63 ctx->xsl_error.count++;64 mutex_unlock(&ctx->xsl_error_lock);65 66 wake_up_all(&ctx->events_wq);67}68 69int ocxl_context_attach(struct ocxl_context *ctx, u64 amr, struct mm_struct *mm)70{71 int rc;72 unsigned long pidr = 0;73 struct pci_dev *dev;74 75 // Locks both status & tidr76 mutex_lock(&ctx->status_mutex);77 if (ctx->status != OPENED) {78 rc = -EIO;79 goto out;80 }81 82 if (mm)83 pidr = mm->context.id;84 85 dev = to_pci_dev(ctx->afu->fn->dev.parent);86 rc = ocxl_link_add_pe(ctx->afu->fn->link, ctx->pasid, pidr, ctx->tidr,87 amr, pci_dev_id(dev), mm, xsl_fault_error, ctx);88 if (rc)89 goto out;90 91 ctx->status = ATTACHED;92out:93 mutex_unlock(&ctx->status_mutex);94 return rc;95}96EXPORT_SYMBOL_GPL(ocxl_context_attach);97 98static vm_fault_t map_afu_irq(struct vm_area_struct *vma, unsigned long address,99 u64 offset, struct ocxl_context *ctx)100{101 u64 trigger_addr;102 int irq_id = ocxl_irq_offset_to_id(ctx, offset);103 104 trigger_addr = ocxl_afu_irq_get_addr(ctx, irq_id);105 if (!trigger_addr)106 return VM_FAULT_SIGBUS;107 108 return vmf_insert_pfn(vma, address, trigger_addr >> PAGE_SHIFT);109}110 111static vm_fault_t map_pp_mmio(struct vm_area_struct *vma, unsigned long address,112 u64 offset, struct ocxl_context *ctx)113{114 u64 pp_mmio_addr;115 int pasid_off;116 vm_fault_t ret;117 118 if (offset >= ctx->afu->config.pp_mmio_stride)119 return VM_FAULT_SIGBUS;120 121 mutex_lock(&ctx->status_mutex);122 if (ctx->status != ATTACHED) {123 mutex_unlock(&ctx->status_mutex);124 pr_debug("%s: Context not attached, failing mmio mmap\n",125 __func__);126 return VM_FAULT_SIGBUS;127 }128 129 pasid_off = ctx->pasid - ctx->afu->pasid_base;130 pp_mmio_addr = ctx->afu->pp_mmio_start +131 pasid_off * ctx->afu->config.pp_mmio_stride +132 offset;133 134 ret = vmf_insert_pfn(vma, address, pp_mmio_addr >> PAGE_SHIFT);135 mutex_unlock(&ctx->status_mutex);136 return ret;137}138 139static vm_fault_t ocxl_mmap_fault(struct vm_fault *vmf)140{141 struct vm_area_struct *vma = vmf->vma;142 struct ocxl_context *ctx = vma->vm_file->private_data;143 u64 offset;144 vm_fault_t ret;145 146 offset = vmf->pgoff << PAGE_SHIFT;147 pr_debug("%s: pasid %d address 0x%lx offset 0x%llx\n", __func__,148 ctx->pasid, vmf->address, offset);149 150 if (offset < ctx->afu->irq_base_offset)151 ret = map_pp_mmio(vma, vmf->address, offset, ctx);152 else153 ret = map_afu_irq(vma, vmf->address, offset, ctx);154 return ret;155}156 157static const struct vm_operations_struct ocxl_vmops = {158 .fault = ocxl_mmap_fault,159};160 161static int check_mmap_afu_irq(struct ocxl_context *ctx,162 struct vm_area_struct *vma)163{164 int irq_id = ocxl_irq_offset_to_id(ctx, vma->vm_pgoff << PAGE_SHIFT);165 166 /* only one page */167 if (vma_pages(vma) != 1)168 return -EINVAL;169 170 /* check offset validty */171 if (!ocxl_afu_irq_get_addr(ctx, irq_id))172 return -EINVAL;173 174 /*175 * trigger page should only be accessible in write mode.176 *177 * It's a bit theoretical, as a page mmaped with only178 * PROT_WRITE is currently readable, but it doesn't hurt.179 */180 if ((vma->vm_flags & VM_READ) || (vma->vm_flags & VM_EXEC) ||181 !(vma->vm_flags & VM_WRITE))182 return -EINVAL;183 vm_flags_clear(vma, VM_MAYREAD | VM_MAYEXEC);184 return 0;185}186 187static int check_mmap_mmio(struct ocxl_context *ctx,188 struct vm_area_struct *vma)189{190 if ((vma_pages(vma) + vma->vm_pgoff) >191 (ctx->afu->config.pp_mmio_stride >> PAGE_SHIFT))192 return -EINVAL;193 return 0;194}195 196int ocxl_context_mmap(struct ocxl_context *ctx, struct vm_area_struct *vma)197{198 int rc;199 200 if ((vma->vm_pgoff << PAGE_SHIFT) < ctx->afu->irq_base_offset)201 rc = check_mmap_mmio(ctx, vma);202 else203 rc = check_mmap_afu_irq(ctx, vma);204 if (rc)205 return rc;206 207 vm_flags_set(vma, VM_IO | VM_PFNMAP);208 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);209 vma->vm_ops = &ocxl_vmops;210 return 0;211}212 213int ocxl_context_detach(struct ocxl_context *ctx)214{215 struct pci_dev *dev;216 int afu_control_pos;217 enum ocxl_context_status status;218 int rc;219 220 mutex_lock(&ctx->status_mutex);221 status = ctx->status;222 ctx->status = CLOSED;223 mutex_unlock(&ctx->status_mutex);224 if (status != ATTACHED)225 return 0;226 227 dev = to_pci_dev(ctx->afu->fn->dev.parent);228 afu_control_pos = ctx->afu->config.dvsec_afu_control_pos;229 230 mutex_lock(&ctx->afu->afu_control_lock);231 rc = ocxl_config_terminate_pasid(dev, afu_control_pos, ctx->pasid);232 mutex_unlock(&ctx->afu->afu_control_lock);233 trace_ocxl_terminate_pasid(ctx->pasid, rc);234 if (rc) {235 /*236 * If we timeout waiting for the AFU to terminate the237 * pasid, then it's dangerous to clean up the Process238 * Element entry in the SPA, as it may be referenced239 * in the future by the AFU. In which case, we would240 * checkstop because of an invalid PE access (FIR241 * register 2, bit 42). So leave the PE242 * defined. Caller shouldn't free the context so that243 * PASID remains allocated.244 *245 * A link reset will be required to cleanup the AFU246 * and the SPA.247 */248 if (rc == -EBUSY)249 return rc;250 }251 rc = ocxl_link_remove_pe(ctx->afu->fn->link, ctx->pasid);252 if (rc) {253 dev_warn(&dev->dev,254 "Couldn't remove PE entry cleanly: %d\n", rc);255 }256 return 0;257}258EXPORT_SYMBOL_GPL(ocxl_context_detach);259 260void ocxl_context_detach_all(struct ocxl_afu *afu)261{262 struct ocxl_context *ctx;263 int tmp;264 265 mutex_lock(&afu->contexts_lock);266 idr_for_each_entry(&afu->contexts_idr, ctx, tmp) {267 ocxl_context_detach(ctx);268 /*269 * We are force detaching - remove any active mmio270 * mappings so userspace cannot interfere with the271 * card if it comes back. Easiest way to exercise272 * this is to unbind and rebind the driver via sysfs273 * while it is in use.274 */275 mutex_lock(&ctx->mapping_lock);276 if (ctx->mapping)277 unmap_mapping_range(ctx->mapping, 0, 0, 1);278 mutex_unlock(&ctx->mapping_lock);279 }280 mutex_unlock(&afu->contexts_lock);281}282 283void ocxl_context_free(struct ocxl_context *ctx)284{285 mutex_lock(&ctx->afu->contexts_lock);286 ctx->afu->pasid_count--;287 idr_remove(&ctx->afu->contexts_idr, ctx->pasid);288 mutex_unlock(&ctx->afu->contexts_lock);289 290 ocxl_afu_irq_free_all(ctx);291 idr_destroy(&ctx->irq_idr);292 /* reference to the AFU taken in ocxl_context_alloc() */293 ocxl_afu_put(ctx->afu);294 kfree(ctx);295}296EXPORT_SYMBOL_GPL(ocxl_context_free);297