2509 lines · c
1// SPDX-License-Identifier: GPL-2.02// Copyright (c) 2011-2018, The Linux Foundation. All rights reserved.3// Copyright (c) 2018, Linaro Limited4 5#include <linux/completion.h>6#include <linux/device.h>7#include <linux/dma-buf.h>8#include <linux/dma-mapping.h>9#include <linux/dma-resv.h>10#include <linux/idr.h>11#include <linux/list.h>12#include <linux/miscdevice.h>13#include <linux/module.h>14#include <linux/of_address.h>15#include <linux/of.h>16#include <linux/platform_device.h>17#include <linux/sort.h>18#include <linux/of_platform.h>19#include <linux/rpmsg.h>20#include <linux/scatterlist.h>21#include <linux/slab.h>22#include <linux/firmware/qcom/qcom_scm.h>23#include <uapi/misc/fastrpc.h>24#include <linux/of_reserved_mem.h>25 26#define ADSP_DOMAIN_ID (0)27#define MDSP_DOMAIN_ID (1)28#define SDSP_DOMAIN_ID (2)29#define CDSP_DOMAIN_ID (3)30#define CDSP1_DOMAIN_ID (4)31#define FASTRPC_DEV_MAX 5 /* adsp, mdsp, slpi, cdsp, cdsp1 */32#define FASTRPC_MAX_SESSIONS 1433#define FASTRPC_MAX_VMIDS 1634#define FASTRPC_ALIGN 12835#define FASTRPC_MAX_FDLIST 1636#define FASTRPC_MAX_CRCLIST 6437#define FASTRPC_PHYS(p) ((p) & 0xffffffff)38#define FASTRPC_CTX_MAX (256)39#define FASTRPC_INIT_HANDLE 140#define FASTRPC_DSP_UTILITIES_HANDLE 241#define FASTRPC_CTXID_MASK (0xFF0)42#define INIT_FILELEN_MAX (2 * 1024 * 1024)43#define INIT_FILE_NAMELEN_MAX (128)44#define FASTRPC_DEVICE_NAME "fastrpc"45 46/* Add memory to static PD pool, protection thru XPU */47#define ADSP_MMAP_HEAP_ADDR 448/* MAP static DMA buffer on DSP User PD */49#define ADSP_MMAP_DMA_BUFFER 650/* Add memory to static PD pool protection thru hypervisor */51#define ADSP_MMAP_REMOTE_HEAP_ADDR 852/* Add memory to userPD pool, for user heap */53#define ADSP_MMAP_ADD_PAGES 0x100054/* Add memory to userPD pool, for LLC heap */55#define ADSP_MMAP_ADD_PAGES_LLC 0x3000,56 57#define DSP_UNSUPPORTED_API (0x80000414)58/* MAX NUMBER of DSP ATTRIBUTES SUPPORTED */59#define FASTRPC_MAX_DSP_ATTRIBUTES (256)60#define FASTRPC_MAX_DSP_ATTRIBUTES_LEN (sizeof(u32) * FASTRPC_MAX_DSP_ATTRIBUTES)61 62/* Retrives number of input buffers from the scalars parameter */63#define REMOTE_SCALARS_INBUFS(sc) (((sc) >> 16) & 0x0ff)64 65/* Retrives number of output buffers from the scalars parameter */66#define REMOTE_SCALARS_OUTBUFS(sc) (((sc) >> 8) & 0x0ff)67 68/* Retrives number of input handles from the scalars parameter */69#define REMOTE_SCALARS_INHANDLES(sc) (((sc) >> 4) & 0x0f)70 71/* Retrives number of output handles from the scalars parameter */72#define REMOTE_SCALARS_OUTHANDLES(sc) ((sc) & 0x0f)73 74#define REMOTE_SCALARS_LENGTH(sc) (REMOTE_SCALARS_INBUFS(sc) + \75 REMOTE_SCALARS_OUTBUFS(sc) + \76 REMOTE_SCALARS_INHANDLES(sc)+ \77 REMOTE_SCALARS_OUTHANDLES(sc))78#define FASTRPC_BUILD_SCALARS(attr, method, in, out, oin, oout) \79 (((attr & 0x07) << 29) | \80 ((method & 0x1f) << 24) | \81 ((in & 0xff) << 16) | \82 ((out & 0xff) << 8) | \83 ((oin & 0x0f) << 4) | \84 (oout & 0x0f))85 86#define FASTRPC_SCALARS(method, in, out) \87 FASTRPC_BUILD_SCALARS(0, method, in, out, 0, 0)88 89#define FASTRPC_CREATE_PROCESS_NARGS 690#define FASTRPC_CREATE_STATIC_PROCESS_NARGS 391/* Remote Method id table */92#define FASTRPC_RMID_INIT_ATTACH 093#define FASTRPC_RMID_INIT_RELEASE 194#define FASTRPC_RMID_INIT_MMAP 495#define FASTRPC_RMID_INIT_MUNMAP 596#define FASTRPC_RMID_INIT_CREATE 697#define FASTRPC_RMID_INIT_CREATE_ATTR 798#define FASTRPC_RMID_INIT_CREATE_STATIC 899#define FASTRPC_RMID_INIT_MEM_MAP 10100#define FASTRPC_RMID_INIT_MEM_UNMAP 11101 102/* Protection Domain(PD) ids */103#define ROOT_PD (0)104#define USER_PD (1)105#define SENSORS_PD (2)106 107#define miscdev_to_fdevice(d) container_of(d, struct fastrpc_device, miscdev)108 109static const char *domains[FASTRPC_DEV_MAX] = { "adsp", "mdsp",110 "sdsp", "cdsp", "cdsp1" };111struct fastrpc_phy_page {112 u64 addr; /* physical address */113 u64 size; /* size of contiguous region */114};115 116struct fastrpc_invoke_buf {117 u32 num; /* number of contiguous regions */118 u32 pgidx; /* index to start of contiguous region */119};120 121struct fastrpc_remote_dmahandle {122 s32 fd; /* dma handle fd */123 u32 offset; /* dma handle offset */124 u32 len; /* dma handle length */125};126 127struct fastrpc_remote_buf {128 u64 pv; /* buffer pointer */129 u64 len; /* length of buffer */130};131 132union fastrpc_remote_arg {133 struct fastrpc_remote_buf buf;134 struct fastrpc_remote_dmahandle dma;135};136 137struct fastrpc_mmap_rsp_msg {138 u64 vaddr;139};140 141struct fastrpc_mmap_req_msg {142 s32 pgid;143 u32 flags;144 u64 vaddr;145 s32 num;146};147 148struct fastrpc_mem_map_req_msg {149 s32 pgid;150 s32 fd;151 s32 offset;152 u32 flags;153 u64 vaddrin;154 s32 num;155 s32 data_len;156};157 158struct fastrpc_munmap_req_msg {159 s32 pgid;160 u64 vaddr;161 u64 size;162};163 164struct fastrpc_mem_unmap_req_msg {165 s32 pgid;166 s32 fd;167 u64 vaddrin;168 u64 len;169};170 171struct fastrpc_msg {172 int pid; /* process group id */173 int tid; /* thread id */174 u64 ctx; /* invoke caller context */175 u32 handle; /* handle to invoke */176 u32 sc; /* scalars structure describing the data */177 u64 addr; /* physical address */178 u64 size; /* size of contiguous region */179};180 181struct fastrpc_invoke_rsp {182 u64 ctx; /* invoke caller context */183 int retval; /* invoke return value */184};185 186struct fastrpc_buf_overlap {187 u64 start;188 u64 end;189 int raix;190 u64 mstart;191 u64 mend;192 u64 offset;193};194 195struct fastrpc_buf {196 struct fastrpc_user *fl;197 struct dma_buf *dmabuf;198 struct device *dev;199 void *virt;200 u64 phys;201 u64 size;202 /* Lock for dma buf attachments */203 struct mutex lock;204 struct list_head attachments;205 /* mmap support */206 struct list_head node; /* list of user requested mmaps */207 uintptr_t raddr;208};209 210struct fastrpc_dma_buf_attachment {211 struct device *dev;212 struct sg_table sgt;213 struct list_head node;214};215 216struct fastrpc_map {217 struct list_head node;218 struct fastrpc_user *fl;219 int fd;220 struct dma_buf *buf;221 struct sg_table *table;222 struct dma_buf_attachment *attach;223 u64 phys;224 u64 size;225 void *va;226 u64 len;227 u64 raddr;228 u32 attr;229 struct kref refcount;230};231 232struct fastrpc_invoke_ctx {233 int nscalars;234 int nbufs;235 int retval;236 int pid;237 int tgid;238 u32 sc;239 u32 *crc;240 u64 ctxid;241 u64 msg_sz;242 struct kref refcount;243 struct list_head node; /* list of ctxs */244 struct completion work;245 struct work_struct put_work;246 struct fastrpc_msg msg;247 struct fastrpc_user *fl;248 union fastrpc_remote_arg *rpra;249 struct fastrpc_map **maps;250 struct fastrpc_buf *buf;251 struct fastrpc_invoke_args *args;252 struct fastrpc_buf_overlap *olaps;253 struct fastrpc_channel_ctx *cctx;254};255 256struct fastrpc_session_ctx {257 struct device *dev;258 int sid;259 bool used;260 bool valid;261};262 263struct fastrpc_channel_ctx {264 int domain_id;265 int sesscount;266 int vmcount;267 struct qcom_scm_vmperm vmperms[FASTRPC_MAX_VMIDS];268 struct rpmsg_device *rpdev;269 struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS];270 spinlock_t lock;271 struct idr ctx_idr;272 struct list_head users;273 struct kref refcount;274 /* Flag if dsp attributes are cached */275 bool valid_attributes;276 u32 dsp_attributes[FASTRPC_MAX_DSP_ATTRIBUTES];277 struct fastrpc_device *secure_fdevice;278 struct fastrpc_device *fdevice;279 struct fastrpc_buf *remote_heap;280 struct list_head invoke_interrupted_mmaps;281 bool secure;282 bool unsigned_support;283 u64 dma_mask;284};285 286struct fastrpc_device {287 struct fastrpc_channel_ctx *cctx;288 struct miscdevice miscdev;289 bool secure;290};291 292struct fastrpc_user {293 struct list_head user;294 struct list_head maps;295 struct list_head pending;296 struct list_head mmaps;297 298 struct fastrpc_channel_ctx *cctx;299 struct fastrpc_session_ctx *sctx;300 struct fastrpc_buf *init_mem;301 302 int tgid;303 int pd;304 bool is_secure_dev;305 /* Lock for lists */306 spinlock_t lock;307 /* lock for allocations */308 struct mutex mutex;309};310 311static void fastrpc_free_map(struct kref *ref)312{313 struct fastrpc_map *map;314 315 map = container_of(ref, struct fastrpc_map, refcount);316 317 if (map->table) {318 if (map->attr & FASTRPC_ATTR_SECUREMAP) {319 struct qcom_scm_vmperm perm;320 int vmid = map->fl->cctx->vmperms[0].vmid;321 u64 src_perms = BIT(QCOM_SCM_VMID_HLOS) | BIT(vmid);322 int err = 0;323 324 perm.vmid = QCOM_SCM_VMID_HLOS;325 perm.perm = QCOM_SCM_PERM_RWX;326 err = qcom_scm_assign_mem(map->phys, map->size,327 &src_perms, &perm, 1);328 if (err) {329 dev_err(map->fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d\n",330 map->phys, map->size, err);331 return;332 }333 }334 dma_buf_unmap_attachment_unlocked(map->attach, map->table,335 DMA_BIDIRECTIONAL);336 dma_buf_detach(map->buf, map->attach);337 dma_buf_put(map->buf);338 }339 340 if (map->fl) {341 spin_lock(&map->fl->lock);342 list_del(&map->node);343 spin_unlock(&map->fl->lock);344 map->fl = NULL;345 }346 347 kfree(map);348}349 350static void fastrpc_map_put(struct fastrpc_map *map)351{352 if (map)353 kref_put(&map->refcount, fastrpc_free_map);354}355 356static int fastrpc_map_get(struct fastrpc_map *map)357{358 if (!map)359 return -ENOENT;360 361 return kref_get_unless_zero(&map->refcount) ? 0 : -ENOENT;362}363 364 365static int fastrpc_map_lookup(struct fastrpc_user *fl, int fd,366 struct fastrpc_map **ppmap, bool take_ref)367{368 struct fastrpc_session_ctx *sess = fl->sctx;369 struct fastrpc_map *map = NULL;370 int ret = -ENOENT;371 372 spin_lock(&fl->lock);373 list_for_each_entry(map, &fl->maps, node) {374 if (map->fd != fd)375 continue;376 377 if (take_ref) {378 ret = fastrpc_map_get(map);379 if (ret) {380 dev_dbg(sess->dev, "%s: Failed to get map fd=%d ret=%d\n",381 __func__, fd, ret);382 break;383 }384 }385 386 *ppmap = map;387 ret = 0;388 break;389 }390 spin_unlock(&fl->lock);391 392 return ret;393}394 395static void fastrpc_buf_free(struct fastrpc_buf *buf)396{397 dma_free_coherent(buf->dev, buf->size, buf->virt,398 FASTRPC_PHYS(buf->phys));399 kfree(buf);400}401 402static int __fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev,403 u64 size, struct fastrpc_buf **obuf)404{405 struct fastrpc_buf *buf;406 407 buf = kzalloc(sizeof(*buf), GFP_KERNEL);408 if (!buf)409 return -ENOMEM;410 411 INIT_LIST_HEAD(&buf->attachments);412 INIT_LIST_HEAD(&buf->node);413 mutex_init(&buf->lock);414 415 buf->fl = fl;416 buf->virt = NULL;417 buf->phys = 0;418 buf->size = size;419 buf->dev = dev;420 buf->raddr = 0;421 422 buf->virt = dma_alloc_coherent(dev, buf->size, (dma_addr_t *)&buf->phys,423 GFP_KERNEL);424 if (!buf->virt) {425 mutex_destroy(&buf->lock);426 kfree(buf);427 return -ENOMEM;428 }429 430 *obuf = buf;431 432 return 0;433}434 435static int fastrpc_buf_alloc(struct fastrpc_user *fl, struct device *dev,436 u64 size, struct fastrpc_buf **obuf)437{438 int ret;439 struct fastrpc_buf *buf;440 441 ret = __fastrpc_buf_alloc(fl, dev, size, obuf);442 if (ret)443 return ret;444 445 buf = *obuf;446 447 if (fl->sctx && fl->sctx->sid)448 buf->phys += ((u64)fl->sctx->sid << 32);449 450 return 0;451}452 453static int fastrpc_remote_heap_alloc(struct fastrpc_user *fl, struct device *dev,454 u64 size, struct fastrpc_buf **obuf)455{456 struct device *rdev = &fl->cctx->rpdev->dev;457 458 return __fastrpc_buf_alloc(fl, rdev, size, obuf);459}460 461static void fastrpc_channel_ctx_free(struct kref *ref)462{463 struct fastrpc_channel_ctx *cctx;464 465 cctx = container_of(ref, struct fastrpc_channel_ctx, refcount);466 467 kfree(cctx);468}469 470static void fastrpc_channel_ctx_get(struct fastrpc_channel_ctx *cctx)471{472 kref_get(&cctx->refcount);473}474 475static void fastrpc_channel_ctx_put(struct fastrpc_channel_ctx *cctx)476{477 kref_put(&cctx->refcount, fastrpc_channel_ctx_free);478}479 480static void fastrpc_context_free(struct kref *ref)481{482 struct fastrpc_invoke_ctx *ctx;483 struct fastrpc_channel_ctx *cctx;484 unsigned long flags;485 int i;486 487 ctx = container_of(ref, struct fastrpc_invoke_ctx, refcount);488 cctx = ctx->cctx;489 490 for (i = 0; i < ctx->nbufs; i++)491 fastrpc_map_put(ctx->maps[i]);492 493 if (ctx->buf)494 fastrpc_buf_free(ctx->buf);495 496 spin_lock_irqsave(&cctx->lock, flags);497 idr_remove(&cctx->ctx_idr, ctx->ctxid >> 4);498 spin_unlock_irqrestore(&cctx->lock, flags);499 500 kfree(ctx->maps);501 kfree(ctx->olaps);502 kfree(ctx);503 504 fastrpc_channel_ctx_put(cctx);505}506 507static void fastrpc_context_get(struct fastrpc_invoke_ctx *ctx)508{509 kref_get(&ctx->refcount);510}511 512static void fastrpc_context_put(struct fastrpc_invoke_ctx *ctx)513{514 kref_put(&ctx->refcount, fastrpc_context_free);515}516 517static void fastrpc_context_put_wq(struct work_struct *work)518{519 struct fastrpc_invoke_ctx *ctx =520 container_of(work, struct fastrpc_invoke_ctx, put_work);521 522 fastrpc_context_put(ctx);523}524 525#define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)526static int olaps_cmp(const void *a, const void *b)527{528 struct fastrpc_buf_overlap *pa = (struct fastrpc_buf_overlap *)a;529 struct fastrpc_buf_overlap *pb = (struct fastrpc_buf_overlap *)b;530 /* sort with lowest starting buffer first */531 int st = CMP(pa->start, pb->start);532 /* sort with highest ending buffer first */533 int ed = CMP(pb->end, pa->end);534 535 return st == 0 ? ed : st;536}537 538static void fastrpc_get_buff_overlaps(struct fastrpc_invoke_ctx *ctx)539{540 u64 max_end = 0;541 int i;542 543 for (i = 0; i < ctx->nbufs; ++i) {544 ctx->olaps[i].start = ctx->args[i].ptr;545 ctx->olaps[i].end = ctx->olaps[i].start + ctx->args[i].length;546 ctx->olaps[i].raix = i;547 }548 549 sort(ctx->olaps, ctx->nbufs, sizeof(*ctx->olaps), olaps_cmp, NULL);550 551 for (i = 0; i < ctx->nbufs; ++i) {552 /* Falling inside previous range */553 if (ctx->olaps[i].start < max_end) {554 ctx->olaps[i].mstart = max_end;555 ctx->olaps[i].mend = ctx->olaps[i].end;556 ctx->olaps[i].offset = max_end - ctx->olaps[i].start;557 558 if (ctx->olaps[i].end > max_end) {559 max_end = ctx->olaps[i].end;560 } else {561 ctx->olaps[i].mend = 0;562 ctx->olaps[i].mstart = 0;563 }564 565 } else {566 ctx->olaps[i].mend = ctx->olaps[i].end;567 ctx->olaps[i].mstart = ctx->olaps[i].start;568 ctx->olaps[i].offset = 0;569 max_end = ctx->olaps[i].end;570 }571 }572}573 574static struct fastrpc_invoke_ctx *fastrpc_context_alloc(575 struct fastrpc_user *user, u32 kernel, u32 sc,576 struct fastrpc_invoke_args *args)577{578 struct fastrpc_channel_ctx *cctx = user->cctx;579 struct fastrpc_invoke_ctx *ctx = NULL;580 unsigned long flags;581 int ret;582 583 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);584 if (!ctx)585 return ERR_PTR(-ENOMEM);586 587 INIT_LIST_HEAD(&ctx->node);588 ctx->fl = user;589 ctx->nscalars = REMOTE_SCALARS_LENGTH(sc);590 ctx->nbufs = REMOTE_SCALARS_INBUFS(sc) +591 REMOTE_SCALARS_OUTBUFS(sc);592 593 if (ctx->nscalars) {594 ctx->maps = kcalloc(ctx->nscalars,595 sizeof(*ctx->maps), GFP_KERNEL);596 if (!ctx->maps) {597 kfree(ctx);598 return ERR_PTR(-ENOMEM);599 }600 ctx->olaps = kcalloc(ctx->nscalars,601 sizeof(*ctx->olaps), GFP_KERNEL);602 if (!ctx->olaps) {603 kfree(ctx->maps);604 kfree(ctx);605 return ERR_PTR(-ENOMEM);606 }607 ctx->args = args;608 fastrpc_get_buff_overlaps(ctx);609 }610 611 /* Released in fastrpc_context_put() */612 fastrpc_channel_ctx_get(cctx);613 614 ctx->sc = sc;615 ctx->retval = -1;616 ctx->pid = current->pid;617 ctx->tgid = user->tgid;618 ctx->cctx = cctx;619 init_completion(&ctx->work);620 INIT_WORK(&ctx->put_work, fastrpc_context_put_wq);621 622 spin_lock(&user->lock);623 list_add_tail(&ctx->node, &user->pending);624 spin_unlock(&user->lock);625 626 spin_lock_irqsave(&cctx->lock, flags);627 ret = idr_alloc_cyclic(&cctx->ctx_idr, ctx, 1,628 FASTRPC_CTX_MAX, GFP_ATOMIC);629 if (ret < 0) {630 spin_unlock_irqrestore(&cctx->lock, flags);631 goto err_idr;632 }633 ctx->ctxid = ret << 4;634 spin_unlock_irqrestore(&cctx->lock, flags);635 636 kref_init(&ctx->refcount);637 638 return ctx;639err_idr:640 spin_lock(&user->lock);641 list_del(&ctx->node);642 spin_unlock(&user->lock);643 fastrpc_channel_ctx_put(cctx);644 kfree(ctx->maps);645 kfree(ctx->olaps);646 kfree(ctx);647 648 return ERR_PTR(ret);649}650 651static struct sg_table *652fastrpc_map_dma_buf(struct dma_buf_attachment *attachment,653 enum dma_data_direction dir)654{655 struct fastrpc_dma_buf_attachment *a = attachment->priv;656 struct sg_table *table;657 int ret;658 659 table = &a->sgt;660 661 ret = dma_map_sgtable(attachment->dev, table, dir, 0);662 if (ret)663 table = ERR_PTR(ret);664 return table;665}666 667static void fastrpc_unmap_dma_buf(struct dma_buf_attachment *attach,668 struct sg_table *table,669 enum dma_data_direction dir)670{671 dma_unmap_sgtable(attach->dev, table, dir, 0);672}673 674static void fastrpc_release(struct dma_buf *dmabuf)675{676 struct fastrpc_buf *buffer = dmabuf->priv;677 678 fastrpc_buf_free(buffer);679}680 681static int fastrpc_dma_buf_attach(struct dma_buf *dmabuf,682 struct dma_buf_attachment *attachment)683{684 struct fastrpc_dma_buf_attachment *a;685 struct fastrpc_buf *buffer = dmabuf->priv;686 int ret;687 688 a = kzalloc(sizeof(*a), GFP_KERNEL);689 if (!a)690 return -ENOMEM;691 692 ret = dma_get_sgtable(buffer->dev, &a->sgt, buffer->virt,693 FASTRPC_PHYS(buffer->phys), buffer->size);694 if (ret < 0) {695 dev_err(buffer->dev, "failed to get scatterlist from DMA API\n");696 kfree(a);697 return -EINVAL;698 }699 700 a->dev = attachment->dev;701 INIT_LIST_HEAD(&a->node);702 attachment->priv = a;703 704 mutex_lock(&buffer->lock);705 list_add(&a->node, &buffer->attachments);706 mutex_unlock(&buffer->lock);707 708 return 0;709}710 711static void fastrpc_dma_buf_detatch(struct dma_buf *dmabuf,712 struct dma_buf_attachment *attachment)713{714 struct fastrpc_dma_buf_attachment *a = attachment->priv;715 struct fastrpc_buf *buffer = dmabuf->priv;716 717 mutex_lock(&buffer->lock);718 list_del(&a->node);719 mutex_unlock(&buffer->lock);720 sg_free_table(&a->sgt);721 kfree(a);722}723 724static int fastrpc_vmap(struct dma_buf *dmabuf, struct iosys_map *map)725{726 struct fastrpc_buf *buf = dmabuf->priv;727 728 iosys_map_set_vaddr(map, buf->virt);729 730 return 0;731}732 733static int fastrpc_mmap(struct dma_buf *dmabuf,734 struct vm_area_struct *vma)735{736 struct fastrpc_buf *buf = dmabuf->priv;737 size_t size = vma->vm_end - vma->vm_start;738 739 dma_resv_assert_held(dmabuf->resv);740 741 return dma_mmap_coherent(buf->dev, vma, buf->virt,742 FASTRPC_PHYS(buf->phys), size);743}744 745static const struct dma_buf_ops fastrpc_dma_buf_ops = {746 .attach = fastrpc_dma_buf_attach,747 .detach = fastrpc_dma_buf_detatch,748 .map_dma_buf = fastrpc_map_dma_buf,749 .unmap_dma_buf = fastrpc_unmap_dma_buf,750 .mmap = fastrpc_mmap,751 .vmap = fastrpc_vmap,752 .release = fastrpc_release,753};754 755static int fastrpc_map_create(struct fastrpc_user *fl, int fd,756 u64 len, u32 attr, struct fastrpc_map **ppmap)757{758 struct fastrpc_session_ctx *sess = fl->sctx;759 struct fastrpc_map *map = NULL;760 struct sg_table *table;761 int err = 0;762 763 if (!fastrpc_map_lookup(fl, fd, ppmap, true))764 return 0;765 766 map = kzalloc(sizeof(*map), GFP_KERNEL);767 if (!map)768 return -ENOMEM;769 770 INIT_LIST_HEAD(&map->node);771 kref_init(&map->refcount);772 773 map->fl = fl;774 map->fd = fd;775 map->buf = dma_buf_get(fd);776 if (IS_ERR(map->buf)) {777 err = PTR_ERR(map->buf);778 goto get_err;779 }780 781 map->attach = dma_buf_attach(map->buf, sess->dev);782 if (IS_ERR(map->attach)) {783 dev_err(sess->dev, "Failed to attach dmabuf\n");784 err = PTR_ERR(map->attach);785 goto attach_err;786 }787 788 table = dma_buf_map_attachment_unlocked(map->attach, DMA_BIDIRECTIONAL);789 if (IS_ERR(table)) {790 err = PTR_ERR(table);791 goto map_err;792 }793 map->table = table;794 795 if (attr & FASTRPC_ATTR_SECUREMAP) {796 map->phys = sg_phys(map->table->sgl);797 } else {798 map->phys = sg_dma_address(map->table->sgl);799 map->phys += ((u64)fl->sctx->sid << 32);800 }801 map->size = len;802 map->va = sg_virt(map->table->sgl);803 map->len = len;804 805 if (attr & FASTRPC_ATTR_SECUREMAP) {806 /*807 * If subsystem VMIDs are defined in DTSI, then do808 * hyp_assign from HLOS to those VM(s)809 */810 u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);811 struct qcom_scm_vmperm dst_perms[2] = {0};812 813 dst_perms[0].vmid = QCOM_SCM_VMID_HLOS;814 dst_perms[0].perm = QCOM_SCM_PERM_RW;815 dst_perms[1].vmid = fl->cctx->vmperms[0].vmid;816 dst_perms[1].perm = QCOM_SCM_PERM_RWX;817 map->attr = attr;818 err = qcom_scm_assign_mem(map->phys, (u64)map->size, &src_perms, dst_perms, 2);819 if (err) {820 dev_err(sess->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d\n",821 map->phys, map->size, err);822 goto map_err;823 }824 }825 spin_lock(&fl->lock);826 list_add_tail(&map->node, &fl->maps);827 spin_unlock(&fl->lock);828 *ppmap = map;829 830 return 0;831 832map_err:833 dma_buf_detach(map->buf, map->attach);834attach_err:835 dma_buf_put(map->buf);836get_err:837 fastrpc_map_put(map);838 839 return err;840}841 842/*843 * Fastrpc payload buffer with metadata looks like:844 *845 * >>>>>> START of METADATA <<<<<<<<<846 * +---------------------------------+847 * | Arguments |848 * | type:(union fastrpc_remote_arg)|849 * | (0 - N) |850 * +---------------------------------+851 * | Invoke Buffer list |852 * | type:(struct fastrpc_invoke_buf)|853 * | (0 - N) |854 * +---------------------------------+855 * | Page info list |856 * | type:(struct fastrpc_phy_page) |857 * | (0 - N) |858 * +---------------------------------+859 * | Optional info |860 * |(can be specific to SoC/Firmware)|861 * +---------------------------------+862 * >>>>>>>> END of METADATA <<<<<<<<<863 * +---------------------------------+864 * | Inline ARGS |865 * | (0-N) |866 * +---------------------------------+867 */868 869static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx *ctx)870{871 int size = 0;872 873 size = (sizeof(struct fastrpc_remote_buf) +874 sizeof(struct fastrpc_invoke_buf) +875 sizeof(struct fastrpc_phy_page)) * ctx->nscalars +876 sizeof(u64) * FASTRPC_MAX_FDLIST +877 sizeof(u32) * FASTRPC_MAX_CRCLIST;878 879 return size;880}881 882static u64 fastrpc_get_payload_size(struct fastrpc_invoke_ctx *ctx, int metalen)883{884 u64 size = 0;885 int oix;886 887 size = ALIGN(metalen, FASTRPC_ALIGN);888 for (oix = 0; oix < ctx->nbufs; oix++) {889 int i = ctx->olaps[oix].raix;890 891 if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1) {892 893 if (ctx->olaps[oix].offset == 0)894 size = ALIGN(size, FASTRPC_ALIGN);895 896 size += (ctx->olaps[oix].mend - ctx->olaps[oix].mstart);897 }898 }899 900 return size;901}902 903static int fastrpc_create_maps(struct fastrpc_invoke_ctx *ctx)904{905 struct device *dev = ctx->fl->sctx->dev;906 int i, err;907 908 for (i = 0; i < ctx->nscalars; ++i) {909 910 if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1 ||911 ctx->args[i].length == 0)912 continue;913 914 err = fastrpc_map_create(ctx->fl, ctx->args[i].fd,915 ctx->args[i].length, ctx->args[i].attr, &ctx->maps[i]);916 if (err) {917 dev_err(dev, "Error Creating map %d\n", err);918 return -EINVAL;919 }920 921 }922 return 0;923}924 925static struct fastrpc_invoke_buf *fastrpc_invoke_buf_start(union fastrpc_remote_arg *pra, int len)926{927 return (struct fastrpc_invoke_buf *)(&pra[len]);928}929 930static struct fastrpc_phy_page *fastrpc_phy_page_start(struct fastrpc_invoke_buf *buf, int len)931{932 return (struct fastrpc_phy_page *)(&buf[len]);933}934 935static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx)936{937 struct device *dev = ctx->fl->sctx->dev;938 union fastrpc_remote_arg *rpra;939 struct fastrpc_invoke_buf *list;940 struct fastrpc_phy_page *pages;941 int inbufs, i, oix, err = 0;942 u64 len, rlen, pkt_size;943 u64 pg_start, pg_end;944 uintptr_t args;945 int metalen;946 947 inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);948 metalen = fastrpc_get_meta_size(ctx);949 pkt_size = fastrpc_get_payload_size(ctx, metalen);950 951 err = fastrpc_create_maps(ctx);952 if (err)953 return err;954 955 ctx->msg_sz = pkt_size;956 957 if (ctx->fl->sctx->sid)958 err = fastrpc_buf_alloc(ctx->fl, dev, pkt_size, &ctx->buf);959 else960 err = fastrpc_remote_heap_alloc(ctx->fl, dev, pkt_size, &ctx->buf);961 if (err)962 return err;963 964 memset(ctx->buf->virt, 0, pkt_size);965 rpra = ctx->buf->virt;966 list = fastrpc_invoke_buf_start(rpra, ctx->nscalars);967 pages = fastrpc_phy_page_start(list, ctx->nscalars);968 args = (uintptr_t)ctx->buf->virt + metalen;969 rlen = pkt_size - metalen;970 ctx->rpra = rpra;971 972 for (oix = 0; oix < ctx->nbufs; ++oix) {973 int mlen;974 975 i = ctx->olaps[oix].raix;976 len = ctx->args[i].length;977 978 rpra[i].buf.pv = 0;979 rpra[i].buf.len = len;980 list[i].num = len ? 1 : 0;981 list[i].pgidx = i;982 983 if (!len)984 continue;985 986 if (ctx->maps[i]) {987 struct vm_area_struct *vma = NULL;988 989 rpra[i].buf.pv = (u64) ctx->args[i].ptr;990 pages[i].addr = ctx->maps[i]->phys;991 992 mmap_read_lock(current->mm);993 vma = find_vma(current->mm, ctx->args[i].ptr);994 if (vma)995 pages[i].addr += ctx->args[i].ptr -996 vma->vm_start;997 mmap_read_unlock(current->mm);998 999 pg_start = (ctx->args[i].ptr & PAGE_MASK) >> PAGE_SHIFT;1000 pg_end = ((ctx->args[i].ptr + len - 1) & PAGE_MASK) >>1001 PAGE_SHIFT;1002 pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;1003 1004 } else {1005 1006 if (ctx->olaps[oix].offset == 0) {1007 rlen -= ALIGN(args, FASTRPC_ALIGN) - args;1008 args = ALIGN(args, FASTRPC_ALIGN);1009 }1010 1011 mlen = ctx->olaps[oix].mend - ctx->olaps[oix].mstart;1012 1013 if (rlen < mlen)1014 goto bail;1015 1016 rpra[i].buf.pv = args - ctx->olaps[oix].offset;1017 pages[i].addr = ctx->buf->phys -1018 ctx->olaps[oix].offset +1019 (pkt_size - rlen);1020 pages[i].addr = pages[i].addr & PAGE_MASK;1021 1022 pg_start = (args & PAGE_MASK) >> PAGE_SHIFT;1023 pg_end = ((args + len - 1) & PAGE_MASK) >> PAGE_SHIFT;1024 pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE;1025 args = args + mlen;1026 rlen -= mlen;1027 }1028 1029 if (i < inbufs && !ctx->maps[i]) {1030 void *dst = (void *)(uintptr_t)rpra[i].buf.pv;1031 void *src = (void *)(uintptr_t)ctx->args[i].ptr;1032 1033 if (!kernel) {1034 if (copy_from_user(dst, (void __user *)src,1035 len)) {1036 err = -EFAULT;1037 goto bail;1038 }1039 } else {1040 memcpy(dst, src, len);1041 }1042 }1043 }1044 1045 for (i = ctx->nbufs; i < ctx->nscalars; ++i) {1046 list[i].num = ctx->args[i].length ? 1 : 0;1047 list[i].pgidx = i;1048 if (ctx->maps[i]) {1049 pages[i].addr = ctx->maps[i]->phys;1050 pages[i].size = ctx->maps[i]->size;1051 }1052 rpra[i].dma.fd = ctx->args[i].fd;1053 rpra[i].dma.len = ctx->args[i].length;1054 rpra[i].dma.offset = (u64) ctx->args[i].ptr;1055 }1056 1057bail:1058 if (err)1059 dev_err(dev, "Error: get invoke args failed:%d\n", err);1060 1061 return err;1062}1063 1064static int fastrpc_put_args(struct fastrpc_invoke_ctx *ctx,1065 u32 kernel)1066{1067 union fastrpc_remote_arg *rpra = ctx->rpra;1068 struct fastrpc_user *fl = ctx->fl;1069 struct fastrpc_map *mmap = NULL;1070 struct fastrpc_invoke_buf *list;1071 struct fastrpc_phy_page *pages;1072 u64 *fdlist;1073 int i, inbufs, outbufs, handles;1074 1075 inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);1076 outbufs = REMOTE_SCALARS_OUTBUFS(ctx->sc);1077 handles = REMOTE_SCALARS_INHANDLES(ctx->sc) + REMOTE_SCALARS_OUTHANDLES(ctx->sc);1078 list = fastrpc_invoke_buf_start(rpra, ctx->nscalars);1079 pages = fastrpc_phy_page_start(list, ctx->nscalars);1080 fdlist = (uint64_t *)(pages + inbufs + outbufs + handles);1081 1082 for (i = inbufs; i < ctx->nbufs; ++i) {1083 if (!ctx->maps[i]) {1084 void *src = (void *)(uintptr_t)rpra[i].buf.pv;1085 void *dst = (void *)(uintptr_t)ctx->args[i].ptr;1086 u64 len = rpra[i].buf.len;1087 1088 if (!kernel) {1089 if (copy_to_user((void __user *)dst, src, len))1090 return -EFAULT;1091 } else {1092 memcpy(dst, src, len);1093 }1094 }1095 }1096 1097 /* Clean up fdlist which is updated by DSP */1098 for (i = 0; i < FASTRPC_MAX_FDLIST; i++) {1099 if (!fdlist[i])1100 break;1101 if (!fastrpc_map_lookup(fl, (int)fdlist[i], &mmap, false))1102 fastrpc_map_put(mmap);1103 }1104 1105 return 0;1106}1107 1108static int fastrpc_invoke_send(struct fastrpc_session_ctx *sctx,1109 struct fastrpc_invoke_ctx *ctx,1110 u32 kernel, uint32_t handle)1111{1112 struct fastrpc_channel_ctx *cctx;1113 struct fastrpc_user *fl = ctx->fl;1114 struct fastrpc_msg *msg = &ctx->msg;1115 int ret;1116 1117 cctx = fl->cctx;1118 msg->pid = fl->tgid;1119 msg->tid = current->pid;1120 1121 if (kernel)1122 msg->pid = 0;1123 1124 msg->ctx = ctx->ctxid | fl->pd;1125 msg->handle = handle;1126 msg->sc = ctx->sc;1127 msg->addr = ctx->buf ? ctx->buf->phys : 0;1128 msg->size = roundup(ctx->msg_sz, PAGE_SIZE);1129 fastrpc_context_get(ctx);1130 1131 ret = rpmsg_send(cctx->rpdev->ept, (void *)msg, sizeof(*msg));1132 1133 if (ret)1134 fastrpc_context_put(ctx);1135 1136 return ret;1137 1138}1139 1140static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel,1141 u32 handle, u32 sc,1142 struct fastrpc_invoke_args *args)1143{1144 struct fastrpc_invoke_ctx *ctx = NULL;1145 struct fastrpc_buf *buf, *b;1146 1147 int err = 0;1148 1149 if (!fl->sctx)1150 return -EINVAL;1151 1152 if (!fl->cctx->rpdev)1153 return -EPIPE;1154 1155 if (handle == FASTRPC_INIT_HANDLE && !kernel) {1156 dev_warn_ratelimited(fl->sctx->dev, "user app trying to send a kernel RPC message (%d)\n", handle);1157 return -EPERM;1158 }1159 1160 ctx = fastrpc_context_alloc(fl, kernel, sc, args);1161 if (IS_ERR(ctx))1162 return PTR_ERR(ctx);1163 1164 err = fastrpc_get_args(kernel, ctx);1165 if (err)1166 goto bail;1167 1168 /* make sure that all CPU memory writes are seen by DSP */1169 dma_wmb();1170 /* Send invoke buffer to remote dsp */1171 err = fastrpc_invoke_send(fl->sctx, ctx, kernel, handle);1172 if (err)1173 goto bail;1174 1175 if (kernel) {1176 if (!wait_for_completion_timeout(&ctx->work, 10 * HZ))1177 err = -ETIMEDOUT;1178 } else {1179 err = wait_for_completion_interruptible(&ctx->work);1180 }1181 1182 if (err)1183 goto bail;1184 1185 /* make sure that all memory writes by DSP are seen by CPU */1186 dma_rmb();1187 /* populate all the output buffers with results */1188 err = fastrpc_put_args(ctx, kernel);1189 if (err)1190 goto bail;1191 1192 /* Check the response from remote dsp */1193 err = ctx->retval;1194 if (err)1195 goto bail;1196 1197bail:1198 if (err != -ERESTARTSYS && err != -ETIMEDOUT) {1199 /* We are done with this compute context */1200 spin_lock(&fl->lock);1201 list_del(&ctx->node);1202 spin_unlock(&fl->lock);1203 fastrpc_context_put(ctx);1204 }1205 1206 if (err == -ERESTARTSYS) {1207 list_for_each_entry_safe(buf, b, &fl->mmaps, node) {1208 list_del(&buf->node);1209 list_add_tail(&buf->node, &fl->cctx->invoke_interrupted_mmaps);1210 }1211 }1212 1213 if (err)1214 dev_dbg(fl->sctx->dev, "Error: Invoke Failed %d\n", err);1215 1216 return err;1217}1218 1219static bool is_session_rejected(struct fastrpc_user *fl, bool unsigned_pd_request)1220{1221 /* Check if the device node is non-secure and channel is secure*/1222 if (!fl->is_secure_dev && fl->cctx->secure) {1223 /*1224 * Allow untrusted applications to offload only to Unsigned PD when1225 * channel is configured as secure and block untrusted apps on channel1226 * that does not support unsigned PD offload1227 */1228 if (!fl->cctx->unsigned_support || !unsigned_pd_request) {1229 dev_err(&fl->cctx->rpdev->dev, "Error: Untrusted application trying to offload to signed PD\n");1230 return true;1231 }1232 }1233 1234 return false;1235}1236 1237static int fastrpc_init_create_static_process(struct fastrpc_user *fl,1238 char __user *argp)1239{1240 struct fastrpc_init_create_static init;1241 struct fastrpc_invoke_args *args;1242 struct fastrpc_phy_page pages[1];1243 char *name;1244 int err;1245 bool scm_done = false;1246 struct {1247 int pgid;1248 u32 namelen;1249 u32 pageslen;1250 } inbuf;1251 u32 sc;1252 1253 args = kcalloc(FASTRPC_CREATE_STATIC_PROCESS_NARGS, sizeof(*args), GFP_KERNEL);1254 if (!args)1255 return -ENOMEM;1256 1257 if (copy_from_user(&init, argp, sizeof(init))) {1258 err = -EFAULT;1259 goto err;1260 }1261 1262 if (init.namelen > INIT_FILE_NAMELEN_MAX) {1263 err = -EINVAL;1264 goto err;1265 }1266 1267 name = memdup_user(u64_to_user_ptr(init.name), init.namelen);1268 if (IS_ERR(name)) {1269 err = PTR_ERR(name);1270 goto err;1271 }1272 1273 if (!fl->cctx->remote_heap) {1274 err = fastrpc_remote_heap_alloc(fl, fl->sctx->dev, init.memlen,1275 &fl->cctx->remote_heap);1276 if (err)1277 goto err_name;1278 1279 /* Map if we have any heap VMIDs associated with this ADSP Static Process. */1280 if (fl->cctx->vmcount) {1281 u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);1282 1283 err = qcom_scm_assign_mem(fl->cctx->remote_heap->phys,1284 (u64)fl->cctx->remote_heap->size,1285 &src_perms,1286 fl->cctx->vmperms, fl->cctx->vmcount);1287 if (err) {1288 dev_err(fl->sctx->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d\n",1289 fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err);1290 goto err_map;1291 }1292 scm_done = true;1293 }1294 }1295 1296 inbuf.pgid = fl->tgid;1297 inbuf.namelen = init.namelen;1298 inbuf.pageslen = 0;1299 fl->pd = USER_PD;1300 1301 args[0].ptr = (u64)(uintptr_t)&inbuf;1302 args[0].length = sizeof(inbuf);1303 args[0].fd = -1;1304 1305 args[1].ptr = (u64)(uintptr_t)name;1306 args[1].length = inbuf.namelen;1307 args[1].fd = -1;1308 1309 pages[0].addr = fl->cctx->remote_heap->phys;1310 pages[0].size = fl->cctx->remote_heap->size;1311 1312 args[2].ptr = (u64)(uintptr_t) pages;1313 args[2].length = sizeof(*pages);1314 args[2].fd = -1;1315 1316 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_STATIC, 3, 0);1317 1318 err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,1319 sc, args);1320 if (err)1321 goto err_invoke;1322 1323 kfree(args);1324 kfree(name);1325 1326 return 0;1327err_invoke:1328 if (fl->cctx->vmcount && scm_done) {1329 u64 src_perms = 0;1330 struct qcom_scm_vmperm dst_perms;1331 u32 i;1332 1333 for (i = 0; i < fl->cctx->vmcount; i++)1334 src_perms |= BIT(fl->cctx->vmperms[i].vmid);1335 1336 dst_perms.vmid = QCOM_SCM_VMID_HLOS;1337 dst_perms.perm = QCOM_SCM_PERM_RWX;1338 err = qcom_scm_assign_mem(fl->cctx->remote_heap->phys,1339 (u64)fl->cctx->remote_heap->size,1340 &src_perms, &dst_perms, 1);1341 if (err)1342 dev_err(fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d\n",1343 fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err);1344 }1345err_map:1346 fastrpc_buf_free(fl->cctx->remote_heap);1347err_name:1348 kfree(name);1349err:1350 kfree(args);1351 1352 return err;1353}1354 1355static int fastrpc_init_create_process(struct fastrpc_user *fl,1356 char __user *argp)1357{1358 struct fastrpc_init_create init;1359 struct fastrpc_invoke_args *args;1360 struct fastrpc_phy_page pages[1];1361 struct fastrpc_map *map = NULL;1362 struct fastrpc_buf *imem = NULL;1363 int memlen;1364 int err;1365 struct {1366 int pgid;1367 u32 namelen;1368 u32 filelen;1369 u32 pageslen;1370 u32 attrs;1371 u32 siglen;1372 } inbuf;1373 u32 sc;1374 bool unsigned_module = false;1375 1376 args = kcalloc(FASTRPC_CREATE_PROCESS_NARGS, sizeof(*args), GFP_KERNEL);1377 if (!args)1378 return -ENOMEM;1379 1380 if (copy_from_user(&init, argp, sizeof(init))) {1381 err = -EFAULT;1382 goto err;1383 }1384 1385 if (init.attrs & FASTRPC_MODE_UNSIGNED_MODULE)1386 unsigned_module = true;1387 1388 if (is_session_rejected(fl, unsigned_module)) {1389 err = -ECONNREFUSED;1390 goto err;1391 }1392 1393 if (init.filelen > INIT_FILELEN_MAX) {1394 err = -EINVAL;1395 goto err;1396 }1397 1398 inbuf.pgid = fl->tgid;1399 inbuf.namelen = strlen(current->comm) + 1;1400 inbuf.filelen = init.filelen;1401 inbuf.pageslen = 1;1402 inbuf.attrs = init.attrs;1403 inbuf.siglen = init.siglen;1404 fl->pd = USER_PD;1405 1406 if (init.filelen && init.filefd) {1407 err = fastrpc_map_create(fl, init.filefd, init.filelen, 0, &map);1408 if (err)1409 goto err;1410 }1411 1412 memlen = ALIGN(max(INIT_FILELEN_MAX, (int)init.filelen * 4),1413 1024 * 1024);1414 err = fastrpc_buf_alloc(fl, fl->sctx->dev, memlen,1415 &imem);1416 if (err)1417 goto err_alloc;1418 1419 fl->init_mem = imem;1420 args[0].ptr = (u64)(uintptr_t)&inbuf;1421 args[0].length = sizeof(inbuf);1422 args[0].fd = -1;1423 1424 args[1].ptr = (u64)(uintptr_t)current->comm;1425 args[1].length = inbuf.namelen;1426 args[1].fd = -1;1427 1428 args[2].ptr = (u64) init.file;1429 args[2].length = inbuf.filelen;1430 args[2].fd = init.filefd;1431 1432 pages[0].addr = imem->phys;1433 pages[0].size = imem->size;1434 1435 args[3].ptr = (u64)(uintptr_t) pages;1436 args[3].length = 1 * sizeof(*pages);1437 args[3].fd = -1;1438 1439 args[4].ptr = (u64)(uintptr_t)&inbuf.attrs;1440 args[4].length = sizeof(inbuf.attrs);1441 args[4].fd = -1;1442 1443 args[5].ptr = (u64)(uintptr_t) &inbuf.siglen;1444 args[5].length = sizeof(inbuf.siglen);1445 args[5].fd = -1;1446 1447 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE, 4, 0);1448 if (init.attrs)1449 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_ATTR, 4, 0);1450 1451 err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,1452 sc, args);1453 if (err)1454 goto err_invoke;1455 1456 kfree(args);1457 1458 return 0;1459 1460err_invoke:1461 fl->init_mem = NULL;1462 fastrpc_buf_free(imem);1463err_alloc:1464 fastrpc_map_put(map);1465err:1466 kfree(args);1467 1468 return err;1469}1470 1471static struct fastrpc_session_ctx *fastrpc_session_alloc(1472 struct fastrpc_channel_ctx *cctx)1473{1474 struct fastrpc_session_ctx *session = NULL;1475 unsigned long flags;1476 int i;1477 1478 spin_lock_irqsave(&cctx->lock, flags);1479 for (i = 0; i < cctx->sesscount; i++) {1480 if (!cctx->session[i].used && cctx->session[i].valid) {1481 cctx->session[i].used = true;1482 session = &cctx->session[i];1483 break;1484 }1485 }1486 spin_unlock_irqrestore(&cctx->lock, flags);1487 1488 return session;1489}1490 1491static void fastrpc_session_free(struct fastrpc_channel_ctx *cctx,1492 struct fastrpc_session_ctx *session)1493{1494 unsigned long flags;1495 1496 spin_lock_irqsave(&cctx->lock, flags);1497 session->used = false;1498 spin_unlock_irqrestore(&cctx->lock, flags);1499}1500 1501static int fastrpc_release_current_dsp_process(struct fastrpc_user *fl)1502{1503 struct fastrpc_invoke_args args[1];1504 int tgid = 0;1505 u32 sc;1506 1507 tgid = fl->tgid;1508 args[0].ptr = (u64)(uintptr_t) &tgid;1509 args[0].length = sizeof(tgid);1510 args[0].fd = -1;1511 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_RELEASE, 1, 0);1512 1513 return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,1514 sc, &args[0]);1515}1516 1517static int fastrpc_device_release(struct inode *inode, struct file *file)1518{1519 struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;1520 struct fastrpc_channel_ctx *cctx = fl->cctx;1521 struct fastrpc_invoke_ctx *ctx, *n;1522 struct fastrpc_map *map, *m;1523 struct fastrpc_buf *buf, *b;1524 unsigned long flags;1525 1526 fastrpc_release_current_dsp_process(fl);1527 1528 spin_lock_irqsave(&cctx->lock, flags);1529 list_del(&fl->user);1530 spin_unlock_irqrestore(&cctx->lock, flags);1531 1532 if (fl->init_mem)1533 fastrpc_buf_free(fl->init_mem);1534 1535 list_for_each_entry_safe(ctx, n, &fl->pending, node) {1536 list_del(&ctx->node);1537 fastrpc_context_put(ctx);1538 }1539 1540 list_for_each_entry_safe(map, m, &fl->maps, node)1541 fastrpc_map_put(map);1542 1543 list_for_each_entry_safe(buf, b, &fl->mmaps, node) {1544 list_del(&buf->node);1545 fastrpc_buf_free(buf);1546 }1547 1548 fastrpc_session_free(cctx, fl->sctx);1549 fastrpc_channel_ctx_put(cctx);1550 1551 mutex_destroy(&fl->mutex);1552 kfree(fl);1553 file->private_data = NULL;1554 1555 return 0;1556}1557 1558static int fastrpc_device_open(struct inode *inode, struct file *filp)1559{1560 struct fastrpc_channel_ctx *cctx;1561 struct fastrpc_device *fdevice;1562 struct fastrpc_user *fl = NULL;1563 unsigned long flags;1564 1565 fdevice = miscdev_to_fdevice(filp->private_data);1566 cctx = fdevice->cctx;1567 1568 fl = kzalloc(sizeof(*fl), GFP_KERNEL);1569 if (!fl)1570 return -ENOMEM;1571 1572 /* Released in fastrpc_device_release() */1573 fastrpc_channel_ctx_get(cctx);1574 1575 filp->private_data = fl;1576 spin_lock_init(&fl->lock);1577 mutex_init(&fl->mutex);1578 INIT_LIST_HEAD(&fl->pending);1579 INIT_LIST_HEAD(&fl->maps);1580 INIT_LIST_HEAD(&fl->mmaps);1581 INIT_LIST_HEAD(&fl->user);1582 fl->tgid = current->tgid;1583 fl->cctx = cctx;1584 fl->is_secure_dev = fdevice->secure;1585 1586 fl->sctx = fastrpc_session_alloc(cctx);1587 if (!fl->sctx) {1588 dev_err(&cctx->rpdev->dev, "No session available\n");1589 mutex_destroy(&fl->mutex);1590 kfree(fl);1591 1592 return -EBUSY;1593 }1594 1595 spin_lock_irqsave(&cctx->lock, flags);1596 list_add_tail(&fl->user, &cctx->users);1597 spin_unlock_irqrestore(&cctx->lock, flags);1598 1599 return 0;1600}1601 1602static int fastrpc_dmabuf_alloc(struct fastrpc_user *fl, char __user *argp)1603{1604 struct fastrpc_alloc_dma_buf bp;1605 DEFINE_DMA_BUF_EXPORT_INFO(exp_info);1606 struct fastrpc_buf *buf = NULL;1607 int err;1608 1609 if (copy_from_user(&bp, argp, sizeof(bp)))1610 return -EFAULT;1611 1612 err = fastrpc_buf_alloc(fl, fl->sctx->dev, bp.size, &buf);1613 if (err)1614 return err;1615 exp_info.ops = &fastrpc_dma_buf_ops;1616 exp_info.size = bp.size;1617 exp_info.flags = O_RDWR;1618 exp_info.priv = buf;1619 buf->dmabuf = dma_buf_export(&exp_info);1620 if (IS_ERR(buf->dmabuf)) {1621 err = PTR_ERR(buf->dmabuf);1622 fastrpc_buf_free(buf);1623 return err;1624 }1625 1626 bp.fd = dma_buf_fd(buf->dmabuf, O_ACCMODE);1627 if (bp.fd < 0) {1628 dma_buf_put(buf->dmabuf);1629 return -EINVAL;1630 }1631 1632 if (copy_to_user(argp, &bp, sizeof(bp))) {1633 /*1634 * The usercopy failed, but we can't do much about it, as1635 * dma_buf_fd() already called fd_install() and made the1636 * file descriptor accessible for the current process. It1637 * might already be closed and dmabuf no longer valid when1638 * we reach this point. Therefore "leak" the fd and rely on1639 * the process exit path to do any required cleanup.1640 */1641 return -EFAULT;1642 }1643 1644 return 0;1645}1646 1647static int fastrpc_init_attach(struct fastrpc_user *fl, int pd)1648{1649 struct fastrpc_invoke_args args[1];1650 int tgid = fl->tgid;1651 u32 sc;1652 1653 args[0].ptr = (u64)(uintptr_t) &tgid;1654 args[0].length = sizeof(tgid);1655 args[0].fd = -1;1656 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_ATTACH, 1, 0);1657 fl->pd = pd;1658 1659 return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE,1660 sc, &args[0]);1661}1662 1663static int fastrpc_invoke(struct fastrpc_user *fl, char __user *argp)1664{1665 struct fastrpc_invoke_args *args = NULL;1666 struct fastrpc_invoke inv;1667 u32 nscalars;1668 int err;1669 1670 if (copy_from_user(&inv, argp, sizeof(inv)))1671 return -EFAULT;1672 1673 /* nscalars is truncated here to max supported value */1674 nscalars = REMOTE_SCALARS_LENGTH(inv.sc);1675 if (nscalars) {1676 args = kcalloc(nscalars, sizeof(*args), GFP_KERNEL);1677 if (!args)1678 return -ENOMEM;1679 1680 if (copy_from_user(args, (void __user *)(uintptr_t)inv.args,1681 nscalars * sizeof(*args))) {1682 kfree(args);1683 return -EFAULT;1684 }1685 }1686 1687 err = fastrpc_internal_invoke(fl, false, inv.handle, inv.sc, args);1688 kfree(args);1689 1690 return err;1691}1692 1693static int fastrpc_get_info_from_dsp(struct fastrpc_user *fl, uint32_t *dsp_attr_buf,1694 uint32_t dsp_attr_buf_len)1695{1696 struct fastrpc_invoke_args args[2] = { 0 };1697 1698 /*1699 * Capability filled in userspace. This carries the information1700 * about the remoteproc support which is fetched from the remoteproc1701 * sysfs node by userspace.1702 */1703 dsp_attr_buf[0] = 0;1704 dsp_attr_buf_len -= 1;1705 1706 args[0].ptr = (u64)(uintptr_t)&dsp_attr_buf_len;1707 args[0].length = sizeof(dsp_attr_buf_len);1708 args[0].fd = -1;1709 args[1].ptr = (u64)(uintptr_t)&dsp_attr_buf[1];1710 args[1].length = dsp_attr_buf_len * sizeof(u32);1711 args[1].fd = -1;1712 1713 return fastrpc_internal_invoke(fl, true, FASTRPC_DSP_UTILITIES_HANDLE,1714 FASTRPC_SCALARS(0, 1, 1), args);1715}1716 1717static int fastrpc_get_info_from_kernel(struct fastrpc_ioctl_capability *cap,1718 struct fastrpc_user *fl)1719{1720 struct fastrpc_channel_ctx *cctx = fl->cctx;1721 uint32_t attribute_id = cap->attribute_id;1722 uint32_t *dsp_attributes;1723 unsigned long flags;1724 uint32_t domain = cap->domain;1725 int err;1726 1727 spin_lock_irqsave(&cctx->lock, flags);1728 /* check if we already have queried dsp for attributes */1729 if (cctx->valid_attributes) {1730 spin_unlock_irqrestore(&cctx->lock, flags);1731 goto done;1732 }1733 spin_unlock_irqrestore(&cctx->lock, flags);1734 1735 dsp_attributes = kzalloc(FASTRPC_MAX_DSP_ATTRIBUTES_LEN, GFP_KERNEL);1736 if (!dsp_attributes)1737 return -ENOMEM;1738 1739 err = fastrpc_get_info_from_dsp(fl, dsp_attributes, FASTRPC_MAX_DSP_ATTRIBUTES);1740 if (err == DSP_UNSUPPORTED_API) {1741 dev_info(&cctx->rpdev->dev,1742 "Warning: DSP capabilities not supported on domain: %d\n", domain);1743 kfree(dsp_attributes);1744 return -EOPNOTSUPP;1745 } else if (err) {1746 dev_err(&cctx->rpdev->dev, "Error: dsp information is incorrect err: %d\n", err);1747 kfree(dsp_attributes);1748 return err;1749 }1750 1751 spin_lock_irqsave(&cctx->lock, flags);1752 memcpy(cctx->dsp_attributes, dsp_attributes, FASTRPC_MAX_DSP_ATTRIBUTES_LEN);1753 cctx->valid_attributes = true;1754 spin_unlock_irqrestore(&cctx->lock, flags);1755 kfree(dsp_attributes);1756done:1757 cap->capability = cctx->dsp_attributes[attribute_id];1758 return 0;1759}1760 1761static int fastrpc_get_dsp_info(struct fastrpc_user *fl, char __user *argp)1762{1763 struct fastrpc_ioctl_capability cap = {0};1764 int err = 0;1765 1766 if (copy_from_user(&cap, argp, sizeof(cap)))1767 return -EFAULT;1768 1769 cap.capability = 0;1770 if (cap.domain >= FASTRPC_DEV_MAX) {1771 dev_err(&fl->cctx->rpdev->dev, "Error: Invalid domain id:%d, err:%d\n",1772 cap.domain, err);1773 return -ECHRNG;1774 }1775 1776 /* Fastrpc Capablities does not support modem domain */1777 if (cap.domain == MDSP_DOMAIN_ID) {1778 dev_err(&fl->cctx->rpdev->dev, "Error: modem not supported %d\n", err);1779 return -ECHRNG;1780 }1781 1782 if (cap.attribute_id >= FASTRPC_MAX_DSP_ATTRIBUTES) {1783 dev_err(&fl->cctx->rpdev->dev, "Error: invalid attribute: %d, err: %d\n",1784 cap.attribute_id, err);1785 return -EOVERFLOW;1786 }1787 1788 err = fastrpc_get_info_from_kernel(&cap, fl);1789 if (err)1790 return err;1791 1792 if (copy_to_user(argp, &cap, sizeof(cap)))1793 return -EFAULT;1794 1795 return 0;1796}1797 1798static int fastrpc_req_munmap_impl(struct fastrpc_user *fl, struct fastrpc_buf *buf)1799{1800 struct fastrpc_invoke_args args[1] = { [0] = { 0 } };1801 struct fastrpc_munmap_req_msg req_msg;1802 struct device *dev = fl->sctx->dev;1803 int err;1804 u32 sc;1805 1806 req_msg.pgid = fl->tgid;1807 req_msg.size = buf->size;1808 req_msg.vaddr = buf->raddr;1809 1810 args[0].ptr = (u64) (uintptr_t) &req_msg;1811 args[0].length = sizeof(req_msg);1812 1813 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MUNMAP, 1, 0);1814 err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,1815 &args[0]);1816 if (!err) {1817 dev_dbg(dev, "unmmap\tpt 0x%09lx OK\n", buf->raddr);1818 spin_lock(&fl->lock);1819 list_del(&buf->node);1820 spin_unlock(&fl->lock);1821 fastrpc_buf_free(buf);1822 } else {1823 dev_err(dev, "unmmap\tpt 0x%09lx ERROR\n", buf->raddr);1824 }1825 1826 return err;1827}1828 1829static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp)1830{1831 struct fastrpc_buf *buf = NULL, *iter, *b;1832 struct fastrpc_req_munmap req;1833 struct device *dev = fl->sctx->dev;1834 1835 if (copy_from_user(&req, argp, sizeof(req)))1836 return -EFAULT;1837 1838 spin_lock(&fl->lock);1839 list_for_each_entry_safe(iter, b, &fl->mmaps, node) {1840 if ((iter->raddr == req.vaddrout) && (iter->size == req.size)) {1841 buf = iter;1842 break;1843 }1844 }1845 spin_unlock(&fl->lock);1846 1847 if (!buf) {1848 dev_err(dev, "mmap\t\tpt 0x%09llx [len 0x%08llx] not in list\n",1849 req.vaddrout, req.size);1850 return -EINVAL;1851 }1852 1853 return fastrpc_req_munmap_impl(fl, buf);1854}1855 1856static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp)1857{1858 struct fastrpc_invoke_args args[3] = { [0 ... 2] = { 0 } };1859 struct fastrpc_buf *buf = NULL;1860 struct fastrpc_mmap_req_msg req_msg;1861 struct fastrpc_mmap_rsp_msg rsp_msg;1862 struct fastrpc_phy_page pages;1863 struct fastrpc_req_mmap req;1864 struct device *dev = fl->sctx->dev;1865 int err;1866 u32 sc;1867 1868 if (copy_from_user(&req, argp, sizeof(req)))1869 return -EFAULT;1870 1871 if (req.flags != ADSP_MMAP_ADD_PAGES && req.flags != ADSP_MMAP_REMOTE_HEAP_ADDR) {1872 dev_err(dev, "flag not supported 0x%x\n", req.flags);1873 1874 return -EINVAL;1875 }1876 1877 if (req.vaddrin) {1878 dev_err(dev, "adding user allocated pages is not supported\n");1879 return -EINVAL;1880 }1881 1882 if (req.flags == ADSP_MMAP_REMOTE_HEAP_ADDR)1883 err = fastrpc_remote_heap_alloc(fl, dev, req.size, &buf);1884 else1885 err = fastrpc_buf_alloc(fl, dev, req.size, &buf);1886 1887 if (err) {1888 dev_err(dev, "failed to allocate buffer\n");1889 return err;1890 }1891 1892 req_msg.pgid = fl->tgid;1893 req_msg.flags = req.flags;1894 req_msg.vaddr = req.vaddrin;1895 req_msg.num = sizeof(pages);1896 1897 args[0].ptr = (u64) (uintptr_t) &req_msg;1898 args[0].length = sizeof(req_msg);1899 1900 pages.addr = buf->phys;1901 pages.size = buf->size;1902 1903 args[1].ptr = (u64) (uintptr_t) &pages;1904 args[1].length = sizeof(pages);1905 1906 args[2].ptr = (u64) (uintptr_t) &rsp_msg;1907 args[2].length = sizeof(rsp_msg);1908 1909 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MMAP, 2, 1);1910 err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,1911 &args[0]);1912 if (err) {1913 dev_err(dev, "mmap error (len 0x%08llx)\n", buf->size);1914 fastrpc_buf_free(buf);1915 return err;1916 }1917 1918 /* update the buffer to be able to deallocate the memory on the DSP */1919 buf->raddr = (uintptr_t) rsp_msg.vaddr;1920 1921 /* let the client know the address to use */1922 req.vaddrout = rsp_msg.vaddr;1923 1924 /* Add memory to static PD pool, protection thru hypervisor */1925 if (req.flags == ADSP_MMAP_REMOTE_HEAP_ADDR && fl->cctx->vmcount) {1926 u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);1927 1928 err = qcom_scm_assign_mem(buf->phys, (u64)buf->size,1929 &src_perms, fl->cctx->vmperms, fl->cctx->vmcount);1930 if (err) {1931 dev_err(fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d",1932 buf->phys, buf->size, err);1933 goto err_assign;1934 }1935 }1936 1937 spin_lock(&fl->lock);1938 list_add_tail(&buf->node, &fl->mmaps);1939 spin_unlock(&fl->lock);1940 1941 if (copy_to_user((void __user *)argp, &req, sizeof(req))) {1942 err = -EFAULT;1943 goto err_assign;1944 }1945 1946 dev_dbg(dev, "mmap\t\tpt 0x%09lx OK [len 0x%08llx]\n",1947 buf->raddr, buf->size);1948 1949 return 0;1950 1951err_assign:1952 fastrpc_req_munmap_impl(fl, buf);1953 1954 return err;1955}1956 1957static int fastrpc_req_mem_unmap_impl(struct fastrpc_user *fl, struct fastrpc_mem_unmap *req)1958{1959 struct fastrpc_invoke_args args[1] = { [0] = { 0 } };1960 struct fastrpc_map *map = NULL, *iter, *m;1961 struct fastrpc_mem_unmap_req_msg req_msg = { 0 };1962 int err = 0;1963 u32 sc;1964 struct device *dev = fl->sctx->dev;1965 1966 spin_lock(&fl->lock);1967 list_for_each_entry_safe(iter, m, &fl->maps, node) {1968 if ((req->fd < 0 || iter->fd == req->fd) && (iter->raddr == req->vaddr)) {1969 map = iter;1970 break;1971 }1972 }1973 1974 spin_unlock(&fl->lock);1975 1976 if (!map) {1977 dev_err(dev, "map not in list\n");1978 return -EINVAL;1979 }1980 1981 req_msg.pgid = fl->tgid;1982 req_msg.len = map->len;1983 req_msg.vaddrin = map->raddr;1984 req_msg.fd = map->fd;1985 1986 args[0].ptr = (u64) (uintptr_t) &req_msg;1987 args[0].length = sizeof(req_msg);1988 1989 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_UNMAP, 1, 0);1990 err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc,1991 &args[0]);1992 if (err) {1993 dev_err(dev, "unmmap\tpt fd = %d, 0x%09llx error\n", map->fd, map->raddr);1994 return err;1995 }1996 fastrpc_map_put(map);1997 1998 return 0;1999}2000 2001static int fastrpc_req_mem_unmap(struct fastrpc_user *fl, char __user *argp)2002{2003 struct fastrpc_mem_unmap req;2004 2005 if (copy_from_user(&req, argp, sizeof(req)))2006 return -EFAULT;2007 2008 return fastrpc_req_mem_unmap_impl(fl, &req);2009}2010 2011static int fastrpc_req_mem_map(struct fastrpc_user *fl, char __user *argp)2012{2013 struct fastrpc_invoke_args args[4] = { [0 ... 3] = { 0 } };2014 struct fastrpc_mem_map_req_msg req_msg = { 0 };2015 struct fastrpc_mmap_rsp_msg rsp_msg = { 0 };2016 struct fastrpc_mem_unmap req_unmap = { 0 };2017 struct fastrpc_phy_page pages = { 0 };2018 struct fastrpc_mem_map req;2019 struct device *dev = fl->sctx->dev;2020 struct fastrpc_map *map = NULL;2021 int err;2022 u32 sc;2023 2024 if (copy_from_user(&req, argp, sizeof(req)))2025 return -EFAULT;2026 2027 /* create SMMU mapping */2028 err = fastrpc_map_create(fl, req.fd, req.length, 0, &map);2029 if (err) {2030 dev_err(dev, "failed to map buffer, fd = %d\n", req.fd);2031 return err;2032 }2033 2034 req_msg.pgid = fl->tgid;2035 req_msg.fd = req.fd;2036 req_msg.offset = req.offset;2037 req_msg.vaddrin = req.vaddrin;2038 map->va = (void *) (uintptr_t) req.vaddrin;2039 req_msg.flags = req.flags;2040 req_msg.num = sizeof(pages);2041 req_msg.data_len = 0;2042 2043 args[0].ptr = (u64) (uintptr_t) &req_msg;2044 args[0].length = sizeof(req_msg);2045 2046 pages.addr = map->phys;2047 pages.size = map->size;2048 2049 args[1].ptr = (u64) (uintptr_t) &pages;2050 args[1].length = sizeof(pages);2051 2052 args[2].ptr = (u64) (uintptr_t) &pages;2053 args[2].length = 0;2054 2055 args[3].ptr = (u64) (uintptr_t) &rsp_msg;2056 args[3].length = sizeof(rsp_msg);2057 2058 sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_MAP, 3, 1);2059 err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc, &args[0]);2060 if (err) {2061 dev_err(dev, "mem mmap error, fd %d, vaddr %llx, size %lld\n",2062 req.fd, req.vaddrin, map->size);2063 goto err_invoke;2064 }2065 2066 /* update the buffer to be able to deallocate the memory on the DSP */2067 map->raddr = rsp_msg.vaddr;2068 2069 /* let the client know the address to use */2070 req.vaddrout = rsp_msg.vaddr;2071 2072 if (copy_to_user((void __user *)argp, &req, sizeof(req))) {2073 /* unmap the memory and release the buffer */2074 req_unmap.vaddr = (uintptr_t) rsp_msg.vaddr;2075 req_unmap.length = map->size;2076 fastrpc_req_mem_unmap_impl(fl, &req_unmap);2077 return -EFAULT;2078 }2079 2080 return 0;2081 2082err_invoke:2083 fastrpc_map_put(map);2084 2085 return err;2086}2087 2088static long fastrpc_device_ioctl(struct file *file, unsigned int cmd,2089 unsigned long arg)2090{2091 struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;2092 char __user *argp = (char __user *)arg;2093 int err;2094 2095 switch (cmd) {2096 case FASTRPC_IOCTL_INVOKE:2097 err = fastrpc_invoke(fl, argp);2098 break;2099 case FASTRPC_IOCTL_INIT_ATTACH:2100 err = fastrpc_init_attach(fl, ROOT_PD);2101 break;2102 case FASTRPC_IOCTL_INIT_ATTACH_SNS:2103 err = fastrpc_init_attach(fl, SENSORS_PD);2104 break;2105 case FASTRPC_IOCTL_INIT_CREATE_STATIC:2106 err = fastrpc_init_create_static_process(fl, argp);2107 break;2108 case FASTRPC_IOCTL_INIT_CREATE:2109 err = fastrpc_init_create_process(fl, argp);2110 break;2111 case FASTRPC_IOCTL_ALLOC_DMA_BUFF:2112 err = fastrpc_dmabuf_alloc(fl, argp);2113 break;2114 case FASTRPC_IOCTL_MMAP:2115 err = fastrpc_req_mmap(fl, argp);2116 break;2117 case FASTRPC_IOCTL_MUNMAP:2118 err = fastrpc_req_munmap(fl, argp);2119 break;2120 case FASTRPC_IOCTL_MEM_MAP:2121 err = fastrpc_req_mem_map(fl, argp);2122 break;2123 case FASTRPC_IOCTL_MEM_UNMAP:2124 err = fastrpc_req_mem_unmap(fl, argp);2125 break;2126 case FASTRPC_IOCTL_GET_DSP_INFO:2127 err = fastrpc_get_dsp_info(fl, argp);2128 break;2129 default:2130 err = -ENOTTY;2131 break;2132 }2133 2134 return err;2135}2136 2137static const struct file_operations fastrpc_fops = {2138 .open = fastrpc_device_open,2139 .release = fastrpc_device_release,2140 .unlocked_ioctl = fastrpc_device_ioctl,2141 .compat_ioctl = fastrpc_device_ioctl,2142};2143 2144static int fastrpc_cb_probe(struct platform_device *pdev)2145{2146 struct fastrpc_channel_ctx *cctx;2147 struct fastrpc_session_ctx *sess;2148 struct device *dev = &pdev->dev;2149 int i, sessions = 0;2150 unsigned long flags;2151 int rc;2152 2153 cctx = dev_get_drvdata(dev->parent);2154 if (!cctx)2155 return -EINVAL;2156 2157 of_property_read_u32(dev->of_node, "qcom,nsessions", &sessions);2158 2159 spin_lock_irqsave(&cctx->lock, flags);2160 if (cctx->sesscount >= FASTRPC_MAX_SESSIONS) {2161 dev_err(&pdev->dev, "too many sessions\n");2162 spin_unlock_irqrestore(&cctx->lock, flags);2163 return -ENOSPC;2164 }2165 sess = &cctx->session[cctx->sesscount++];2166 sess->used = false;2167 sess->valid = true;2168 sess->dev = dev;2169 dev_set_drvdata(dev, sess);2170 2171 if (of_property_read_u32(dev->of_node, "reg", &sess->sid))2172 dev_info(dev, "FastRPC Session ID not specified in DT\n");2173 2174 if (sessions > 0) {2175 struct fastrpc_session_ctx *dup_sess;2176 2177 for (i = 1; i < sessions; i++) {2178 if (cctx->sesscount >= FASTRPC_MAX_SESSIONS)2179 break;2180 dup_sess = &cctx->session[cctx->sesscount++];2181 memcpy(dup_sess, sess, sizeof(*dup_sess));2182 }2183 }2184 spin_unlock_irqrestore(&cctx->lock, flags);2185 rc = dma_set_mask(dev, DMA_BIT_MASK(32));2186 if (rc) {2187 dev_err(dev, "32-bit DMA enable failed\n");2188 return rc;2189 }2190 2191 return 0;2192}2193 2194static void fastrpc_cb_remove(struct platform_device *pdev)2195{2196 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(pdev->dev.parent);2197 struct fastrpc_session_ctx *sess = dev_get_drvdata(&pdev->dev);2198 unsigned long flags;2199 int i;2200 2201 spin_lock_irqsave(&cctx->lock, flags);2202 for (i = 0; i < FASTRPC_MAX_SESSIONS; i++) {2203 if (cctx->session[i].sid == sess->sid) {2204 cctx->session[i].valid = false;2205 cctx->sesscount--;2206 }2207 }2208 spin_unlock_irqrestore(&cctx->lock, flags);2209}2210 2211static const struct of_device_id fastrpc_match_table[] = {2212 { .compatible = "qcom,fastrpc-compute-cb", },2213 {}2214};2215 2216static struct platform_driver fastrpc_cb_driver = {2217 .probe = fastrpc_cb_probe,2218 .remove_new = fastrpc_cb_remove,2219 .driver = {2220 .name = "qcom,fastrpc-cb",2221 .of_match_table = fastrpc_match_table,2222 .suppress_bind_attrs = true,2223 },2224};2225 2226static int fastrpc_device_register(struct device *dev, struct fastrpc_channel_ctx *cctx,2227 bool is_secured, const char *domain)2228{2229 struct fastrpc_device *fdev;2230 int err;2231 2232 fdev = devm_kzalloc(dev, sizeof(*fdev), GFP_KERNEL);2233 if (!fdev)2234 return -ENOMEM;2235 2236 fdev->secure = is_secured;2237 fdev->cctx = cctx;2238 fdev->miscdev.minor = MISC_DYNAMIC_MINOR;2239 fdev->miscdev.fops = &fastrpc_fops;2240 fdev->miscdev.name = devm_kasprintf(dev, GFP_KERNEL, "fastrpc-%s%s",2241 domain, is_secured ? "-secure" : "");2242 if (!fdev->miscdev.name)2243 return -ENOMEM;2244 2245 err = misc_register(&fdev->miscdev);2246 if (!err) {2247 if (is_secured)2248 cctx->secure_fdevice = fdev;2249 else2250 cctx->fdevice = fdev;2251 }2252 2253 return err;2254}2255 2256static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)2257{2258 struct device *rdev = &rpdev->dev;2259 struct fastrpc_channel_ctx *data;2260 int i, err, domain_id = -1, vmcount;2261 const char *domain;2262 bool secure_dsp;2263 struct device_node *rmem_node;2264 struct reserved_mem *rmem;2265 unsigned int vmids[FASTRPC_MAX_VMIDS];2266 2267 err = of_property_read_string(rdev->of_node, "label", &domain);2268 if (err) {2269 dev_info(rdev, "FastRPC Domain not specified in DT\n");2270 return err;2271 }2272 2273 for (i = 0; i < FASTRPC_DEV_MAX; i++) {2274 if (!strcmp(domains[i], domain)) {2275 domain_id = i;2276 break;2277 }2278 }2279 2280 if (domain_id < 0) {2281 dev_info(rdev, "FastRPC Invalid Domain ID %d\n", domain_id);2282 return -EINVAL;2283 }2284 2285 if (of_reserved_mem_device_init_by_idx(rdev, rdev->of_node, 0))2286 dev_info(rdev, "no reserved DMA memory for FASTRPC\n");2287 2288 vmcount = of_property_read_variable_u32_array(rdev->of_node,2289 "qcom,vmids", &vmids[0], 0, FASTRPC_MAX_VMIDS);2290 if (vmcount < 0)2291 vmcount = 0;2292 else if (!qcom_scm_is_available())2293 return -EPROBE_DEFER;2294 2295 data = kzalloc(sizeof(*data), GFP_KERNEL);2296 if (!data)2297 return -ENOMEM;2298 2299 if (vmcount) {2300 data->vmcount = vmcount;2301 for (i = 0; i < data->vmcount; i++) {2302 data->vmperms[i].vmid = vmids[i];2303 data->vmperms[i].perm = QCOM_SCM_PERM_RWX;2304 }2305 }2306 2307 rmem_node = of_parse_phandle(rdev->of_node, "memory-region", 0);2308 if (domain_id == SDSP_DOMAIN_ID && rmem_node) {2309 u64 src_perms;2310 2311 rmem = of_reserved_mem_lookup(rmem_node);2312 if (!rmem) {2313 err = -EINVAL;2314 goto fdev_error;2315 }2316 2317 src_perms = BIT(QCOM_SCM_VMID_HLOS);2318 2319 qcom_scm_assign_mem(rmem->base, rmem->size, &src_perms,2320 data->vmperms, data->vmcount);2321 2322 }2323 2324 secure_dsp = !(of_property_read_bool(rdev->of_node, "qcom,non-secure-domain"));2325 data->secure = secure_dsp;2326 2327 switch (domain_id) {2328 case ADSP_DOMAIN_ID:2329 case MDSP_DOMAIN_ID:2330 case SDSP_DOMAIN_ID:2331 /* Unsigned PD offloading is only supported on CDSP and CDSP1 */2332 data->unsigned_support = false;2333 err = fastrpc_device_register(rdev, data, secure_dsp, domains[domain_id]);2334 if (err)2335 goto fdev_error;2336 break;2337 case CDSP_DOMAIN_ID:2338 case CDSP1_DOMAIN_ID:2339 data->unsigned_support = true;2340 /* Create both device nodes so that we can allow both Signed and Unsigned PD */2341 err = fastrpc_device_register(rdev, data, true, domains[domain_id]);2342 if (err)2343 goto fdev_error;2344 2345 err = fastrpc_device_register(rdev, data, false, domains[domain_id]);2346 if (err)2347 goto fdev_error;2348 break;2349 default:2350 err = -EINVAL;2351 goto fdev_error;2352 }2353 2354 kref_init(&data->refcount);2355 2356 dev_set_drvdata(&rpdev->dev, data);2357 rdev->dma_mask = &data->dma_mask;2358 dma_set_mask_and_coherent(rdev, DMA_BIT_MASK(32));2359 INIT_LIST_HEAD(&data->users);2360 INIT_LIST_HEAD(&data->invoke_interrupted_mmaps);2361 spin_lock_init(&data->lock);2362 idr_init(&data->ctx_idr);2363 data->domain_id = domain_id;2364 data->rpdev = rpdev;2365 2366 err = of_platform_populate(rdev->of_node, NULL, NULL, rdev);2367 if (err)2368 goto populate_error;2369 2370 return 0;2371 2372populate_error:2373 if (data->fdevice)2374 misc_deregister(&data->fdevice->miscdev);2375 if (data->secure_fdevice)2376 misc_deregister(&data->secure_fdevice->miscdev);2377 2378fdev_error:2379 kfree(data);2380 return err;2381}2382 2383static void fastrpc_notify_users(struct fastrpc_user *user)2384{2385 struct fastrpc_invoke_ctx *ctx;2386 2387 spin_lock(&user->lock);2388 list_for_each_entry(ctx, &user->pending, node) {2389 ctx->retval = -EPIPE;2390 complete(&ctx->work);2391 }2392 spin_unlock(&user->lock);2393}2394 2395static void fastrpc_rpmsg_remove(struct rpmsg_device *rpdev)2396{2397 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev);2398 struct fastrpc_buf *buf, *b;2399 struct fastrpc_user *user;2400 unsigned long flags;2401 2402 /* No invocations past this point */2403 spin_lock_irqsave(&cctx->lock, flags);2404 cctx->rpdev = NULL;2405 list_for_each_entry(user, &cctx->users, user)2406 fastrpc_notify_users(user);2407 spin_unlock_irqrestore(&cctx->lock, flags);2408 2409 if (cctx->fdevice)2410 misc_deregister(&cctx->fdevice->miscdev);2411 2412 if (cctx->secure_fdevice)2413 misc_deregister(&cctx->secure_fdevice->miscdev);2414 2415 list_for_each_entry_safe(buf, b, &cctx->invoke_interrupted_mmaps, node)2416 list_del(&buf->node);2417 2418 if (cctx->remote_heap)2419 fastrpc_buf_free(cctx->remote_heap);2420 2421 of_platform_depopulate(&rpdev->dev);2422 2423 fastrpc_channel_ctx_put(cctx);2424}2425 2426static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data,2427 int len, void *priv, u32 addr)2428{2429 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev);2430 struct fastrpc_invoke_rsp *rsp = data;2431 struct fastrpc_invoke_ctx *ctx;2432 unsigned long flags;2433 unsigned long ctxid;2434 2435 if (len < sizeof(*rsp))2436 return -EINVAL;2437 2438 ctxid = ((rsp->ctx & FASTRPC_CTXID_MASK) >> 4);2439 2440 spin_lock_irqsave(&cctx->lock, flags);2441 ctx = idr_find(&cctx->ctx_idr, ctxid);2442 spin_unlock_irqrestore(&cctx->lock, flags);2443 2444 if (!ctx) {2445 dev_err(&rpdev->dev, "No context ID matches response\n");2446 return -ENOENT;2447 }2448 2449 ctx->retval = rsp->retval;2450 complete(&ctx->work);2451 2452 /*2453 * The DMA buffer associated with the context cannot be freed in2454 * interrupt context so schedule it through a worker thread to2455 * avoid a kernel BUG.2456 */2457 schedule_work(&ctx->put_work);2458 2459 return 0;2460}2461 2462static const struct of_device_id fastrpc_rpmsg_of_match[] = {2463 { .compatible = "qcom,fastrpc" },2464 { },2465};2466MODULE_DEVICE_TABLE(of, fastrpc_rpmsg_of_match);2467 2468static struct rpmsg_driver fastrpc_driver = {2469 .probe = fastrpc_rpmsg_probe,2470 .remove = fastrpc_rpmsg_remove,2471 .callback = fastrpc_rpmsg_callback,2472 .drv = {2473 .name = "qcom,fastrpc",2474 .of_match_table = fastrpc_rpmsg_of_match,2475 },2476};2477 2478static int fastrpc_init(void)2479{2480 int ret;2481 2482 ret = platform_driver_register(&fastrpc_cb_driver);2483 if (ret < 0) {2484 pr_err("fastrpc: failed to register cb driver\n");2485 return ret;2486 }2487 2488 ret = register_rpmsg_driver(&fastrpc_driver);2489 if (ret < 0) {2490 pr_err("fastrpc: failed to register rpmsg driver\n");2491 platform_driver_unregister(&fastrpc_cb_driver);2492 return ret;2493 }2494 2495 return 0;2496}2497module_init(fastrpc_init);2498 2499static void fastrpc_exit(void)2500{2501 platform_driver_unregister(&fastrpc_cb_driver);2502 unregister_rpmsg_driver(&fastrpc_driver);2503}2504module_exit(fastrpc_exit);2505 2506MODULE_DESCRIPTION("Qualcomm FastRPC");2507MODULE_LICENSE("GPL v2");2508MODULE_IMPORT_NS(DMA_BUF);2509