849 lines · c
1// SPDX-License-Identifier: GPL-2.02 3/*4 * Xen dma-buf functionality for gntdev.5 *6 * DMA buffer implementation is based on drivers/gpu/drm/drm_prime.c.7 *8 * Copyright (c) 2018 Oleksandr Andrushchenko, EPAM Systems Inc.9 */10 11#include <linux/kernel.h>12#include <linux/errno.h>13#include <linux/dma-buf.h>14#include <linux/dma-direct.h>15#include <linux/slab.h>16#include <linux/types.h>17#include <linux/uaccess.h>18#include <linux/module.h>19 20#include <xen/xen.h>21#include <xen/grant_table.h>22 23#include "gntdev-common.h"24#include "gntdev-dmabuf.h"25 26MODULE_IMPORT_NS(DMA_BUF);27 28struct gntdev_dmabuf {29 struct gntdev_dmabuf_priv *priv;30 struct dma_buf *dmabuf;31 struct list_head next;32 int fd;33 34 union {35 struct {36 /* Exported buffers are reference counted. */37 struct kref refcount;38 39 struct gntdev_priv *priv;40 struct gntdev_grant_map *map;41 } exp;42 struct {43 /* Granted references of the imported buffer. */44 grant_ref_t *refs;45 /* Scatter-gather table of the imported buffer. */46 struct sg_table *sgt;47 /* dma-buf attachment of the imported buffer. */48 struct dma_buf_attachment *attach;49 } imp;50 } u;51 52 /* Number of pages this buffer has. */53 int nr_pages;54 /* Pages of this buffer (only for dma-buf export). */55 struct page **pages;56};57 58struct gntdev_dmabuf_wait_obj {59 struct list_head next;60 struct gntdev_dmabuf *gntdev_dmabuf;61 struct completion completion;62};63 64struct gntdev_dmabuf_attachment {65 struct sg_table *sgt;66 enum dma_data_direction dir;67};68 69struct gntdev_dmabuf_priv {70 /* List of exported DMA buffers. */71 struct list_head exp_list;72 /* List of wait objects. */73 struct list_head exp_wait_list;74 /* List of imported DMA buffers. */75 struct list_head imp_list;76 /* This is the lock which protects dma_buf_xxx lists. */77 struct mutex lock;78 /*79 * We reference this file while exporting dma-bufs, so80 * the grant device context is not destroyed while there are81 * external users alive.82 */83 struct file *filp;84};85 86/* DMA buffer export support. */87 88/* Implementation of wait for exported DMA buffer to be released. */89 90static void dmabuf_exp_release(struct kref *kref);91 92static struct gntdev_dmabuf_wait_obj *93dmabuf_exp_wait_obj_new(struct gntdev_dmabuf_priv *priv,94 struct gntdev_dmabuf *gntdev_dmabuf)95{96 struct gntdev_dmabuf_wait_obj *obj;97 98 obj = kzalloc(sizeof(*obj), GFP_KERNEL);99 if (!obj)100 return ERR_PTR(-ENOMEM);101 102 init_completion(&obj->completion);103 obj->gntdev_dmabuf = gntdev_dmabuf;104 105 mutex_lock(&priv->lock);106 list_add(&obj->next, &priv->exp_wait_list);107 /* Put our reference and wait for gntdev_dmabuf's release to fire. */108 kref_put(&gntdev_dmabuf->u.exp.refcount, dmabuf_exp_release);109 mutex_unlock(&priv->lock);110 return obj;111}112 113static void dmabuf_exp_wait_obj_free(struct gntdev_dmabuf_priv *priv,114 struct gntdev_dmabuf_wait_obj *obj)115{116 mutex_lock(&priv->lock);117 list_del(&obj->next);118 mutex_unlock(&priv->lock);119 kfree(obj);120}121 122static int dmabuf_exp_wait_obj_wait(struct gntdev_dmabuf_wait_obj *obj,123 u32 wait_to_ms)124{125 if (wait_for_completion_timeout(&obj->completion,126 msecs_to_jiffies(wait_to_ms)) <= 0)127 return -ETIMEDOUT;128 129 return 0;130}131 132static void dmabuf_exp_wait_obj_signal(struct gntdev_dmabuf_priv *priv,133 struct gntdev_dmabuf *gntdev_dmabuf)134{135 struct gntdev_dmabuf_wait_obj *obj;136 137 list_for_each_entry(obj, &priv->exp_wait_list, next)138 if (obj->gntdev_dmabuf == gntdev_dmabuf) {139 pr_debug("Found gntdev_dmabuf in the wait list, wake\n");140 complete_all(&obj->completion);141 break;142 }143}144 145static struct gntdev_dmabuf *146dmabuf_exp_wait_obj_get_dmabuf(struct gntdev_dmabuf_priv *priv, int fd)147{148 struct gntdev_dmabuf *gntdev_dmabuf, *ret = ERR_PTR(-ENOENT);149 150 mutex_lock(&priv->lock);151 list_for_each_entry(gntdev_dmabuf, &priv->exp_list, next)152 if (gntdev_dmabuf->fd == fd) {153 pr_debug("Found gntdev_dmabuf in the wait list\n");154 kref_get(&gntdev_dmabuf->u.exp.refcount);155 ret = gntdev_dmabuf;156 break;157 }158 mutex_unlock(&priv->lock);159 return ret;160}161 162static int dmabuf_exp_wait_released(struct gntdev_dmabuf_priv *priv, int fd,163 int wait_to_ms)164{165 struct gntdev_dmabuf *gntdev_dmabuf;166 struct gntdev_dmabuf_wait_obj *obj;167 int ret;168 169 pr_debug("Will wait for dma-buf with fd %d\n", fd);170 /*171 * Try to find the DMA buffer: if not found means that172 * either the buffer has already been released or file descriptor173 * provided is wrong.174 */175 gntdev_dmabuf = dmabuf_exp_wait_obj_get_dmabuf(priv, fd);176 if (IS_ERR(gntdev_dmabuf))177 return PTR_ERR(gntdev_dmabuf);178 179 /*180 * gntdev_dmabuf still exists and is reference count locked by us now,181 * so prepare to wait: allocate wait object and add it to the wait list,182 * so we can find it on release.183 */184 obj = dmabuf_exp_wait_obj_new(priv, gntdev_dmabuf);185 if (IS_ERR(obj))186 return PTR_ERR(obj);187 188 ret = dmabuf_exp_wait_obj_wait(obj, wait_to_ms);189 dmabuf_exp_wait_obj_free(priv, obj);190 return ret;191}192 193/* DMA buffer export support. */194 195static struct sg_table *196dmabuf_pages_to_sgt(struct page **pages, unsigned int nr_pages)197{198 struct sg_table *sgt;199 int ret;200 201 sgt = kmalloc(sizeof(*sgt), GFP_KERNEL);202 if (!sgt) {203 ret = -ENOMEM;204 goto out;205 }206 207 ret = sg_alloc_table_from_pages(sgt, pages, nr_pages, 0,208 nr_pages << PAGE_SHIFT,209 GFP_KERNEL);210 if (ret)211 goto out;212 213 return sgt;214 215out:216 kfree(sgt);217 return ERR_PTR(ret);218}219 220static int dmabuf_exp_ops_attach(struct dma_buf *dma_buf,221 struct dma_buf_attachment *attach)222{223 struct gntdev_dmabuf_attachment *gntdev_dmabuf_attach;224 225 gntdev_dmabuf_attach = kzalloc(sizeof(*gntdev_dmabuf_attach),226 GFP_KERNEL);227 if (!gntdev_dmabuf_attach)228 return -ENOMEM;229 230 gntdev_dmabuf_attach->dir = DMA_NONE;231 attach->priv = gntdev_dmabuf_attach;232 return 0;233}234 235static void dmabuf_exp_ops_detach(struct dma_buf *dma_buf,236 struct dma_buf_attachment *attach)237{238 struct gntdev_dmabuf_attachment *gntdev_dmabuf_attach = attach->priv;239 240 if (gntdev_dmabuf_attach) {241 struct sg_table *sgt = gntdev_dmabuf_attach->sgt;242 243 if (sgt) {244 if (gntdev_dmabuf_attach->dir != DMA_NONE)245 dma_unmap_sgtable(attach->dev, sgt,246 gntdev_dmabuf_attach->dir,247 DMA_ATTR_SKIP_CPU_SYNC);248 sg_free_table(sgt);249 }250 251 kfree(sgt);252 kfree(gntdev_dmabuf_attach);253 attach->priv = NULL;254 }255}256 257static struct sg_table *258dmabuf_exp_ops_map_dma_buf(struct dma_buf_attachment *attach,259 enum dma_data_direction dir)260{261 struct gntdev_dmabuf_attachment *gntdev_dmabuf_attach = attach->priv;262 struct gntdev_dmabuf *gntdev_dmabuf = attach->dmabuf->priv;263 struct sg_table *sgt;264 265 pr_debug("Mapping %d pages for dev %p\n", gntdev_dmabuf->nr_pages,266 attach->dev);267 268 if (dir == DMA_NONE || !gntdev_dmabuf_attach)269 return ERR_PTR(-EINVAL);270 271 /* Return the cached mapping when possible. */272 if (gntdev_dmabuf_attach->dir == dir)273 return gntdev_dmabuf_attach->sgt;274 275 /*276 * Two mappings with different directions for the same attachment are277 * not allowed.278 */279 if (gntdev_dmabuf_attach->dir != DMA_NONE)280 return ERR_PTR(-EBUSY);281 282 sgt = dmabuf_pages_to_sgt(gntdev_dmabuf->pages,283 gntdev_dmabuf->nr_pages);284 if (!IS_ERR(sgt)) {285 if (dma_map_sgtable(attach->dev, sgt, dir,286 DMA_ATTR_SKIP_CPU_SYNC)) {287 sg_free_table(sgt);288 kfree(sgt);289 sgt = ERR_PTR(-ENOMEM);290 } else {291 gntdev_dmabuf_attach->sgt = sgt;292 gntdev_dmabuf_attach->dir = dir;293 }294 }295 if (IS_ERR(sgt))296 pr_debug("Failed to map sg table for dev %p\n", attach->dev);297 return sgt;298}299 300static void dmabuf_exp_ops_unmap_dma_buf(struct dma_buf_attachment *attach,301 struct sg_table *sgt,302 enum dma_data_direction dir)303{304 /* Not implemented. The unmap is done at dmabuf_exp_ops_detach(). */305}306 307static void dmabuf_exp_release(struct kref *kref)308{309 struct gntdev_dmabuf *gntdev_dmabuf =310 container_of(kref, struct gntdev_dmabuf, u.exp.refcount);311 312 dmabuf_exp_wait_obj_signal(gntdev_dmabuf->priv, gntdev_dmabuf);313 list_del(&gntdev_dmabuf->next);314 fput(gntdev_dmabuf->priv->filp);315 kfree(gntdev_dmabuf);316}317 318static void dmabuf_exp_remove_map(struct gntdev_priv *priv,319 struct gntdev_grant_map *map)320{321 mutex_lock(&priv->lock);322 list_del(&map->next);323 gntdev_put_map(NULL /* already removed */, map);324 mutex_unlock(&priv->lock);325}326 327static void dmabuf_exp_ops_release(struct dma_buf *dma_buf)328{329 struct gntdev_dmabuf *gntdev_dmabuf = dma_buf->priv;330 struct gntdev_dmabuf_priv *priv = gntdev_dmabuf->priv;331 332 dmabuf_exp_remove_map(gntdev_dmabuf->u.exp.priv,333 gntdev_dmabuf->u.exp.map);334 mutex_lock(&priv->lock);335 kref_put(&gntdev_dmabuf->u.exp.refcount, dmabuf_exp_release);336 mutex_unlock(&priv->lock);337}338 339static const struct dma_buf_ops dmabuf_exp_ops = {340 .attach = dmabuf_exp_ops_attach,341 .detach = dmabuf_exp_ops_detach,342 .map_dma_buf = dmabuf_exp_ops_map_dma_buf,343 .unmap_dma_buf = dmabuf_exp_ops_unmap_dma_buf,344 .release = dmabuf_exp_ops_release,345};346 347struct gntdev_dmabuf_export_args {348 struct gntdev_priv *priv;349 struct gntdev_grant_map *map;350 struct gntdev_dmabuf_priv *dmabuf_priv;351 struct device *dev;352 int count;353 struct page **pages;354 u32 fd;355};356 357static int dmabuf_exp_from_pages(struct gntdev_dmabuf_export_args *args)358{359 DEFINE_DMA_BUF_EXPORT_INFO(exp_info);360 struct gntdev_dmabuf *gntdev_dmabuf;361 int ret;362 363 gntdev_dmabuf = kzalloc(sizeof(*gntdev_dmabuf), GFP_KERNEL);364 if (!gntdev_dmabuf)365 return -ENOMEM;366 367 kref_init(&gntdev_dmabuf->u.exp.refcount);368 369 gntdev_dmabuf->priv = args->dmabuf_priv;370 gntdev_dmabuf->nr_pages = args->count;371 gntdev_dmabuf->pages = args->pages;372 gntdev_dmabuf->u.exp.priv = args->priv;373 gntdev_dmabuf->u.exp.map = args->map;374 375 exp_info.exp_name = KBUILD_MODNAME;376 if (args->dev->driver && args->dev->driver->owner)377 exp_info.owner = args->dev->driver->owner;378 else379 exp_info.owner = THIS_MODULE;380 exp_info.ops = &dmabuf_exp_ops;381 exp_info.size = args->count << PAGE_SHIFT;382 exp_info.flags = O_RDWR;383 exp_info.priv = gntdev_dmabuf;384 385 gntdev_dmabuf->dmabuf = dma_buf_export(&exp_info);386 if (IS_ERR(gntdev_dmabuf->dmabuf)) {387 ret = PTR_ERR(gntdev_dmabuf->dmabuf);388 gntdev_dmabuf->dmabuf = NULL;389 goto fail;390 }391 392 ret = dma_buf_fd(gntdev_dmabuf->dmabuf, O_CLOEXEC);393 if (ret < 0)394 goto fail;395 396 gntdev_dmabuf->fd = ret;397 args->fd = ret;398 399 pr_debug("Exporting DMA buffer with fd %d\n", ret);400 401 mutex_lock(&args->dmabuf_priv->lock);402 list_add(&gntdev_dmabuf->next, &args->dmabuf_priv->exp_list);403 mutex_unlock(&args->dmabuf_priv->lock);404 get_file(gntdev_dmabuf->priv->filp);405 return 0;406 407fail:408 if (gntdev_dmabuf->dmabuf)409 dma_buf_put(gntdev_dmabuf->dmabuf);410 kfree(gntdev_dmabuf);411 return ret;412}413 414static struct gntdev_grant_map *415dmabuf_exp_alloc_backing_storage(struct gntdev_priv *priv, int dmabuf_flags,416 int count)417{418 struct gntdev_grant_map *map;419 420 if (unlikely(gntdev_test_page_count(count)))421 return ERR_PTR(-EINVAL);422 423 if ((dmabuf_flags & GNTDEV_DMA_FLAG_WC) &&424 (dmabuf_flags & GNTDEV_DMA_FLAG_COHERENT)) {425 pr_debug("Wrong dma-buf flags: 0x%x\n", dmabuf_flags);426 return ERR_PTR(-EINVAL);427 }428 429 map = gntdev_alloc_map(priv, count, dmabuf_flags);430 if (!map)431 return ERR_PTR(-ENOMEM);432 433 return map;434}435 436static int dmabuf_exp_from_refs(struct gntdev_priv *priv, int flags,437 int count, u32 domid, u32 *refs, u32 *fd)438{439 struct gntdev_grant_map *map;440 struct gntdev_dmabuf_export_args args;441 int i, ret;442 443 map = dmabuf_exp_alloc_backing_storage(priv, flags, count);444 if (IS_ERR(map))445 return PTR_ERR(map);446 447 for (i = 0; i < count; i++) {448 map->grants[i].domid = domid;449 map->grants[i].ref = refs[i];450 }451 452 mutex_lock(&priv->lock);453 gntdev_add_map(priv, map);454 mutex_unlock(&priv->lock);455 456 map->flags |= GNTMAP_host_map;457#if defined(CONFIG_X86)458 map->flags |= GNTMAP_device_map;459#endif460 461 ret = gntdev_map_grant_pages(map);462 if (ret < 0)463 goto out;464 465 args.priv = priv;466 args.map = map;467 args.dev = priv->dma_dev;468 args.dmabuf_priv = priv->dmabuf_priv;469 args.count = map->count;470 args.pages = map->pages;471 args.fd = -1; /* Shut up unnecessary gcc warning for i386 */472 473 ret = dmabuf_exp_from_pages(&args);474 if (ret < 0)475 goto out;476 477 *fd = args.fd;478 return 0;479 480out:481 dmabuf_exp_remove_map(priv, map);482 return ret;483}484 485/* DMA buffer import support. */486 487static int488dmabuf_imp_grant_foreign_access(unsigned long *gfns, u32 *refs,489 int count, int domid)490{491 grant_ref_t priv_gref_head;492 int i, ret;493 494 ret = gnttab_alloc_grant_references(count, &priv_gref_head);495 if (ret < 0) {496 pr_debug("Cannot allocate grant references, ret %d\n", ret);497 return ret;498 }499 500 for (i = 0; i < count; i++) {501 int cur_ref;502 503 cur_ref = gnttab_claim_grant_reference(&priv_gref_head);504 if (cur_ref < 0) {505 ret = cur_ref;506 pr_debug("Cannot claim grant reference, ret %d\n", ret);507 goto out;508 }509 510 gnttab_grant_foreign_access_ref(cur_ref, domid,511 gfns[i], 0);512 refs[i] = cur_ref;513 }514 515 return 0;516 517out:518 gnttab_free_grant_references(priv_gref_head);519 return ret;520}521 522static void dmabuf_imp_end_foreign_access(u32 *refs, int count)523{524 int i;525 526 for (i = 0; i < count; i++)527 if (refs[i] != INVALID_GRANT_REF)528 gnttab_end_foreign_access(refs[i], NULL);529}530 531static void dmabuf_imp_free_storage(struct gntdev_dmabuf *gntdev_dmabuf)532{533 kfree(gntdev_dmabuf->u.imp.refs);534 kfree(gntdev_dmabuf);535}536 537static struct gntdev_dmabuf *dmabuf_imp_alloc_storage(int count)538{539 struct gntdev_dmabuf *gntdev_dmabuf;540 int i;541 542 gntdev_dmabuf = kzalloc(sizeof(*gntdev_dmabuf), GFP_KERNEL);543 if (!gntdev_dmabuf)544 goto fail_no_free;545 546 gntdev_dmabuf->u.imp.refs = kcalloc(count,547 sizeof(gntdev_dmabuf->u.imp.refs[0]),548 GFP_KERNEL);549 if (!gntdev_dmabuf->u.imp.refs)550 goto fail;551 552 gntdev_dmabuf->nr_pages = count;553 554 for (i = 0; i < count; i++)555 gntdev_dmabuf->u.imp.refs[i] = INVALID_GRANT_REF;556 557 return gntdev_dmabuf;558 559fail:560 dmabuf_imp_free_storage(gntdev_dmabuf);561fail_no_free:562 return ERR_PTR(-ENOMEM);563}564 565static struct gntdev_dmabuf *566dmabuf_imp_to_refs(struct gntdev_dmabuf_priv *priv, struct device *dev,567 int fd, int count, int domid)568{569 struct gntdev_dmabuf *gntdev_dmabuf, *ret;570 struct dma_buf *dma_buf;571 struct dma_buf_attachment *attach;572 struct sg_table *sgt;573 struct sg_dma_page_iter sg_iter;574 unsigned long *gfns;575 int i;576 577 dma_buf = dma_buf_get(fd);578 if (IS_ERR(dma_buf))579 return ERR_CAST(dma_buf);580 581 gntdev_dmabuf = dmabuf_imp_alloc_storage(count);582 if (IS_ERR(gntdev_dmabuf)) {583 ret = gntdev_dmabuf;584 goto fail_put;585 }586 587 gntdev_dmabuf->priv = priv;588 gntdev_dmabuf->fd = fd;589 590 attach = dma_buf_attach(dma_buf, dev);591 if (IS_ERR(attach)) {592 ret = ERR_CAST(attach);593 goto fail_free_obj;594 }595 596 gntdev_dmabuf->u.imp.attach = attach;597 598 sgt = dma_buf_map_attachment_unlocked(attach, DMA_BIDIRECTIONAL);599 if (IS_ERR(sgt)) {600 ret = ERR_CAST(sgt);601 goto fail_detach;602 }603 604 /* Check that we have zero offset. */605 if (sgt->sgl->offset) {606 ret = ERR_PTR(-EINVAL);607 pr_debug("DMA buffer has %d bytes offset, user-space expects 0\n",608 sgt->sgl->offset);609 goto fail_unmap;610 }611 612 /* Check number of pages that imported buffer has. */613 if (attach->dmabuf->size != gntdev_dmabuf->nr_pages << PAGE_SHIFT) {614 ret = ERR_PTR(-EINVAL);615 pr_debug("DMA buffer has %zu pages, user-space expects %d\n",616 attach->dmabuf->size, gntdev_dmabuf->nr_pages);617 goto fail_unmap;618 }619 620 gntdev_dmabuf->u.imp.sgt = sgt;621 622 gfns = kcalloc(count, sizeof(*gfns), GFP_KERNEL);623 if (!gfns) {624 ret = ERR_PTR(-ENOMEM);625 goto fail_unmap;626 }627 628 /*629 * Now convert sgt to array of gfns without accessing underlying pages.630 * It is not allowed to access the underlying struct page of an sg table631 * exported by DMA-buf, but since we deal with special Xen dma device here632 * (not a normal physical one) look at the dma addresses in the sg table633 * and then calculate gfns directly from them.634 */635 i = 0;636 for_each_sgtable_dma_page(sgt, &sg_iter, 0) {637 dma_addr_t addr = sg_page_iter_dma_address(&sg_iter);638 unsigned long pfn = bfn_to_pfn(XEN_PFN_DOWN(dma_to_phys(dev, addr)));639 640 gfns[i++] = pfn_to_gfn(pfn);641 }642 643 ret = ERR_PTR(dmabuf_imp_grant_foreign_access(gfns,644 gntdev_dmabuf->u.imp.refs,645 count, domid));646 kfree(gfns);647 if (IS_ERR(ret))648 goto fail_end_access;649 650 pr_debug("Imported DMA buffer with fd %d\n", fd);651 652 mutex_lock(&priv->lock);653 list_add(&gntdev_dmabuf->next, &priv->imp_list);654 mutex_unlock(&priv->lock);655 656 return gntdev_dmabuf;657 658fail_end_access:659 dmabuf_imp_end_foreign_access(gntdev_dmabuf->u.imp.refs, count);660fail_unmap:661 dma_buf_unmap_attachment_unlocked(attach, sgt, DMA_BIDIRECTIONAL);662fail_detach:663 dma_buf_detach(dma_buf, attach);664fail_free_obj:665 dmabuf_imp_free_storage(gntdev_dmabuf);666fail_put:667 dma_buf_put(dma_buf);668 return ret;669}670 671/*672 * Find the hyper dma-buf by its file descriptor and remove673 * it from the buffer's list.674 */675static struct gntdev_dmabuf *676dmabuf_imp_find_unlink(struct gntdev_dmabuf_priv *priv, int fd)677{678 struct gntdev_dmabuf *q, *gntdev_dmabuf, *ret = ERR_PTR(-ENOENT);679 680 mutex_lock(&priv->lock);681 list_for_each_entry_safe(gntdev_dmabuf, q, &priv->imp_list, next) {682 if (gntdev_dmabuf->fd == fd) {683 pr_debug("Found gntdev_dmabuf in the import list\n");684 ret = gntdev_dmabuf;685 list_del(&gntdev_dmabuf->next);686 break;687 }688 }689 mutex_unlock(&priv->lock);690 return ret;691}692 693static int dmabuf_imp_release(struct gntdev_dmabuf_priv *priv, u32 fd)694{695 struct gntdev_dmabuf *gntdev_dmabuf;696 struct dma_buf_attachment *attach;697 struct dma_buf *dma_buf;698 699 gntdev_dmabuf = dmabuf_imp_find_unlink(priv, fd);700 if (IS_ERR(gntdev_dmabuf))701 return PTR_ERR(gntdev_dmabuf);702 703 pr_debug("Releasing DMA buffer with fd %d\n", fd);704 705 dmabuf_imp_end_foreign_access(gntdev_dmabuf->u.imp.refs,706 gntdev_dmabuf->nr_pages);707 708 attach = gntdev_dmabuf->u.imp.attach;709 710 if (gntdev_dmabuf->u.imp.sgt)711 dma_buf_unmap_attachment_unlocked(attach, gntdev_dmabuf->u.imp.sgt,712 DMA_BIDIRECTIONAL);713 dma_buf = attach->dmabuf;714 dma_buf_detach(attach->dmabuf, attach);715 dma_buf_put(dma_buf);716 717 dmabuf_imp_free_storage(gntdev_dmabuf);718 return 0;719}720 721static void dmabuf_imp_release_all(struct gntdev_dmabuf_priv *priv)722{723 struct gntdev_dmabuf *q, *gntdev_dmabuf;724 725 list_for_each_entry_safe(gntdev_dmabuf, q, &priv->imp_list, next)726 dmabuf_imp_release(priv, gntdev_dmabuf->fd);727}728 729/* DMA buffer IOCTL support. */730 731long gntdev_ioctl_dmabuf_exp_from_refs(struct gntdev_priv *priv, int use_ptemod,732 struct ioctl_gntdev_dmabuf_exp_from_refs __user *u)733{734 struct ioctl_gntdev_dmabuf_exp_from_refs op;735 u32 *refs;736 long ret;737 738 if (use_ptemod) {739 pr_debug("Cannot provide dma-buf: use_ptemode %d\n",740 use_ptemod);741 return -EINVAL;742 }743 744 if (copy_from_user(&op, u, sizeof(op)) != 0)745 return -EFAULT;746 747 if (unlikely(gntdev_test_page_count(op.count)))748 return -EINVAL;749 750 refs = kcalloc(op.count, sizeof(*refs), GFP_KERNEL);751 if (!refs)752 return -ENOMEM;753 754 if (copy_from_user(refs, u->refs, sizeof(*refs) * op.count) != 0) {755 ret = -EFAULT;756 goto out;757 }758 759 ret = dmabuf_exp_from_refs(priv, op.flags, op.count,760 op.domid, refs, &op.fd);761 if (ret)762 goto out;763 764 if (copy_to_user(u, &op, sizeof(op)) != 0)765 ret = -EFAULT;766 767out:768 kfree(refs);769 return ret;770}771 772long gntdev_ioctl_dmabuf_exp_wait_released(struct gntdev_priv *priv,773 struct ioctl_gntdev_dmabuf_exp_wait_released __user *u)774{775 struct ioctl_gntdev_dmabuf_exp_wait_released op;776 777 if (copy_from_user(&op, u, sizeof(op)) != 0)778 return -EFAULT;779 780 return dmabuf_exp_wait_released(priv->dmabuf_priv, op.fd,781 op.wait_to_ms);782}783 784long gntdev_ioctl_dmabuf_imp_to_refs(struct gntdev_priv *priv,785 struct ioctl_gntdev_dmabuf_imp_to_refs __user *u)786{787 struct ioctl_gntdev_dmabuf_imp_to_refs op;788 struct gntdev_dmabuf *gntdev_dmabuf;789 long ret;790 791 if (copy_from_user(&op, u, sizeof(op)) != 0)792 return -EFAULT;793 794 if (unlikely(gntdev_test_page_count(op.count)))795 return -EINVAL;796 797 gntdev_dmabuf = dmabuf_imp_to_refs(priv->dmabuf_priv,798 priv->dma_dev, op.fd,799 op.count, op.domid);800 if (IS_ERR(gntdev_dmabuf))801 return PTR_ERR(gntdev_dmabuf);802 803 if (copy_to_user(u->refs, gntdev_dmabuf->u.imp.refs,804 sizeof(*u->refs) * op.count) != 0) {805 ret = -EFAULT;806 goto out_release;807 }808 return 0;809 810out_release:811 dmabuf_imp_release(priv->dmabuf_priv, op.fd);812 return ret;813}814 815long gntdev_ioctl_dmabuf_imp_release(struct gntdev_priv *priv,816 struct ioctl_gntdev_dmabuf_imp_release __user *u)817{818 struct ioctl_gntdev_dmabuf_imp_release op;819 820 if (copy_from_user(&op, u, sizeof(op)) != 0)821 return -EFAULT;822 823 return dmabuf_imp_release(priv->dmabuf_priv, op.fd);824}825 826struct gntdev_dmabuf_priv *gntdev_dmabuf_init(struct file *filp)827{828 struct gntdev_dmabuf_priv *priv;829 830 priv = kzalloc(sizeof(*priv), GFP_KERNEL);831 if (!priv)832 return ERR_PTR(-ENOMEM);833 834 mutex_init(&priv->lock);835 INIT_LIST_HEAD(&priv->exp_list);836 INIT_LIST_HEAD(&priv->exp_wait_list);837 INIT_LIST_HEAD(&priv->imp_list);838 839 priv->filp = filp;840 841 return priv;842}843 844void gntdev_dmabuf_fini(struct gntdev_dmabuf_priv *priv)845{846 dmabuf_imp_release_all(priv);847 kfree(priv);848}849