510 lines · c
1// SPDX-License-Identifier: GPL-2.02#include <linux/cred.h>3#include <linux/device.h>4#include <linux/dma-buf.h>5#include <linux/dma-resv.h>6#include <linux/highmem.h>7#include <linux/init.h>8#include <linux/kernel.h>9#include <linux/memfd.h>10#include <linux/miscdevice.h>11#include <linux/module.h>12#include <linux/shmem_fs.h>13#include <linux/hugetlb.h>14#include <linux/slab.h>15#include <linux/udmabuf.h>16#include <linux/vmalloc.h>17#include <linux/iosys-map.h>18 19static int list_limit = 1024;20module_param(list_limit, int, 0644);21MODULE_PARM_DESC(list_limit, "udmabuf_create_list->count limit. Default is 1024.");22 23static int size_limit_mb = 64;24module_param(size_limit_mb, int, 0644);25MODULE_PARM_DESC(size_limit_mb, "Max size of a dmabuf, in megabytes. Default is 64.");26 27struct udmabuf {28 pgoff_t pagecount;29 struct folio **folios;30 struct sg_table *sg;31 struct miscdevice *device;32 pgoff_t *offsets;33 struct list_head unpin_list;34};35 36struct udmabuf_folio {37 struct folio *folio;38 struct list_head list;39};40 41static vm_fault_t udmabuf_vm_fault(struct vm_fault *vmf)42{43 struct vm_area_struct *vma = vmf->vma;44 struct udmabuf *ubuf = vma->vm_private_data;45 pgoff_t pgoff = vmf->pgoff;46 unsigned long pfn;47 48 if (pgoff >= ubuf->pagecount)49 return VM_FAULT_SIGBUS;50 51 pfn = folio_pfn(ubuf->folios[pgoff]);52 pfn += ubuf->offsets[pgoff] >> PAGE_SHIFT;53 54 return vmf_insert_pfn(vma, vmf->address, pfn);55}56 57static const struct vm_operations_struct udmabuf_vm_ops = {58 .fault = udmabuf_vm_fault,59};60 61static int mmap_udmabuf(struct dma_buf *buf, struct vm_area_struct *vma)62{63 struct udmabuf *ubuf = buf->priv;64 65 if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0)66 return -EINVAL;67 68 vma->vm_ops = &udmabuf_vm_ops;69 vma->vm_private_data = ubuf;70 vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);71 return 0;72}73 74static int vmap_udmabuf(struct dma_buf *buf, struct iosys_map *map)75{76 struct udmabuf *ubuf = buf->priv;77 struct page **pages;78 void *vaddr;79 pgoff_t pg;80 81 dma_resv_assert_held(buf->resv);82 83 pages = kmalloc_array(ubuf->pagecount, sizeof(*pages), GFP_KERNEL);84 if (!pages)85 return -ENOMEM;86 87 for (pg = 0; pg < ubuf->pagecount; pg++)88 pages[pg] = &ubuf->folios[pg]->page;89 90 vaddr = vm_map_ram(pages, ubuf->pagecount, -1);91 kfree(pages);92 if (!vaddr)93 return -EINVAL;94 95 iosys_map_set_vaddr(map, vaddr);96 return 0;97}98 99static void vunmap_udmabuf(struct dma_buf *buf, struct iosys_map *map)100{101 struct udmabuf *ubuf = buf->priv;102 103 dma_resv_assert_held(buf->resv);104 105 vm_unmap_ram(map->vaddr, ubuf->pagecount);106}107 108static struct sg_table *get_sg_table(struct device *dev, struct dma_buf *buf,109 enum dma_data_direction direction)110{111 struct udmabuf *ubuf = buf->priv;112 struct sg_table *sg;113 struct scatterlist *sgl;114 unsigned int i = 0;115 int ret;116 117 sg = kzalloc(sizeof(*sg), GFP_KERNEL);118 if (!sg)119 return ERR_PTR(-ENOMEM);120 121 ret = sg_alloc_table(sg, ubuf->pagecount, GFP_KERNEL);122 if (ret < 0)123 goto err_alloc;124 125 for_each_sg(sg->sgl, sgl, ubuf->pagecount, i)126 sg_set_folio(sgl, ubuf->folios[i], PAGE_SIZE,127 ubuf->offsets[i]);128 129 ret = dma_map_sgtable(dev, sg, direction, 0);130 if (ret < 0)131 goto err_map;132 return sg;133 134err_map:135 sg_free_table(sg);136err_alloc:137 kfree(sg);138 return ERR_PTR(ret);139}140 141static void put_sg_table(struct device *dev, struct sg_table *sg,142 enum dma_data_direction direction)143{144 dma_unmap_sgtable(dev, sg, direction, 0);145 sg_free_table(sg);146 kfree(sg);147}148 149static struct sg_table *map_udmabuf(struct dma_buf_attachment *at,150 enum dma_data_direction direction)151{152 return get_sg_table(at->dev, at->dmabuf, direction);153}154 155static void unmap_udmabuf(struct dma_buf_attachment *at,156 struct sg_table *sg,157 enum dma_data_direction direction)158{159 return put_sg_table(at->dev, sg, direction);160}161 162static void unpin_all_folios(struct list_head *unpin_list)163{164 struct udmabuf_folio *ubuf_folio;165 166 while (!list_empty(unpin_list)) {167 ubuf_folio = list_first_entry(unpin_list,168 struct udmabuf_folio, list);169 unpin_folio(ubuf_folio->folio);170 171 list_del(&ubuf_folio->list);172 kfree(ubuf_folio);173 }174}175 176static int add_to_unpin_list(struct list_head *unpin_list,177 struct folio *folio)178{179 struct udmabuf_folio *ubuf_folio;180 181 ubuf_folio = kzalloc(sizeof(*ubuf_folio), GFP_KERNEL);182 if (!ubuf_folio)183 return -ENOMEM;184 185 ubuf_folio->folio = folio;186 list_add_tail(&ubuf_folio->list, unpin_list);187 return 0;188}189 190static void release_udmabuf(struct dma_buf *buf)191{192 struct udmabuf *ubuf = buf->priv;193 struct device *dev = ubuf->device->this_device;194 195 if (ubuf->sg)196 put_sg_table(dev, ubuf->sg, DMA_BIDIRECTIONAL);197 198 unpin_all_folios(&ubuf->unpin_list);199 kfree(ubuf->offsets);200 kfree(ubuf->folios);201 kfree(ubuf);202}203 204static int begin_cpu_udmabuf(struct dma_buf *buf,205 enum dma_data_direction direction)206{207 struct udmabuf *ubuf = buf->priv;208 struct device *dev = ubuf->device->this_device;209 int ret = 0;210 211 if (!ubuf->sg) {212 ubuf->sg = get_sg_table(dev, buf, direction);213 if (IS_ERR(ubuf->sg)) {214 ret = PTR_ERR(ubuf->sg);215 ubuf->sg = NULL;216 }217 } else {218 dma_sync_sg_for_cpu(dev, ubuf->sg->sgl, ubuf->sg->nents,219 direction);220 }221 222 return ret;223}224 225static int end_cpu_udmabuf(struct dma_buf *buf,226 enum dma_data_direction direction)227{228 struct udmabuf *ubuf = buf->priv;229 struct device *dev = ubuf->device->this_device;230 231 if (!ubuf->sg)232 return -EINVAL;233 234 dma_sync_sg_for_device(dev, ubuf->sg->sgl, ubuf->sg->nents, direction);235 return 0;236}237 238static const struct dma_buf_ops udmabuf_ops = {239 .cache_sgt_mapping = true,240 .map_dma_buf = map_udmabuf,241 .unmap_dma_buf = unmap_udmabuf,242 .release = release_udmabuf,243 .mmap = mmap_udmabuf,244 .vmap = vmap_udmabuf,245 .vunmap = vunmap_udmabuf,246 .begin_cpu_access = begin_cpu_udmabuf,247 .end_cpu_access = end_cpu_udmabuf,248};249 250#define SEALS_WANTED (F_SEAL_SHRINK)251#define SEALS_DENIED (F_SEAL_WRITE)252 253static int check_memfd_seals(struct file *memfd)254{255 int seals;256 257 if (!memfd)258 return -EBADFD;259 260 if (!shmem_file(memfd) && !is_file_hugepages(memfd))261 return -EBADFD;262 263 seals = memfd_fcntl(memfd, F_GET_SEALS, 0);264 if (seals == -EINVAL)265 return -EBADFD;266 267 if ((seals & SEALS_WANTED) != SEALS_WANTED ||268 (seals & SEALS_DENIED) != 0)269 return -EINVAL;270 271 return 0;272}273 274static int export_udmabuf(struct udmabuf *ubuf,275 struct miscdevice *device,276 u32 flags)277{278 DEFINE_DMA_BUF_EXPORT_INFO(exp_info);279 struct dma_buf *buf;280 281 ubuf->device = device;282 exp_info.ops = &udmabuf_ops;283 exp_info.size = ubuf->pagecount << PAGE_SHIFT;284 exp_info.priv = ubuf;285 exp_info.flags = O_RDWR;286 287 buf = dma_buf_export(&exp_info);288 if (IS_ERR(buf))289 return PTR_ERR(buf);290 291 return dma_buf_fd(buf, flags);292}293 294static long udmabuf_create(struct miscdevice *device,295 struct udmabuf_create_list *head,296 struct udmabuf_create_item *list)297{298 pgoff_t pgoff, pgcnt, pglimit, pgbuf = 0;299 long nr_folios, ret = -EINVAL;300 struct file *memfd = NULL;301 struct folio **folios;302 struct udmabuf *ubuf;303 u32 i, j, k, flags;304 loff_t end;305 306 ubuf = kzalloc(sizeof(*ubuf), GFP_KERNEL);307 if (!ubuf)308 return -ENOMEM;309 310 INIT_LIST_HEAD(&ubuf->unpin_list);311 pglimit = (size_limit_mb * 1024 * 1024) >> PAGE_SHIFT;312 for (i = 0; i < head->count; i++) {313 if (!IS_ALIGNED(list[i].offset, PAGE_SIZE))314 goto err;315 if (!IS_ALIGNED(list[i].size, PAGE_SIZE))316 goto err;317 ubuf->pagecount += list[i].size >> PAGE_SHIFT;318 if (ubuf->pagecount > pglimit)319 goto err;320 }321 322 if (!ubuf->pagecount)323 goto err;324 325 ubuf->folios = kmalloc_array(ubuf->pagecount, sizeof(*ubuf->folios),326 GFP_KERNEL);327 if (!ubuf->folios) {328 ret = -ENOMEM;329 goto err;330 }331 ubuf->offsets = kcalloc(ubuf->pagecount, sizeof(*ubuf->offsets),332 GFP_KERNEL);333 if (!ubuf->offsets) {334 ret = -ENOMEM;335 goto err;336 }337 338 pgbuf = 0;339 for (i = 0; i < head->count; i++) {340 memfd = fget(list[i].memfd);341 ret = check_memfd_seals(memfd);342 if (ret < 0)343 goto err;344 345 pgcnt = list[i].size >> PAGE_SHIFT;346 folios = kmalloc_array(pgcnt, sizeof(*folios), GFP_KERNEL);347 if (!folios) {348 ret = -ENOMEM;349 goto err;350 }351 352 end = list[i].offset + (pgcnt << PAGE_SHIFT) - 1;353 ret = memfd_pin_folios(memfd, list[i].offset, end,354 folios, pgcnt, &pgoff);355 if (ret <= 0) {356 kfree(folios);357 if (!ret)358 ret = -EINVAL;359 goto err;360 }361 362 nr_folios = ret;363 pgoff >>= PAGE_SHIFT;364 for (j = 0, k = 0; j < pgcnt; j++) {365 ubuf->folios[pgbuf] = folios[k];366 ubuf->offsets[pgbuf] = pgoff << PAGE_SHIFT;367 368 if (j == 0 || ubuf->folios[pgbuf-1] != folios[k]) {369 ret = add_to_unpin_list(&ubuf->unpin_list,370 folios[k]);371 if (ret < 0) {372 kfree(folios);373 goto err;374 }375 }376 377 pgbuf++;378 if (++pgoff == folio_nr_pages(folios[k])) {379 pgoff = 0;380 if (++k == nr_folios)381 break;382 }383 }384 385 kfree(folios);386 fput(memfd);387 memfd = NULL;388 }389 390 flags = head->flags & UDMABUF_FLAGS_CLOEXEC ? O_CLOEXEC : 0;391 ret = export_udmabuf(ubuf, device, flags);392 if (ret < 0)393 goto err;394 395 return ret;396 397err:398 if (memfd)399 fput(memfd);400 unpin_all_folios(&ubuf->unpin_list);401 kfree(ubuf->offsets);402 kfree(ubuf->folios);403 kfree(ubuf);404 return ret;405}406 407static long udmabuf_ioctl_create(struct file *filp, unsigned long arg)408{409 struct udmabuf_create create;410 struct udmabuf_create_list head;411 struct udmabuf_create_item list;412 413 if (copy_from_user(&create, (void __user *)arg,414 sizeof(create)))415 return -EFAULT;416 417 head.flags = create.flags;418 head.count = 1;419 list.memfd = create.memfd;420 list.offset = create.offset;421 list.size = create.size;422 423 return udmabuf_create(filp->private_data, &head, &list);424}425 426static long udmabuf_ioctl_create_list(struct file *filp, unsigned long arg)427{428 struct udmabuf_create_list head;429 struct udmabuf_create_item *list;430 int ret = -EINVAL;431 u32 lsize;432 433 if (copy_from_user(&head, (void __user *)arg, sizeof(head)))434 return -EFAULT;435 if (head.count > list_limit)436 return -EINVAL;437 lsize = sizeof(struct udmabuf_create_item) * head.count;438 list = memdup_user((void __user *)(arg + sizeof(head)), lsize);439 if (IS_ERR(list))440 return PTR_ERR(list);441 442 ret = udmabuf_create(filp->private_data, &head, list);443 kfree(list);444 return ret;445}446 447static long udmabuf_ioctl(struct file *filp, unsigned int ioctl,448 unsigned long arg)449{450 long ret;451 452 switch (ioctl) {453 case UDMABUF_CREATE:454 ret = udmabuf_ioctl_create(filp, arg);455 break;456 case UDMABUF_CREATE_LIST:457 ret = udmabuf_ioctl_create_list(filp, arg);458 break;459 default:460 ret = -ENOTTY;461 break;462 }463 return ret;464}465 466static const struct file_operations udmabuf_fops = {467 .owner = THIS_MODULE,468 .unlocked_ioctl = udmabuf_ioctl,469#ifdef CONFIG_COMPAT470 .compat_ioctl = udmabuf_ioctl,471#endif472};473 474static struct miscdevice udmabuf_misc = {475 .minor = MISC_DYNAMIC_MINOR,476 .name = "udmabuf",477 .fops = &udmabuf_fops,478};479 480static int __init udmabuf_dev_init(void)481{482 int ret;483 484 ret = misc_register(&udmabuf_misc);485 if (ret < 0) {486 pr_err("Could not initialize udmabuf device\n");487 return ret;488 }489 490 ret = dma_coerce_mask_and_coherent(udmabuf_misc.this_device,491 DMA_BIT_MASK(64));492 if (ret < 0) {493 pr_err("Could not setup DMA mask for udmabuf device\n");494 misc_deregister(&udmabuf_misc);495 return ret;496 }497 498 return 0;499}500 501static void __exit udmabuf_dev_exit(void)502{503 misc_deregister(&udmabuf_misc);504}505 506module_init(udmabuf_dev_init)507module_exit(udmabuf_dev_exit)508 509MODULE_AUTHOR("Gerd Hoffmann <kraxel@redhat.com>");510