165 lines · c
1/*2 * Copyright © 2014 Intel Corporation3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice (including the next12 * paragraph) shall be included in all copies or substantial portions of the13 * Software.14 *15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS21 * IN THE SOFTWARE.22 *23 */24 25#include <linux/mm.h>26#include <linux/io-mapping.h>27 28 29#include "i915_drv.h"30#include "i915_mm.h"31 32struct remap_pfn {33 struct mm_struct *mm;34 unsigned long pfn;35 pgprot_t prot;36 37 struct sgt_iter sgt;38 resource_size_t iobase;39};40 41#define use_dma(io) ((io) != -1)42 43static inline unsigned long sgt_pfn(const struct remap_pfn *r)44{45 if (use_dma(r->iobase))46 return (r->sgt.dma + r->sgt.curr + r->iobase) >> PAGE_SHIFT;47 else48 return r->sgt.pfn + (r->sgt.curr >> PAGE_SHIFT);49}50 51static int remap_sg(pte_t *pte, unsigned long addr, void *data)52{53 struct remap_pfn *r = data;54 55 if (GEM_WARN_ON(!r->sgt.sgp))56 return -EINVAL;57 58 /* Special PTE are not associated with any struct page */59 set_pte_at(r->mm, addr, pte,60 pte_mkspecial(pfn_pte(sgt_pfn(r), r->prot)));61 r->pfn++; /* track insertions in case we need to unwind later */62 63 r->sgt.curr += PAGE_SIZE;64 if (r->sgt.curr >= r->sgt.max)65 r->sgt = __sgt_iter(__sg_next(r->sgt.sgp), use_dma(r->iobase));66 67 return 0;68}69 70#define EXPECTED_FLAGS (VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP)71 72#if IS_ENABLED(CONFIG_X86)73static int remap_pfn(pte_t *pte, unsigned long addr, void *data)74{75 struct remap_pfn *r = data;76 77 /* Special PTE are not associated with any struct page */78 set_pte_at(r->mm, addr, pte, pte_mkspecial(pfn_pte(r->pfn, r->prot)));79 r->pfn++;80 81 return 0;82}83 84/**85 * remap_io_mapping - remap an IO mapping to userspace86 * @vma: user vma to map to87 * @addr: target user address to start at88 * @pfn: physical address of kernel memory89 * @size: size of map area90 * @iomap: the source io_mapping91 *92 * Note: this is only safe if the mm semaphore is held when called.93 */94int remap_io_mapping(struct vm_area_struct *vma,95 unsigned long addr, unsigned long pfn, unsigned long size,96 struct io_mapping *iomap)97{98 struct remap_pfn r;99 int err;100 101 GEM_BUG_ON((vma->vm_flags & EXPECTED_FLAGS) != EXPECTED_FLAGS);102 103 /* We rely on prevalidation of the io-mapping to skip track_pfn(). */104 r.mm = vma->vm_mm;105 r.pfn = pfn;106 r.prot = __pgprot((pgprot_val(iomap->prot) & _PAGE_CACHE_MASK) |107 (pgprot_val(vma->vm_page_prot) & ~_PAGE_CACHE_MASK));108 109 err = apply_to_page_range(r.mm, addr, size, remap_pfn, &r);110 if (unlikely(err)) {111 zap_vma_ptes(vma, addr, (r.pfn - pfn) << PAGE_SHIFT);112 return err;113 }114 115 return 0;116}117#endif118 119/**120 * remap_io_sg - remap an IO mapping to userspace121 * @vma: user vma to map to122 * @addr: target user address to start at123 * @size: size of map area124 * @sgl: Start sg entry125 * @offset: offset from the start of the page126 * @iobase: Use stored dma address offset by this address or pfn if -1127 *128 * Note: this is only safe if the mm semaphore is held when called.129 */130int remap_io_sg(struct vm_area_struct *vma,131 unsigned long addr, unsigned long size,132 struct scatterlist *sgl, unsigned long offset,133 resource_size_t iobase)134{135 struct remap_pfn r = {136 .mm = vma->vm_mm,137 .prot = vma->vm_page_prot,138 .sgt = __sgt_iter(sgl, use_dma(iobase)),139 .iobase = iobase,140 };141 int err;142 143 /* We rely on prevalidation of the io-mapping to skip track_pfn(). */144 GEM_BUG_ON((vma->vm_flags & EXPECTED_FLAGS) != EXPECTED_FLAGS);145 146 while (offset >= sg_dma_len(r.sgt.sgp) >> PAGE_SHIFT) {147 offset -= sg_dma_len(r.sgt.sgp) >> PAGE_SHIFT;148 r.sgt = __sgt_iter(__sg_next(r.sgt.sgp), use_dma(iobase));149 if (!r.sgt.sgp)150 return -EINVAL;151 }152 r.sgt.curr = offset << PAGE_SHIFT;153 154 if (!use_dma(iobase))155 flush_cache_range(vma, addr, size);156 157 err = apply_to_page_range(r.mm, addr, size, remap_sg, &r);158 if (unlikely(err)) {159 zap_vma_ptes(vma, addr, r.pfn << PAGE_SHIFT);160 return err;161 }162 163 return 0;164}165