254 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2024 Intel Corporation4 */5 6#include <linux/scatterlist.h>7#include <linux/mmu_notifier.h>8#include <linux/dma-mapping.h>9#include <linux/memremap.h>10#include <linux/swap.h>11#include <linux/hmm.h>12#include <linux/mm.h>13#include "xe_hmm.h"14#include "xe_vm.h"15#include "xe_bo.h"16 17static u64 xe_npages_in_range(unsigned long start, unsigned long end)18{19 return (end - start) >> PAGE_SHIFT;20}21 22/*23 * xe_mark_range_accessed() - mark a range is accessed, so core mm24 * have such information for memory eviction or write back to25 * hard disk26 *27 * @range: the range to mark28 * @write: if write to this range, we mark pages in this range29 * as dirty30 */31static void xe_mark_range_accessed(struct hmm_range *range, bool write)32{33 struct page *page;34 u64 i, npages;35 36 npages = xe_npages_in_range(range->start, range->end);37 for (i = 0; i < npages; i++) {38 page = hmm_pfn_to_page(range->hmm_pfns[i]);39 if (write)40 set_page_dirty_lock(page);41 42 mark_page_accessed(page);43 }44}45 46/*47 * xe_build_sg() - build a scatter gather table for all the physical pages/pfn48 * in a hmm_range. dma-map pages if necessary. dma-address is save in sg table49 * and will be used to program GPU page table later.50 *51 * @xe: the xe device who will access the dma-address in sg table52 * @range: the hmm range that we build the sg table from. range->hmm_pfns[]53 * has the pfn numbers of pages that back up this hmm address range.54 * @st: pointer to the sg table.55 * @write: whether we write to this range. This decides dma map direction56 * for system pages. If write we map it bi-diretional; otherwise57 * DMA_TO_DEVICE58 *59 * All the contiguous pfns will be collapsed into one entry in60 * the scatter gather table. This is for the purpose of efficiently61 * programming GPU page table.62 *63 * The dma_address in the sg table will later be used by GPU to64 * access memory. So if the memory is system memory, we need to65 * do a dma-mapping so it can be accessed by GPU/DMA.66 *67 * FIXME: This function currently only support pages in system68 * memory. If the memory is GPU local memory (of the GPU who69 * is going to access memory), we need gpu dpa (device physical70 * address), and there is no need of dma-mapping. This is TBD.71 *72 * FIXME: dma-mapping for peer gpu device to access remote gpu's73 * memory. Add this when you support p2p74 *75 * This function allocates the storage of the sg table. It is76 * caller's responsibility to free it calling sg_free_table.77 *78 * Returns 0 if successful; -ENOMEM if fails to allocate memory79 */80static int xe_build_sg(struct xe_device *xe, struct hmm_range *range,81 struct sg_table *st, bool write)82{83 struct device *dev = xe->drm.dev;84 struct page **pages;85 u64 i, npages;86 int ret;87 88 npages = xe_npages_in_range(range->start, range->end);89 pages = kvmalloc_array(npages, sizeof(*pages), GFP_KERNEL);90 if (!pages)91 return -ENOMEM;92 93 for (i = 0; i < npages; i++) {94 pages[i] = hmm_pfn_to_page(range->hmm_pfns[i]);95 xe_assert(xe, !is_device_private_page(pages[i]));96 }97 98 ret = sg_alloc_table_from_pages_segment(st, pages, npages, 0, npages << PAGE_SHIFT,99 xe_sg_segment_size(dev), GFP_KERNEL);100 if (ret)101 goto free_pages;102 103 ret = dma_map_sgtable(dev, st, write ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE,104 DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_NO_KERNEL_MAPPING);105 if (ret) {106 sg_free_table(st);107 st = NULL;108 }109 110free_pages:111 kvfree(pages);112 return ret;113}114 115/*116 * xe_hmm_userptr_free_sg() - Free the scatter gather table of userptr117 *118 * @uvma: the userptr vma which hold the scatter gather table119 *120 * With function xe_userptr_populate_range, we allocate storage of121 * the userptr sg table. This is a helper function to free this122 * sg table, and dma unmap the address in the table.123 */124void xe_hmm_userptr_free_sg(struct xe_userptr_vma *uvma)125{126 struct xe_userptr *userptr = &uvma->userptr;127 struct xe_vma *vma = &uvma->vma;128 bool write = !xe_vma_read_only(vma);129 struct xe_vm *vm = xe_vma_vm(vma);130 struct xe_device *xe = vm->xe;131 struct device *dev = xe->drm.dev;132 133 xe_assert(xe, userptr->sg);134 dma_unmap_sgtable(dev, userptr->sg,135 write ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE, 0);136 137 sg_free_table(userptr->sg);138 userptr->sg = NULL;139}140 141/**142 * xe_hmm_userptr_populate_range() - Populate physical pages of a virtual143 * address range144 *145 * @uvma: userptr vma which has information of the range to populate.146 * @is_mm_mmap_locked: True if mmap_read_lock is already acquired by caller.147 *148 * This function populate the physical pages of a virtual149 * address range. The populated physical pages is saved in150 * userptr's sg table. It is similar to get_user_pages but call151 * hmm_range_fault.152 *153 * This function also read mmu notifier sequence # (154 * mmu_interval_read_begin), for the purpose of later155 * comparison (through mmu_interval_read_retry).156 *157 * This must be called with mmap read or write lock held.158 *159 * This function allocates the storage of the userptr sg table.160 * It is caller's responsibility to free it calling sg_free_table.161 *162 * returns: 0 for succuss; negative error no on failure163 */164int xe_hmm_userptr_populate_range(struct xe_userptr_vma *uvma,165 bool is_mm_mmap_locked)166{167 unsigned long timeout =168 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);169 unsigned long *pfns, flags = HMM_PFN_REQ_FAULT;170 struct xe_userptr *userptr;171 struct xe_vma *vma = &uvma->vma;172 u64 userptr_start = xe_vma_userptr(vma);173 u64 userptr_end = userptr_start + xe_vma_size(vma);174 struct xe_vm *vm = xe_vma_vm(vma);175 struct hmm_range hmm_range;176 bool write = !xe_vma_read_only(vma);177 unsigned long notifier_seq;178 u64 npages;179 int ret;180 181 userptr = &uvma->userptr;182 183 if (is_mm_mmap_locked)184 mmap_assert_locked(userptr->notifier.mm);185 186 if (vma->gpuva.flags & XE_VMA_DESTROYED)187 return 0;188 189 notifier_seq = mmu_interval_read_begin(&userptr->notifier);190 if (notifier_seq == userptr->notifier_seq)191 return 0;192 193 if (userptr->sg)194 xe_hmm_userptr_free_sg(uvma);195 196 npages = xe_npages_in_range(userptr_start, userptr_end);197 pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL);198 if (unlikely(!pfns))199 return -ENOMEM;200 201 if (write)202 flags |= HMM_PFN_REQ_WRITE;203 204 if (!mmget_not_zero(userptr->notifier.mm)) {205 ret = -EFAULT;206 goto free_pfns;207 }208 209 hmm_range.default_flags = flags;210 hmm_range.hmm_pfns = pfns;211 hmm_range.notifier = &userptr->notifier;212 hmm_range.start = userptr_start;213 hmm_range.end = userptr_end;214 hmm_range.dev_private_owner = vm->xe;215 216 while (true) {217 hmm_range.notifier_seq = mmu_interval_read_begin(&userptr->notifier);218 219 if (!is_mm_mmap_locked)220 mmap_read_lock(userptr->notifier.mm);221 222 ret = hmm_range_fault(&hmm_range);223 224 if (!is_mm_mmap_locked)225 mmap_read_unlock(userptr->notifier.mm);226 227 if (ret == -EBUSY) {228 if (time_after(jiffies, timeout))229 break;230 231 continue;232 }233 break;234 }235 236 mmput(userptr->notifier.mm);237 238 if (ret)239 goto free_pfns;240 241 ret = xe_build_sg(vm->xe, &hmm_range, &userptr->sgt, write);242 if (ret)243 goto free_pfns;244 245 xe_mark_range_accessed(&hmm_range, write);246 userptr->sg = &userptr->sgt;247 userptr->notifier_seq = hmm_range.notifier_seq;248 249free_pfns:250 kvfree(pfns);251 return ret;252}253 254