469 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR MIT2/**************************************************************************3 *4 * Copyright 2019-2023 VMware, Inc., Palo Alto, CA., USA5 *6 * Permission is hereby granted, free of charge, to any person obtaining a7 * copy of this software and associated documentation files (the8 * "Software"), to deal in the Software without restriction, including9 * without limitation the rights to use, copy, modify, merge, publish,10 * distribute, sub license, and/or sell copies of the Software, and to11 * permit persons to whom the Software is furnished to do so, subject to12 * the following conditions:13 *14 * The above copyright notice and this permission notice (including the15 * next paragraph) shall be included in all copies or substantial portions16 * of the Software.17 *18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE24 * USE OR OTHER DEALINGS IN THE SOFTWARE.25 *26 **************************************************************************/27#include "vmwgfx_bo.h"28#include "vmwgfx_drv.h"29 30/*31 * Different methods for tracking dirty:32 * VMW_BO_DIRTY_PAGETABLE - Scan the pagetable for hardware dirty bits33 * VMW_BO_DIRTY_MKWRITE - Write-protect page table entries and record write-34 * accesses in the VM mkwrite() callback35 */36enum vmw_bo_dirty_method {37 VMW_BO_DIRTY_PAGETABLE,38 VMW_BO_DIRTY_MKWRITE,39};40 41/*42 * No dirtied pages at scan trigger a transition to the _MKWRITE method,43 * similarly a certain percentage of dirty pages trigger a transition to44 * the _PAGETABLE method. How many triggers should we wait for before45 * changing method?46 */47#define VMW_DIRTY_NUM_CHANGE_TRIGGERS 248 49/* Percentage to trigger a transition to the _PAGETABLE method */50#define VMW_DIRTY_PERCENTAGE 1051 52/**53 * struct vmw_bo_dirty - Dirty information for buffer objects54 * @start: First currently dirty bit55 * @end: Last currently dirty bit + 156 * @method: The currently used dirty method57 * @change_count: Number of consecutive method change triggers58 * @ref_count: Reference count for this structure59 * @bitmap_size: The size of the bitmap in bits. Typically equal to the60 * nuber of pages in the bo.61 * @bitmap: A bitmap where each bit represents a page. A set bit means a62 * dirty page.63 */64struct vmw_bo_dirty {65 unsigned long start;66 unsigned long end;67 enum vmw_bo_dirty_method method;68 unsigned int change_count;69 unsigned int ref_count;70 unsigned long bitmap_size;71 unsigned long bitmap[];72};73 74/**75 * vmw_bo_dirty_scan_pagetable - Perform a pagetable scan for dirty bits76 * @vbo: The buffer object to scan77 *78 * Scans the pagetable for dirty bits. Clear those bits and modify the79 * dirty structure with the results. This function may change the80 * dirty-tracking method.81 */82static void vmw_bo_dirty_scan_pagetable(struct vmw_bo *vbo)83{84 struct vmw_bo_dirty *dirty = vbo->dirty;85 pgoff_t offset = drm_vma_node_start(&vbo->tbo.base.vma_node);86 struct address_space *mapping = vbo->tbo.bdev->dev_mapping;87 pgoff_t num_marked;88 89 num_marked = clean_record_shared_mapping_range90 (mapping,91 offset, dirty->bitmap_size,92 offset, &dirty->bitmap[0],93 &dirty->start, &dirty->end);94 if (num_marked == 0)95 dirty->change_count++;96 else97 dirty->change_count = 0;98 99 if (dirty->change_count > VMW_DIRTY_NUM_CHANGE_TRIGGERS) {100 dirty->change_count = 0;101 dirty->method = VMW_BO_DIRTY_MKWRITE;102 wp_shared_mapping_range(mapping,103 offset, dirty->bitmap_size);104 clean_record_shared_mapping_range(mapping,105 offset, dirty->bitmap_size,106 offset, &dirty->bitmap[0],107 &dirty->start, &dirty->end);108 }109}110 111/**112 * vmw_bo_dirty_scan_mkwrite - Reset the mkwrite dirty-tracking method113 * @vbo: The buffer object to scan114 *115 * Write-protect pages written to so that consecutive write accesses will116 * trigger a call to mkwrite.117 *118 * This function may change the dirty-tracking method.119 */120static void vmw_bo_dirty_scan_mkwrite(struct vmw_bo *vbo)121{122 struct vmw_bo_dirty *dirty = vbo->dirty;123 unsigned long offset = drm_vma_node_start(&vbo->tbo.base.vma_node);124 struct address_space *mapping = vbo->tbo.bdev->dev_mapping;125 pgoff_t num_marked;126 127 if (dirty->end <= dirty->start)128 return;129 130 num_marked = wp_shared_mapping_range(vbo->tbo.bdev->dev_mapping,131 dirty->start + offset,132 dirty->end - dirty->start);133 134 if (100UL * num_marked / dirty->bitmap_size >135 VMW_DIRTY_PERCENTAGE)136 dirty->change_count++;137 else138 dirty->change_count = 0;139 140 if (dirty->change_count > VMW_DIRTY_NUM_CHANGE_TRIGGERS) {141 pgoff_t start = 0;142 pgoff_t end = dirty->bitmap_size;143 144 dirty->method = VMW_BO_DIRTY_PAGETABLE;145 clean_record_shared_mapping_range(mapping, offset, end, offset,146 &dirty->bitmap[0],147 &start, &end);148 bitmap_clear(&dirty->bitmap[0], 0, dirty->bitmap_size);149 if (dirty->start < dirty->end)150 bitmap_set(&dirty->bitmap[0], dirty->start,151 dirty->end - dirty->start);152 dirty->change_count = 0;153 }154}155 156/**157 * vmw_bo_dirty_scan - Scan for dirty pages and add them to the dirty158 * tracking structure159 * @vbo: The buffer object to scan160 *161 * This function may change the dirty tracking method.162 */163void vmw_bo_dirty_scan(struct vmw_bo *vbo)164{165 struct vmw_bo_dirty *dirty = vbo->dirty;166 167 if (dirty->method == VMW_BO_DIRTY_PAGETABLE)168 vmw_bo_dirty_scan_pagetable(vbo);169 else170 vmw_bo_dirty_scan_mkwrite(vbo);171}172 173/**174 * vmw_bo_dirty_pre_unmap - write-protect and pick up dirty pages before175 * an unmap_mapping_range operation.176 * @vbo: The buffer object,177 * @start: First page of the range within the buffer object.178 * @end: Last page of the range within the buffer object + 1.179 *180 * If we're using the _PAGETABLE scan method, we may leak dirty pages181 * when calling unmap_mapping_range(). This function makes sure we pick182 * up all dirty pages.183 */184static void vmw_bo_dirty_pre_unmap(struct vmw_bo *vbo,185 pgoff_t start, pgoff_t end)186{187 struct vmw_bo_dirty *dirty = vbo->dirty;188 unsigned long offset = drm_vma_node_start(&vbo->tbo.base.vma_node);189 struct address_space *mapping = vbo->tbo.bdev->dev_mapping;190 191 if (dirty->method != VMW_BO_DIRTY_PAGETABLE || start >= end)192 return;193 194 wp_shared_mapping_range(mapping, start + offset, end - start);195 clean_record_shared_mapping_range(mapping, start + offset,196 end - start, offset,197 &dirty->bitmap[0], &dirty->start,198 &dirty->end);199}200 201/**202 * vmw_bo_dirty_unmap - Clear all ptes pointing to a range within a bo203 * @vbo: The buffer object,204 * @start: First page of the range within the buffer object.205 * @end: Last page of the range within the buffer object + 1.206 *207 * This is similar to ttm_bo_unmap_virtual() except it takes a subrange.208 */209void vmw_bo_dirty_unmap(struct vmw_bo *vbo,210 pgoff_t start, pgoff_t end)211{212 unsigned long offset = drm_vma_node_start(&vbo->tbo.base.vma_node);213 struct address_space *mapping = vbo->tbo.bdev->dev_mapping;214 215 vmw_bo_dirty_pre_unmap(vbo, start, end);216 unmap_shared_mapping_range(mapping, (offset + start) << PAGE_SHIFT,217 (loff_t) (end - start) << PAGE_SHIFT);218}219 220/**221 * vmw_bo_dirty_add - Add a dirty-tracking user to a buffer object222 * @vbo: The buffer object223 *224 * This function registers a dirty-tracking user to a buffer object.225 * A user can be for example a resource or a vma in a special user-space226 * mapping.227 *228 * Return: Zero on success, -ENOMEM on memory allocation failure.229 */230int vmw_bo_dirty_add(struct vmw_bo *vbo)231{232 struct vmw_bo_dirty *dirty = vbo->dirty;233 pgoff_t num_pages = PFN_UP(vbo->tbo.resource->size);234 size_t size;235 int ret;236 237 if (dirty) {238 dirty->ref_count++;239 return 0;240 }241 242 size = sizeof(*dirty) + BITS_TO_LONGS(num_pages) * sizeof(long);243 dirty = kvzalloc(size, GFP_KERNEL);244 if (!dirty) {245 ret = -ENOMEM;246 goto out_no_dirty;247 }248 249 dirty->bitmap_size = num_pages;250 dirty->start = dirty->bitmap_size;251 dirty->end = 0;252 dirty->ref_count = 1;253 if (num_pages < PAGE_SIZE / sizeof(pte_t)) {254 dirty->method = VMW_BO_DIRTY_PAGETABLE;255 } else {256 struct address_space *mapping = vbo->tbo.bdev->dev_mapping;257 pgoff_t offset = drm_vma_node_start(&vbo->tbo.base.vma_node);258 259 dirty->method = VMW_BO_DIRTY_MKWRITE;260 261 /* Write-protect and then pick up already dirty bits */262 wp_shared_mapping_range(mapping, offset, num_pages);263 clean_record_shared_mapping_range(mapping, offset, num_pages,264 offset,265 &dirty->bitmap[0],266 &dirty->start, &dirty->end);267 }268 269 vbo->dirty = dirty;270 271 return 0;272 273out_no_dirty:274 return ret;275}276 277/**278 * vmw_bo_dirty_release - Release a dirty-tracking user from a buffer object279 * @vbo: The buffer object280 *281 * This function releases a dirty-tracking user from a buffer object.282 * If the reference count reaches zero, then the dirty-tracking object is283 * freed and the pointer to it cleared.284 *285 * Return: Zero on success, -ENOMEM on memory allocation failure.286 */287void vmw_bo_dirty_release(struct vmw_bo *vbo)288{289 struct vmw_bo_dirty *dirty = vbo->dirty;290 291 if (dirty && --dirty->ref_count == 0) {292 kvfree(dirty);293 vbo->dirty = NULL;294 }295}296 297/**298 * vmw_bo_dirty_transfer_to_res - Pick up a resource's dirty region from299 * its backing mob.300 * @res: The resource301 *302 * This function will pick up all dirty ranges affecting the resource from303 * it's backup mob, and call vmw_resource_dirty_update() once for each304 * range. The transferred ranges will be cleared from the backing mob's305 * dirty tracking.306 */307void vmw_bo_dirty_transfer_to_res(struct vmw_resource *res)308{309 struct vmw_bo *vbo = res->guest_memory_bo;310 struct vmw_bo_dirty *dirty = vbo->dirty;311 pgoff_t start, cur, end;312 unsigned long res_start = res->guest_memory_offset;313 unsigned long res_end = res->guest_memory_offset + res->guest_memory_size;314 315 WARN_ON_ONCE(res_start & ~PAGE_MASK);316 res_start >>= PAGE_SHIFT;317 res_end = DIV_ROUND_UP(res_end, PAGE_SIZE);318 319 if (res_start >= dirty->end || res_end <= dirty->start)320 return;321 322 cur = max(res_start, dirty->start);323 res_end = max(res_end, dirty->end);324 while (cur < res_end) {325 unsigned long num;326 327 start = find_next_bit(&dirty->bitmap[0], res_end, cur);328 if (start >= res_end)329 break;330 331 end = find_next_zero_bit(&dirty->bitmap[0], res_end, start + 1);332 cur = end + 1;333 num = end - start;334 bitmap_clear(&dirty->bitmap[0], start, num);335 vmw_resource_dirty_update(res, start, end);336 }337 338 if (res_start <= dirty->start && res_end > dirty->start)339 dirty->start = res_end;340 if (res_start < dirty->end && res_end >= dirty->end)341 dirty->end = res_start;342}343 344/**345 * vmw_bo_dirty_clear_res - Clear a resource's dirty region from346 * its backing mob.347 * @res: The resource348 *349 * This function will clear all dirty ranges affecting the resource from350 * it's backup mob's dirty tracking.351 */352void vmw_bo_dirty_clear_res(struct vmw_resource *res)353{354 unsigned long res_start = res->guest_memory_offset;355 unsigned long res_end = res->guest_memory_offset + res->guest_memory_size;356 struct vmw_bo *vbo = res->guest_memory_bo;357 struct vmw_bo_dirty *dirty = vbo->dirty;358 359 res_start >>= PAGE_SHIFT;360 res_end = DIV_ROUND_UP(res_end, PAGE_SIZE);361 362 if (res_start >= dirty->end || res_end <= dirty->start)363 return;364 365 res_start = max(res_start, dirty->start);366 res_end = min(res_end, dirty->end);367 bitmap_clear(&dirty->bitmap[0], res_start, res_end - res_start);368 369 if (res_start <= dirty->start && res_end > dirty->start)370 dirty->start = res_end;371 if (res_start < dirty->end && res_end >= dirty->end)372 dirty->end = res_start;373}374 375vm_fault_t vmw_bo_vm_mkwrite(struct vm_fault *vmf)376{377 struct vm_area_struct *vma = vmf->vma;378 struct ttm_buffer_object *bo = (struct ttm_buffer_object *)379 vma->vm_private_data;380 vm_fault_t ret;381 unsigned long page_offset;382 unsigned int save_flags;383 struct vmw_bo *vbo = to_vmw_bo(&bo->base);384 385 /*386 * mkwrite() doesn't handle the VM_FAULT_RETRY return value correctly.387 * So make sure the TTM helpers are aware.388 */389 save_flags = vmf->flags;390 vmf->flags &= ~FAULT_FLAG_ALLOW_RETRY;391 ret = ttm_bo_vm_reserve(bo, vmf);392 vmf->flags = save_flags;393 if (ret)394 return ret;395 396 page_offset = vmf->pgoff - drm_vma_node_start(&bo->base.vma_node);397 if (unlikely(page_offset >= PFN_UP(bo->resource->size))) {398 ret = VM_FAULT_SIGBUS;399 goto out_unlock;400 }401 402 if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE &&403 !test_bit(page_offset, &vbo->dirty->bitmap[0])) {404 struct vmw_bo_dirty *dirty = vbo->dirty;405 406 __set_bit(page_offset, &dirty->bitmap[0]);407 dirty->start = min(dirty->start, page_offset);408 dirty->end = max(dirty->end, page_offset + 1);409 }410 411out_unlock:412 dma_resv_unlock(bo->base.resv);413 return ret;414}415 416vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf)417{418 struct vm_area_struct *vma = vmf->vma;419 struct ttm_buffer_object *bo = (struct ttm_buffer_object *)420 vma->vm_private_data;421 struct vmw_bo *vbo = to_vmw_bo(&bo->base);422 pgoff_t num_prefault;423 pgprot_t prot;424 vm_fault_t ret;425 426 ret = ttm_bo_vm_reserve(bo, vmf);427 if (ret)428 return ret;429 430 num_prefault = (vma->vm_flags & VM_RAND_READ) ? 1 :431 TTM_BO_VM_NUM_PREFAULT;432 433 if (vbo->dirty) {434 pgoff_t allowed_prefault;435 unsigned long page_offset;436 437 page_offset = vmf->pgoff -438 drm_vma_node_start(&bo->base.vma_node);439 if (page_offset >= PFN_UP(bo->resource->size) ||440 vmw_resources_clean(vbo, page_offset,441 page_offset + PAGE_SIZE,442 &allowed_prefault)) {443 ret = VM_FAULT_SIGBUS;444 goto out_unlock;445 }446 447 num_prefault = min(num_prefault, allowed_prefault);448 }449 450 /*451 * If we don't track dirty using the MKWRITE method, make sure452 * sure the page protection is write-enabled so we don't get453 * a lot of unnecessary write faults.454 */455 if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE)456 prot = vm_get_page_prot(vma->vm_flags & ~VM_SHARED);457 else458 prot = vm_get_page_prot(vma->vm_flags);459 460 ret = ttm_bo_vm_fault_reserved(vmf, prot, num_prefault);461 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))462 return ret;463 464out_unlock:465 dma_resv_unlock(bo->base.resv);466 467 return ret;468}469