626 lines · c
1// SPDX-License-Identifier: GPL-2.0-only OR MIT2/* Copyright (c) 2023 Imagination Technologies Ltd. */3 4#include "pvr_free_list.h"5#include "pvr_gem.h"6#include "pvr_hwrt.h"7#include "pvr_rogue_fwif.h"8#include "pvr_vm.h"9 10#include <drm/drm_gem.h>11#include <linux/slab.h>12#include <linux/xarray.h>13#include <uapi/drm/pvr_drm.h>14 15#define FREE_LIST_ENTRY_SIZE sizeof(u32)16 17#define FREE_LIST_ALIGNMENT \18 ((ROGUE_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE / FREE_LIST_ENTRY_SIZE) - 1)19 20#define FREE_LIST_MIN_PAGES 5021#define FREE_LIST_MIN_PAGES_BRN66011 4022#define FREE_LIST_MIN_PAGES_ROGUEXE 2523 24/**25 * pvr_get_free_list_min_pages() - Get minimum free list size for this device26 * @pvr_dev: Device pointer.27 *28 * Returns:29 * * Minimum free list size, in PM physical pages.30 */31u3232pvr_get_free_list_min_pages(struct pvr_device *pvr_dev)33{34 u32 value;35 36 if (PVR_HAS_FEATURE(pvr_dev, roguexe)) {37 if (PVR_HAS_QUIRK(pvr_dev, 66011))38 value = FREE_LIST_MIN_PAGES_BRN66011;39 else40 value = FREE_LIST_MIN_PAGES_ROGUEXE;41 } else {42 value = FREE_LIST_MIN_PAGES;43 }44 45 return value;46}47 48static int49free_list_create_kernel_structure(struct pvr_file *pvr_file,50 struct drm_pvr_ioctl_create_free_list_args *args,51 struct pvr_free_list *free_list)52{53 struct pvr_gem_object *free_list_obj;54 struct pvr_vm_context *vm_ctx;55 u64 free_list_size;56 int err;57 58 if (args->grow_threshold > 100 ||59 args->initial_num_pages > args->max_num_pages ||60 args->grow_num_pages > args->max_num_pages ||61 args->max_num_pages == 0 ||62 (args->initial_num_pages < args->max_num_pages && !args->grow_num_pages) ||63 (args->initial_num_pages == args->max_num_pages && args->grow_num_pages))64 return -EINVAL;65 66 if ((args->initial_num_pages & FREE_LIST_ALIGNMENT) ||67 (args->max_num_pages & FREE_LIST_ALIGNMENT) ||68 (args->grow_num_pages & FREE_LIST_ALIGNMENT))69 return -EINVAL;70 71 vm_ctx = pvr_vm_context_lookup(pvr_file, args->vm_context_handle);72 if (!vm_ctx)73 return -EINVAL;74 75 free_list_obj = pvr_vm_find_gem_object(vm_ctx, args->free_list_gpu_addr,76 NULL, &free_list_size);77 if (!free_list_obj) {78 err = -EINVAL;79 goto err_put_vm_context;80 }81 82 if ((free_list_obj->flags & DRM_PVR_BO_ALLOW_CPU_USERSPACE_ACCESS) ||83 !(free_list_obj->flags & DRM_PVR_BO_PM_FW_PROTECT) ||84 free_list_size < (args->max_num_pages * FREE_LIST_ENTRY_SIZE)) {85 err = -EINVAL;86 goto err_put_free_list_obj;87 }88 89 free_list->pvr_dev = pvr_file->pvr_dev;90 free_list->current_pages = 0;91 free_list->max_pages = args->max_num_pages;92 free_list->grow_pages = args->grow_num_pages;93 free_list->grow_threshold = args->grow_threshold;94 free_list->obj = free_list_obj;95 free_list->free_list_gpu_addr = args->free_list_gpu_addr;96 free_list->initial_num_pages = args->initial_num_pages;97 98 pvr_vm_context_put(vm_ctx);99 100 return 0;101 102err_put_free_list_obj:103 pvr_gem_object_put(free_list_obj);104 105err_put_vm_context:106 pvr_vm_context_put(vm_ctx);107 108 return err;109}110 111static void112free_list_destroy_kernel_structure(struct pvr_free_list *free_list)113{114 WARN_ON(!list_empty(&free_list->hwrt_list));115 116 pvr_gem_object_put(free_list->obj);117}118 119/**120 * calculate_free_list_ready_pages_locked() - Function to work out the number of free121 * list pages to reserve for growing within122 * the FW without having to wait for the123 * host to progress a grow request124 * @free_list: Pointer to free list.125 * @pages: Total pages currently in free list.126 *127 * If the threshold or grow size means less than the alignment size (4 pages on128 * Rogue), then the feature is not used.129 *130 * Caller must hold &free_list->lock.131 *132 * Return: number of pages to reserve.133 */134static u32135calculate_free_list_ready_pages_locked(struct pvr_free_list *free_list, u32 pages)136{137 u32 ready_pages;138 139 lockdep_assert_held(&free_list->lock);140 141 ready_pages = ((pages * free_list->grow_threshold) / 100);142 143 /* The number of pages must be less than the grow size. */144 ready_pages = min(ready_pages, free_list->grow_pages);145 146 /*147 * The number of pages must be a multiple of the free list align size.148 */149 ready_pages &= ~FREE_LIST_ALIGNMENT;150 151 return ready_pages;152}153 154static u32155calculate_free_list_ready_pages(struct pvr_free_list *free_list, u32 pages)156{157 u32 ret;158 159 mutex_lock(&free_list->lock);160 161 ret = calculate_free_list_ready_pages_locked(free_list, pages);162 163 mutex_unlock(&free_list->lock);164 165 return ret;166}167 168static void169free_list_fw_init(void *cpu_ptr, void *priv)170{171 struct rogue_fwif_freelist *fw_data = cpu_ptr;172 struct pvr_free_list *free_list = priv;173 u32 ready_pages;174 175 /* Fill out FW structure */176 ready_pages = calculate_free_list_ready_pages(free_list,177 free_list->initial_num_pages);178 179 fw_data->max_pages = free_list->max_pages;180 fw_data->current_pages = free_list->initial_num_pages - ready_pages;181 fw_data->grow_pages = free_list->grow_pages;182 fw_data->ready_pages = ready_pages;183 fw_data->freelist_id = free_list->fw_id;184 fw_data->grow_pending = false;185 fw_data->current_stack_top = fw_data->current_pages - 1;186 fw_data->freelist_dev_addr = free_list->free_list_gpu_addr;187 fw_data->current_dev_addr = (fw_data->freelist_dev_addr +188 ((fw_data->max_pages - fw_data->current_pages) *189 FREE_LIST_ENTRY_SIZE)) &190 ~((u64)ROGUE_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE - 1);191}192 193static int194free_list_create_fw_structure(struct pvr_file *pvr_file,195 struct drm_pvr_ioctl_create_free_list_args *args,196 struct pvr_free_list *free_list)197{198 struct pvr_device *pvr_dev = pvr_file->pvr_dev;199 200 /*201 * Create and map the FW structure so we can initialise it. This is not202 * accessed on the CPU side post-initialisation so the mapping lifetime203 * is only for this function.204 */205 free_list->fw_data = pvr_fw_object_create_and_map(pvr_dev, sizeof(*free_list->fw_data),206 PVR_BO_FW_FLAGS_DEVICE_UNCACHED,207 free_list_fw_init, free_list,208 &free_list->fw_obj);209 if (IS_ERR(free_list->fw_data))210 return PTR_ERR(free_list->fw_data);211 212 return 0;213}214 215static void216free_list_destroy_fw_structure(struct pvr_free_list *free_list)217{218 pvr_fw_object_unmap_and_destroy(free_list->fw_obj);219}220 221static int222pvr_free_list_insert_pages_locked(struct pvr_free_list *free_list,223 struct sg_table *sgt, u32 offset, u32 num_pages)224{225 struct sg_dma_page_iter dma_iter;226 u32 *page_list;227 228 lockdep_assert_held(&free_list->lock);229 230 page_list = pvr_gem_object_vmap(free_list->obj);231 if (IS_ERR(page_list))232 return PTR_ERR(page_list);233 234 offset /= FREE_LIST_ENTRY_SIZE;235 /* clang-format off */236 for_each_sgtable_dma_page(sgt, &dma_iter, 0) {237 dma_addr_t dma_addr = sg_page_iter_dma_address(&dma_iter);238 u64 dma_pfn = dma_addr >>239 ROGUE_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT;240 u32 dma_addr_offset;241 242 BUILD_BUG_ON(ROGUE_BIF_PM_PHYSICAL_PAGE_SIZE > PAGE_SIZE);243 244 for (dma_addr_offset = 0; dma_addr_offset < PAGE_SIZE;245 dma_addr_offset += ROGUE_BIF_PM_PHYSICAL_PAGE_SIZE) {246 WARN_ON_ONCE(dma_pfn >> 32);247 248 page_list[offset++] = (u32)dma_pfn;249 dma_pfn++;250 251 num_pages--;252 if (!num_pages)253 break;254 }255 256 if (!num_pages)257 break;258 }259 /* clang-format on */260 261 /* Make sure our free_list update is flushed. */262 wmb();263 264 pvr_gem_object_vunmap(free_list->obj);265 266 return 0;267}268 269static int270pvr_free_list_insert_node_locked(struct pvr_free_list_node *free_list_node)271{272 struct pvr_free_list *free_list = free_list_node->free_list;273 struct sg_table *sgt;274 u32 start_page;275 u32 offset;276 int err;277 278 lockdep_assert_held(&free_list->lock);279 280 start_page = free_list->max_pages - free_list->current_pages -281 free_list_node->num_pages;282 offset = (start_page * FREE_LIST_ENTRY_SIZE) &283 ~((u64)ROGUE_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE - 1);284 285 sgt = drm_gem_shmem_get_pages_sgt(&free_list_node->mem_obj->base);286 if (WARN_ON(IS_ERR(sgt)))287 return PTR_ERR(sgt);288 289 err = pvr_free_list_insert_pages_locked(free_list, sgt,290 offset, free_list_node->num_pages);291 if (!err)292 free_list->current_pages += free_list_node->num_pages;293 294 return err;295}296 297static int298pvr_free_list_grow(struct pvr_free_list *free_list, u32 num_pages)299{300 struct pvr_device *pvr_dev = free_list->pvr_dev;301 struct pvr_free_list_node *free_list_node;302 int err;303 304 mutex_lock(&free_list->lock);305 306 if (num_pages & FREE_LIST_ALIGNMENT) {307 err = -EINVAL;308 goto err_unlock;309 }310 311 free_list_node = kzalloc(sizeof(*free_list_node), GFP_KERNEL);312 if (!free_list_node) {313 err = -ENOMEM;314 goto err_unlock;315 }316 317 free_list_node->num_pages = num_pages;318 free_list_node->free_list = free_list;319 320 free_list_node->mem_obj = pvr_gem_object_create(pvr_dev,321 num_pages <<322 ROGUE_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT,323 PVR_BO_FW_FLAGS_DEVICE_CACHED);324 if (IS_ERR(free_list_node->mem_obj)) {325 err = PTR_ERR(free_list_node->mem_obj);326 goto err_free;327 }328 329 err = pvr_free_list_insert_node_locked(free_list_node);330 if (err)331 goto err_destroy_gem_object;332 333 list_add_tail(&free_list_node->node, &free_list->mem_block_list);334 335 /*336 * Reserve a number ready pages to allow the FW to process OOM quickly337 * and asynchronously request a grow.338 */339 free_list->ready_pages =340 calculate_free_list_ready_pages_locked(free_list,341 free_list->current_pages);342 free_list->current_pages -= free_list->ready_pages;343 344 mutex_unlock(&free_list->lock);345 346 return 0;347 348err_destroy_gem_object:349 pvr_gem_object_put(free_list_node->mem_obj);350 351err_free:352 kfree(free_list_node);353 354err_unlock:355 mutex_unlock(&free_list->lock);356 357 return err;358}359 360void pvr_free_list_process_grow_req(struct pvr_device *pvr_dev,361 struct rogue_fwif_fwccb_cmd_freelist_gs_data *req)362{363 struct pvr_free_list *free_list = pvr_free_list_lookup_id(pvr_dev, req->freelist_id);364 struct rogue_fwif_kccb_cmd resp_cmd = {365 .cmd_type = ROGUE_FWIF_KCCB_CMD_FREELIST_GROW_UPDATE,366 };367 struct rogue_fwif_freelist_gs_data *resp = &resp_cmd.cmd_data.free_list_gs_data;368 u32 grow_pages = 0;369 370 /* If we don't have a freelist registered for this ID, we can't do much. */371 if (WARN_ON(!free_list))372 return;373 374 /* Since the FW made the request, it has already consumed the ready pages,375 * update the host struct.376 */377 free_list->current_pages += free_list->ready_pages;378 free_list->ready_pages = 0;379 380 /* If the grow succeeds, update the grow_pages argument. */381 if (!pvr_free_list_grow(free_list, free_list->grow_pages))382 grow_pages = free_list->grow_pages;383 384 /* Now prepare the response and send it back to the FW. */385 pvr_fw_object_get_fw_addr(free_list->fw_obj, &resp->freelist_fw_addr);386 resp->delta_pages = grow_pages;387 resp->new_pages = free_list->current_pages + free_list->ready_pages;388 resp->ready_pages = free_list->ready_pages;389 pvr_free_list_put(free_list);390 391 WARN_ON(pvr_kccb_send_cmd(pvr_dev, &resp_cmd, NULL));392}393 394static void395pvr_free_list_free_node(struct pvr_free_list_node *free_list_node)396{397 pvr_gem_object_put(free_list_node->mem_obj);398 399 kfree(free_list_node);400}401 402/**403 * pvr_free_list_create() - Create a new free list and return an object pointer404 * @pvr_file: Pointer to pvr_file structure.405 * @args: Creation arguments from userspace.406 *407 * Return:408 * * Pointer to new free_list, or409 * * ERR_PTR(-%ENOMEM) on out of memory.410 */411struct pvr_free_list *412pvr_free_list_create(struct pvr_file *pvr_file,413 struct drm_pvr_ioctl_create_free_list_args *args)414{415 struct pvr_free_list *free_list;416 int err;417 418 /* Create and fill out the kernel structure */419 free_list = kzalloc(sizeof(*free_list), GFP_KERNEL);420 421 if (!free_list)422 return ERR_PTR(-ENOMEM);423 424 kref_init(&free_list->ref_count);425 INIT_LIST_HEAD(&free_list->mem_block_list);426 INIT_LIST_HEAD(&free_list->hwrt_list);427 mutex_init(&free_list->lock);428 429 err = free_list_create_kernel_structure(pvr_file, args, free_list);430 if (err < 0)431 goto err_free;432 433 /* Allocate global object ID for firmware. */434 err = xa_alloc(&pvr_file->pvr_dev->free_list_ids,435 &free_list->fw_id,436 free_list,437 xa_limit_32b,438 GFP_KERNEL);439 if (err)440 goto err_destroy_kernel_structure;441 442 err = free_list_create_fw_structure(pvr_file, args, free_list);443 if (err < 0)444 goto err_free_fw_id;445 446 err = pvr_free_list_grow(free_list, args->initial_num_pages);447 if (err < 0)448 goto err_fw_struct_cleanup;449 450 return free_list;451 452err_fw_struct_cleanup:453 WARN_ON(pvr_fw_structure_cleanup(free_list->pvr_dev,454 ROGUE_FWIF_CLEANUP_FREELIST,455 free_list->fw_obj, 0));456 457err_free_fw_id:458 xa_erase(&free_list->pvr_dev->free_list_ids, free_list->fw_id);459 460err_destroy_kernel_structure:461 free_list_destroy_kernel_structure(free_list);462 463err_free:464 mutex_destroy(&free_list->lock);465 kfree(free_list);466 467 return ERR_PTR(err);468}469 470static void471pvr_free_list_release(struct kref *ref_count)472{473 struct pvr_free_list *free_list =474 container_of(ref_count, struct pvr_free_list, ref_count);475 struct list_head *pos, *n;476 int err;477 478 xa_erase(&free_list->pvr_dev->free_list_ids, free_list->fw_id);479 480 err = pvr_fw_structure_cleanup(free_list->pvr_dev,481 ROGUE_FWIF_CLEANUP_FREELIST,482 free_list->fw_obj, 0);483 if (err == -EBUSY) {484 /* Flush the FWCCB to process any HWR or freelist reconstruction485 * request that might keep the freelist busy, and try again.486 */487 pvr_fwccb_process(free_list->pvr_dev);488 err = pvr_fw_structure_cleanup(free_list->pvr_dev,489 ROGUE_FWIF_CLEANUP_FREELIST,490 free_list->fw_obj, 0);491 }492 493 WARN_ON(err);494 495 /* clang-format off */496 list_for_each_safe(pos, n, &free_list->mem_block_list) {497 struct pvr_free_list_node *free_list_node =498 container_of(pos, struct pvr_free_list_node, node);499 500 list_del(pos);501 pvr_free_list_free_node(free_list_node);502 }503 /* clang-format on */504 505 free_list_destroy_kernel_structure(free_list);506 free_list_destroy_fw_structure(free_list);507 mutex_destroy(&free_list->lock);508 kfree(free_list);509}510 511/**512 * pvr_destroy_free_lists_for_file: Destroy any free lists associated with the513 * given file.514 * @pvr_file: Pointer to pvr_file structure.515 *516 * Removes all free lists associated with @pvr_file from the device free_list517 * list and drops initial references. Free lists will then be destroyed once518 * all outstanding references are dropped.519 */520void pvr_destroy_free_lists_for_file(struct pvr_file *pvr_file)521{522 struct pvr_free_list *free_list;523 unsigned long handle;524 525 xa_for_each(&pvr_file->free_list_handles, handle, free_list) {526 (void)free_list;527 pvr_free_list_put(xa_erase(&pvr_file->free_list_handles, handle));528 }529}530 531/**532 * pvr_free_list_put() - Release reference on free list533 * @free_list: Pointer to list to release reference on534 */535void536pvr_free_list_put(struct pvr_free_list *free_list)537{538 if (free_list)539 kref_put(&free_list->ref_count, pvr_free_list_release);540}541 542void pvr_free_list_add_hwrt(struct pvr_free_list *free_list, struct pvr_hwrt_data *hwrt_data)543{544 mutex_lock(&free_list->lock);545 546 list_add_tail(&hwrt_data->freelist_node, &free_list->hwrt_list);547 548 mutex_unlock(&free_list->lock);549}550 551void pvr_free_list_remove_hwrt(struct pvr_free_list *free_list, struct pvr_hwrt_data *hwrt_data)552{553 mutex_lock(&free_list->lock);554 555 list_del(&hwrt_data->freelist_node);556 557 mutex_unlock(&free_list->lock);558}559 560static void561pvr_free_list_reconstruct(struct pvr_device *pvr_dev, u32 freelist_id)562{563 struct pvr_free_list *free_list = pvr_free_list_lookup_id(pvr_dev, freelist_id);564 struct pvr_free_list_node *free_list_node;565 struct rogue_fwif_freelist *fw_data;566 struct pvr_hwrt_data *hwrt_data;567 568 if (!free_list)569 return;570 571 mutex_lock(&free_list->lock);572 573 /* Rebuild the free list based on the memory block list. */574 free_list->current_pages = 0;575 576 list_for_each_entry(free_list_node, &free_list->mem_block_list, node)577 WARN_ON(pvr_free_list_insert_node_locked(free_list_node));578 579 /*580 * Remove the ready pages, which are reserved to allow the FW to process OOM quickly and581 * asynchronously request a grow.582 */583 free_list->current_pages -= free_list->ready_pages;584 585 fw_data = free_list->fw_data;586 fw_data->current_stack_top = fw_data->current_pages - 1;587 fw_data->allocated_page_count = 0;588 fw_data->allocated_mmu_page_count = 0;589 590 /* Reset the state of any associated HWRTs. */591 list_for_each_entry(hwrt_data, &free_list->hwrt_list, freelist_node) {592 struct rogue_fwif_hwrtdata *hwrt_fw_data = pvr_fw_object_vmap(hwrt_data->fw_obj);593 594 if (!WARN_ON(IS_ERR(hwrt_fw_data))) {595 hwrt_fw_data->state = ROGUE_FWIF_RTDATA_STATE_HWR;596 hwrt_fw_data->hwrt_data_flags &= ~HWRTDATA_HAS_LAST_GEOM;597 }598 599 pvr_fw_object_vunmap(hwrt_data->fw_obj);600 }601 602 mutex_unlock(&free_list->lock);603 604 pvr_free_list_put(free_list);605}606 607void608pvr_free_list_process_reconstruct_req(struct pvr_device *pvr_dev,609 struct rogue_fwif_fwccb_cmd_freelists_reconstruction_data *req)610{611 struct rogue_fwif_kccb_cmd resp_cmd = {612 .cmd_type = ROGUE_FWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE,613 };614 struct rogue_fwif_freelists_reconstruction_data *resp =615 &resp_cmd.cmd_data.free_lists_reconstruction_data;616 617 for (u32 i = 0; i < req->freelist_count; i++)618 pvr_free_list_reconstruct(pvr_dev, req->freelist_ids[i]);619 620 resp->freelist_count = req->freelist_count;621 memcpy(resp->freelist_ids, req->freelist_ids,622 req->freelist_count * sizeof(resp->freelist_ids[0]));623 624 WARN_ON(pvr_kccb_send_cmd(pvr_dev, &resp_cmd, NULL));625}626