1162 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR MIT2/**************************************************************************3 *4 * Copyright (c) 2009-2024 Broadcom. All Rights Reserved. The term5 * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.6 *7 * Permission is hereby granted, free of charge, to any person obtaining a8 * copy of this software and associated documentation files (the9 * "Software"), to deal in the Software without restriction, including10 * without limitation the rights to use, copy, modify, merge, publish,11 * distribute, sub license, and/or sell copies of the Software, and to12 * permit persons to whom the Software is furnished to do so, subject to13 * the following conditions:14 *15 * The above copyright notice and this permission notice (including the16 * next paragraph) shall be included in all copies or substantial portions17 * of the Software.18 *19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,21 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL22 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,23 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR24 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE25 * USE OR OTHER DEALINGS IN THE SOFTWARE.26 *27 **************************************************************************/28 29#include <drm/ttm/ttm_placement.h>30 31#include "vmwgfx_binding.h"32#include "vmwgfx_bo.h"33#include "vmwgfx_drv.h"34#include "vmwgfx_resource_priv.h"35 36#define VMW_RES_EVICT_ERR_COUNT 1037 38/**39 * vmw_resource_mob_attach - Mark a resource as attached to its backing mob40 * @res: The resource41 */42void vmw_resource_mob_attach(struct vmw_resource *res)43{44 struct vmw_bo *gbo = res->guest_memory_bo;45 struct rb_node **new = &gbo->res_tree.rb_node, *parent = NULL;46 47 dma_resv_assert_held(gbo->tbo.base.resv);48 res->used_prio = (res->res_dirty) ? res->func->dirty_prio :49 res->func->prio;50 51 while (*new) {52 struct vmw_resource *this =53 container_of(*new, struct vmw_resource, mob_node);54 55 parent = *new;56 new = (res->guest_memory_offset < this->guest_memory_offset) ?57 &((*new)->rb_left) : &((*new)->rb_right);58 }59 60 rb_link_node(&res->mob_node, parent, new);61 rb_insert_color(&res->mob_node, &gbo->res_tree);62 vmw_bo_del_detached_resource(gbo, res);63 64 vmw_bo_prio_add(gbo, res->used_prio);65}66 67/**68 * vmw_resource_mob_detach - Mark a resource as detached from its backing mob69 * @res: The resource70 */71void vmw_resource_mob_detach(struct vmw_resource *res)72{73 struct vmw_bo *gbo = res->guest_memory_bo;74 75 dma_resv_assert_held(gbo->tbo.base.resv);76 if (vmw_resource_mob_attached(res)) {77 rb_erase(&res->mob_node, &gbo->res_tree);78 RB_CLEAR_NODE(&res->mob_node);79 vmw_bo_prio_del(gbo, res->used_prio);80 }81}82 83struct vmw_resource *vmw_resource_reference(struct vmw_resource *res)84{85 kref_get(&res->kref);86 return res;87}88 89struct vmw_resource *90vmw_resource_reference_unless_doomed(struct vmw_resource *res)91{92 return kref_get_unless_zero(&res->kref) ? res : NULL;93}94 95/**96 * vmw_resource_release_id - release a resource id to the id manager.97 *98 * @res: Pointer to the resource.99 *100 * Release the resource id to the resource id manager and set it to -1101 */102void vmw_resource_release_id(struct vmw_resource *res)103{104 struct vmw_private *dev_priv = res->dev_priv;105 struct idr *idr = &dev_priv->res_idr[res->func->res_type];106 107 spin_lock(&dev_priv->resource_lock);108 if (res->id != -1)109 idr_remove(idr, res->id);110 res->id = -1;111 spin_unlock(&dev_priv->resource_lock);112}113 114static void vmw_resource_release(struct kref *kref)115{116 struct vmw_resource *res =117 container_of(kref, struct vmw_resource, kref);118 struct vmw_private *dev_priv = res->dev_priv;119 int id;120 int ret;121 struct idr *idr = &dev_priv->res_idr[res->func->res_type];122 123 spin_lock(&dev_priv->resource_lock);124 list_del_init(&res->lru_head);125 spin_unlock(&dev_priv->resource_lock);126 if (res->guest_memory_bo) {127 struct ttm_buffer_object *bo = &res->guest_memory_bo->tbo;128 129 ret = ttm_bo_reserve(bo, false, false, NULL);130 BUG_ON(ret);131 if (vmw_resource_mob_attached(res) &&132 res->func->unbind != NULL) {133 struct ttm_validate_buffer val_buf;134 135 val_buf.bo = bo;136 val_buf.num_shared = 0;137 res->func->unbind(res, false, &val_buf);138 }139 res->guest_memory_size = false;140 vmw_resource_mob_detach(res);141 if (res->dirty)142 res->func->dirty_free(res);143 if (res->coherent)144 vmw_bo_dirty_release(res->guest_memory_bo);145 ttm_bo_unreserve(bo);146 vmw_user_bo_unref(&res->guest_memory_bo);147 }148 149 if (likely(res->hw_destroy != NULL)) {150 mutex_lock(&dev_priv->binding_mutex);151 vmw_binding_res_list_kill(&res->binding_head);152 mutex_unlock(&dev_priv->binding_mutex);153 res->hw_destroy(res);154 }155 156 id = res->id;157 if (res->res_free != NULL)158 res->res_free(res);159 else160 kfree(res);161 162 spin_lock(&dev_priv->resource_lock);163 if (id != -1)164 idr_remove(idr, id);165 spin_unlock(&dev_priv->resource_lock);166}167 168void vmw_resource_unreference(struct vmw_resource **p_res)169{170 struct vmw_resource *res = *p_res;171 172 *p_res = NULL;173 kref_put(&res->kref, vmw_resource_release);174}175 176 177/**178 * vmw_resource_alloc_id - release a resource id to the id manager.179 *180 * @res: Pointer to the resource.181 *182 * Allocate the lowest free resource from the resource manager, and set183 * @res->id to that id. Returns 0 on success and -ENOMEM on failure.184 */185int vmw_resource_alloc_id(struct vmw_resource *res)186{187 struct vmw_private *dev_priv = res->dev_priv;188 int ret;189 struct idr *idr = &dev_priv->res_idr[res->func->res_type];190 191 BUG_ON(res->id != -1);192 193 idr_preload(GFP_KERNEL);194 spin_lock(&dev_priv->resource_lock);195 196 ret = idr_alloc(idr, res, 1, 0, GFP_NOWAIT);197 if (ret >= 0)198 res->id = ret;199 200 spin_unlock(&dev_priv->resource_lock);201 idr_preload_end();202 return ret < 0 ? ret : 0;203}204 205/**206 * vmw_resource_init - initialize a struct vmw_resource207 *208 * @dev_priv: Pointer to a device private struct.209 * @res: The struct vmw_resource to initialize.210 * @delay_id: Boolean whether to defer device id allocation until211 * the first validation.212 * @res_free: Resource destructor.213 * @func: Resource function table.214 */215int vmw_resource_init(struct vmw_private *dev_priv, struct vmw_resource *res,216 bool delay_id,217 void (*res_free) (struct vmw_resource *res),218 const struct vmw_res_func *func)219{220 kref_init(&res->kref);221 res->hw_destroy = NULL;222 res->res_free = res_free;223 res->dev_priv = dev_priv;224 res->func = func;225 RB_CLEAR_NODE(&res->mob_node);226 INIT_LIST_HEAD(&res->lru_head);227 INIT_LIST_HEAD(&res->binding_head);228 res->id = -1;229 res->guest_memory_bo = NULL;230 res->guest_memory_offset = 0;231 res->guest_memory_dirty = false;232 res->res_dirty = false;233 res->coherent = false;234 res->used_prio = 3;235 res->dirty = NULL;236 if (delay_id)237 return 0;238 else239 return vmw_resource_alloc_id(res);240}241 242 243/**244 * vmw_user_resource_lookup_handle - lookup a struct resource from a245 * TTM user-space handle and perform basic type checks246 *247 * @dev_priv: Pointer to a device private struct248 * @tfile: Pointer to a struct ttm_object_file identifying the caller249 * @handle: The TTM user-space handle250 * @converter: Pointer to an object describing the resource type251 * @p_res: On successful return the location pointed to will contain252 * a pointer to a refcounted struct vmw_resource.253 *254 * If the handle can't be found or is associated with an incorrect resource255 * type, -EINVAL will be returned.256 */257int vmw_user_resource_lookup_handle(struct vmw_private *dev_priv,258 struct ttm_object_file *tfile,259 uint32_t handle,260 const struct vmw_user_resource_conv261 *converter,262 struct vmw_resource **p_res)263{264 struct ttm_base_object *base;265 struct vmw_resource *res;266 int ret = -EINVAL;267 268 base = ttm_base_object_lookup(tfile, handle);269 if (unlikely(!base))270 return -EINVAL;271 272 if (unlikely(ttm_base_object_type(base) != converter->object_type))273 goto out_bad_resource;274 275 res = converter->base_obj_to_res(base);276 kref_get(&res->kref);277 278 *p_res = res;279 ret = 0;280 281out_bad_resource:282 ttm_base_object_unref(&base);283 284 return ret;285}286 287/*288 * Helper function that looks either a surface or bo.289 *290 * The pointer this pointed at by out_surf and out_buf needs to be null.291 */292int vmw_user_object_lookup(struct vmw_private *dev_priv,293 struct drm_file *filp,294 u32 handle,295 struct vmw_user_object *uo)296{297 struct ttm_object_file *tfile = vmw_fpriv(filp)->tfile;298 struct vmw_resource *res;299 int ret;300 301 WARN_ON(uo->surface || uo->buffer);302 303 ret = vmw_user_resource_lookup_handle(dev_priv, tfile, handle,304 user_surface_converter,305 &res);306 if (!ret) {307 uo->surface = vmw_res_to_srf(res);308 return 0;309 }310 311 uo->surface = NULL;312 ret = vmw_user_bo_lookup(filp, handle, &uo->buffer);313 if (!ret && !uo->buffer->is_dumb) {314 uo->surface = vmw_lookup_surface_for_buffer(dev_priv,315 uo->buffer,316 handle);317 if (uo->surface)318 vmw_user_bo_unref(&uo->buffer);319 }320 321 return ret;322}323 324/**325 * vmw_resource_buf_alloc - Allocate a guest memory buffer for a resource.326 *327 * @res: The resource for which to allocate a gbo buffer.328 * @interruptible: Whether any sleeps during allocation should be329 * performed while interruptible.330 */331static int vmw_resource_buf_alloc(struct vmw_resource *res,332 bool interruptible)333{334 unsigned long size = PFN_ALIGN(res->guest_memory_size);335 struct vmw_bo *gbo;336 struct vmw_bo_params bo_params = {337 .domain = res->func->domain,338 .busy_domain = res->func->busy_domain,339 .bo_type = ttm_bo_type_device,340 .size = res->guest_memory_size,341 .pin = false342 };343 int ret;344 345 if (likely(res->guest_memory_bo)) {346 BUG_ON(res->guest_memory_bo->tbo.base.size < size);347 return 0;348 }349 350 ret = vmw_gem_object_create(res->dev_priv, &bo_params, &gbo);351 if (unlikely(ret != 0))352 goto out_no_bo;353 354 res->guest_memory_bo = gbo;355 356out_no_bo:357 return ret;358}359 360/**361 * vmw_resource_do_validate - Make a resource up-to-date and visible362 * to the device.363 *364 * @res: The resource to make visible to the device.365 * @val_buf: Information about a buffer possibly366 * containing backup data if a bind operation is needed.367 * @dirtying: Transfer dirty regions.368 *369 * On hardware resource shortage, this function returns -EBUSY and370 * should be retried once resources have been freed up.371 */372static int vmw_resource_do_validate(struct vmw_resource *res,373 struct ttm_validate_buffer *val_buf,374 bool dirtying)375{376 int ret = 0;377 const struct vmw_res_func *func = res->func;378 379 if (unlikely(res->id == -1)) {380 ret = func->create(res);381 if (unlikely(ret != 0))382 return ret;383 }384 385 if (func->bind &&386 ((func->needs_guest_memory && !vmw_resource_mob_attached(res) &&387 val_buf->bo) ||388 (!func->needs_guest_memory && val_buf->bo))) {389 ret = func->bind(res, val_buf);390 if (unlikely(ret != 0))391 goto out_bind_failed;392 if (func->needs_guest_memory)393 vmw_resource_mob_attach(res);394 }395 396 /*397 * Handle the case where the backup mob is marked coherent but398 * the resource isn't.399 */400 if (func->dirty_alloc && vmw_resource_mob_attached(res) &&401 !res->coherent) {402 if (res->guest_memory_bo->dirty && !res->dirty) {403 ret = func->dirty_alloc(res);404 if (ret)405 return ret;406 } else if (!res->guest_memory_bo->dirty && res->dirty) {407 func->dirty_free(res);408 }409 }410 411 /*412 * Transfer the dirty regions to the resource and update413 * the resource.414 */415 if (res->dirty) {416 if (dirtying && !res->res_dirty) {417 pgoff_t start = res->guest_memory_offset >> PAGE_SHIFT;418 pgoff_t end = __KERNEL_DIV_ROUND_UP419 (res->guest_memory_offset + res->guest_memory_size,420 PAGE_SIZE);421 422 vmw_bo_dirty_unmap(res->guest_memory_bo, start, end);423 }424 425 vmw_bo_dirty_transfer_to_res(res);426 return func->dirty_sync(res);427 }428 429 return 0;430 431out_bind_failed:432 func->destroy(res);433 434 return ret;435}436 437/**438 * vmw_resource_unreserve - Unreserve a resource previously reserved for439 * command submission.440 *441 * @res: Pointer to the struct vmw_resource to unreserve.442 * @dirty_set: Change dirty status of the resource.443 * @dirty: When changing dirty status indicates the new status.444 * @switch_guest_memory: Guest memory buffer has been switched.445 * @new_guest_memory_bo: Pointer to new guest memory buffer if command submission446 * switched. May be NULL.447 * @new_guest_memory_offset: New gbo offset if @switch_guest_memory is true.448 *449 * Currently unreserving a resource means putting it back on the device's450 * resource lru list, so that it can be evicted if necessary.451 */452void vmw_resource_unreserve(struct vmw_resource *res,453 bool dirty_set,454 bool dirty,455 bool switch_guest_memory,456 struct vmw_bo *new_guest_memory_bo,457 unsigned long new_guest_memory_offset)458{459 struct vmw_private *dev_priv = res->dev_priv;460 461 if (!list_empty(&res->lru_head))462 return;463 464 if (switch_guest_memory && new_guest_memory_bo != res->guest_memory_bo) {465 if (res->guest_memory_bo) {466 vmw_resource_mob_detach(res);467 if (res->coherent)468 vmw_bo_dirty_release(res->guest_memory_bo);469 vmw_user_bo_unref(&res->guest_memory_bo);470 }471 472 if (new_guest_memory_bo) {473 res->guest_memory_bo = vmw_user_bo_ref(new_guest_memory_bo);474 475 /*476 * The validation code should already have added a477 * dirty tracker here.478 */479 WARN_ON(res->coherent && !new_guest_memory_bo->dirty);480 481 vmw_resource_mob_attach(res);482 } else {483 res->guest_memory_bo = NULL;484 }485 } else if (switch_guest_memory && res->coherent) {486 vmw_bo_dirty_release(res->guest_memory_bo);487 }488 489 if (switch_guest_memory)490 res->guest_memory_offset = new_guest_memory_offset;491 492 if (dirty_set)493 res->res_dirty = dirty;494 495 if (!res->func->may_evict || res->id == -1 || res->pin_count)496 return;497 498 spin_lock(&dev_priv->resource_lock);499 list_add_tail(&res->lru_head,500 &res->dev_priv->res_lru[res->func->res_type]);501 spin_unlock(&dev_priv->resource_lock);502}503 504/**505 * vmw_resource_check_buffer - Check whether a backup buffer is needed506 * for a resource and in that case, allocate507 * one, reserve and validate it.508 *509 * @ticket: The ww acquire context to use, or NULL if trylocking.510 * @res: The resource for which to allocate a backup buffer.511 * @interruptible: Whether any sleeps during allocation should be512 * performed while interruptible.513 * @val_buf: On successful return contains data about the514 * reserved and validated backup buffer.515 */516static int517vmw_resource_check_buffer(struct ww_acquire_ctx *ticket,518 struct vmw_resource *res,519 bool interruptible,520 struct ttm_validate_buffer *val_buf)521{522 struct ttm_operation_ctx ctx = { true, false };523 struct list_head val_list;524 bool guest_memory_dirty = false;525 int ret;526 527 if (unlikely(!res->guest_memory_bo)) {528 ret = vmw_resource_buf_alloc(res, interruptible);529 if (unlikely(ret != 0))530 return ret;531 }532 533 INIT_LIST_HEAD(&val_list);534 ttm_bo_get(&res->guest_memory_bo->tbo);535 val_buf->bo = &res->guest_memory_bo->tbo;536 val_buf->num_shared = 0;537 list_add_tail(&val_buf->head, &val_list);538 ret = ttm_eu_reserve_buffers(ticket, &val_list, interruptible, NULL);539 if (unlikely(ret != 0))540 goto out_no_reserve;541 542 if (res->func->needs_guest_memory && !vmw_resource_mob_attached(res))543 return 0;544 545 guest_memory_dirty = res->guest_memory_dirty;546 vmw_bo_placement_set(res->guest_memory_bo, res->func->domain,547 res->func->busy_domain);548 ret = ttm_bo_validate(&res->guest_memory_bo->tbo,549 &res->guest_memory_bo->placement,550 &ctx);551 552 if (unlikely(ret != 0))553 goto out_no_validate;554 555 return 0;556 557out_no_validate:558 ttm_eu_backoff_reservation(ticket, &val_list);559out_no_reserve:560 ttm_bo_put(val_buf->bo);561 val_buf->bo = NULL;562 if (guest_memory_dirty)563 vmw_user_bo_unref(&res->guest_memory_bo);564 565 return ret;566}567 568/*569 * vmw_resource_reserve - Reserve a resource for command submission570 *571 * @res: The resource to reserve.572 *573 * This function takes the resource off the LRU list and make sure574 * a guest memory buffer is present for guest-backed resources.575 * However, the buffer may not be bound to the resource at this576 * point.577 *578 */579int vmw_resource_reserve(struct vmw_resource *res, bool interruptible,580 bool no_guest_memory)581{582 struct vmw_private *dev_priv = res->dev_priv;583 int ret;584 585 spin_lock(&dev_priv->resource_lock);586 list_del_init(&res->lru_head);587 spin_unlock(&dev_priv->resource_lock);588 589 if (res->func->needs_guest_memory && !res->guest_memory_bo &&590 !no_guest_memory) {591 ret = vmw_resource_buf_alloc(res, interruptible);592 if (unlikely(ret != 0)) {593 DRM_ERROR("Failed to allocate a guest memory buffer "594 "of size %lu. bytes\n",595 (unsigned long) res->guest_memory_size);596 return ret;597 }598 }599 600 return 0;601}602 603/**604 * vmw_resource_backoff_reservation - Unreserve and unreference a605 * guest memory buffer606 *.607 * @ticket: The ww acquire ctx used for reservation.608 * @val_buf: Guest memory buffer information.609 */610static void611vmw_resource_backoff_reservation(struct ww_acquire_ctx *ticket,612 struct ttm_validate_buffer *val_buf)613{614 struct list_head val_list;615 616 if (likely(val_buf->bo == NULL))617 return;618 619 INIT_LIST_HEAD(&val_list);620 list_add_tail(&val_buf->head, &val_list);621 ttm_eu_backoff_reservation(ticket, &val_list);622 ttm_bo_put(val_buf->bo);623 val_buf->bo = NULL;624}625 626/**627 * vmw_resource_do_evict - Evict a resource, and transfer its data628 * to a backup buffer.629 *630 * @ticket: The ww acquire ticket to use, or NULL if trylocking.631 * @res: The resource to evict.632 * @interruptible: Whether to wait interruptible.633 */634static int vmw_resource_do_evict(struct ww_acquire_ctx *ticket,635 struct vmw_resource *res, bool interruptible)636{637 struct ttm_validate_buffer val_buf;638 const struct vmw_res_func *func = res->func;639 int ret;640 641 BUG_ON(!func->may_evict);642 643 val_buf.bo = NULL;644 val_buf.num_shared = 0;645 ret = vmw_resource_check_buffer(ticket, res, interruptible, &val_buf);646 if (unlikely(ret != 0))647 return ret;648 649 if (unlikely(func->unbind != NULL &&650 (!func->needs_guest_memory || vmw_resource_mob_attached(res)))) {651 ret = func->unbind(res, res->res_dirty, &val_buf);652 if (unlikely(ret != 0))653 goto out_no_unbind;654 vmw_resource_mob_detach(res);655 }656 ret = func->destroy(res);657 res->guest_memory_dirty = true;658 res->res_dirty = false;659out_no_unbind:660 vmw_resource_backoff_reservation(ticket, &val_buf);661 662 return ret;663}664 665 666/**667 * vmw_resource_validate - Make a resource up-to-date and visible668 * to the device.669 * @res: The resource to make visible to the device.670 * @intr: Perform waits interruptible if possible.671 * @dirtying: Pending GPU operation will dirty the resource672 *673 * On successful return, any backup DMA buffer pointed to by @res->backup will674 * be reserved and validated.675 * On hardware resource shortage, this function will repeatedly evict676 * resources of the same type until the validation succeeds.677 *678 * Return: Zero on success, -ERESTARTSYS if interrupted, negative error code679 * on failure.680 */681int vmw_resource_validate(struct vmw_resource *res, bool intr,682 bool dirtying)683{684 int ret;685 struct vmw_resource *evict_res;686 struct vmw_private *dev_priv = res->dev_priv;687 struct list_head *lru_list = &dev_priv->res_lru[res->func->res_type];688 struct ttm_validate_buffer val_buf;689 unsigned err_count = 0;690 691 if (!res->func->create)692 return 0;693 694 val_buf.bo = NULL;695 val_buf.num_shared = 0;696 if (res->guest_memory_bo)697 val_buf.bo = &res->guest_memory_bo->tbo;698 do {699 ret = vmw_resource_do_validate(res, &val_buf, dirtying);700 if (likely(ret != -EBUSY))701 break;702 703 spin_lock(&dev_priv->resource_lock);704 if (list_empty(lru_list) || !res->func->may_evict) {705 DRM_ERROR("Out of device device resources "706 "for %s.\n", res->func->type_name);707 ret = -EBUSY;708 spin_unlock(&dev_priv->resource_lock);709 break;710 }711 712 evict_res = vmw_resource_reference713 (list_first_entry(lru_list, struct vmw_resource,714 lru_head));715 list_del_init(&evict_res->lru_head);716 717 spin_unlock(&dev_priv->resource_lock);718 719 /* Trylock backup buffers with a NULL ticket. */720 ret = vmw_resource_do_evict(NULL, evict_res, intr);721 if (unlikely(ret != 0)) {722 spin_lock(&dev_priv->resource_lock);723 list_add_tail(&evict_res->lru_head, lru_list);724 spin_unlock(&dev_priv->resource_lock);725 if (ret == -ERESTARTSYS ||726 ++err_count > VMW_RES_EVICT_ERR_COUNT) {727 vmw_resource_unreference(&evict_res);728 goto out_no_validate;729 }730 }731 732 vmw_resource_unreference(&evict_res);733 } while (1);734 735 if (unlikely(ret != 0))736 goto out_no_validate;737 else if (!res->func->needs_guest_memory && res->guest_memory_bo) {738 WARN_ON_ONCE(vmw_resource_mob_attached(res));739 vmw_user_bo_unref(&res->guest_memory_bo);740 }741 742 return 0;743 744out_no_validate:745 return ret;746}747 748 749/**750 * vmw_resource_unbind_list751 *752 * @vbo: Pointer to the current backing MOB.753 *754 * Evicts the Guest Backed hardware resource if the backup755 * buffer is being moved out of MOB memory.756 * Note that this function will not race with the resource757 * validation code, since resource validation and eviction758 * both require the backup buffer to be reserved.759 */760void vmw_resource_unbind_list(struct vmw_bo *vbo)761{762 struct ttm_validate_buffer val_buf = {763 .bo = &vbo->tbo,764 .num_shared = 0765 };766 767 dma_resv_assert_held(vbo->tbo.base.resv);768 while (!RB_EMPTY_ROOT(&vbo->res_tree)) {769 struct rb_node *node = vbo->res_tree.rb_node;770 struct vmw_resource *res =771 container_of(node, struct vmw_resource, mob_node);772 773 if (!WARN_ON_ONCE(!res->func->unbind))774 (void) res->func->unbind(res, res->res_dirty, &val_buf);775 776 res->guest_memory_size = true;777 res->res_dirty = false;778 vmw_resource_mob_detach(res);779 }780 781 (void) ttm_bo_wait(&vbo->tbo, false, false);782}783 784 785/**786 * vmw_query_readback_all - Read back cached query states787 *788 * @dx_query_mob: Buffer containing the DX query MOB789 *790 * Read back cached states from the device if they exist. This function791 * assumes binding_mutex is held.792 */793int vmw_query_readback_all(struct vmw_bo *dx_query_mob)794{795 struct vmw_resource *dx_query_ctx;796 struct vmw_private *dev_priv;797 struct {798 SVGA3dCmdHeader header;799 SVGA3dCmdDXReadbackAllQuery body;800 } *cmd;801 802 803 /* No query bound, so do nothing */804 if (!dx_query_mob || !dx_query_mob->dx_query_ctx)805 return 0;806 807 dx_query_ctx = dx_query_mob->dx_query_ctx;808 dev_priv = dx_query_ctx->dev_priv;809 810 cmd = VMW_CMD_CTX_RESERVE(dev_priv, sizeof(*cmd), dx_query_ctx->id);811 if (unlikely(cmd == NULL))812 return -ENOMEM;813 814 cmd->header.id = SVGA_3D_CMD_DX_READBACK_ALL_QUERY;815 cmd->header.size = sizeof(cmd->body);816 cmd->body.cid = dx_query_ctx->id;817 818 vmw_cmd_commit(dev_priv, sizeof(*cmd));819 820 /* Triggers a rebind the next time affected context is bound */821 dx_query_mob->dx_query_ctx = NULL;822 823 return 0;824}825 826 827 828/**829 * vmw_query_move_notify - Read back cached query states830 *831 * @bo: The TTM buffer object about to move.832 * @old_mem: The memory region @bo is moving from.833 * @new_mem: The memory region @bo is moving to.834 *835 * Called before the query MOB is swapped out to read back cached query836 * states from the device.837 */838void vmw_query_move_notify(struct ttm_buffer_object *bo,839 struct ttm_resource *old_mem,840 struct ttm_resource *new_mem)841{842 struct vmw_bo *dx_query_mob;843 struct ttm_device *bdev = bo->bdev;844 struct vmw_private *dev_priv = vmw_priv_from_ttm(bdev);845 846 mutex_lock(&dev_priv->binding_mutex);847 848 /* If BO is being moved from MOB to system memory */849 if (old_mem &&850 new_mem->mem_type == TTM_PL_SYSTEM &&851 old_mem->mem_type == VMW_PL_MOB) {852 struct vmw_fence_obj *fence;853 854 dx_query_mob = to_vmw_bo(&bo->base);855 if (!dx_query_mob || !dx_query_mob->dx_query_ctx) {856 mutex_unlock(&dev_priv->binding_mutex);857 return;858 }859 860 (void) vmw_query_readback_all(dx_query_mob);861 mutex_unlock(&dev_priv->binding_mutex);862 863 /* Create a fence and attach the BO to it */864 (void) vmw_execbuf_fence_commands(NULL, dev_priv, &fence, NULL);865 vmw_bo_fence_single(bo, fence);866 867 if (fence != NULL)868 vmw_fence_obj_unreference(&fence);869 870 (void) ttm_bo_wait(bo, false, false);871 } else872 mutex_unlock(&dev_priv->binding_mutex);873}874 875/**876 * vmw_resource_needs_backup - Return whether a resource needs a backup buffer.877 *878 * @res: The resource being queried.879 */880bool vmw_resource_needs_backup(const struct vmw_resource *res)881{882 return res->func->needs_guest_memory;883}884 885/**886 * vmw_resource_evict_type - Evict all resources of a specific type887 *888 * @dev_priv: Pointer to a device private struct889 * @type: The resource type to evict890 *891 * To avoid thrashing starvation or as part of the hibernation sequence,892 * try to evict all evictable resources of a specific type.893 */894static void vmw_resource_evict_type(struct vmw_private *dev_priv,895 enum vmw_res_type type)896{897 struct list_head *lru_list = &dev_priv->res_lru[type];898 struct vmw_resource *evict_res;899 unsigned err_count = 0;900 int ret;901 struct ww_acquire_ctx ticket;902 903 do {904 spin_lock(&dev_priv->resource_lock);905 906 if (list_empty(lru_list))907 goto out_unlock;908 909 evict_res = vmw_resource_reference(910 list_first_entry(lru_list, struct vmw_resource,911 lru_head));912 list_del_init(&evict_res->lru_head);913 spin_unlock(&dev_priv->resource_lock);914 915 /* Wait lock backup buffers with a ticket. */916 ret = vmw_resource_do_evict(&ticket, evict_res, false);917 if (unlikely(ret != 0)) {918 spin_lock(&dev_priv->resource_lock);919 list_add_tail(&evict_res->lru_head, lru_list);920 spin_unlock(&dev_priv->resource_lock);921 if (++err_count > VMW_RES_EVICT_ERR_COUNT) {922 vmw_resource_unreference(&evict_res);923 return;924 }925 }926 927 vmw_resource_unreference(&evict_res);928 } while (1);929 930out_unlock:931 spin_unlock(&dev_priv->resource_lock);932}933 934/**935 * vmw_resource_evict_all - Evict all evictable resources936 *937 * @dev_priv: Pointer to a device private struct938 *939 * To avoid thrashing starvation or as part of the hibernation sequence,940 * evict all evictable resources. In particular this means that all941 * guest-backed resources that are registered with the device are942 * evicted and the OTable becomes clean.943 */944void vmw_resource_evict_all(struct vmw_private *dev_priv)945{946 enum vmw_res_type type;947 948 mutex_lock(&dev_priv->cmdbuf_mutex);949 950 for (type = 0; type < vmw_res_max; ++type)951 vmw_resource_evict_type(dev_priv, type);952 953 mutex_unlock(&dev_priv->cmdbuf_mutex);954}955 956/*957 * vmw_resource_pin - Add a pin reference on a resource958 *959 * @res: The resource to add a pin reference on960 *961 * This function adds a pin reference, and if needed validates the resource.962 * Having a pin reference means that the resource can never be evicted, and963 * its id will never change as long as there is a pin reference.964 * This function returns 0 on success and a negative error code on failure.965 */966int vmw_resource_pin(struct vmw_resource *res, bool interruptible)967{968 struct ttm_operation_ctx ctx = { interruptible, false };969 struct vmw_private *dev_priv = res->dev_priv;970 int ret;971 972 mutex_lock(&dev_priv->cmdbuf_mutex);973 ret = vmw_resource_reserve(res, interruptible, false);974 if (ret)975 goto out_no_reserve;976 977 if (res->pin_count == 0) {978 struct vmw_bo *vbo = NULL;979 980 if (res->guest_memory_bo) {981 vbo = res->guest_memory_bo;982 983 ret = ttm_bo_reserve(&vbo->tbo, interruptible, false, NULL);984 if (ret)985 goto out_no_validate;986 if (!vbo->tbo.pin_count) {987 vmw_bo_placement_set(vbo,988 res->func->domain,989 res->func->busy_domain);990 ret = ttm_bo_validate991 (&vbo->tbo,992 &vbo->placement,993 &ctx);994 if (ret) {995 ttm_bo_unreserve(&vbo->tbo);996 goto out_no_validate;997 }998 }999 1000 /* Do we really need to pin the MOB as well? */1001 vmw_bo_pin_reserved(vbo, true);1002 }1003 ret = vmw_resource_validate(res, interruptible, true);1004 if (vbo)1005 ttm_bo_unreserve(&vbo->tbo);1006 if (ret)1007 goto out_no_validate;1008 }1009 res->pin_count++;1010 1011out_no_validate:1012 vmw_resource_unreserve(res, false, false, false, NULL, 0UL);1013out_no_reserve:1014 mutex_unlock(&dev_priv->cmdbuf_mutex);1015 1016 return ret;1017}1018 1019/**1020 * vmw_resource_unpin - Remove a pin reference from a resource1021 *1022 * @res: The resource to remove a pin reference from1023 *1024 * Having a pin reference means that the resource can never be evicted, and1025 * its id will never change as long as there is a pin reference.1026 */1027void vmw_resource_unpin(struct vmw_resource *res)1028{1029 struct vmw_private *dev_priv = res->dev_priv;1030 int ret;1031 1032 mutex_lock(&dev_priv->cmdbuf_mutex);1033 1034 ret = vmw_resource_reserve(res, false, true);1035 WARN_ON(ret);1036 1037 WARN_ON(res->pin_count == 0);1038 if (--res->pin_count == 0 && res->guest_memory_bo) {1039 struct vmw_bo *vbo = res->guest_memory_bo;1040 1041 (void) ttm_bo_reserve(&vbo->tbo, false, false, NULL);1042 vmw_bo_pin_reserved(vbo, false);1043 ttm_bo_unreserve(&vbo->tbo);1044 }1045 1046 vmw_resource_unreserve(res, false, false, false, NULL, 0UL);1047 1048 mutex_unlock(&dev_priv->cmdbuf_mutex);1049}1050 1051/**1052 * vmw_res_type - Return the resource type1053 *1054 * @res: Pointer to the resource1055 */1056enum vmw_res_type vmw_res_type(const struct vmw_resource *res)1057{1058 return res->func->res_type;1059}1060 1061/**1062 * vmw_resource_dirty_update - Update a resource's dirty tracker with a1063 * sequential range of touched backing store memory.1064 * @res: The resource.1065 * @start: The first page touched.1066 * @end: The last page touched + 1.1067 */1068void vmw_resource_dirty_update(struct vmw_resource *res, pgoff_t start,1069 pgoff_t end)1070{1071 if (res->dirty)1072 res->func->dirty_range_add(res, start << PAGE_SHIFT,1073 end << PAGE_SHIFT);1074}1075 1076int vmw_resource_clean(struct vmw_resource *res)1077{1078 int ret = 0;1079 1080 if (res->res_dirty) {1081 if (!res->func->clean)1082 return -EINVAL;1083 1084 ret = res->func->clean(res);1085 if (ret)1086 return ret;1087 res->res_dirty = false;1088 }1089 return ret;1090}1091 1092/**1093 * vmw_resources_clean - Clean resources intersecting a mob range1094 * @vbo: The mob buffer object1095 * @start: The mob page offset starting the range1096 * @end: The mob page offset ending the range1097 * @num_prefault: Returns how many pages including the first have been1098 * cleaned and are ok to prefault1099 */1100int vmw_resources_clean(struct vmw_bo *vbo, pgoff_t start,1101 pgoff_t end, pgoff_t *num_prefault)1102{1103 struct rb_node *cur = vbo->res_tree.rb_node;1104 struct vmw_resource *found = NULL;1105 unsigned long res_start = start << PAGE_SHIFT;1106 unsigned long res_end = end << PAGE_SHIFT;1107 unsigned long last_cleaned = 0;1108 int ret;1109 1110 /*1111 * Find the resource with lowest backup_offset that intersects the1112 * range.1113 */1114 while (cur) {1115 struct vmw_resource *cur_res =1116 container_of(cur, struct vmw_resource, mob_node);1117 1118 if (cur_res->guest_memory_offset >= res_end) {1119 cur = cur->rb_left;1120 } else if (cur_res->guest_memory_offset + cur_res->guest_memory_size <=1121 res_start) {1122 cur = cur->rb_right;1123 } else {1124 found = cur_res;1125 cur = cur->rb_left;1126 /* Continue to look for resources with lower offsets */1127 }1128 }1129 1130 /*1131 * In order of increasing guest_memory_offset, clean dirty resources1132 * intersecting the range.1133 */1134 while (found) {1135 ret = vmw_resource_clean(found);1136 if (ret)1137 return ret;1138 last_cleaned = found->guest_memory_offset + found->guest_memory_size;1139 cur = rb_next(&found->mob_node);1140 if (!cur)1141 break;1142 1143 found = container_of(cur, struct vmw_resource, mob_node);1144 if (found->guest_memory_offset >= res_end)1145 break;1146 }1147 1148 /*1149 * Set number of pages allowed prefaulting and fence the buffer object1150 */1151 *num_prefault = 1;1152 if (last_cleaned > res_start) {1153 struct ttm_buffer_object *bo = &vbo->tbo;1154 1155 *num_prefault = __KERNEL_DIV_ROUND_UP(last_cleaned - res_start,1156 PAGE_SIZE);1157 vmw_bo_fence_single(bo, NULL);1158 }1159 1160 return 0;1161}1162