189 lines · c
1/*2 * Copyright 2016-2018 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 */22 23#include <linux/dma-fence.h>24#include <linux/spinlock.h>25#include <linux/atomic.h>26#include <linux/stacktrace.h>27#include <linux/sched.h>28#include <linux/slab.h>29#include <linux/sched/mm.h>30#include "amdgpu_amdkfd.h"31#include "kfd_svm.h"32 33static const struct dma_fence_ops amdkfd_fence_ops;34static atomic_t fence_seq = ATOMIC_INIT(0);35 36/* Eviction Fence37 * Fence helper functions to deal with KFD memory eviction.38 * Big Idea - Since KFD submissions are done by user queues, a BO cannot be39 * evicted unless all the user queues for that process are evicted.40 *41 * All the BOs in a process share an eviction fence. When process X wants42 * to map VRAM memory but TTM can't find enough space, TTM will attempt to43 * evict BOs from its LRU list. TTM checks if the BO is valuable to evict44 * by calling ttm_device_funcs->eviction_valuable().45 *46 * ttm_device_funcs->eviction_valuable() - will return false if the BO belongs47 * to process X. Otherwise, it will return true to indicate BO can be48 * evicted by TTM.49 *50 * If ttm_device_funcs->eviction_valuable returns true, then TTM will continue51 * the evcition process for that BO by calling ttm_bo_evict --> amdgpu_bo_move52 * --> amdgpu_copy_buffer(). This sets up job in GPU scheduler.53 *54 * GPU Scheduler (amd_sched_main) - sets up a cb (fence_add_callback) to55 * nofity when the BO is free to move. fence_add_callback --> enable_signaling56 * --> amdgpu_amdkfd_fence.enable_signaling57 *58 * amdgpu_amdkfd_fence.enable_signaling - Start a work item that will quiesce59 * user queues and signal fence. The work item will also start another delayed60 * work item to restore BOs61 */62 63struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,64 struct mm_struct *mm,65 struct svm_range_bo *svm_bo)66{67 struct amdgpu_amdkfd_fence *fence;68 69 fence = kzalloc(sizeof(*fence), GFP_KERNEL);70 if (fence == NULL)71 return NULL;72 73 /* This reference gets released in amdkfd_fence_release */74 mmgrab(mm);75 fence->mm = mm;76 get_task_comm(fence->timeline_name, current);77 spin_lock_init(&fence->lock);78 fence->svm_bo = svm_bo;79 dma_fence_init(&fence->base, &amdkfd_fence_ops, &fence->lock,80 context, atomic_inc_return(&fence_seq));81 82 return fence;83}84 85struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f)86{87 struct amdgpu_amdkfd_fence *fence;88 89 if (!f)90 return NULL;91 92 fence = container_of(f, struct amdgpu_amdkfd_fence, base);93 if (f->ops == &amdkfd_fence_ops)94 return fence;95 96 return NULL;97}98 99static const char *amdkfd_fence_get_driver_name(struct dma_fence *f)100{101 return "amdgpu_amdkfd_fence";102}103 104static const char *amdkfd_fence_get_timeline_name(struct dma_fence *f)105{106 struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);107 108 return fence->timeline_name;109}110 111/**112 * amdkfd_fence_enable_signaling - This gets called when TTM wants to evict113 * a KFD BO and schedules a job to move the BO.114 * If fence is already signaled return true.115 * If fence is not signaled schedule a evict KFD process work item.116 *117 * @f: dma_fence118 */119static bool amdkfd_fence_enable_signaling(struct dma_fence *f)120{121 struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);122 123 if (!fence)124 return false;125 126 if (dma_fence_is_signaled(f))127 return true;128 129 if (!fence->svm_bo) {130 if (!kgd2kfd_schedule_evict_and_restore_process(fence->mm, f))131 return true;132 } else {133 if (!svm_range_schedule_evict_svm_bo(fence))134 return true;135 }136 return false;137}138 139/**140 * amdkfd_fence_release - callback that fence can be freed141 *142 * @f: dma_fence143 *144 * This function is called when the reference count becomes zero.145 * Drops the mm_struct reference and RCU schedules freeing up the fence.146 */147static void amdkfd_fence_release(struct dma_fence *f)148{149 struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);150 151 /* Unconditionally signal the fence. The process is getting152 * terminated.153 */154 if (WARN_ON(!fence))155 return; /* Not an amdgpu_amdkfd_fence */156 157 mmdrop(fence->mm);158 kfree_rcu(f, rcu);159}160 161/**162 * amdkfd_fence_check_mm - Check whether to prevent eviction of @f by @mm163 *164 * @f: [IN] fence165 * @mm: [IN] mm that needs to be verified166 *167 * Check if @mm is same as that of the fence @f, if same return TRUE else168 * return FALSE.169 * For svm bo, which support vram overcommitment, always return FALSE.170 */171bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm)172{173 struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f);174 175 if (!fence)176 return false;177 else if (fence->mm == mm && !fence->svm_bo)178 return true;179 180 return false;181}182 183static const struct dma_fence_ops amdkfd_fence_ops = {184 .get_driver_name = amdkfd_fence_get_driver_name,185 .get_timeline_name = amdkfd_fence_get_timeline_name,186 .enable_signaling = amdkfd_fence_enable_signaling,187 .release = amdkfd_fence_release,188};189