490 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright 2014 Advanced Micro Devices, Inc.4 * All Rights Reserved.5 *6 * Permission is hereby granted, free of charge, to any person obtaining a7 * copy of this software and associated documentation files (the8 * "Software"), to deal in the Software without restriction, including9 * without limitation the rights to use, copy, modify, merge, publish,10 * distribute, sub license, and/or sell copies of the Software, and to11 * permit persons to whom the Software is furnished to do so, subject to12 * the following conditions:13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,18 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR19 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE20 * USE OR OTHER DEALINGS IN THE SOFTWARE.21 *22 * The above copyright notice and this permission notice (including the23 * next paragraph) shall be included in all copies or substantial portions24 * of the Software.25 *26 */27/*28 * Authors:29 * Christian König <christian.koenig@amd.com>30 */31 32#include <linux/dma-fence-chain.h>33 34#include "amdgpu.h"35#include "amdgpu_trace.h"36#include "amdgpu_amdkfd.h"37 38struct amdgpu_sync_entry {39 struct hlist_node node;40 struct dma_fence *fence;41};42 43static struct kmem_cache *amdgpu_sync_slab;44 45/**46 * amdgpu_sync_create - zero init sync object47 *48 * @sync: sync object to initialize49 *50 * Just clear the sync object for now.51 */52void amdgpu_sync_create(struct amdgpu_sync *sync)53{54 hash_init(sync->fences);55}56 57/**58 * amdgpu_sync_same_dev - test if fence belong to us59 *60 * @adev: amdgpu device to use for the test61 * @f: fence to test62 *63 * Test if the fence was issued by us.64 */65static bool amdgpu_sync_same_dev(struct amdgpu_device *adev,66 struct dma_fence *f)67{68 struct drm_sched_fence *s_fence = to_drm_sched_fence(f);69 70 if (s_fence) {71 struct amdgpu_ring *ring;72 73 ring = container_of(s_fence->sched, struct amdgpu_ring, sched);74 return ring->adev == adev;75 }76 77 return false;78}79 80/**81 * amdgpu_sync_get_owner - extract the owner of a fence82 *83 * @f: fence get the owner from84 *85 * Extract who originally created the fence.86 */87static void *amdgpu_sync_get_owner(struct dma_fence *f)88{89 struct drm_sched_fence *s_fence;90 struct amdgpu_amdkfd_fence *kfd_fence;91 92 if (!f)93 return AMDGPU_FENCE_OWNER_UNDEFINED;94 95 s_fence = to_drm_sched_fence(f);96 if (s_fence)97 return s_fence->owner;98 99 kfd_fence = to_amdgpu_amdkfd_fence(f);100 if (kfd_fence)101 return AMDGPU_FENCE_OWNER_KFD;102 103 return AMDGPU_FENCE_OWNER_UNDEFINED;104}105 106/**107 * amdgpu_sync_keep_later - Keep the later fence108 *109 * @keep: existing fence to test110 * @fence: new fence111 *112 * Either keep the existing fence or the new one, depending which one is later.113 */114static void amdgpu_sync_keep_later(struct dma_fence **keep,115 struct dma_fence *fence)116{117 if (*keep && dma_fence_is_later(*keep, fence))118 return;119 120 dma_fence_put(*keep);121 *keep = dma_fence_get(fence);122}123 124/**125 * amdgpu_sync_add_later - add the fence to the hash126 *127 * @sync: sync object to add the fence to128 * @f: fence to add129 *130 * Tries to add the fence to an existing hash entry. Returns true when an entry131 * was found, false otherwise.132 */133static bool amdgpu_sync_add_later(struct amdgpu_sync *sync, struct dma_fence *f)134{135 struct amdgpu_sync_entry *e;136 137 hash_for_each_possible(sync->fences, e, node, f->context) {138 if (unlikely(e->fence->context != f->context))139 continue;140 141 amdgpu_sync_keep_later(&e->fence, f);142 return true;143 }144 return false;145}146 147/**148 * amdgpu_sync_fence - remember to sync to this fence149 *150 * @sync: sync object to add fence to151 * @f: fence to sync to152 *153 * Add the fence to the sync object.154 */155int amdgpu_sync_fence(struct amdgpu_sync *sync, struct dma_fence *f)156{157 struct amdgpu_sync_entry *e;158 159 if (!f)160 return 0;161 162 if (amdgpu_sync_add_later(sync, f))163 return 0;164 165 e = kmem_cache_alloc(amdgpu_sync_slab, GFP_KERNEL);166 if (!e)167 return -ENOMEM;168 169 hash_add(sync->fences, &e->node, f->context);170 e->fence = dma_fence_get(f);171 return 0;172}173 174/* Determine based on the owner and mode if we should sync to a fence or not */175static bool amdgpu_sync_test_fence(struct amdgpu_device *adev,176 enum amdgpu_sync_mode mode,177 void *owner, struct dma_fence *f)178{179 void *fence_owner = amdgpu_sync_get_owner(f);180 181 /* Always sync to moves, no matter what */182 if (fence_owner == AMDGPU_FENCE_OWNER_UNDEFINED)183 return true;184 185 /* We only want to trigger KFD eviction fences on186 * evict or move jobs. Skip KFD fences otherwise.187 */188 if (fence_owner == AMDGPU_FENCE_OWNER_KFD &&189 owner != AMDGPU_FENCE_OWNER_UNDEFINED)190 return false;191 192 /* Never sync to VM updates either. */193 if (fence_owner == AMDGPU_FENCE_OWNER_VM &&194 owner != AMDGPU_FENCE_OWNER_UNDEFINED &&195 owner != AMDGPU_FENCE_OWNER_KFD)196 return false;197 198 /* Ignore fences depending on the sync mode */199 switch (mode) {200 case AMDGPU_SYNC_ALWAYS:201 return true;202 203 case AMDGPU_SYNC_NE_OWNER:204 if (amdgpu_sync_same_dev(adev, f) &&205 fence_owner == owner)206 return false;207 break;208 209 case AMDGPU_SYNC_EQ_OWNER:210 if (amdgpu_sync_same_dev(adev, f) &&211 fence_owner != owner)212 return false;213 break;214 215 case AMDGPU_SYNC_EXPLICIT:216 return false;217 }218 219 WARN(debug_evictions && fence_owner == AMDGPU_FENCE_OWNER_KFD,220 "Adding eviction fence to sync obj");221 return true;222}223 224/**225 * amdgpu_sync_resv - sync to a reservation object226 *227 * @adev: amdgpu device228 * @sync: sync object to add fences from reservation object to229 * @resv: reservation object with embedded fence230 * @mode: how owner affects which fences we sync to231 * @owner: owner of the planned job submission232 *233 * Sync to the fence234 */235int amdgpu_sync_resv(struct amdgpu_device *adev, struct amdgpu_sync *sync,236 struct dma_resv *resv, enum amdgpu_sync_mode mode,237 void *owner)238{239 struct dma_resv_iter cursor;240 struct dma_fence *f;241 int r;242 243 if (resv == NULL)244 return -EINVAL;245 246 /* TODO: Use DMA_RESV_USAGE_READ here */247 dma_resv_for_each_fence(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP, f) {248 dma_fence_chain_for_each(f, f) {249 struct dma_fence *tmp = dma_fence_chain_contained(f);250 251 if (amdgpu_sync_test_fence(adev, mode, owner, tmp)) {252 r = amdgpu_sync_fence(sync, f);253 dma_fence_put(f);254 if (r)255 return r;256 break;257 }258 }259 }260 return 0;261}262 263/**264 * amdgpu_sync_kfd - sync to KFD fences265 *266 * @sync: sync object to add KFD fences to267 * @resv: reservation object with KFD fences268 *269 * Extract all KFD fences and add them to the sync object.270 */271int amdgpu_sync_kfd(struct amdgpu_sync *sync, struct dma_resv *resv)272{273 struct dma_resv_iter cursor;274 struct dma_fence *f;275 int r = 0;276 277 dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP);278 dma_resv_for_each_fence_unlocked(&cursor, f) {279 void *fence_owner = amdgpu_sync_get_owner(f);280 281 if (fence_owner != AMDGPU_FENCE_OWNER_KFD)282 continue;283 284 r = amdgpu_sync_fence(sync, f);285 if (r)286 break;287 }288 dma_resv_iter_end(&cursor);289 290 return r;291}292 293/* Free the entry back to the slab */294static void amdgpu_sync_entry_free(struct amdgpu_sync_entry *e)295{296 hash_del(&e->node);297 dma_fence_put(e->fence);298 kmem_cache_free(amdgpu_sync_slab, e);299}300 301/**302 * amdgpu_sync_peek_fence - get the next fence not signaled yet303 *304 * @sync: the sync object305 * @ring: optional ring to use for test306 *307 * Returns the next fence not signaled yet without removing it from the sync308 * object.309 */310struct dma_fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,311 struct amdgpu_ring *ring)312{313 struct amdgpu_sync_entry *e;314 struct hlist_node *tmp;315 int i;316 317 hash_for_each_safe(sync->fences, i, tmp, e, node) {318 struct dma_fence *f = e->fence;319 struct drm_sched_fence *s_fence = to_drm_sched_fence(f);320 321 if (dma_fence_is_signaled(f)) {322 amdgpu_sync_entry_free(e);323 continue;324 }325 if (ring && s_fence) {326 /* For fences from the same ring it is sufficient327 * when they are scheduled.328 */329 if (s_fence->sched == &ring->sched) {330 if (dma_fence_is_signaled(&s_fence->scheduled))331 continue;332 333 return &s_fence->scheduled;334 }335 }336 337 return f;338 }339 340 return NULL;341}342 343/**344 * amdgpu_sync_get_fence - get the next fence from the sync object345 *346 * @sync: sync object to use347 *348 * Get and removes the next fence from the sync object not signaled yet.349 */350struct dma_fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync)351{352 struct amdgpu_sync_entry *e;353 struct hlist_node *tmp;354 struct dma_fence *f;355 int i;356 357 hash_for_each_safe(sync->fences, i, tmp, e, node) {358 359 f = e->fence;360 361 hash_del(&e->node);362 kmem_cache_free(amdgpu_sync_slab, e);363 364 if (!dma_fence_is_signaled(f))365 return f;366 367 dma_fence_put(f);368 }369 return NULL;370}371 372/**373 * amdgpu_sync_clone - clone a sync object374 *375 * @source: sync object to clone376 * @clone: pointer to destination sync object377 *378 * Adds references to all unsignaled fences in @source to @clone. Also379 * removes signaled fences from @source while at it.380 */381int amdgpu_sync_clone(struct amdgpu_sync *source, struct amdgpu_sync *clone)382{383 struct amdgpu_sync_entry *e;384 struct hlist_node *tmp;385 struct dma_fence *f;386 int i, r;387 388 hash_for_each_safe(source->fences, i, tmp, e, node) {389 f = e->fence;390 if (!dma_fence_is_signaled(f)) {391 r = amdgpu_sync_fence(clone, f);392 if (r)393 return r;394 } else {395 amdgpu_sync_entry_free(e);396 }397 }398 399 return 0;400}401 402/**403 * amdgpu_sync_push_to_job - push fences into job404 * @sync: sync object to get the fences from405 * @job: job to push the fences into406 *407 * Add all unsignaled fences from sync to job.408 */409int amdgpu_sync_push_to_job(struct amdgpu_sync *sync, struct amdgpu_job *job)410{411 struct amdgpu_sync_entry *e;412 struct hlist_node *tmp;413 struct dma_fence *f;414 int i, r;415 416 hash_for_each_safe(sync->fences, i, tmp, e, node) {417 f = e->fence;418 if (dma_fence_is_signaled(f)) {419 amdgpu_sync_entry_free(e);420 continue;421 }422 423 dma_fence_get(f);424 r = drm_sched_job_add_dependency(&job->base, f);425 if (r) {426 dma_fence_put(f);427 return r;428 }429 }430 return 0;431}432 433int amdgpu_sync_wait(struct amdgpu_sync *sync, bool intr)434{435 struct amdgpu_sync_entry *e;436 struct hlist_node *tmp;437 int i, r;438 439 hash_for_each_safe(sync->fences, i, tmp, e, node) {440 r = dma_fence_wait(e->fence, intr);441 if (r)442 return r;443 444 amdgpu_sync_entry_free(e);445 }446 447 return 0;448}449 450/**451 * amdgpu_sync_free - free the sync object452 *453 * @sync: sync object to use454 *455 * Free the sync object.456 */457void amdgpu_sync_free(struct amdgpu_sync *sync)458{459 struct amdgpu_sync_entry *e;460 struct hlist_node *tmp;461 unsigned int i;462 463 hash_for_each_safe(sync->fences, i, tmp, e, node)464 amdgpu_sync_entry_free(e);465}466 467/**468 * amdgpu_sync_init - init sync object subsystem469 *470 * Allocate the slab allocator.471 */472int amdgpu_sync_init(void)473{474 amdgpu_sync_slab = KMEM_CACHE(amdgpu_sync_entry, SLAB_HWCACHE_ALIGN);475 if (!amdgpu_sync_slab)476 return -ENOMEM;477 478 return 0;479}480 481/**482 * amdgpu_sync_fini - fini sync object subsystem483 *484 * Free the slab allocator.485 */486void amdgpu_sync_fini(void)487{488 kmem_cache_destroy(amdgpu_sync_slab);489}490