1008 lines · c
1/*2 * Copyright 2009 Jerome Glisse.3 * All Rights Reserved.4 *5 * Permission is hereby granted, free of charge, to any person obtaining a6 * copy of this software and associated documentation files (the7 * "Software"), to deal in the Software without restriction, including8 * without limitation the rights to use, copy, modify, merge, publish,9 * distribute, sub license, and/or sell copies of the Software, and to10 * permit persons to whom the Software is furnished to do so, subject to11 * the following conditions:12 *13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE19 * USE OR OTHER DEALINGS IN THE SOFTWARE.20 *21 * The above copyright notice and this permission notice (including the22 * next paragraph) shall be included in all copies or substantial portions23 * of the Software.24 *25 */26/*27 * Authors:28 * Jerome Glisse <glisse@freedesktop.org>29 * Dave Airlie30 */31#include <linux/seq_file.h>32#include <linux/atomic.h>33#include <linux/wait.h>34#include <linux/kref.h>35#include <linux/slab.h>36#include <linux/firmware.h>37#include <linux/pm_runtime.h>38 39#include <drm/drm_drv.h>40#include "amdgpu.h"41#include "amdgpu_trace.h"42#include "amdgpu_reset.h"43 44/*45 * Fences mark an event in the GPUs pipeline and are used46 * for GPU/CPU synchronization. When the fence is written,47 * it is expected that all buffers associated with that fence48 * are no longer in use by the associated ring on the GPU and49 * that the relevant GPU caches have been flushed.50 */51 52struct amdgpu_fence {53 struct dma_fence base;54 55 /* RB, DMA, etc. */56 struct amdgpu_ring *ring;57 ktime_t start_timestamp;58};59 60static struct kmem_cache *amdgpu_fence_slab;61 62int amdgpu_fence_slab_init(void)63{64 amdgpu_fence_slab = KMEM_CACHE(amdgpu_fence, SLAB_HWCACHE_ALIGN);65 if (!amdgpu_fence_slab)66 return -ENOMEM;67 return 0;68}69 70void amdgpu_fence_slab_fini(void)71{72 rcu_barrier();73 kmem_cache_destroy(amdgpu_fence_slab);74}75/*76 * Cast helper77 */78static const struct dma_fence_ops amdgpu_fence_ops;79static const struct dma_fence_ops amdgpu_job_fence_ops;80static inline struct amdgpu_fence *to_amdgpu_fence(struct dma_fence *f)81{82 struct amdgpu_fence *__f = container_of(f, struct amdgpu_fence, base);83 84 if (__f->base.ops == &amdgpu_fence_ops ||85 __f->base.ops == &amdgpu_job_fence_ops)86 return __f;87 88 return NULL;89}90 91/**92 * amdgpu_fence_write - write a fence value93 *94 * @ring: ring the fence is associated with95 * @seq: sequence number to write96 *97 * Writes a fence value to memory (all asics).98 */99static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)100{101 struct amdgpu_fence_driver *drv = &ring->fence_drv;102 103 if (drv->cpu_addr)104 *drv->cpu_addr = cpu_to_le32(seq);105}106 107/**108 * amdgpu_fence_read - read a fence value109 *110 * @ring: ring the fence is associated with111 *112 * Reads a fence value from memory (all asics).113 * Returns the value of the fence read from memory.114 */115static u32 amdgpu_fence_read(struct amdgpu_ring *ring)116{117 struct amdgpu_fence_driver *drv = &ring->fence_drv;118 u32 seq = 0;119 120 if (drv->cpu_addr)121 seq = le32_to_cpu(*drv->cpu_addr);122 else123 seq = atomic_read(&drv->last_seq);124 125 return seq;126}127 128/**129 * amdgpu_fence_emit - emit a fence on the requested ring130 *131 * @ring: ring the fence is associated with132 * @f: resulting fence object133 * @job: job the fence is embedded in134 * @flags: flags to pass into the subordinate .emit_fence() call135 *136 * Emits a fence command on the requested ring (all asics).137 * Returns 0 on success, -ENOMEM on failure.138 */139int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, struct amdgpu_job *job,140 unsigned int flags)141{142 struct amdgpu_device *adev = ring->adev;143 struct dma_fence *fence;144 struct amdgpu_fence *am_fence;145 struct dma_fence __rcu **ptr;146 uint32_t seq;147 int r;148 149 if (job == NULL) {150 /* create a sperate hw fence */151 am_fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_ATOMIC);152 if (am_fence == NULL)153 return -ENOMEM;154 fence = &am_fence->base;155 am_fence->ring = ring;156 } else {157 /* take use of job-embedded fence */158 fence = &job->hw_fence;159 }160 161 seq = ++ring->fence_drv.sync_seq;162 if (job && job->job_run_counter) {163 /* reinit seq for resubmitted jobs */164 fence->seqno = seq;165 /* TO be inline with external fence creation and other drivers */166 dma_fence_get(fence);167 } else {168 if (job) {169 dma_fence_init(fence, &amdgpu_job_fence_ops,170 &ring->fence_drv.lock,171 adev->fence_context + ring->idx, seq);172 /* Against remove in amdgpu_job_{free, free_cb} */173 dma_fence_get(fence);174 } else {175 dma_fence_init(fence, &amdgpu_fence_ops,176 &ring->fence_drv.lock,177 adev->fence_context + ring->idx, seq);178 }179 }180 181 amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,182 seq, flags | AMDGPU_FENCE_FLAG_INT);183 pm_runtime_get_noresume(adev_to_drm(adev)->dev);184 ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];185 if (unlikely(rcu_dereference_protected(*ptr, 1))) {186 struct dma_fence *old;187 188 rcu_read_lock();189 old = dma_fence_get_rcu_safe(ptr);190 rcu_read_unlock();191 192 if (old) {193 r = dma_fence_wait(old, false);194 dma_fence_put(old);195 if (r)196 return r;197 }198 }199 200 to_amdgpu_fence(fence)->start_timestamp = ktime_get();201 202 /* This function can't be called concurrently anyway, otherwise203 * emitting the fence would mess up the hardware ring buffer.204 */205 rcu_assign_pointer(*ptr, dma_fence_get(fence));206 207 *f = fence;208 209 return 0;210}211 212/**213 * amdgpu_fence_emit_polling - emit a fence on the requeste ring214 *215 * @ring: ring the fence is associated with216 * @s: resulting sequence number217 * @timeout: the timeout for waiting in usecs218 *219 * Emits a fence command on the requested ring (all asics).220 * Used For polling fence.221 * Returns 0 on success, -ENOMEM on failure.222 */223int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s,224 uint32_t timeout)225{226 uint32_t seq;227 signed long r;228 229 if (!s)230 return -EINVAL;231 232 seq = ++ring->fence_drv.sync_seq;233 r = amdgpu_fence_wait_polling(ring,234 seq - ring->fence_drv.num_fences_mask,235 timeout);236 if (r < 1)237 return -ETIMEDOUT;238 239 amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,240 seq, 0);241 242 *s = seq;243 244 return 0;245}246 247/**248 * amdgpu_fence_schedule_fallback - schedule fallback check249 *250 * @ring: pointer to struct amdgpu_ring251 *252 * Start a timer as fallback to our interrupts.253 */254static void amdgpu_fence_schedule_fallback(struct amdgpu_ring *ring)255{256 mod_timer(&ring->fence_drv.fallback_timer,257 jiffies + AMDGPU_FENCE_JIFFIES_TIMEOUT);258}259 260/**261 * amdgpu_fence_process - check for fence activity262 *263 * @ring: pointer to struct amdgpu_ring264 *265 * Checks the current fence value and calculates the last266 * signalled fence value. Wakes the fence queue if the267 * sequence number has increased.268 *269 * Returns true if fence was processed270 */271bool amdgpu_fence_process(struct amdgpu_ring *ring)272{273 struct amdgpu_fence_driver *drv = &ring->fence_drv;274 struct amdgpu_device *adev = ring->adev;275 uint32_t seq, last_seq;276 277 do {278 last_seq = atomic_read(&ring->fence_drv.last_seq);279 seq = amdgpu_fence_read(ring);280 281 } while (atomic_cmpxchg(&drv->last_seq, last_seq, seq) != last_seq);282 283 if (del_timer(&ring->fence_drv.fallback_timer) &&284 seq != ring->fence_drv.sync_seq)285 amdgpu_fence_schedule_fallback(ring);286 287 if (unlikely(seq == last_seq))288 return false;289 290 last_seq &= drv->num_fences_mask;291 seq &= drv->num_fences_mask;292 293 do {294 struct dma_fence *fence, **ptr;295 296 ++last_seq;297 last_seq &= drv->num_fences_mask;298 ptr = &drv->fences[last_seq];299 300 /* There is always exactly one thread signaling this fence slot */301 fence = rcu_dereference_protected(*ptr, 1);302 RCU_INIT_POINTER(*ptr, NULL);303 304 if (!fence)305 continue;306 307 dma_fence_signal(fence);308 dma_fence_put(fence);309 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);310 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);311 } while (last_seq != seq);312 313 return true;314}315 316/**317 * amdgpu_fence_fallback - fallback for hardware interrupts318 *319 * @t: timer context used to obtain the pointer to ring structure320 *321 * Checks for fence activity.322 */323static void amdgpu_fence_fallback(struct timer_list *t)324{325 struct amdgpu_ring *ring = from_timer(ring, t,326 fence_drv.fallback_timer);327 328 if (amdgpu_fence_process(ring))329 DRM_WARN("Fence fallback timer expired on ring %s\n", ring->name);330}331 332/**333 * amdgpu_fence_wait_empty - wait for all fences to signal334 *335 * @ring: ring index the fence is associated with336 *337 * Wait for all fences on the requested ring to signal (all asics).338 * Returns 0 if the fences have passed, error for all other cases.339 */340int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)341{342 uint64_t seq = READ_ONCE(ring->fence_drv.sync_seq);343 struct dma_fence *fence, **ptr;344 int r;345 346 if (!seq)347 return 0;348 349 ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];350 rcu_read_lock();351 fence = rcu_dereference(*ptr);352 if (!fence || !dma_fence_get_rcu(fence)) {353 rcu_read_unlock();354 return 0;355 }356 rcu_read_unlock();357 358 r = dma_fence_wait(fence, false);359 dma_fence_put(fence);360 return r;361}362 363/**364 * amdgpu_fence_wait_polling - busy wait for givn sequence number365 *366 * @ring: ring index the fence is associated with367 * @wait_seq: sequence number to wait368 * @timeout: the timeout for waiting in usecs369 *370 * Wait for all fences on the requested ring to signal (all asics).371 * Returns left time if no timeout, 0 or minus if timeout.372 */373signed long amdgpu_fence_wait_polling(struct amdgpu_ring *ring,374 uint32_t wait_seq,375 signed long timeout)376{377 378 while ((int32_t)(wait_seq - amdgpu_fence_read(ring)) > 0 && timeout > 0) {379 udelay(2);380 timeout -= 2;381 }382 return timeout > 0 ? timeout : 0;383}384/**385 * amdgpu_fence_count_emitted - get the count of emitted fences386 *387 * @ring: ring the fence is associated with388 *389 * Get the number of fences emitted on the requested ring (all asics).390 * Returns the number of emitted fences on the ring. Used by the391 * dynpm code to ring track activity.392 */393unsigned int amdgpu_fence_count_emitted(struct amdgpu_ring *ring)394{395 uint64_t emitted;396 397 /* We are not protected by ring lock when reading the last sequence398 * but it's ok to report slightly wrong fence count here.399 */400 emitted = 0x100000000ull;401 emitted -= atomic_read(&ring->fence_drv.last_seq);402 emitted += READ_ONCE(ring->fence_drv.sync_seq);403 return lower_32_bits(emitted);404}405 406/**407 * amdgpu_fence_last_unsignaled_time_us - the time fence emitted until now408 * @ring: ring the fence is associated with409 *410 * Find the earliest fence unsignaled until now, calculate the time delta411 * between the time fence emitted and now.412 */413u64 amdgpu_fence_last_unsignaled_time_us(struct amdgpu_ring *ring)414{415 struct amdgpu_fence_driver *drv = &ring->fence_drv;416 struct dma_fence *fence;417 uint32_t last_seq, sync_seq;418 419 last_seq = atomic_read(&ring->fence_drv.last_seq);420 sync_seq = READ_ONCE(ring->fence_drv.sync_seq);421 if (last_seq == sync_seq)422 return 0;423 424 ++last_seq;425 last_seq &= drv->num_fences_mask;426 fence = drv->fences[last_seq];427 if (!fence)428 return 0;429 430 return ktime_us_delta(ktime_get(),431 to_amdgpu_fence(fence)->start_timestamp);432}433 434/**435 * amdgpu_fence_update_start_timestamp - update the timestamp of the fence436 * @ring: ring the fence is associated with437 * @seq: the fence seq number to update.438 * @timestamp: the start timestamp to update.439 *440 * The function called at the time the fence and related ib is about to441 * resubmit to gpu in MCBP scenario. Thus we do not consider race condition442 * with amdgpu_fence_process to modify the same fence.443 */444void amdgpu_fence_update_start_timestamp(struct amdgpu_ring *ring, uint32_t seq, ktime_t timestamp)445{446 struct amdgpu_fence_driver *drv = &ring->fence_drv;447 struct dma_fence *fence;448 449 seq &= drv->num_fences_mask;450 fence = drv->fences[seq];451 if (!fence)452 return;453 454 to_amdgpu_fence(fence)->start_timestamp = timestamp;455}456 457/**458 * amdgpu_fence_driver_start_ring - make the fence driver459 * ready for use on the requested ring.460 *461 * @ring: ring to start the fence driver on462 * @irq_src: interrupt source to use for this ring463 * @irq_type: interrupt type to use for this ring464 *465 * Make the fence driver ready for processing (all asics).466 * Not all asics have all rings, so each asic will only467 * start the fence driver on the rings it has.468 * Returns 0 for success, errors for failure.469 */470int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,471 struct amdgpu_irq_src *irq_src,472 unsigned int irq_type)473{474 struct amdgpu_device *adev = ring->adev;475 uint64_t index;476 477 if (ring->funcs->type != AMDGPU_RING_TYPE_UVD) {478 ring->fence_drv.cpu_addr = ring->fence_cpu_addr;479 ring->fence_drv.gpu_addr = ring->fence_gpu_addr;480 } else {481 /* put fence directly behind firmware */482 index = ALIGN(adev->uvd.fw->size, 8);483 ring->fence_drv.cpu_addr = adev->uvd.inst[ring->me].cpu_addr + index;484 ring->fence_drv.gpu_addr = adev->uvd.inst[ring->me].gpu_addr + index;485 }486 amdgpu_fence_write(ring, atomic_read(&ring->fence_drv.last_seq));487 488 ring->fence_drv.irq_src = irq_src;489 ring->fence_drv.irq_type = irq_type;490 ring->fence_drv.initialized = true;491 492 DRM_DEV_DEBUG(adev->dev, "fence driver on ring %s use gpu addr 0x%016llx\n",493 ring->name, ring->fence_drv.gpu_addr);494 return 0;495}496 497/**498 * amdgpu_fence_driver_init_ring - init the fence driver499 * for the requested ring.500 *501 * @ring: ring to init the fence driver on502 *503 * Init the fence driver for the requested ring (all asics).504 * Helper function for amdgpu_fence_driver_init().505 */506int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring)507{508 struct amdgpu_device *adev = ring->adev;509 510 if (!adev)511 return -EINVAL;512 513 if (!is_power_of_2(ring->num_hw_submission))514 return -EINVAL;515 516 ring->fence_drv.cpu_addr = NULL;517 ring->fence_drv.gpu_addr = 0;518 ring->fence_drv.sync_seq = 0;519 atomic_set(&ring->fence_drv.last_seq, 0);520 ring->fence_drv.initialized = false;521 522 timer_setup(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback, 0);523 524 ring->fence_drv.num_fences_mask = ring->num_hw_submission * 2 - 1;525 spin_lock_init(&ring->fence_drv.lock);526 ring->fence_drv.fences = kcalloc(ring->num_hw_submission * 2, sizeof(void *),527 GFP_KERNEL);528 529 if (!ring->fence_drv.fences)530 return -ENOMEM;531 532 return 0;533}534 535/**536 * amdgpu_fence_driver_sw_init - init the fence driver537 * for all possible rings.538 *539 * @adev: amdgpu device pointer540 *541 * Init the fence driver for all possible rings (all asics).542 * Not all asics have all rings, so each asic will only543 * start the fence driver on the rings it has using544 * amdgpu_fence_driver_start_ring().545 * Returns 0 for success.546 */547int amdgpu_fence_driver_sw_init(struct amdgpu_device *adev)548{549 return 0;550}551 552/**553 * amdgpu_fence_need_ring_interrupt_restore - helper function to check whether554 * fence driver interrupts need to be restored.555 *556 * @ring: ring that to be checked557 *558 * Interrupts for rings that belong to GFX IP don't need to be restored559 * when the target power state is s0ix.560 *561 * Return true if need to restore interrupts, false otherwise.562 */563static bool amdgpu_fence_need_ring_interrupt_restore(struct amdgpu_ring *ring)564{565 struct amdgpu_device *adev = ring->adev;566 bool is_gfx_power_domain = false;567 568 switch (ring->funcs->type) {569 case AMDGPU_RING_TYPE_SDMA:570 /* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */571 if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >=572 IP_VERSION(5, 0, 0))573 is_gfx_power_domain = true;574 break;575 case AMDGPU_RING_TYPE_GFX:576 case AMDGPU_RING_TYPE_COMPUTE:577 case AMDGPU_RING_TYPE_KIQ:578 case AMDGPU_RING_TYPE_MES:579 is_gfx_power_domain = true;580 break;581 default:582 break;583 }584 585 return !(adev->in_s0ix && is_gfx_power_domain);586}587 588/**589 * amdgpu_fence_driver_hw_fini - tear down the fence driver590 * for all possible rings.591 *592 * @adev: amdgpu device pointer593 *594 * Tear down the fence driver for all possible rings (all asics).595 */596void amdgpu_fence_driver_hw_fini(struct amdgpu_device *adev)597{598 int i, r;599 600 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {601 struct amdgpu_ring *ring = adev->rings[i];602 603 if (!ring || !ring->fence_drv.initialized)604 continue;605 606 /* You can't wait for HW to signal if it's gone */607 if (!drm_dev_is_unplugged(adev_to_drm(adev)))608 r = amdgpu_fence_wait_empty(ring);609 else610 r = -ENODEV;611 /* no need to trigger GPU reset as we are unloading */612 if (r)613 amdgpu_fence_driver_force_completion(ring);614 615 if (!drm_dev_is_unplugged(adev_to_drm(adev)) &&616 ring->fence_drv.irq_src &&617 amdgpu_fence_need_ring_interrupt_restore(ring))618 amdgpu_irq_put(adev, ring->fence_drv.irq_src,619 ring->fence_drv.irq_type);620 621 del_timer_sync(&ring->fence_drv.fallback_timer);622 }623}624 625/* Will either stop and flush handlers for amdgpu interrupt or reanble it */626void amdgpu_fence_driver_isr_toggle(struct amdgpu_device *adev, bool stop)627{628 int i;629 630 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {631 struct amdgpu_ring *ring = adev->rings[i];632 633 if (!ring || !ring->fence_drv.initialized || !ring->fence_drv.irq_src)634 continue;635 636 if (stop)637 disable_irq(adev->irq.irq);638 else639 enable_irq(adev->irq.irq);640 }641}642 643void amdgpu_fence_driver_sw_fini(struct amdgpu_device *adev)644{645 unsigned int i, j;646 647 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {648 struct amdgpu_ring *ring = adev->rings[i];649 650 if (!ring || !ring->fence_drv.initialized)651 continue;652 653 /*654 * Notice we check for sched.ops since there's some655 * override on the meaning of sched.ready by amdgpu.656 * The natural check would be sched.ready, which is657 * set as drm_sched_init() finishes...658 */659 if (ring->sched.ops)660 drm_sched_fini(&ring->sched);661 662 for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j)663 dma_fence_put(ring->fence_drv.fences[j]);664 kfree(ring->fence_drv.fences);665 ring->fence_drv.fences = NULL;666 ring->fence_drv.initialized = false;667 }668}669 670/**671 * amdgpu_fence_driver_hw_init - enable the fence driver672 * for all possible rings.673 *674 * @adev: amdgpu device pointer675 *676 * Enable the fence driver for all possible rings (all asics).677 * Not all asics have all rings, so each asic will only678 * start the fence driver on the rings it has using679 * amdgpu_fence_driver_start_ring().680 * Returns 0 for success.681 */682void amdgpu_fence_driver_hw_init(struct amdgpu_device *adev)683{684 int i;685 686 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {687 struct amdgpu_ring *ring = adev->rings[i];688 689 if (!ring || !ring->fence_drv.initialized)690 continue;691 692 /* enable the interrupt */693 if (ring->fence_drv.irq_src &&694 amdgpu_fence_need_ring_interrupt_restore(ring))695 amdgpu_irq_get(adev, ring->fence_drv.irq_src,696 ring->fence_drv.irq_type);697 }698}699 700/**701 * amdgpu_fence_driver_clear_job_fences - clear job embedded fences of ring702 *703 * @ring: fence of the ring to be cleared704 *705 */706void amdgpu_fence_driver_clear_job_fences(struct amdgpu_ring *ring)707{708 int i;709 struct dma_fence *old, **ptr;710 711 for (i = 0; i <= ring->fence_drv.num_fences_mask; i++) {712 ptr = &ring->fence_drv.fences[i];713 old = rcu_dereference_protected(*ptr, 1);714 if (old && old->ops == &amdgpu_job_fence_ops) {715 struct amdgpu_job *job;716 717 /* For non-scheduler bad job, i.e. failed ib test, we need to signal718 * it right here or we won't be able to track them in fence_drv719 * and they will remain unsignaled during sa_bo free.720 */721 job = container_of(old, struct amdgpu_job, hw_fence);722 if (!job->base.s_fence && !dma_fence_is_signaled(old))723 dma_fence_signal(old);724 RCU_INIT_POINTER(*ptr, NULL);725 dma_fence_put(old);726 }727 }728}729 730/**731 * amdgpu_fence_driver_set_error - set error code on fences732 * @ring: the ring which contains the fences733 * @error: the error code to set734 *735 * Set an error code to all the fences pending on the ring.736 */737void amdgpu_fence_driver_set_error(struct amdgpu_ring *ring, int error)738{739 struct amdgpu_fence_driver *drv = &ring->fence_drv;740 unsigned long flags;741 742 spin_lock_irqsave(&drv->lock, flags);743 for (unsigned int i = 0; i <= drv->num_fences_mask; ++i) {744 struct dma_fence *fence;745 746 fence = rcu_dereference_protected(drv->fences[i],747 lockdep_is_held(&drv->lock));748 if (fence && !dma_fence_is_signaled_locked(fence))749 dma_fence_set_error(fence, error);750 }751 spin_unlock_irqrestore(&drv->lock, flags);752}753 754/**755 * amdgpu_fence_driver_force_completion - force signal latest fence of ring756 *757 * @ring: fence of the ring to signal758 *759 */760void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring)761{762 amdgpu_fence_driver_set_error(ring, -ECANCELED);763 amdgpu_fence_write(ring, ring->fence_drv.sync_seq);764 amdgpu_fence_process(ring);765}766 767/*768 * Common fence implementation769 */770 771static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence)772{773 return "amdgpu";774}775 776static const char *amdgpu_fence_get_timeline_name(struct dma_fence *f)777{778 return (const char *)to_amdgpu_fence(f)->ring->name;779}780 781static const char *amdgpu_job_fence_get_timeline_name(struct dma_fence *f)782{783 struct amdgpu_job *job = container_of(f, struct amdgpu_job, hw_fence);784 785 return (const char *)to_amdgpu_ring(job->base.sched)->name;786}787 788/**789 * amdgpu_fence_enable_signaling - enable signalling on fence790 * @f: fence791 *792 * This function is called with fence_queue lock held, and adds a callback793 * to fence_queue that checks if this fence is signaled, and if so it794 * signals the fence and removes itself.795 */796static bool amdgpu_fence_enable_signaling(struct dma_fence *f)797{798 if (!timer_pending(&to_amdgpu_fence(f)->ring->fence_drv.fallback_timer))799 amdgpu_fence_schedule_fallback(to_amdgpu_fence(f)->ring);800 801 return true;802}803 804/**805 * amdgpu_job_fence_enable_signaling - enable signalling on job fence806 * @f: fence807 *808 * This is the simliar function with amdgpu_fence_enable_signaling above, it809 * only handles the job embedded fence.810 */811static bool amdgpu_job_fence_enable_signaling(struct dma_fence *f)812{813 struct amdgpu_job *job = container_of(f, struct amdgpu_job, hw_fence);814 815 if (!timer_pending(&to_amdgpu_ring(job->base.sched)->fence_drv.fallback_timer))816 amdgpu_fence_schedule_fallback(to_amdgpu_ring(job->base.sched));817 818 return true;819}820 821/**822 * amdgpu_fence_free - free up the fence memory823 *824 * @rcu: RCU callback head825 *826 * Free up the fence memory after the RCU grace period.827 */828static void amdgpu_fence_free(struct rcu_head *rcu)829{830 struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);831 832 /* free fence_slab if it's separated fence*/833 kmem_cache_free(amdgpu_fence_slab, to_amdgpu_fence(f));834}835 836/**837 * amdgpu_job_fence_free - free up the job with embedded fence838 *839 * @rcu: RCU callback head840 *841 * Free up the job with embedded fence after the RCU grace period.842 */843static void amdgpu_job_fence_free(struct rcu_head *rcu)844{845 struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);846 847 /* free job if fence has a parent job */848 kfree(container_of(f, struct amdgpu_job, hw_fence));849}850 851/**852 * amdgpu_fence_release - callback that fence can be freed853 *854 * @f: fence855 *856 * This function is called when the reference count becomes zero.857 * It just RCU schedules freeing up the fence.858 */859static void amdgpu_fence_release(struct dma_fence *f)860{861 call_rcu(&f->rcu, amdgpu_fence_free);862}863 864/**865 * amdgpu_job_fence_release - callback that job embedded fence can be freed866 *867 * @f: fence868 *869 * This is the simliar function with amdgpu_fence_release above, it870 * only handles the job embedded fence.871 */872static void amdgpu_job_fence_release(struct dma_fence *f)873{874 call_rcu(&f->rcu, amdgpu_job_fence_free);875}876 877static const struct dma_fence_ops amdgpu_fence_ops = {878 .get_driver_name = amdgpu_fence_get_driver_name,879 .get_timeline_name = amdgpu_fence_get_timeline_name,880 .enable_signaling = amdgpu_fence_enable_signaling,881 .release = amdgpu_fence_release,882};883 884static const struct dma_fence_ops amdgpu_job_fence_ops = {885 .get_driver_name = amdgpu_fence_get_driver_name,886 .get_timeline_name = amdgpu_job_fence_get_timeline_name,887 .enable_signaling = amdgpu_job_fence_enable_signaling,888 .release = amdgpu_job_fence_release,889};890 891/*892 * Fence debugfs893 */894#if defined(CONFIG_DEBUG_FS)895static int amdgpu_debugfs_fence_info_show(struct seq_file *m, void *unused)896{897 struct amdgpu_device *adev = m->private;898 int i;899 900 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {901 struct amdgpu_ring *ring = adev->rings[i];902 903 if (!ring || !ring->fence_drv.initialized)904 continue;905 906 amdgpu_fence_process(ring);907 908 seq_printf(m, "--- ring %d (%s) ---\n", i, ring->name);909 seq_printf(m, "Last signaled fence 0x%08x\n",910 atomic_read(&ring->fence_drv.last_seq));911 seq_printf(m, "Last emitted 0x%08x\n",912 ring->fence_drv.sync_seq);913 914 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX ||915 ring->funcs->type == AMDGPU_RING_TYPE_SDMA) {916 seq_printf(m, "Last signaled trailing fence 0x%08x\n",917 le32_to_cpu(*ring->trail_fence_cpu_addr));918 seq_printf(m, "Last emitted 0x%08x\n",919 ring->trail_seq);920 }921 922 if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)923 continue;924 925 /* set in CP_VMID_PREEMPT and preemption occurred */926 seq_printf(m, "Last preempted 0x%08x\n",927 le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));928 /* set in CP_VMID_RESET and reset occurred */929 seq_printf(m, "Last reset 0x%08x\n",930 le32_to_cpu(*(ring->fence_drv.cpu_addr + 4)));931 /* Both preemption and reset occurred */932 seq_printf(m, "Last both 0x%08x\n",933 le32_to_cpu(*(ring->fence_drv.cpu_addr + 6)));934 }935 return 0;936}937 938/*939 * amdgpu_debugfs_gpu_recover - manually trigger a gpu reset & recover940 *941 * Manually trigger a gpu reset at the next fence wait.942 */943static int gpu_recover_get(void *data, u64 *val)944{945 struct amdgpu_device *adev = (struct amdgpu_device *)data;946 struct drm_device *dev = adev_to_drm(adev);947 int r;948 949 r = pm_runtime_get_sync(dev->dev);950 if (r < 0) {951 pm_runtime_put_autosuspend(dev->dev);952 return 0;953 }954 955 if (amdgpu_reset_domain_schedule(adev->reset_domain, &adev->reset_work))956 flush_work(&adev->reset_work);957 958 *val = atomic_read(&adev->reset_domain->reset_res);959 960 pm_runtime_mark_last_busy(dev->dev);961 pm_runtime_put_autosuspend(dev->dev);962 963 return 0;964}965 966DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_fence_info);967DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_gpu_recover_fops, gpu_recover_get, NULL,968 "%lld\n");969 970static void amdgpu_debugfs_reset_work(struct work_struct *work)971{972 struct amdgpu_device *adev = container_of(work, struct amdgpu_device,973 reset_work);974 975 struct amdgpu_reset_context reset_context;976 977 memset(&reset_context, 0, sizeof(reset_context));978 979 reset_context.method = AMD_RESET_METHOD_NONE;980 reset_context.reset_req_dev = adev;981 reset_context.src = AMDGPU_RESET_SRC_USER;982 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);983 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);984 985 amdgpu_device_gpu_recover(adev, NULL, &reset_context);986}987 988#endif989 990void amdgpu_debugfs_fence_init(struct amdgpu_device *adev)991{992#if defined(CONFIG_DEBUG_FS)993 struct drm_minor *minor = adev_to_drm(adev)->primary;994 struct dentry *root = minor->debugfs_root;995 996 debugfs_create_file("amdgpu_fence_info", 0444, root, adev,997 &amdgpu_debugfs_fence_info_fops);998 999 if (!amdgpu_sriov_vf(adev)) {1000 1001 INIT_WORK(&adev->reset_work, amdgpu_debugfs_reset_work);1002 debugfs_create_file("amdgpu_gpu_recover", 0444, root, adev,1003 &amdgpu_debugfs_gpu_recover_fops);1004 }1005#endif1006}1007 1008