263 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2019 Intel Corporation4 */5 6#include <linux/workqueue.h>7 8#include "i915_drv.h" /* for_each_engine() */9#include "i915_request.h"10#include "intel_engine_heartbeat.h"11#include "intel_execlists_submission.h"12#include "intel_gt.h"13#include "intel_gt_pm.h"14#include "intel_gt_requests.h"15#include "intel_timeline.h"16 17static bool retire_requests(struct intel_timeline *tl)18{19 struct i915_request *rq, *rn;20 21 list_for_each_entry_safe(rq, rn, &tl->requests, link)22 if (!i915_request_retire(rq))23 return false;24 25 /* And check nothing new was submitted */26 return !i915_active_fence_isset(&tl->last_request);27}28 29static bool engine_active(const struct intel_engine_cs *engine)30{31 return !list_empty(&engine->kernel_context->timeline->requests);32}33 34static bool flush_submission(struct intel_gt *gt, long timeout)35{36 struct intel_engine_cs *engine;37 enum intel_engine_id id;38 bool active = false;39 40 if (!timeout)41 return false;42 43 if (!intel_gt_pm_is_awake(gt))44 return false;45 46 for_each_engine(engine, gt, id) {47 intel_engine_flush_submission(engine);48 49 /* Flush the background retirement and idle barriers */50 flush_work(&engine->retire_work);51 flush_delayed_work(&engine->wakeref.work);52 53 /* Is the idle barrier still outstanding? */54 active |= engine_active(engine);55 }56 57 return active;58}59 60static void engine_retire(struct work_struct *work)61{62 struct intel_engine_cs *engine =63 container_of(work, typeof(*engine), retire_work);64 struct intel_timeline *tl = xchg(&engine->retire, NULL);65 66 do {67 struct intel_timeline *next = xchg(&tl->retire, NULL);68 69 /*70 * Our goal here is to retire _idle_ timelines as soon as71 * possible (as they are idle, we do not expect userspace72 * to be cleaning up anytime soon).73 *74 * If the timeline is currently locked, either it is being75 * retired elsewhere or about to be!76 */77 if (mutex_trylock(&tl->mutex)) {78 retire_requests(tl);79 mutex_unlock(&tl->mutex);80 }81 intel_timeline_put(tl);82 83 GEM_BUG_ON(!next);84 tl = ptr_mask_bits(next, 1);85 } while (tl);86}87 88static bool add_retire(struct intel_engine_cs *engine,89 struct intel_timeline *tl)90{91#define STUB ((struct intel_timeline *)1)92 struct intel_timeline *first;93 94 /*95 * We open-code a llist here to include the additional tag [BIT(0)]96 * so that we know when the timeline is already on a97 * retirement queue: either this engine or another.98 */99 100 if (cmpxchg(&tl->retire, NULL, STUB)) /* already queued */101 return false;102 103 intel_timeline_get(tl);104 first = READ_ONCE(engine->retire);105 do106 tl->retire = ptr_pack_bits(first, 1, 1);107 while (!try_cmpxchg(&engine->retire, &first, tl));108 109 return !first;110}111 112void intel_engine_add_retire(struct intel_engine_cs *engine,113 struct intel_timeline *tl)114{115 /* We don't deal well with the engine disappearing beneath us */116 GEM_BUG_ON(intel_engine_is_virtual(engine));117 118 if (add_retire(engine, tl))119 queue_work(engine->i915->unordered_wq, &engine->retire_work);120}121 122void intel_engine_init_retire(struct intel_engine_cs *engine)123{124 INIT_WORK(&engine->retire_work, engine_retire);125}126 127void intel_engine_fini_retire(struct intel_engine_cs *engine)128{129 flush_work(&engine->retire_work);130 GEM_BUG_ON(engine->retire);131}132 133long intel_gt_retire_requests_timeout(struct intel_gt *gt, long timeout,134 long *remaining_timeout)135{136 struct intel_gt_timelines *timelines = >->timelines;137 struct intel_timeline *tl, *tn;138 unsigned long active_count = 0;139 LIST_HEAD(free);140 141 flush_submission(gt, timeout); /* kick the ksoftirqd tasklets */142 spin_lock(&timelines->lock);143 list_for_each_entry_safe(tl, tn, &timelines->active_list, link) {144 if (!mutex_trylock(&tl->mutex)) {145 active_count++; /* report busy to caller, try again? */146 continue;147 }148 149 intel_timeline_get(tl);150 GEM_BUG_ON(!atomic_read(&tl->active_count));151 atomic_inc(&tl->active_count); /* pin the list element */152 spin_unlock(&timelines->lock);153 154 if (timeout > 0) {155 struct dma_fence *fence;156 157 fence = i915_active_fence_get(&tl->last_request);158 if (fence) {159 mutex_unlock(&tl->mutex);160 161 timeout = dma_fence_wait_timeout(fence,162 true,163 timeout);164 dma_fence_put(fence);165 166 /* Retirement is best effort */167 if (!mutex_trylock(&tl->mutex)) {168 active_count++;169 goto out_active;170 }171 }172 }173 174 if (!retire_requests(tl))175 active_count++;176 mutex_unlock(&tl->mutex);177 178out_active: spin_lock(&timelines->lock);179 180 /* Resume list iteration after reacquiring spinlock */181 list_safe_reset_next(tl, tn, link);182 if (atomic_dec_and_test(&tl->active_count))183 list_del(&tl->link);184 185 /* Defer the final release to after the spinlock */186 if (refcount_dec_and_test(&tl->kref.refcount)) {187 GEM_BUG_ON(atomic_read(&tl->active_count));188 list_add(&tl->link, &free);189 }190 }191 spin_unlock(&timelines->lock);192 193 list_for_each_entry_safe(tl, tn, &free, link)194 __intel_timeline_free(&tl->kref);195 196 if (flush_submission(gt, timeout)) /* Wait, there's more! */197 active_count++;198 199 if (remaining_timeout)200 *remaining_timeout = timeout;201 202 return active_count ? timeout ?: -ETIME : 0;203}204 205static void retire_work_handler(struct work_struct *work)206{207 struct intel_gt *gt =208 container_of(work, typeof(*gt), requests.retire_work.work);209 210 queue_delayed_work(gt->i915->unordered_wq, >->requests.retire_work,211 round_jiffies_up_relative(HZ));212 intel_gt_retire_requests(gt);213}214 215void intel_gt_init_requests(struct intel_gt *gt)216{217 INIT_DELAYED_WORK(>->requests.retire_work, retire_work_handler);218}219 220void intel_gt_park_requests(struct intel_gt *gt)221{222 cancel_delayed_work(>->requests.retire_work);223}224 225void intel_gt_unpark_requests(struct intel_gt *gt)226{227 queue_delayed_work(gt->i915->unordered_wq, >->requests.retire_work,228 round_jiffies_up_relative(HZ));229}230 231void intel_gt_fini_requests(struct intel_gt *gt)232{233 /* Wait until the work is marked as finished before unloading! */234 cancel_delayed_work_sync(>->requests.retire_work);235 236 flush_work(>->watchdog.work);237}238 239void intel_gt_watchdog_work(struct work_struct *work)240{241 struct intel_gt *gt =242 container_of(work, typeof(*gt), watchdog.work);243 struct i915_request *rq, *rn;244 struct llist_node *first;245 246 first = llist_del_all(>->watchdog.list);247 if (!first)248 return;249 250 llist_for_each_entry_safe(rq, rn, first, watchdog.link) {251 if (!i915_request_completed(rq)) {252 struct dma_fence *f = &rq->fence;253 254 pr_notice("Fence expiration time out i915-%s:%s:%llx!\n",255 f->ops->get_driver_name(f),256 f->ops->get_timeline_name(f),257 f->seqno);258 i915_request_cancel(rq, -EINTR);259 }260 i915_request_put(rq);261 }262}263