296 lines · c
1/*2 * SPDX-License-Identifier: MIT3 *4 * Copyright © 2016 Intel Corporation5 */6 7#include <linux/dma-fence-array.h>8#include <linux/dma-fence-chain.h>9#include <linux/jiffies.h>10 11#include "gt/intel_engine.h"12#include "gt/intel_rps.h"13 14#include "i915_gem_ioctls.h"15#include "i915_gem_object.h"16 17static long18i915_gem_object_wait_fence(struct dma_fence *fence,19 unsigned int flags,20 long timeout)21{22 BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1);23 24 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))25 return timeout;26 27 if (dma_fence_is_i915(fence))28 return i915_request_wait_timeout(to_request(fence), flags, timeout);29 30 return dma_fence_wait_timeout(fence,31 flags & I915_WAIT_INTERRUPTIBLE,32 timeout);33}34 35static void36i915_gem_object_boost(struct dma_resv *resv, unsigned int flags)37{38 struct dma_resv_iter cursor;39 struct dma_fence *fence;40 41 /*42 * Prescan all fences for potential boosting before we begin waiting.43 *44 * When we wait, we wait on outstanding fences serially. If the45 * dma-resv contains a sequence such as 1:1, 1:2 instead of a reduced46 * form 1:2, then as we look at each wait in turn we see that each47 * request is currently executing and not worthy of boosting. But if48 * we only happen to look at the final fence in the sequence (because49 * of request coalescing or splitting between read/write arrays by50 * the iterator), then we would boost. As such our decision to boost51 * or not is delicately balanced on the order we wait on fences.52 *53 * So instead of looking for boosts sequentially, look for all boosts54 * upfront and then wait on the outstanding fences.55 */56 57 dma_resv_iter_begin(&cursor, resv,58 dma_resv_usage_rw(flags & I915_WAIT_ALL));59 dma_resv_for_each_fence_unlocked(&cursor, fence)60 if (dma_fence_is_i915(fence) &&61 !i915_request_started(to_request(fence)))62 intel_rps_boost(to_request(fence));63 dma_resv_iter_end(&cursor);64}65 66static long67i915_gem_object_wait_reservation(struct dma_resv *resv,68 unsigned int flags,69 long timeout)70{71 struct dma_resv_iter cursor;72 struct dma_fence *fence;73 long ret = timeout ?: 1;74 75 i915_gem_object_boost(resv, flags);76 77 dma_resv_iter_begin(&cursor, resv,78 dma_resv_usage_rw(flags & I915_WAIT_ALL));79 dma_resv_for_each_fence_unlocked(&cursor, fence) {80 ret = i915_gem_object_wait_fence(fence, flags, timeout);81 if (ret <= 0)82 break;83 84 if (timeout)85 timeout = ret;86 }87 dma_resv_iter_end(&cursor);88 89 return ret;90}91 92static void fence_set_priority(struct dma_fence *fence,93 const struct i915_sched_attr *attr)94{95 struct i915_request *rq;96 struct intel_engine_cs *engine;97 98 if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence))99 return;100 101 rq = to_request(fence);102 engine = rq->engine;103 104 rcu_read_lock(); /* RCU serialisation for set-wedged protection */105 if (engine->sched_engine->schedule)106 engine->sched_engine->schedule(rq, attr);107 rcu_read_unlock();108}109 110static inline bool __dma_fence_is_chain(const struct dma_fence *fence)111{112 return fence->ops == &dma_fence_chain_ops;113}114 115void i915_gem_fence_wait_priority(struct dma_fence *fence,116 const struct i915_sched_attr *attr)117{118 if (dma_fence_is_signaled(fence))119 return;120 121 local_bh_disable();122 123 /* Recurse once into a fence-array */124 if (dma_fence_is_array(fence)) {125 struct dma_fence_array *array = to_dma_fence_array(fence);126 int i;127 128 for (i = 0; i < array->num_fences; i++)129 fence_set_priority(array->fences[i], attr);130 } else if (__dma_fence_is_chain(fence)) {131 struct dma_fence *iter;132 133 /* The chain is ordered; if we boost the last, we boost all */134 dma_fence_chain_for_each(iter, fence) {135 fence_set_priority(to_dma_fence_chain(iter)->fence,136 attr);137 break;138 }139 dma_fence_put(iter);140 } else {141 fence_set_priority(fence, attr);142 }143 144 local_bh_enable(); /* kick the tasklets if queues were reprioritised */145}146 147int148i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,149 unsigned int flags,150 const struct i915_sched_attr *attr)151{152 struct dma_resv_iter cursor;153 struct dma_fence *fence;154 155 dma_resv_iter_begin(&cursor, obj->base.resv,156 dma_resv_usage_rw(flags & I915_WAIT_ALL));157 dma_resv_for_each_fence_unlocked(&cursor, fence)158 i915_gem_fence_wait_priority(fence, attr);159 dma_resv_iter_end(&cursor);160 return 0;161}162 163/**164 * i915_gem_object_wait - Waits for rendering to the object to be completed165 * @obj: i915 gem object166 * @flags: how to wait (under a lock, for all rendering or just for writes etc)167 * @timeout: how long to wait168 */169int170i915_gem_object_wait(struct drm_i915_gem_object *obj,171 unsigned int flags,172 long timeout)173{174 might_sleep();175 GEM_BUG_ON(timeout < 0);176 177 timeout = i915_gem_object_wait_reservation(obj->base.resv,178 flags, timeout);179 180 if (timeout < 0)181 return timeout;182 183 return !timeout ? -ETIME : 0;184}185 186static inline unsigned long nsecs_to_jiffies_timeout(const u64 n)187{188 /* nsecs_to_jiffies64() does not guard against overflow */189 if ((NSEC_PER_SEC % HZ) != 0 &&190 div_u64(n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ)191 return MAX_JIFFY_OFFSET;192 193 return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1);194}195 196static unsigned long to_wait_timeout(s64 timeout_ns)197{198 if (timeout_ns < 0)199 return MAX_SCHEDULE_TIMEOUT;200 201 if (timeout_ns == 0)202 return 0;203 204 return nsecs_to_jiffies_timeout(timeout_ns);205}206 207/**208 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT209 * @dev: drm device pointer210 * @data: ioctl data blob211 * @file: drm file pointer212 *213 * Returns 0 if successful, else an error is returned with the remaining time in214 * the timeout parameter.215 * -ETIME: object is still busy after timeout216 * -ERESTARTSYS: signal interrupted the wait217 * -ENONENT: object doesn't exist218 * Also possible, but rare:219 * -EAGAIN: incomplete, restart syscall220 * -ENOMEM: damn221 * -ENODEV: Internal IRQ fail222 * -E?: The add request failed223 *224 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any225 * non-zero timeout parameter the wait ioctl will wait for the given number of226 * nanoseconds on an object becoming unbusy. Since the wait itself does so227 * without holding struct_mutex the object may become re-busied before this228 * function completes. A similar but shorter * race condition exists in the busy229 * ioctl230 */231int232i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)233{234 struct drm_i915_gem_wait *args = data;235 struct drm_i915_gem_object *obj;236 ktime_t start;237 long ret;238 239 if (args->flags != 0)240 return -EINVAL;241 242 obj = i915_gem_object_lookup(file, args->bo_handle);243 if (!obj)244 return -ENOENT;245 246 start = ktime_get();247 248 ret = i915_gem_object_wait(obj,249 I915_WAIT_INTERRUPTIBLE |250 I915_WAIT_PRIORITY |251 I915_WAIT_ALL,252 to_wait_timeout(args->timeout_ns));253 254 if (args->timeout_ns > 0) {255 args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start));256 if (args->timeout_ns < 0)257 args->timeout_ns = 0;258 259 /*260 * Apparently ktime isn't accurate enough and occasionally has a261 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch262 * things up to make the test happy. We allow up to 1 jiffy.263 *264 * This is a regression from the timespec->ktime conversion.265 */266 if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns))267 args->timeout_ns = 0;268 269 /* Asked to wait beyond the jiffy/scheduler precision? */270 if (ret == -ETIME && args->timeout_ns)271 ret = -EAGAIN;272 }273 274 i915_gem_object_put(obj);275 return ret;276}277 278/**279 * i915_gem_object_wait_migration - Sync an accelerated migration operation280 * @obj: The migrating object.281 * @flags: waiting flags. Currently supports only I915_WAIT_INTERRUPTIBLE.282 *283 * Wait for any pending async migration operation on the object,284 * whether it's explicitly (i915_gem_object_migrate()) or implicitly285 * (swapin, initial clearing) initiated.286 *287 * Return: 0 if successful, -ERESTARTSYS if a signal was hit during waiting.288 */289int i915_gem_object_wait_migration(struct drm_i915_gem_object *obj,290 unsigned int flags)291{292 might_sleep();293 294 return i915_gem_object_wait_moving_fence(obj, !!(flags & I915_WAIT_INTERRUPTIBLE));295}296