338 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2019 Intel Corporation4 */5 6#include "gem/i915_gem_internal.h"7#include "gem/i915_gem_lmem.h"8#include "gem/i915_gem_object.h"9 10#include "i915_drv.h"11#include "i915_vma.h"12#include "intel_engine.h"13#include "intel_engine_regs.h"14#include "intel_gpu_commands.h"15#include "intel_ring.h"16#include "intel_gt.h"17#include "intel_timeline.h"18 19unsigned int intel_ring_update_space(struct intel_ring *ring)20{21 unsigned int space;22 23 space = __intel_ring_space(ring->head, ring->emit, ring->size);24 25 ring->space = space;26 return space;27}28 29void __intel_ring_pin(struct intel_ring *ring)30{31 GEM_BUG_ON(!atomic_read(&ring->pin_count));32 atomic_inc(&ring->pin_count);33}34 35int intel_ring_pin(struct intel_ring *ring, struct i915_gem_ww_ctx *ww)36{37 struct i915_vma *vma = ring->vma;38 unsigned int flags;39 void *addr;40 int ret;41 42 if (atomic_fetch_inc(&ring->pin_count))43 return 0;44 45 /* Ring wraparound at offset 0 sometimes hangs. No idea why. */46 flags = PIN_OFFSET_BIAS | i915_ggtt_pin_bias(vma);47 48 if (i915_gem_object_is_stolen(vma->obj))49 flags |= PIN_MAPPABLE;50 else51 flags |= PIN_HIGH;52 53 ret = i915_ggtt_pin(vma, ww, 0, flags);54 if (unlikely(ret))55 goto err_unpin;56 57 if (i915_vma_is_map_and_fenceable(vma) && !HAS_LLC(vma->vm->i915)) {58 addr = (void __force *)i915_vma_pin_iomap(vma);59 } else {60 int type = intel_gt_coherent_map_type(vma->vm->gt, vma->obj, false);61 62 addr = i915_gem_object_pin_map(vma->obj, type);63 }64 65 if (IS_ERR(addr)) {66 ret = PTR_ERR(addr);67 goto err_ring;68 }69 70 i915_vma_make_unshrinkable(vma);71 72 /* Discard any unused bytes beyond that submitted to hw. */73 intel_ring_reset(ring, ring->emit);74 75 ring->vaddr = addr;76 return 0;77 78err_ring:79 i915_vma_unpin(vma);80err_unpin:81 atomic_dec(&ring->pin_count);82 return ret;83}84 85void intel_ring_reset(struct intel_ring *ring, u32 tail)86{87 tail = intel_ring_wrap(ring, tail);88 ring->tail = tail;89 ring->head = tail;90 ring->emit = tail;91 intel_ring_update_space(ring);92}93 94void intel_ring_unpin(struct intel_ring *ring)95{96 struct i915_vma *vma = ring->vma;97 98 if (!atomic_dec_and_test(&ring->pin_count))99 return;100 101 i915_vma_unset_ggtt_write(vma);102 if (i915_vma_is_map_and_fenceable(vma) && !HAS_LLC(vma->vm->i915))103 i915_vma_unpin_iomap(vma);104 else105 i915_gem_object_unpin_map(vma->obj);106 107 i915_vma_make_purgeable(vma);108 i915_vma_unpin(vma);109}110 111static struct i915_vma *create_ring_vma(struct i915_ggtt *ggtt, int size)112{113 struct i915_address_space *vm = &ggtt->vm;114 struct drm_i915_private *i915 = vm->i915;115 struct drm_i915_gem_object *obj;116 struct i915_vma *vma;117 118 obj = i915_gem_object_create_lmem(i915, size, I915_BO_ALLOC_VOLATILE |119 I915_BO_ALLOC_PM_VOLATILE);120 if (IS_ERR(obj) && i915_ggtt_has_aperture(ggtt) && !HAS_LLC(i915))121 obj = i915_gem_object_create_stolen(i915, size);122 if (IS_ERR(obj))123 obj = i915_gem_object_create_internal(i915, size);124 if (IS_ERR(obj))125 return ERR_CAST(obj);126 127 /*128 * Mark ring buffers as read-only from GPU side (so no stray overwrites)129 * if supported by the platform's GGTT.130 */131 if (vm->has_read_only)132 i915_gem_object_set_readonly(obj);133 134 vma = i915_vma_instance(obj, vm, NULL);135 if (IS_ERR(vma))136 goto err;137 138 return vma;139 140err:141 i915_gem_object_put(obj);142 return vma;143}144 145struct intel_ring *146intel_engine_create_ring(struct intel_engine_cs *engine, int size)147{148 struct drm_i915_private *i915 = engine->i915;149 struct intel_ring *ring;150 struct i915_vma *vma;151 152 GEM_BUG_ON(!is_power_of_2(size));153 GEM_BUG_ON(RING_CTL_SIZE(size) & ~RING_NR_PAGES);154 155 ring = kzalloc(sizeof(*ring), GFP_KERNEL);156 if (!ring)157 return ERR_PTR(-ENOMEM);158 159 kref_init(&ring->ref);160 ring->size = size;161 ring->wrap = BITS_PER_TYPE(ring->size) - ilog2(size);162 163 /*164 * Workaround an erratum on the i830 which causes a hang if165 * the TAIL pointer points to within the last 2 cachelines166 * of the buffer.167 */168 ring->effective_size = size;169 if (IS_I830(i915) || IS_I845G(i915))170 ring->effective_size -= 2 * CACHELINE_BYTES;171 172 intel_ring_update_space(ring);173 174 vma = create_ring_vma(engine->gt->ggtt, size);175 if (IS_ERR(vma)) {176 kfree(ring);177 return ERR_CAST(vma);178 }179 ring->vma = vma;180 181 return ring;182}183 184void intel_ring_free(struct kref *ref)185{186 struct intel_ring *ring = container_of(ref, typeof(*ring), ref);187 188 i915_vma_put(ring->vma);189 kfree(ring);190}191 192static noinline int193wait_for_space(struct intel_ring *ring,194 struct intel_timeline *tl,195 unsigned int bytes)196{197 struct i915_request *target;198 long timeout;199 200 if (intel_ring_update_space(ring) >= bytes)201 return 0;202 203 GEM_BUG_ON(list_empty(&tl->requests));204 list_for_each_entry(target, &tl->requests, link) {205 if (target->ring != ring)206 continue;207 208 /* Would completion of this request free enough space? */209 if (bytes <= __intel_ring_space(target->postfix,210 ring->emit, ring->size))211 break;212 }213 214 if (GEM_WARN_ON(&target->link == &tl->requests))215 return -ENOSPC;216 217 timeout = i915_request_wait(target,218 I915_WAIT_INTERRUPTIBLE,219 MAX_SCHEDULE_TIMEOUT);220 if (timeout < 0)221 return timeout;222 223 i915_request_retire_upto(target);224 225 intel_ring_update_space(ring);226 GEM_BUG_ON(ring->space < bytes);227 return 0;228}229 230u32 *intel_ring_begin(struct i915_request *rq, unsigned int num_dwords)231{232 struct intel_ring *ring = rq->ring;233 const unsigned int remain_usable = ring->effective_size - ring->emit;234 const unsigned int bytes = num_dwords * sizeof(u32);235 unsigned int need_wrap = 0;236 unsigned int total_bytes;237 u32 *cs;238 239 /* Packets must be qword aligned. */240 GEM_BUG_ON(num_dwords & 1);241 242 total_bytes = bytes + rq->reserved_space;243 GEM_BUG_ON(total_bytes > ring->effective_size);244 245 if (unlikely(total_bytes > remain_usable)) {246 const int remain_actual = ring->size - ring->emit;247 248 if (bytes > remain_usable) {249 /*250 * Not enough space for the basic request. So need to251 * flush out the remainder and then wait for252 * base + reserved.253 */254 total_bytes += remain_actual;255 need_wrap = remain_actual | 1;256 } else {257 /*258 * The base request will fit but the reserved space259 * falls off the end. So we don't need an immediate260 * wrap and only need to effectively wait for the261 * reserved size from the start of ringbuffer.262 */263 total_bytes = rq->reserved_space + remain_actual;264 }265 }266 267 if (unlikely(total_bytes > ring->space)) {268 int ret;269 270 /*271 * Space is reserved in the ringbuffer for finalising the272 * request, as that cannot be allowed to fail. During request273 * finalisation, reserved_space is set to 0 to stop the274 * overallocation and the assumption is that then we never need275 * to wait (which has the risk of failing with EINTR).276 *277 * See also i915_request_alloc() and i915_request_add().278 */279 GEM_BUG_ON(!rq->reserved_space);280 281 ret = wait_for_space(ring,282 i915_request_timeline(rq),283 total_bytes);284 if (unlikely(ret))285 return ERR_PTR(ret);286 }287 288 if (unlikely(need_wrap)) {289 need_wrap &= ~1;290 GEM_BUG_ON(need_wrap > ring->space);291 GEM_BUG_ON(ring->emit + need_wrap > ring->size);292 GEM_BUG_ON(!IS_ALIGNED(need_wrap, sizeof(u64)));293 294 /* Fill the tail with MI_NOOP */295 memset64(ring->vaddr + ring->emit, 0, need_wrap / sizeof(u64));296 ring->space -= need_wrap;297 ring->emit = 0;298 }299 300 GEM_BUG_ON(ring->emit > ring->size - bytes);301 GEM_BUG_ON(ring->space < bytes);302 cs = ring->vaddr + ring->emit;303 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))304 memset32(cs, POISON_INUSE, bytes / sizeof(*cs));305 ring->emit += bytes;306 ring->space -= bytes;307 308 return cs;309}310 311/* Align the ring tail to a cacheline boundary */312int intel_ring_cacheline_align(struct i915_request *rq)313{314 int num_dwords;315 void *cs;316 317 num_dwords = (rq->ring->emit & (CACHELINE_BYTES - 1)) / sizeof(u32);318 if (num_dwords == 0)319 return 0;320 321 num_dwords = CACHELINE_DWORDS - num_dwords;322 GEM_BUG_ON(num_dwords & 1);323 324 cs = intel_ring_begin(rq, num_dwords);325 if (IS_ERR(cs))326 return PTR_ERR(cs);327 328 memset64(cs, (u64)MI_NOOP << 32 | MI_NOOP, num_dwords / 2);329 intel_ring_advance(rq, cs + num_dwords);330 331 GEM_BUG_ON(rq->ring->emit & (CACHELINE_BYTES - 1));332 return 0;333}334 335#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)336#include "selftest_ring.c"337#endif338