435 lines · c
1/*2 * SPDX-License-Identifier: MIT3 *4 * Copyright © 2017 Intel Corporation5 */6 7#include <linux/prime_numbers.h>8 9#include "gt/intel_engine_pm.h"10#include "gt/intel_gpu_commands.h"11#include "gt/intel_gt.h"12#include "gt/intel_gt_pm.h"13#include "gt/intel_ring.h"14 15#include "i915_selftest.h"16#include "selftests/i915_random.h"17 18struct context {19 struct drm_i915_gem_object *obj;20 struct intel_engine_cs *engine;21};22 23static int cpu_set(struct context *ctx, unsigned long offset, u32 v)24{25 unsigned int needs_clflush;26 struct page *page;27 u32 *cpu;28 int err;29 30 i915_gem_object_lock(ctx->obj, NULL);31 err = i915_gem_object_prepare_write(ctx->obj, &needs_clflush);32 if (err)33 goto out;34 35 page = i915_gem_object_get_page(ctx->obj, offset >> PAGE_SHIFT);36 cpu = kmap_local_page(page) + offset_in_page(offset);37 38 if (needs_clflush & CLFLUSH_BEFORE)39 drm_clflush_virt_range(cpu, sizeof(*cpu));40 41 *cpu = v;42 43 if (needs_clflush & CLFLUSH_AFTER)44 drm_clflush_virt_range(cpu, sizeof(*cpu));45 46 kunmap_local(cpu);47 i915_gem_object_finish_access(ctx->obj);48 49out:50 i915_gem_object_unlock(ctx->obj);51 return err;52}53 54static int cpu_get(struct context *ctx, unsigned long offset, u32 *v)55{56 unsigned int needs_clflush;57 struct page *page;58 u32 *cpu;59 int err;60 61 i915_gem_object_lock(ctx->obj, NULL);62 err = i915_gem_object_prepare_read(ctx->obj, &needs_clflush);63 if (err)64 goto out;65 66 page = i915_gem_object_get_page(ctx->obj, offset >> PAGE_SHIFT);67 cpu = kmap_local_page(page) + offset_in_page(offset);68 69 if (needs_clflush & CLFLUSH_BEFORE)70 drm_clflush_virt_range(cpu, sizeof(*cpu));71 72 *v = *cpu;73 74 kunmap_local(cpu);75 i915_gem_object_finish_access(ctx->obj);76 77out:78 i915_gem_object_unlock(ctx->obj);79 return err;80}81 82static int gtt_set(struct context *ctx, unsigned long offset, u32 v)83{84 intel_wakeref_t wakeref;85 struct i915_vma *vma;86 u32 __iomem *map;87 int err = 0;88 89 i915_gem_object_lock(ctx->obj, NULL);90 err = i915_gem_object_set_to_gtt_domain(ctx->obj, true);91 i915_gem_object_unlock(ctx->obj);92 if (err)93 return err;94 95 vma = i915_gem_object_ggtt_pin(ctx->obj, NULL, 0, 0, PIN_MAPPABLE);96 if (IS_ERR(vma))97 return PTR_ERR(vma);98 99 wakeref = intel_gt_pm_get(vma->vm->gt);100 101 map = i915_vma_pin_iomap(vma);102 i915_vma_unpin(vma);103 if (IS_ERR(map)) {104 err = PTR_ERR(map);105 goto out_rpm;106 }107 108 iowrite32(v, &map[offset / sizeof(*map)]);109 i915_vma_unpin_iomap(vma);110 111out_rpm:112 intel_gt_pm_put(vma->vm->gt, wakeref);113 return err;114}115 116static int gtt_get(struct context *ctx, unsigned long offset, u32 *v)117{118 intel_wakeref_t wakeref;119 struct i915_vma *vma;120 u32 __iomem *map;121 int err = 0;122 123 i915_gem_object_lock(ctx->obj, NULL);124 err = i915_gem_object_set_to_gtt_domain(ctx->obj, false);125 i915_gem_object_unlock(ctx->obj);126 if (err)127 return err;128 129 vma = i915_gem_object_ggtt_pin(ctx->obj, NULL, 0, 0, PIN_MAPPABLE);130 if (IS_ERR(vma))131 return PTR_ERR(vma);132 133 wakeref = intel_gt_pm_get(vma->vm->gt);134 135 map = i915_vma_pin_iomap(vma);136 i915_vma_unpin(vma);137 if (IS_ERR(map)) {138 err = PTR_ERR(map);139 goto out_rpm;140 }141 142 *v = ioread32(&map[offset / sizeof(*map)]);143 i915_vma_unpin_iomap(vma);144 145out_rpm:146 intel_gt_pm_put(vma->vm->gt, wakeref);147 return err;148}149 150static int wc_set(struct context *ctx, unsigned long offset, u32 v)151{152 u32 *map;153 int err;154 155 i915_gem_object_lock(ctx->obj, NULL);156 err = i915_gem_object_set_to_wc_domain(ctx->obj, true);157 i915_gem_object_unlock(ctx->obj);158 if (err)159 return err;160 161 map = i915_gem_object_pin_map_unlocked(ctx->obj, I915_MAP_WC);162 if (IS_ERR(map))163 return PTR_ERR(map);164 165 map[offset / sizeof(*map)] = v;166 167 __i915_gem_object_flush_map(ctx->obj, offset, sizeof(*map));168 i915_gem_object_unpin_map(ctx->obj);169 170 return 0;171}172 173static int wc_get(struct context *ctx, unsigned long offset, u32 *v)174{175 u32 *map;176 int err;177 178 i915_gem_object_lock(ctx->obj, NULL);179 err = i915_gem_object_set_to_wc_domain(ctx->obj, false);180 i915_gem_object_unlock(ctx->obj);181 if (err)182 return err;183 184 map = i915_gem_object_pin_map_unlocked(ctx->obj, I915_MAP_WC);185 if (IS_ERR(map))186 return PTR_ERR(map);187 188 *v = map[offset / sizeof(*map)];189 i915_gem_object_unpin_map(ctx->obj);190 191 return 0;192}193 194static int gpu_set(struct context *ctx, unsigned long offset, u32 v)195{196 struct i915_request *rq;197 struct i915_vma *vma;198 u32 *cs;199 int err;200 201 vma = i915_gem_object_ggtt_pin(ctx->obj, NULL, 0, 0, 0);202 if (IS_ERR(vma))203 return PTR_ERR(vma);204 205 i915_gem_object_lock(ctx->obj, NULL);206 err = i915_gem_object_set_to_gtt_domain(ctx->obj, true);207 if (err)208 goto out_unlock;209 210 rq = intel_engine_create_kernel_request(ctx->engine);211 if (IS_ERR(rq)) {212 err = PTR_ERR(rq);213 goto out_unpin;214 }215 216 cs = intel_ring_begin(rq, 4);217 if (IS_ERR(cs)) {218 err = PTR_ERR(cs);219 goto out_rq;220 }221 222 if (GRAPHICS_VER(ctx->engine->i915) >= 8) {223 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;224 *cs++ = lower_32_bits(i915_ggtt_offset(vma) + offset);225 *cs++ = upper_32_bits(i915_ggtt_offset(vma) + offset);226 *cs++ = v;227 } else if (GRAPHICS_VER(ctx->engine->i915) >= 4) {228 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;229 *cs++ = 0;230 *cs++ = i915_ggtt_offset(vma) + offset;231 *cs++ = v;232 } else {233 *cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;234 *cs++ = i915_ggtt_offset(vma) + offset;235 *cs++ = v;236 *cs++ = MI_NOOP;237 }238 intel_ring_advance(rq, cs);239 240 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);241 242out_rq:243 i915_request_add(rq);244out_unpin:245 i915_vma_unpin(vma);246out_unlock:247 i915_gem_object_unlock(ctx->obj);248 249 return err;250}251 252static bool always_valid(struct context *ctx)253{254 return true;255}256 257static bool needs_fence_registers(struct context *ctx)258{259 struct intel_gt *gt = ctx->engine->gt;260 261 if (intel_gt_is_wedged(gt))262 return false;263 264 return gt->ggtt->num_fences;265}266 267static bool needs_mi_store_dword(struct context *ctx)268{269 if (intel_gt_is_wedged(ctx->engine->gt))270 return false;271 272 return intel_engine_can_store_dword(ctx->engine);273}274 275static const struct igt_coherency_mode {276 const char *name;277 int (*set)(struct context *ctx, unsigned long offset, u32 v);278 int (*get)(struct context *ctx, unsigned long offset, u32 *v);279 bool (*valid)(struct context *ctx);280} igt_coherency_mode[] = {281 { "cpu", cpu_set, cpu_get, always_valid },282 { "gtt", gtt_set, gtt_get, needs_fence_registers },283 { "wc", wc_set, wc_get, always_valid },284 { "gpu", gpu_set, NULL, needs_mi_store_dword },285 { },286};287 288static struct intel_engine_cs *289random_engine(struct drm_i915_private *i915, struct rnd_state *prng)290{291 struct intel_engine_cs *engine;292 unsigned int count;293 294 count = 0;295 for_each_uabi_engine(engine, i915)296 count++;297 298 count = i915_prandom_u32_max_state(count, prng);299 for_each_uabi_engine(engine, i915)300 if (count-- == 0)301 return engine;302 303 return NULL;304}305 306static int igt_gem_coherency(void *arg)307{308 const unsigned int ncachelines = PAGE_SIZE/64;309 struct drm_i915_private *i915 = arg;310 const struct igt_coherency_mode *read, *write, *over;311 unsigned long count, n;312 u32 *offsets, *values;313 I915_RND_STATE(prng);314 struct context ctx;315 int err = 0;316 317 /*318 * We repeatedly write, overwrite and read from a sequence of319 * cachelines in order to try and detect incoherency (unflushed writes320 * from either the CPU or GPU). Each setter/getter uses our cache321 * domain API which should prevent incoherency.322 */323 324 offsets = kmalloc_array(ncachelines, 2*sizeof(u32), GFP_KERNEL);325 if (!offsets)326 return -ENOMEM;327 for (count = 0; count < ncachelines; count++)328 offsets[count] = count * 64 + 4 * (count % 16);329 330 values = offsets + ncachelines;331 332 ctx.engine = random_engine(i915, &prng);333 if (!ctx.engine) {334 err = -ENODEV;335 goto out_free;336 }337 pr_info("%s: using %s\n", __func__, ctx.engine->name);338 intel_engine_pm_get(ctx.engine);339 340 for (over = igt_coherency_mode; over->name; over++) {341 if (!over->set)342 continue;343 344 if (!over->valid(&ctx))345 continue;346 347 for (write = igt_coherency_mode; write->name; write++) {348 if (!write->set)349 continue;350 351 if (!write->valid(&ctx))352 continue;353 354 for (read = igt_coherency_mode; read->name; read++) {355 if (!read->get)356 continue;357 358 if (!read->valid(&ctx))359 continue;360 361 for_each_prime_number_from(count, 1, ncachelines) {362 ctx.obj = i915_gem_object_create_internal(i915, PAGE_SIZE);363 if (IS_ERR(ctx.obj)) {364 err = PTR_ERR(ctx.obj);365 goto out_pm;366 }367 368 i915_random_reorder(offsets, ncachelines, &prng);369 for (n = 0; n < count; n++)370 values[n] = prandom_u32_state(&prng);371 372 for (n = 0; n < count; n++) {373 err = over->set(&ctx, offsets[n], ~values[n]);374 if (err) {375 pr_err("Failed to set stale value[%ld/%ld] in object using %s, err=%d\n",376 n, count, over->name, err);377 goto put_object;378 }379 }380 381 for (n = 0; n < count; n++) {382 err = write->set(&ctx, offsets[n], values[n]);383 if (err) {384 pr_err("Failed to set value[%ld/%ld] in object using %s, err=%d\n",385 n, count, write->name, err);386 goto put_object;387 }388 }389 390 for (n = 0; n < count; n++) {391 u32 found;392 393 err = read->get(&ctx, offsets[n], &found);394 if (err) {395 pr_err("Failed to get value[%ld/%ld] in object using %s, err=%d\n",396 n, count, read->name, err);397 goto put_object;398 }399 400 if (found != values[n]) {401 pr_err("Value[%ld/%ld] mismatch, (overwrite with %s) wrote [%s] %x read [%s] %x (inverse %x), at offset %x\n",402 n, count, over->name,403 write->name, values[n],404 read->name, found,405 ~values[n], offsets[n]);406 err = -EINVAL;407 goto put_object;408 }409 }410 411 i915_gem_object_put(ctx.obj);412 }413 }414 }415 }416out_pm:417 intel_engine_pm_put(ctx.engine);418out_free:419 kfree(offsets);420 return err;421 422put_object:423 i915_gem_object_put(ctx.obj);424 goto out_pm;425}426 427int i915_gem_coherency_live_selftests(struct drm_i915_private *i915)428{429 static const struct i915_subtest tests[] = {430 SUBTEST(igt_gem_coherency),431 };432 433 return i915_live_subtests(tests, i915);434}435