572 lines · c
1/*2 * Copyright © 2016 Intel Corporation3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice (including the next12 * paragraph) shall be included in all copies or substantial portions of the13 * Software.14 *15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS21 * IN THE SOFTWARE.22 *23 */24 25#include "gem/i915_gem_internal.h"26#include "gem/i915_gem_pm.h"27#include "gem/selftests/igt_gem_utils.h"28#include "gem/selftests/mock_context.h"29#include "gt/intel_gt.h"30#include "gt/intel_gt_print.h"31 32#include "i915_selftest.h"33 34#include "igt_flush_test.h"35#include "lib_sw_fence.h"36#include "mock_drm.h"37#include "mock_gem_device.h"38 39static void quirk_add(struct drm_i915_gem_object *obj,40 struct list_head *objects)41{42 /* quirk is only for live tiled objects, use it to declare ownership */43 GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj));44 i915_gem_object_set_tiling_quirk(obj);45 list_add(&obj->st_link, objects);46}47 48static int populate_ggtt(struct i915_ggtt *ggtt, struct list_head *objects)49{50 struct drm_i915_gem_object *obj;51 unsigned long count;52 53 count = 0;54 do {55 struct i915_vma *vma;56 57 obj = i915_gem_object_create_internal(ggtt->vm.i915,58 I915_GTT_PAGE_SIZE);59 if (IS_ERR(obj))60 return PTR_ERR(obj);61 62 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);63 if (IS_ERR(vma)) {64 i915_gem_object_put(obj);65 if (vma == ERR_PTR(-ENOSPC))66 break;67 68 return PTR_ERR(vma);69 }70 71 quirk_add(obj, objects);72 count++;73 } while (1);74 pr_debug("Filled GGTT with %lu pages [%llu total]\n",75 count, ggtt->vm.total / PAGE_SIZE);76 77 if (list_empty(&ggtt->vm.bound_list)) {78 pr_err("No objects on the GGTT inactive list!\n");79 return -EINVAL;80 }81 82 return 0;83}84 85static void unpin_ggtt(struct i915_ggtt *ggtt)86{87 struct i915_vma *vma;88 89 list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)90 if (i915_gem_object_has_tiling_quirk(vma->obj))91 i915_vma_unpin(vma);92}93 94static void cleanup_objects(struct i915_ggtt *ggtt, struct list_head *list)95{96 struct drm_i915_gem_object *obj, *on;97 98 list_for_each_entry_safe(obj, on, list, st_link) {99 GEM_BUG_ON(!i915_gem_object_has_tiling_quirk(obj));100 i915_gem_object_set_tiling_quirk(obj);101 i915_gem_object_put(obj);102 }103 104 i915_gem_drain_freed_objects(ggtt->vm.i915);105}106 107static int igt_evict_something(void *arg)108{109 struct intel_gt *gt = arg;110 struct i915_ggtt *ggtt = gt->ggtt;111 LIST_HEAD(objects);112 int err;113 114 /* Fill the GGTT with pinned objects and try to evict one. */115 116 err = populate_ggtt(ggtt, &objects);117 if (err)118 goto cleanup;119 120 /* Everything is pinned, nothing should happen */121 mutex_lock(&ggtt->vm.mutex);122 err = i915_gem_evict_something(&ggtt->vm, NULL,123 I915_GTT_PAGE_SIZE, 0, 0,124 0, U64_MAX,125 0);126 mutex_unlock(&ggtt->vm.mutex);127 if (err != -ENOSPC) {128 pr_err("i915_gem_evict_something failed on a full GGTT with err=%d\n",129 err);130 goto cleanup;131 }132 133 unpin_ggtt(ggtt);134 135 /* Everything is unpinned, we should be able to evict something */136 mutex_lock(&ggtt->vm.mutex);137 err = i915_gem_evict_something(&ggtt->vm, NULL,138 I915_GTT_PAGE_SIZE, 0, 0,139 0, U64_MAX,140 0);141 mutex_unlock(&ggtt->vm.mutex);142 if (err) {143 pr_err("i915_gem_evict_something failed on a full GGTT with err=%d\n",144 err);145 goto cleanup;146 }147 148cleanup:149 cleanup_objects(ggtt, &objects);150 return err;151}152 153static int igt_overcommit(void *arg)154{155 struct intel_gt *gt = arg;156 struct i915_ggtt *ggtt = gt->ggtt;157 struct drm_i915_gem_object *obj;158 struct i915_vma *vma;159 LIST_HEAD(objects);160 int err;161 162 /* Fill the GGTT with pinned objects and then try to pin one more.163 * We expect it to fail.164 */165 166 err = populate_ggtt(ggtt, &objects);167 if (err)168 goto cleanup;169 170 obj = i915_gem_object_create_internal(gt->i915, I915_GTT_PAGE_SIZE);171 if (IS_ERR(obj)) {172 err = PTR_ERR(obj);173 goto cleanup;174 }175 176 quirk_add(obj, &objects);177 178 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);179 if (vma != ERR_PTR(-ENOSPC)) {180 pr_err("Failed to evict+insert, i915_gem_object_ggtt_pin returned err=%d\n", (int)PTR_ERR_OR_ZERO(vma));181 err = -EINVAL;182 goto cleanup;183 }184 185cleanup:186 cleanup_objects(ggtt, &objects);187 return err;188}189 190static int igt_evict_for_vma(void *arg)191{192 struct intel_gt *gt = arg;193 struct i915_ggtt *ggtt = gt->ggtt;194 struct drm_mm_node target = {195 .start = 0,196 .size = 4096,197 };198 LIST_HEAD(objects);199 int err;200 201 /* Fill the GGTT with pinned objects and try to evict a range. */202 203 err = populate_ggtt(ggtt, &objects);204 if (err)205 goto cleanup;206 207 /* Everything is pinned, nothing should happen */208 mutex_lock(&ggtt->vm.mutex);209 err = i915_gem_evict_for_node(&ggtt->vm, NULL, &target, 0);210 mutex_unlock(&ggtt->vm.mutex);211 if (err != -ENOSPC) {212 pr_err("i915_gem_evict_for_node on a full GGTT returned err=%d\n",213 err);214 goto cleanup;215 }216 217 unpin_ggtt(ggtt);218 219 /* Everything is unpinned, we should be able to evict the node */220 mutex_lock(&ggtt->vm.mutex);221 err = i915_gem_evict_for_node(&ggtt->vm, NULL, &target, 0);222 mutex_unlock(&ggtt->vm.mutex);223 if (err) {224 pr_err("i915_gem_evict_for_node returned err=%d\n",225 err);226 goto cleanup;227 }228 229cleanup:230 cleanup_objects(ggtt, &objects);231 return err;232}233 234static void mock_color_adjust(const struct drm_mm_node *node,235 unsigned long color,236 u64 *start,237 u64 *end)238{239}240 241static int igt_evict_for_cache_color(void *arg)242{243 struct intel_gt *gt = arg;244 struct i915_ggtt *ggtt = gt->ggtt;245 const unsigned long flags = PIN_OFFSET_FIXED;246 struct drm_mm_node target = {247 .start = I915_GTT_PAGE_SIZE * 2,248 .size = I915_GTT_PAGE_SIZE,249 .color = i915_gem_get_pat_index(gt->i915, I915_CACHE_LLC),250 };251 struct drm_i915_gem_object *obj;252 struct i915_vma *vma;253 LIST_HEAD(objects);254 int err;255 256 /*257 * Currently the use of color_adjust for the GGTT is limited to cache258 * coloring and guard pages, and so the presence of mm.color_adjust for259 * the GGTT is assumed to be i915_ggtt_color_adjust, hence using a mock260 * color adjust will work just fine for our purposes.261 */262 ggtt->vm.mm.color_adjust = mock_color_adjust;263 GEM_BUG_ON(!i915_vm_has_cache_coloring(&ggtt->vm));264 265 obj = i915_gem_object_create_internal(gt->i915, I915_GTT_PAGE_SIZE);266 if (IS_ERR(obj)) {267 err = PTR_ERR(obj);268 goto cleanup;269 }270 i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);271 quirk_add(obj, &objects);272 273 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,274 I915_GTT_PAGE_SIZE | flags);275 if (IS_ERR(vma)) {276 pr_err("[0]i915_gem_object_ggtt_pin failed\n");277 err = PTR_ERR(vma);278 goto cleanup;279 }280 281 obj = i915_gem_object_create_internal(gt->i915, I915_GTT_PAGE_SIZE);282 if (IS_ERR(obj)) {283 err = PTR_ERR(obj);284 goto cleanup;285 }286 i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);287 quirk_add(obj, &objects);288 289 /* Neighbouring; same colour - should fit */290 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,291 (I915_GTT_PAGE_SIZE * 2) | flags);292 if (IS_ERR(vma)) {293 pr_err("[1]i915_gem_object_ggtt_pin failed\n");294 err = PTR_ERR(vma);295 goto cleanup;296 }297 298 i915_vma_unpin(vma);299 300 /* Remove just the second vma */301 mutex_lock(&ggtt->vm.mutex);302 err = i915_gem_evict_for_node(&ggtt->vm, NULL, &target, 0);303 mutex_unlock(&ggtt->vm.mutex);304 if (err) {305 pr_err("[0]i915_gem_evict_for_node returned err=%d\n", err);306 goto cleanup;307 }308 309 /* Attempt to remove the first *pinned* vma, by removing the (empty)310 * neighbour -- this should fail.311 */312 target.color = i915_gem_get_pat_index(gt->i915, I915_CACHE_L3_LLC);313 314 mutex_lock(&ggtt->vm.mutex);315 err = i915_gem_evict_for_node(&ggtt->vm, NULL, &target, 0);316 mutex_unlock(&ggtt->vm.mutex);317 if (!err) {318 pr_err("[1]i915_gem_evict_for_node returned err=%d\n", err);319 err = -EINVAL;320 goto cleanup;321 }322 323 err = 0;324 325cleanup:326 unpin_ggtt(ggtt);327 cleanup_objects(ggtt, &objects);328 ggtt->vm.mm.color_adjust = NULL;329 return err;330}331 332static int igt_evict_vm(void *arg)333{334 struct intel_gt *gt = arg;335 struct i915_ggtt *ggtt = gt->ggtt;336 struct i915_gem_ww_ctx ww;337 LIST_HEAD(objects);338 int err;339 340 /* Fill the GGTT with pinned objects and try to evict everything. */341 342 err = populate_ggtt(ggtt, &objects);343 if (err)344 goto cleanup;345 346 /* Everything is pinned, nothing should happen */347 mutex_lock(&ggtt->vm.mutex);348 err = i915_gem_evict_vm(&ggtt->vm, NULL, NULL);349 mutex_unlock(&ggtt->vm.mutex);350 if (err) {351 pr_err("i915_gem_evict_vm on a full GGTT returned err=%d]\n",352 err);353 goto cleanup;354 }355 356 unpin_ggtt(ggtt);357 358 for_i915_gem_ww(&ww, err, false) {359 mutex_lock(&ggtt->vm.mutex);360 err = i915_gem_evict_vm(&ggtt->vm, &ww, NULL);361 mutex_unlock(&ggtt->vm.mutex);362 }363 364 if (err) {365 pr_err("i915_gem_evict_vm on a full GGTT returned err=%d]\n",366 err);367 goto cleanup;368 }369 370cleanup:371 cleanup_objects(ggtt, &objects);372 return err;373}374 375static int igt_evict_contexts(void *arg)376{377 const u64 PRETEND_GGTT_SIZE = 16ull << 20;378 struct intel_gt *gt = arg;379 struct i915_ggtt *ggtt = gt->ggtt;380 struct drm_i915_private *i915 = gt->i915;381 struct intel_engine_cs *engine;382 enum intel_engine_id id;383 struct reserved {384 struct drm_mm_node node;385 struct reserved *next;386 } *reserved = NULL;387 intel_wakeref_t wakeref;388 struct drm_mm_node hole;389 unsigned long count;390 int err;391 392 /*393 * The purpose of this test is to verify that we will trigger an394 * eviction in the GGTT when constructing a request that requires395 * additional space in the GGTT for pinning the context. This space396 * is not directly tied to the request so reclaiming it requires397 * extra work.398 *399 * As such this test is only meaningful for full-ppgtt environments400 * where the GTT space of the request is separate from the GGTT401 * allocation required to build the request.402 */403 if (!HAS_FULL_PPGTT(i915))404 return 0;405 406 wakeref = intel_runtime_pm_get(&i915->runtime_pm);407 408 /* Reserve a block so that we know we have enough to fit a few rq */409 memset(&hole, 0, sizeof(hole));410 mutex_lock(&ggtt->vm.mutex);411 err = i915_gem_gtt_insert(&ggtt->vm, NULL, &hole,412 PRETEND_GGTT_SIZE, 0, I915_COLOR_UNEVICTABLE,413 0, ggtt->vm.total,414 PIN_NOEVICT);415 if (err)416 goto out_locked;417 418 /* Make the GGTT appear small by filling it with unevictable nodes */419 count = 0;420 do {421 struct reserved *r;422 423 mutex_unlock(&ggtt->vm.mutex);424 r = kcalloc(1, sizeof(*r), GFP_KERNEL);425 mutex_lock(&ggtt->vm.mutex);426 if (!r) {427 err = -ENOMEM;428 goto out_locked;429 }430 431 if (i915_gem_gtt_insert(&ggtt->vm, NULL, &r->node,432 1ul << 20, 0, I915_COLOR_UNEVICTABLE,433 0, ggtt->vm.total,434 PIN_NOEVICT)) {435 kfree(r);436 break;437 }438 439 r->next = reserved;440 reserved = r;441 442 count++;443 } while (1);444 drm_mm_remove_node(&hole);445 mutex_unlock(&ggtt->vm.mutex);446 pr_info("Filled GGTT with %lu 1MiB nodes\n", count);447 448 /* Overfill the GGTT with context objects and so try to evict one. */449 for_each_engine(engine, gt, id) {450 struct i915_sw_fence fence;451 struct i915_request *last = NULL;452 453 count = 0;454 onstack_fence_init(&fence);455 do {456 struct intel_context *ce;457 struct i915_request *rq;458 459 ce = intel_context_create(engine);460 if (IS_ERR(ce))461 break;462 463 /* We will need some GGTT space for the rq's context */464 igt_evict_ctl.fail_if_busy = true;465 rq = intel_context_create_request(ce);466 igt_evict_ctl.fail_if_busy = false;467 intel_context_put(ce);468 469 if (IS_ERR(rq)) {470 /* When full, fail_if_busy will trigger EBUSY */471 if (PTR_ERR(rq) != -EBUSY) {472 pr_err("Unexpected error from request alloc (on %s): %d\n",473 engine->name,474 (int)PTR_ERR(rq));475 err = PTR_ERR(rq);476 }477 break;478 }479 480 /* Keep every request/ctx pinned until we are full */481 err = i915_sw_fence_await_sw_fence_gfp(&rq->submit,482 &fence,483 GFP_KERNEL);484 if (err < 0)485 break;486 487 i915_request_add(rq);488 count++;489 if (last)490 i915_request_put(last);491 last = i915_request_get(rq);492 err = 0;493 } while(1);494 onstack_fence_fini(&fence);495 pr_info("Submitted %lu contexts/requests on %s\n",496 count, engine->name);497 if (err)498 break;499 if (last) {500 if (i915_request_wait(last, 0, HZ) < 0) {501 err = -EIO;502 i915_request_put(last);503 pr_err("Failed waiting for last request (on %s)",504 engine->name);505 break;506 }507 i915_request_put(last);508 }509 err = intel_gt_wait_for_idle(engine->gt, HZ * 3);510 if (err) {511 gt_err(engine->gt, "Failed to idle GT (on %s)",512 engine->name);513 break;514 }515 }516 517 mutex_lock(&ggtt->vm.mutex);518out_locked:519 if (igt_flush_test(i915))520 err = -EIO;521 while (reserved) {522 struct reserved *next = reserved->next;523 524 drm_mm_remove_node(&reserved->node);525 kfree(reserved);526 527 reserved = next;528 }529 if (drm_mm_node_allocated(&hole))530 drm_mm_remove_node(&hole);531 mutex_unlock(&ggtt->vm.mutex);532 intel_runtime_pm_put(&i915->runtime_pm, wakeref);533 534 return err;535}536 537int i915_gem_evict_mock_selftests(void)538{539 static const struct i915_subtest tests[] = {540 SUBTEST(igt_evict_something),541 SUBTEST(igt_evict_for_vma),542 SUBTEST(igt_evict_for_cache_color),543 SUBTEST(igt_evict_vm),544 SUBTEST(igt_overcommit),545 };546 struct drm_i915_private *i915;547 intel_wakeref_t wakeref;548 int err = 0;549 550 i915 = mock_gem_device();551 if (!i915)552 return -ENOMEM;553 554 with_intel_runtime_pm(&i915->runtime_pm, wakeref)555 err = i915_subtests(tests, to_gt(i915));556 557 mock_destroy_device(i915);558 return err;559}560 561int i915_gem_evict_live_selftests(struct drm_i915_private *i915)562{563 static const struct i915_subtest tests[] = {564 SUBTEST(igt_evict_contexts),565 };566 567 if (intel_gt_is_wedged(to_gt(i915)))568 return 0;569 570 return intel_gt_live_subtests(tests, to_gt(i915));571}572