brintos / linux-shallow public Read only

1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2018 Intel Corporation4 */5 6#include <linux/prime_numbers.h>7 8#include "gem/i915_gem_internal.h"9#include "gem/i915_gem_pm.h"10#include "gt/intel_engine_heartbeat.h"11#include "gt/intel_reset.h"12#include "gt/selftest_engine_heartbeat.h"13 14#include "i915_selftest.h"15#include "selftests/i915_random.h"16#include "selftests/igt_flush_test.h"17#include "selftests/igt_live_test.h"18#include "selftests/igt_spinner.h"19#include "selftests/lib_sw_fence.h"20 21#include "gem/selftests/igt_gem_utils.h"22#include "gem/selftests/mock_context.h"23 24#define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)25#define NUM_GPR 1626#define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */27 28static bool is_active(struct i915_request *rq)29{30	if (i915_request_is_active(rq))31		return true;32 33	if (i915_request_on_hold(rq))34		return true;35 36	if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq))37		return true;38 39	return false;40}41 42static int wait_for_submit(struct intel_engine_cs *engine,43			   struct i915_request *rq,44			   unsigned long timeout)45{46	/* Ignore our own attempts to suppress excess tasklets */47	tasklet_hi_schedule(&engine->sched_engine->tasklet);48 49	timeout += jiffies;50	do {51		bool done = time_after(jiffies, timeout);52 53		if (i915_request_completed(rq)) /* that was quick! */54			return 0;55 56		/* Wait until the HW has acknowleged the submission (or err) */57		intel_engine_flush_submission(engine);58		if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq))59			return 0;60 61		if (done)62			return -ETIME;63 64		cond_resched();65	} while (1);66}67 68static int wait_for_reset(struct intel_engine_cs *engine,69			  struct i915_request *rq,70			  unsigned long timeout)71{72	timeout += jiffies;73 74	do {75		cond_resched();76		intel_engine_flush_submission(engine);77 78		if (READ_ONCE(engine->execlists.pending[0]))79			continue;80 81		if (i915_request_completed(rq))82			break;83 84		if (READ_ONCE(rq->fence.error))85			break;86	} while (time_before(jiffies, timeout));87 88	if (rq->fence.error != -EIO) {89		pr_err("%s: hanging request %llx:%lld not reset\n",90		       engine->name,91		       rq->fence.context,92		       rq->fence.seqno);93		return -EINVAL;94	}95 96	/* Give the request a jiffy to complete after flushing the worker */97	if (i915_request_wait(rq, 0,98			      max(0l, (long)(timeout - jiffies)) + 1) < 0) {99		pr_err("%s: hanging request %llx:%lld did not complete\n",100		       engine->name,101		       rq->fence.context,102		       rq->fence.seqno);103		return -ETIME;104	}105 106	return 0;107}108 109static int live_sanitycheck(void *arg)110{111	struct intel_gt *gt = arg;112	struct intel_engine_cs *engine;113	enum intel_engine_id id;114	struct igt_spinner spin;115	int err = 0;116 117	if (!HAS_LOGICAL_RING_CONTEXTS(gt->i915))118		return 0;119 120	if (igt_spinner_init(&spin, gt))121		return -ENOMEM;122 123	for_each_engine(engine, gt, id) {124		struct intel_context *ce;125		struct i915_request *rq;126 127		ce = intel_context_create(engine);128		if (IS_ERR(ce)) {129			err = PTR_ERR(ce);130			break;131		}132 133		rq = igt_spinner_create_request(&spin, ce, MI_NOOP);134		if (IS_ERR(rq)) {135			err = PTR_ERR(rq);136			goto out_ctx;137		}138 139		i915_request_add(rq);140		if (!igt_wait_for_spinner(&spin, rq)) {141			GEM_TRACE("spinner failed to start\n");142			GEM_TRACE_DUMP();143			intel_gt_set_wedged(gt);144			err = -EIO;145			goto out_ctx;146		}147 148		igt_spinner_end(&spin);149		if (igt_flush_test(gt->i915)) {150			err = -EIO;151			goto out_ctx;152		}153 154out_ctx:155		intel_context_put(ce);156		if (err)157			break;158	}159 160	igt_spinner_fini(&spin);161	return err;162}163 164static int live_unlite_restore(struct intel_gt *gt, int prio)165{166	struct intel_engine_cs *engine;167	enum intel_engine_id id;168	struct igt_spinner spin;169	int err = -ENOMEM;170 171	/*172	 * Check that we can correctly context switch between 2 instances173	 * on the same engine from the same parent context.174	 */175 176	if (igt_spinner_init(&spin, gt))177		return err;178 179	err = 0;180	for_each_engine(engine, gt, id) {181		struct intel_context *ce[2] = {};182		struct i915_request *rq[2];183		struct igt_live_test t;184		int n;185 186		if (prio && !intel_engine_has_preemption(engine))187			continue;188 189		if (!intel_engine_can_store_dword(engine))190			continue;191 192		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {193			err = -EIO;194			break;195		}196		st_engine_heartbeat_disable(engine);197 198		for (n = 0; n < ARRAY_SIZE(ce); n++) {199			struct intel_context *tmp;200 201			tmp = intel_context_create(engine);202			if (IS_ERR(tmp)) {203				err = PTR_ERR(tmp);204				goto err_ce;205			}206 207			err = intel_context_pin(tmp);208			if (err) {209				intel_context_put(tmp);210				goto err_ce;211			}212 213			/*214			 * Setup the pair of contexts such that if we215			 * lite-restore using the RING_TAIL from ce[1] it216			 * will execute garbage from ce[0]->ring.217			 */218			memset(tmp->ring->vaddr,219			       POISON_INUSE, /* IPEHR: 0x5a5a5a5a [hung!] */220			       tmp->ring->vma->size);221 222			ce[n] = tmp;223		}224		GEM_BUG_ON(!ce[1]->ring->size);225		intel_ring_reset(ce[1]->ring, ce[1]->ring->size / 2);226		lrc_update_regs(ce[1], engine, ce[1]->ring->head);227 228		rq[0] = igt_spinner_create_request(&spin, ce[0], MI_ARB_CHECK);229		if (IS_ERR(rq[0])) {230			err = PTR_ERR(rq[0]);231			goto err_ce;232		}233 234		i915_request_get(rq[0]);235		i915_request_add(rq[0]);236		GEM_BUG_ON(rq[0]->postfix > ce[1]->ring->emit);237 238		if (!igt_wait_for_spinner(&spin, rq[0])) {239			i915_request_put(rq[0]);240			goto err_ce;241		}242 243		rq[1] = i915_request_create(ce[1]);244		if (IS_ERR(rq[1])) {245			err = PTR_ERR(rq[1]);246			i915_request_put(rq[0]);247			goto err_ce;248		}249 250		if (!prio) {251			/*252			 * Ensure we do the switch to ce[1] on completion.253			 *254			 * rq[0] is already submitted, so this should reduce255			 * to a no-op (a wait on a request on the same engine256			 * uses the submit fence, not the completion fence),257			 * but it will install a dependency on rq[1] for rq[0]258			 * that will prevent the pair being reordered by259			 * timeslicing.260			 */261			i915_request_await_dma_fence(rq[1], &rq[0]->fence);262		}263 264		i915_request_get(rq[1]);265		i915_request_add(rq[1]);266		GEM_BUG_ON(rq[1]->postfix <= rq[0]->postfix);267		i915_request_put(rq[0]);268 269		if (prio) {270			struct i915_sched_attr attr = {271				.priority = prio,272			};273 274			/* Alternatively preempt the spinner with ce[1] */275			engine->sched_engine->schedule(rq[1], &attr);276		}277 278		/* And switch back to ce[0] for good measure */279		rq[0] = i915_request_create(ce[0]);280		if (IS_ERR(rq[0])) {281			err = PTR_ERR(rq[0]);282			i915_request_put(rq[1]);283			goto err_ce;284		}285 286		i915_request_await_dma_fence(rq[0], &rq[1]->fence);287		i915_request_get(rq[0]);288		i915_request_add(rq[0]);289		GEM_BUG_ON(rq[0]->postfix > rq[1]->postfix);290		i915_request_put(rq[1]);291		i915_request_put(rq[0]);292 293err_ce:294		intel_engine_flush_submission(engine);295		igt_spinner_end(&spin);296		for (n = 0; n < ARRAY_SIZE(ce); n++) {297			if (IS_ERR_OR_NULL(ce[n]))298				break;299 300			intel_context_unpin(ce[n]);301			intel_context_put(ce[n]);302		}303 304		st_engine_heartbeat_enable(engine);305		if (igt_live_test_end(&t))306			err = -EIO;307		if (err)308			break;309	}310 311	igt_spinner_fini(&spin);312	return err;313}314 315static int live_unlite_switch(void *arg)316{317	return live_unlite_restore(arg, 0);318}319 320static int live_unlite_preempt(void *arg)321{322	return live_unlite_restore(arg, I915_PRIORITY_MAX);323}324 325static int live_unlite_ring(void *arg)326{327	struct intel_gt *gt = arg;328	struct intel_engine_cs *engine;329	struct igt_spinner spin;330	enum intel_engine_id id;331	int err = 0;332 333	/*334	 * Setup a preemption event that will cause almost the entire ring335	 * to be unwound, potentially fooling our intel_ring_direction()336	 * into emitting a forward lite-restore instead of the rollback.337	 */338 339	if (igt_spinner_init(&spin, gt))340		return -ENOMEM;341 342	for_each_engine(engine, gt, id) {343		struct intel_context *ce[2] = {};344		struct i915_request *rq;345		struct igt_live_test t;346		int n;347 348		if (!intel_engine_has_preemption(engine))349			continue;350 351		if (!intel_engine_can_store_dword(engine))352			continue;353 354		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {355			err = -EIO;356			break;357		}358		st_engine_heartbeat_disable(engine);359 360		for (n = 0; n < ARRAY_SIZE(ce); n++) {361			struct intel_context *tmp;362 363			tmp = intel_context_create(engine);364			if (IS_ERR(tmp)) {365				err = PTR_ERR(tmp);366				goto err_ce;367			}368 369			err = intel_context_pin(tmp);370			if (err) {371				intel_context_put(tmp);372				goto err_ce;373			}374 375			memset32(tmp->ring->vaddr,376				 0xdeadbeef, /* trigger a hang if executed */377				 tmp->ring->vma->size / sizeof(u32));378 379			ce[n] = tmp;380		}381 382		/* Create max prio spinner, followed by N low prio nops */383		rq = igt_spinner_create_request(&spin, ce[0], MI_ARB_CHECK);384		if (IS_ERR(rq)) {385			err = PTR_ERR(rq);386			goto err_ce;387		}388 389		i915_request_get(rq);390		rq->sched.attr.priority = I915_PRIORITY_BARRIER;391		i915_request_add(rq);392 393		if (!igt_wait_for_spinner(&spin, rq)) {394			intel_gt_set_wedged(gt);395			i915_request_put(rq);396			err = -ETIME;397			goto err_ce;398		}399 400		/* Fill the ring, until we will cause a wrap */401		n = 0;402		while (intel_ring_direction(ce[0]->ring,403					    rq->wa_tail,404					    ce[0]->ring->tail) <= 0) {405			struct i915_request *tmp;406 407			tmp = intel_context_create_request(ce[0]);408			if (IS_ERR(tmp)) {409				err = PTR_ERR(tmp);410				i915_request_put(rq);411				goto err_ce;412			}413 414			i915_request_add(tmp);415			intel_engine_flush_submission(engine);416			n++;417		}418		intel_engine_flush_submission(engine);419		pr_debug("%s: Filled ring with %d nop tails {size:%x, tail:%x, emit:%x, rq.tail:%x}\n",420			 engine->name, n,421			 ce[0]->ring->size,422			 ce[0]->ring->tail,423			 ce[0]->ring->emit,424			 rq->tail);425		GEM_BUG_ON(intel_ring_direction(ce[0]->ring,426						rq->tail,427						ce[0]->ring->tail) <= 0);428		i915_request_put(rq);429 430		/* Create a second ring to preempt the first ring after rq[0] */431		rq = intel_context_create_request(ce[1]);432		if (IS_ERR(rq)) {433			err = PTR_ERR(rq);434			goto err_ce;435		}436 437		rq->sched.attr.priority = I915_PRIORITY_BARRIER;438		i915_request_get(rq);439		i915_request_add(rq);440 441		err = wait_for_submit(engine, rq, HZ / 2);442		i915_request_put(rq);443		if (err) {444			pr_err("%s: preemption request was not submitted\n",445			       engine->name);446			err = -ETIME;447		}448 449		pr_debug("%s: ring[0]:{ tail:%x, emit:%x }, ring[1]:{ tail:%x, emit:%x }\n",450			 engine->name,451			 ce[0]->ring->tail, ce[0]->ring->emit,452			 ce[1]->ring->tail, ce[1]->ring->emit);453 454err_ce:455		intel_engine_flush_submission(engine);456		igt_spinner_end(&spin);457		for (n = 0; n < ARRAY_SIZE(ce); n++) {458			if (IS_ERR_OR_NULL(ce[n]))459				break;460 461			intel_context_unpin(ce[n]);462			intel_context_put(ce[n]);463		}464		st_engine_heartbeat_enable(engine);465		if (igt_live_test_end(&t))466			err = -EIO;467		if (err)468			break;469	}470 471	igt_spinner_fini(&spin);472	return err;473}474 475static int live_pin_rewind(void *arg)476{477	struct intel_gt *gt = arg;478	struct intel_engine_cs *engine;479	enum intel_engine_id id;480	int err = 0;481 482	/*483	 * We have to be careful not to trust intel_ring too much, for example484	 * ring->head is updated upon retire which is out of sync with pinning485	 * the context. Thus we cannot use ring->head to set CTX_RING_HEAD,486	 * or else we risk writing an older, stale value.487	 *488	 * To simulate this, let's apply a bit of deliberate sabotague.489	 */490 491	for_each_engine(engine, gt, id) {492		struct intel_context *ce;493		struct i915_request *rq;494		struct intel_ring *ring;495		struct igt_live_test t;496 497		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {498			err = -EIO;499			break;500		}501 502		ce = intel_context_create(engine);503		if (IS_ERR(ce)) {504			err = PTR_ERR(ce);505			break;506		}507 508		err = intel_context_pin(ce);509		if (err) {510			intel_context_put(ce);511			break;512		}513 514		/* Keep the context awake while we play games */515		err = i915_active_acquire(&ce->active);516		if (err) {517			intel_context_unpin(ce);518			intel_context_put(ce);519			break;520		}521		ring = ce->ring;522 523		/* Poison the ring, and offset the next request from HEAD */524		memset32(ring->vaddr, STACK_MAGIC, ring->size / sizeof(u32));525		ring->emit = ring->size / 2;526		ring->tail = ring->emit;527		GEM_BUG_ON(ring->head);528 529		intel_context_unpin(ce);530 531		/* Submit a simple nop request */532		GEM_BUG_ON(intel_context_is_pinned(ce));533		rq = intel_context_create_request(ce);534		i915_active_release(&ce->active); /* e.g. async retire */535		intel_context_put(ce);536		if (IS_ERR(rq)) {537			err = PTR_ERR(rq);538			break;539		}540		GEM_BUG_ON(!rq->head);541		i915_request_add(rq);542 543		/* Expect not to hang! */544		if (igt_live_test_end(&t)) {545			err = -EIO;546			break;547		}548	}549 550	return err;551}552 553static int engine_lock_reset_tasklet(struct intel_engine_cs *engine)554{555	tasklet_disable(&engine->sched_engine->tasklet);556	local_bh_disable();557 558	if (test_and_set_bit(I915_RESET_ENGINE + engine->id,559			     &engine->gt->reset.flags)) {560		local_bh_enable();561		tasklet_enable(&engine->sched_engine->tasklet);562 563		intel_gt_set_wedged(engine->gt);564		return -EBUSY;565	}566 567	return 0;568}569 570static void engine_unlock_reset_tasklet(struct intel_engine_cs *engine)571{572	clear_and_wake_up_bit(I915_RESET_ENGINE + engine->id,573			      &engine->gt->reset.flags);574 575	local_bh_enable();576	tasklet_enable(&engine->sched_engine->tasklet);577}578 579static int live_hold_reset(void *arg)580{581	struct intel_gt *gt = arg;582	struct intel_engine_cs *engine;583	enum intel_engine_id id;584	struct igt_spinner spin;585	int err = 0;586 587	/*588	 * In order to support offline error capture for fast preempt reset,589	 * we need to decouple the guilty request and ensure that it and its590	 * descendents are not executed while the capture is in progress.591	 */592 593	if (!intel_has_reset_engine(gt))594		return 0;595 596	if (igt_spinner_init(&spin, gt))597		return -ENOMEM;598 599	for_each_engine(engine, gt, id) {600		struct intel_context *ce;601		struct i915_request *rq;602 603		ce = intel_context_create(engine);604		if (IS_ERR(ce)) {605			err = PTR_ERR(ce);606			break;607		}608 609		st_engine_heartbeat_disable(engine);610 611		rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);612		if (IS_ERR(rq)) {613			err = PTR_ERR(rq);614			goto out;615		}616		i915_request_add(rq);617 618		if (!igt_wait_for_spinner(&spin, rq)) {619			intel_gt_set_wedged(gt);620			err = -ETIME;621			goto out;622		}623 624		/* We have our request executing, now remove it and reset */625 626		err = engine_lock_reset_tasklet(engine);627		if (err)628			goto out;629 630		engine->sched_engine->tasklet.callback(&engine->sched_engine->tasklet);631		GEM_BUG_ON(execlists_active(&engine->execlists) != rq);632 633		i915_request_get(rq);634		execlists_hold(engine, rq);635		GEM_BUG_ON(!i915_request_on_hold(rq));636 637		__intel_engine_reset_bh(engine, NULL);638		GEM_BUG_ON(rq->fence.error != -EIO);639 640		engine_unlock_reset_tasklet(engine);641 642		/* Check that we do not resubmit the held request */643		if (!i915_request_wait(rq, 0, HZ / 5)) {644			pr_err("%s: on hold request completed!\n",645			       engine->name);646			i915_request_put(rq);647			err = -EIO;648			goto out;649		}650		GEM_BUG_ON(!i915_request_on_hold(rq));651 652		/* But is resubmitted on release */653		execlists_unhold(engine, rq);654		if (i915_request_wait(rq, 0, HZ / 5) < 0) {655			pr_err("%s: held request did not complete!\n",656			       engine->name);657			intel_gt_set_wedged(gt);658			err = -ETIME;659		}660		i915_request_put(rq);661 662out:663		st_engine_heartbeat_enable(engine);664		intel_context_put(ce);665		if (err)666			break;667	}668 669	igt_spinner_fini(&spin);670	return err;671}672 673static const char *error_repr(int err)674{675	return err ? "bad" : "good";676}677 678static int live_error_interrupt(void *arg)679{680	static const struct error_phase {681		enum { GOOD = 0, BAD = -EIO } error[2];682	} phases[] = {683		{ { BAD,  GOOD } },684		{ { BAD,  BAD  } },685		{ { BAD,  GOOD } },686		{ { GOOD, GOOD } }, /* sentinel */687	};688	struct intel_gt *gt = arg;689	struct intel_engine_cs *engine;690	enum intel_engine_id id;691 692	/*693	 * We hook up the CS_MASTER_ERROR_INTERRUPT to have forewarning694	 * of invalid commands in user batches that will cause a GPU hang.695	 * This is a faster mechanism than using hangcheck/heartbeats, but696	 * only detects problems the HW knows about -- it will not warn when697	 * we kill the HW!698	 *699	 * To verify our detection and reset, we throw some invalid commands700	 * at the HW and wait for the interrupt.701	 */702 703	if (!intel_has_reset_engine(gt))704		return 0;705 706	for_each_engine(engine, gt, id) {707		const struct error_phase *p;708		int err = 0;709 710		st_engine_heartbeat_disable(engine);711 712		for (p = phases; p->error[0] != GOOD; p++) {713			struct i915_request *client[ARRAY_SIZE(phases->error)];714			u32 *cs;715			int i;716 717			memset(client, 0, sizeof(*client));718			for (i = 0; i < ARRAY_SIZE(client); i++) {719				struct intel_context *ce;720				struct i915_request *rq;721 722				ce = intel_context_create(engine);723				if (IS_ERR(ce)) {724					err = PTR_ERR(ce);725					goto out;726				}727 728				rq = intel_context_create_request(ce);729				intel_context_put(ce);730				if (IS_ERR(rq)) {731					err = PTR_ERR(rq);732					goto out;733				}734 735				if (rq->engine->emit_init_breadcrumb) {736					err = rq->engine->emit_init_breadcrumb(rq);737					if (err) {738						i915_request_add(rq);739						goto out;740					}741				}742 743				cs = intel_ring_begin(rq, 2);744				if (IS_ERR(cs)) {745					i915_request_add(rq);746					err = PTR_ERR(cs);747					goto out;748				}749 750				if (p->error[i]) {751					*cs++ = 0xdeadbeef;752					*cs++ = 0xdeadbeef;753				} else {754					*cs++ = MI_NOOP;755					*cs++ = MI_NOOP;756				}757 758				client[i] = i915_request_get(rq);759				i915_request_add(rq);760			}761 762			err = wait_for_submit(engine, client[0], HZ / 2);763			if (err) {764				pr_err("%s: first request did not start within time!\n",765				       engine->name);766				err = -ETIME;767				goto out;768			}769 770			for (i = 0; i < ARRAY_SIZE(client); i++) {771				if (i915_request_wait(client[i], 0, HZ / 5) < 0)772					pr_debug("%s: %s request incomplete!\n",773						 engine->name,774						 error_repr(p->error[i]));775 776				if (!i915_request_started(client[i])) {777					pr_err("%s: %s request not started!\n",778					       engine->name,779					       error_repr(p->error[i]));780					err = -ETIME;781					goto out;782				}783 784				/* Kick the tasklet to process the error */785				intel_engine_flush_submission(engine);786				if (client[i]->fence.error != p->error[i]) {787					pr_err("%s: %s request (%s) with wrong error code: %d\n",788					       engine->name,789					       error_repr(p->error[i]),790					       i915_request_completed(client[i]) ? "completed" : "running",791					       client[i]->fence.error);792					err = -EINVAL;793					goto out;794				}795			}796 797out:798			for (i = 0; i < ARRAY_SIZE(client); i++)799				if (client[i])800					i915_request_put(client[i]);801			if (err) {802				pr_err("%s: failed at phase[%zd] { %d, %d }\n",803				       engine->name, p - phases,804				       p->error[0], p->error[1]);805				break;806			}807		}808 809		st_engine_heartbeat_enable(engine);810		if (err) {811			intel_gt_set_wedged(gt);812			return err;813		}814	}815 816	return 0;817}818 819static int820emit_semaphore_chain(struct i915_request *rq, struct i915_vma *vma, int idx)821{822	u32 *cs;823 824	cs = intel_ring_begin(rq, 10);825	if (IS_ERR(cs))826		return PTR_ERR(cs);827 828	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;829 830	*cs++ = MI_SEMAPHORE_WAIT |831		MI_SEMAPHORE_GLOBAL_GTT |832		MI_SEMAPHORE_POLL |833		MI_SEMAPHORE_SAD_NEQ_SDD;834	*cs++ = 0;835	*cs++ = i915_ggtt_offset(vma) + 4 * idx;836	*cs++ = 0;837 838	if (idx > 0) {839		*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;840		*cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);841		*cs++ = 0;842		*cs++ = 1;843	} else {844		*cs++ = MI_NOOP;845		*cs++ = MI_NOOP;846		*cs++ = MI_NOOP;847		*cs++ = MI_NOOP;848	}849 850	*cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;851 852	intel_ring_advance(rq, cs);853	return 0;854}855 856static struct i915_request *857semaphore_queue(struct intel_engine_cs *engine, struct i915_vma *vma, int idx)858{859	struct intel_context *ce;860	struct i915_request *rq;861	int err;862 863	ce = intel_context_create(engine);864	if (IS_ERR(ce))865		return ERR_CAST(ce);866 867	rq = intel_context_create_request(ce);868	if (IS_ERR(rq))869		goto out_ce;870 871	err = 0;872	if (rq->engine->emit_init_breadcrumb)873		err = rq->engine->emit_init_breadcrumb(rq);874	if (err == 0)875		err = emit_semaphore_chain(rq, vma, idx);876	if (err == 0)877		i915_request_get(rq);878	i915_request_add(rq);879	if (err)880		rq = ERR_PTR(err);881 882out_ce:883	intel_context_put(ce);884	return rq;885}886 887static int888release_queue(struct intel_engine_cs *engine,889	      struct i915_vma *vma,890	      int idx, int prio)891{892	struct i915_sched_attr attr = {893		.priority = prio,894	};895	struct i915_request *rq;896	u32 *cs;897 898	rq = intel_engine_create_kernel_request(engine);899	if (IS_ERR(rq))900		return PTR_ERR(rq);901 902	cs = intel_ring_begin(rq, 4);903	if (IS_ERR(cs)) {904		i915_request_add(rq);905		return PTR_ERR(cs);906	}907 908	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;909	*cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);910	*cs++ = 0;911	*cs++ = 1;912 913	intel_ring_advance(rq, cs);914 915	i915_request_get(rq);916	i915_request_add(rq);917 918	local_bh_disable();919	engine->sched_engine->schedule(rq, &attr);920	local_bh_enable(); /* kick tasklet */921 922	i915_request_put(rq);923 924	return 0;925}926 927static int928slice_semaphore_queue(struct intel_engine_cs *outer,929		      struct i915_vma *vma,930		      int count)931{932	struct intel_engine_cs *engine;933	struct i915_request *head;934	enum intel_engine_id id;935	int err, i, n = 0;936 937	head = semaphore_queue(outer, vma, n++);938	if (IS_ERR(head))939		return PTR_ERR(head);940 941	for_each_engine(engine, outer->gt, id) {942		if (!intel_engine_has_preemption(engine))943			continue;944 945		for (i = 0; i < count; i++) {946			struct i915_request *rq;947 948			rq = semaphore_queue(engine, vma, n++);949			if (IS_ERR(rq)) {950				err = PTR_ERR(rq);951				goto out;952			}953 954			i915_request_put(rq);955		}956	}957 958	err = release_queue(outer, vma, n, I915_PRIORITY_BARRIER);959	if (err)960		goto out;961 962	if (i915_request_wait(head, 0,963			      2 * outer->gt->info.num_engines * (count + 2) * (count + 3)) < 0) {964		pr_err("%s: Failed to slice along semaphore chain of length (%d, %d)!\n",965		       outer->name, count, n);966		GEM_TRACE_DUMP();967		intel_gt_set_wedged(outer->gt);968		err = -EIO;969	}970 971out:972	i915_request_put(head);973	return err;974}975 976static int live_timeslice_preempt(void *arg)977{978	struct intel_gt *gt = arg;979	struct drm_i915_gem_object *obj;980	struct intel_engine_cs *engine;981	enum intel_engine_id id;982	struct i915_vma *vma;983	void *vaddr;984	int err = 0;985 986	/*987	 * If a request takes too long, we would like to give other users988	 * a fair go on the GPU. In particular, users may create batches989	 * that wait upon external input, where that input may even be990	 * supplied by another GPU job. To avoid blocking forever, we991	 * need to preempt the current task and replace it with another992	 * ready task.993	 */994	if (!CONFIG_DRM_I915_TIMESLICE_DURATION)995		return 0;996 997	obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);998	if (IS_ERR(obj))999		return PTR_ERR(obj);1000 1001	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);1002	if (IS_ERR(vma)) {1003		err = PTR_ERR(vma);1004		goto err_obj;1005	}1006 1007	vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);1008	if (IS_ERR(vaddr)) {1009		err = PTR_ERR(vaddr);1010		goto err_obj;1011	}1012 1013	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);1014	if (err)1015		goto err_map;1016 1017	err = i915_vma_sync(vma);1018	if (err)1019		goto err_pin;1020 1021	for_each_engine(engine, gt, id) {1022		if (!intel_engine_has_preemption(engine))1023			continue;1024 1025		memset(vaddr, 0, PAGE_SIZE);1026 1027		st_engine_heartbeat_disable(engine);1028		err = slice_semaphore_queue(engine, vma, 5);1029		st_engine_heartbeat_enable(engine);1030		if (err)1031			goto err_pin;1032 1033		if (igt_flush_test(gt->i915)) {1034			err = -EIO;1035			goto err_pin;1036		}1037	}1038 1039err_pin:1040	i915_vma_unpin(vma);1041err_map:1042	i915_gem_object_unpin_map(obj);1043err_obj:1044	i915_gem_object_put(obj);1045	return err;1046}1047 1048static struct i915_request *1049create_rewinder(struct intel_context *ce,1050		struct i915_request *wait,1051		void *slot, int idx)1052{1053	const u32 offset =1054		i915_ggtt_offset(ce->engine->status_page.vma) +1055		offset_in_page(slot);1056	struct i915_request *rq;1057	u32 *cs;1058	int err;1059 1060	rq = intel_context_create_request(ce);1061	if (IS_ERR(rq))1062		return rq;1063 1064	if (wait) {1065		err = i915_request_await_dma_fence(rq, &wait->fence);1066		if (err)1067			goto err;1068	}1069 1070	cs = intel_ring_begin(rq, 14);1071	if (IS_ERR(cs)) {1072		err = PTR_ERR(cs);1073		goto err;1074	}1075 1076	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;1077	*cs++ = MI_NOOP;1078 1079	*cs++ = MI_SEMAPHORE_WAIT |1080		MI_SEMAPHORE_GLOBAL_GTT |1081		MI_SEMAPHORE_POLL |1082		MI_SEMAPHORE_SAD_GTE_SDD;1083	*cs++ = idx;1084	*cs++ = offset;1085	*cs++ = 0;1086 1087	*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;1088	*cs++ = i915_mmio_reg_offset(RING_TIMESTAMP(rq->engine->mmio_base));1089	*cs++ = offset + idx * sizeof(u32);1090	*cs++ = 0;1091 1092	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;1093	*cs++ = offset;1094	*cs++ = 0;1095	*cs++ = idx + 1;1096 1097	intel_ring_advance(rq, cs);1098 1099	err = 0;1100err:1101	i915_request_get(rq);1102	i915_request_add(rq);1103	if (err) {1104		i915_request_put(rq);1105		return ERR_PTR(err);1106	}1107 1108	return rq;1109}1110 1111static int live_timeslice_rewind(void *arg)1112{1113	struct intel_gt *gt = arg;1114	struct intel_engine_cs *engine;1115	enum intel_engine_id id;1116 1117	/*1118	 * The usual presumption on timeslice expiration is that we replace1119	 * the active context with another. However, given a chain of1120	 * dependencies we may end up with replacing the context with itself,1121	 * but only a few of those requests, forcing us to rewind the1122	 * RING_TAIL of the original request.1123	 */1124	if (!CONFIG_DRM_I915_TIMESLICE_DURATION)1125		return 0;1126 1127	for_each_engine(engine, gt, id) {1128		enum { A1, A2, B1 };1129		enum { X = 1, Z, Y };1130		struct i915_request *rq[3] = {};1131		struct intel_context *ce;1132		unsigned long timeslice;1133		int i, err = 0;1134		u32 *slot;1135 1136		if (!intel_engine_has_timeslices(engine))1137			continue;1138 1139		/*1140		 * A:rq1 -- semaphore wait, timestamp X1141		 * A:rq2 -- write timestamp Y1142		 *1143		 * B:rq1 [await A:rq1] -- write timestamp Z1144		 *1145		 * Force timeslice, release semaphore.1146		 *1147		 * Expect execution/evaluation order XZY1148		 */1149 1150		st_engine_heartbeat_disable(engine);1151		timeslice = xchg(&engine->props.timeslice_duration_ms, 1);1152 1153		slot = memset32(engine->status_page.addr + 1000, 0, 4);1154 1155		ce = intel_context_create(engine);1156		if (IS_ERR(ce)) {1157			err = PTR_ERR(ce);1158			goto err;1159		}1160 1161		rq[A1] = create_rewinder(ce, NULL, slot, X);1162		if (IS_ERR(rq[A1])) {1163			intel_context_put(ce);1164			goto err;1165		}1166 1167		rq[A2] = create_rewinder(ce, NULL, slot, Y);1168		intel_context_put(ce);1169		if (IS_ERR(rq[A2]))1170			goto err;1171 1172		err = wait_for_submit(engine, rq[A2], HZ / 2);1173		if (err) {1174			pr_err("%s: failed to submit first context\n",1175			       engine->name);1176			goto err;1177		}1178 1179		ce = intel_context_create(engine);1180		if (IS_ERR(ce)) {1181			err = PTR_ERR(ce);1182			goto err;1183		}1184 1185		rq[B1] = create_rewinder(ce, rq[A1], slot, Z);1186		intel_context_put(ce);1187		if (IS_ERR(rq[2]))1188			goto err;1189 1190		err = wait_for_submit(engine, rq[B1], HZ / 2);1191		if (err) {1192			pr_err("%s: failed to submit second context\n",1193			       engine->name);1194			goto err;1195		}1196 1197		/* ELSP[] = { { A:rq1, A:rq2 }, { B:rq1 } } */1198		ENGINE_TRACE(engine, "forcing tasklet for rewind\n");1199		while (i915_request_is_active(rq[A2])) { /* semaphore yield! */1200			/* Wait for the timeslice to kick in */1201			del_timer(&engine->execlists.timer);1202			tasklet_hi_schedule(&engine->sched_engine->tasklet);1203			intel_engine_flush_submission(engine);1204		}1205		/* -> ELSP[] = { { A:rq1 }, { B:rq1 } } */1206		GEM_BUG_ON(!i915_request_is_active(rq[A1]));1207		GEM_BUG_ON(!i915_request_is_active(rq[B1]));1208		GEM_BUG_ON(i915_request_is_active(rq[A2]));1209 1210		/* Release the hounds! */1211		slot[0] = 1;1212		wmb(); /* "pairs" with GPU; paranoid kick of internal CPU$ */1213 1214		for (i = 1; i <= 3; i++) {1215			unsigned long timeout = jiffies + HZ / 2;1216 1217			while (!READ_ONCE(slot[i]) &&1218			       time_before(jiffies, timeout))1219				;1220 1221			if (!time_before(jiffies, timeout)) {1222				pr_err("%s: rq[%d] timed out\n",1223				       engine->name, i - 1);1224				err = -ETIME;1225				goto err;1226			}1227 1228			pr_debug("%s: slot[%d]:%x\n", engine->name, i, slot[i]);1229		}1230 1231		/* XZY: XZ < XY */1232		if (slot[Z] - slot[X] >= slot[Y] - slot[X]) {1233			pr_err("%s: timeslicing did not run context B [%u] before A [%u]!\n",1234			       engine->name,1235			       slot[Z] - slot[X],1236			       slot[Y] - slot[X]);1237			err = -EINVAL;1238		}1239 1240err:1241		memset32(&slot[0], -1, 4);1242		wmb();1243 1244		engine->props.timeslice_duration_ms = timeslice;1245		st_engine_heartbeat_enable(engine);1246		for (i = 0; i < 3; i++)1247			i915_request_put(rq[i]);1248		if (igt_flush_test(gt->i915))1249			err = -EIO;1250		if (err)1251			return err;1252	}1253 1254	return 0;1255}1256 1257static struct i915_request *nop_request(struct intel_engine_cs *engine)1258{1259	struct i915_request *rq;1260 1261	rq = intel_engine_create_kernel_request(engine);1262	if (IS_ERR(rq))1263		return rq;1264 1265	i915_request_get(rq);1266	i915_request_add(rq);1267 1268	return rq;1269}1270 1271static long slice_timeout(struct intel_engine_cs *engine)1272{1273	long timeout;1274 1275	/* Enough time for a timeslice to kick in, and kick out */1276	timeout = 2 * msecs_to_jiffies_timeout(timeslice(engine));1277 1278	/* Enough time for the nop request to complete */1279	timeout += HZ / 5;1280 1281	return timeout + 1;1282}1283 1284static int live_timeslice_queue(void *arg)1285{1286	struct intel_gt *gt = arg;1287	struct drm_i915_gem_object *obj;1288	struct intel_engine_cs *engine;1289	enum intel_engine_id id;1290	struct i915_vma *vma;1291	void *vaddr;1292	int err = 0;1293 1294	/*1295	 * Make sure that even if ELSP[0] and ELSP[1] are filled with1296	 * timeslicing between them disabled, we *do* enable timeslicing1297	 * if the queue demands it. (Normally, we do not submit if1298	 * ELSP[1] is already occupied, so must rely on timeslicing to1299	 * eject ELSP[0] in favour of the queue.)1300	 */1301	if (!CONFIG_DRM_I915_TIMESLICE_DURATION)1302		return 0;1303 1304	obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);1305	if (IS_ERR(obj))1306		return PTR_ERR(obj);1307 1308	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);1309	if (IS_ERR(vma)) {1310		err = PTR_ERR(vma);1311		goto err_obj;1312	}1313 1314	vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);1315	if (IS_ERR(vaddr)) {1316		err = PTR_ERR(vaddr);1317		goto err_obj;1318	}1319 1320	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);1321	if (err)1322		goto err_map;1323 1324	err = i915_vma_sync(vma);1325	if (err)1326		goto err_pin;1327 1328	for_each_engine(engine, gt, id) {1329		struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX };1330		struct i915_request *rq, *nop;1331 1332		if (!intel_engine_has_preemption(engine))1333			continue;1334 1335		st_engine_heartbeat_disable(engine);1336		memset(vaddr, 0, PAGE_SIZE);1337 1338		/* ELSP[0]: semaphore wait */1339		rq = semaphore_queue(engine, vma, 0);1340		if (IS_ERR(rq)) {1341			err = PTR_ERR(rq);1342			goto err_heartbeat;1343		}1344		engine->sched_engine->schedule(rq, &attr);1345		err = wait_for_submit(engine, rq, HZ / 2);1346		if (err) {1347			pr_err("%s: Timed out trying to submit semaphores\n",1348			       engine->name);1349			goto err_rq;1350		}1351 1352		/* ELSP[1]: nop request */1353		nop = nop_request(engine);1354		if (IS_ERR(nop)) {1355			err = PTR_ERR(nop);1356			goto err_rq;1357		}1358		err = wait_for_submit(engine, nop, HZ / 2);1359		i915_request_put(nop);1360		if (err) {1361			pr_err("%s: Timed out trying to submit nop\n",1362			       engine->name);1363			goto err_rq;1364		}1365 1366		GEM_BUG_ON(i915_request_completed(rq));1367		GEM_BUG_ON(execlists_active(&engine->execlists) != rq);1368 1369		/* Queue: semaphore signal, matching priority as semaphore */1370		err = release_queue(engine, vma, 1, effective_prio(rq));1371		if (err)1372			goto err_rq;1373 1374		/* Wait until we ack the release_queue and start timeslicing */1375		do {1376			cond_resched();1377			intel_engine_flush_submission(engine);1378		} while (READ_ONCE(engine->execlists.pending[0]));1379 1380		/* Timeslice every jiffy, so within 2 we should signal */1381		if (i915_request_wait(rq, 0, slice_timeout(engine)) < 0) {1382			struct drm_printer p =1383				drm_info_printer(gt->i915->drm.dev);1384 1385			pr_err("%s: Failed to timeslice into queue\n",1386			       engine->name);1387			intel_engine_dump(engine, &p,1388					  "%s\n", engine->name);1389 1390			memset(vaddr, 0xff, PAGE_SIZE);1391			err = -EIO;1392		}1393err_rq:1394		i915_request_put(rq);1395err_heartbeat:1396		st_engine_heartbeat_enable(engine);1397		if (err)1398			break;1399	}1400 1401err_pin:1402	i915_vma_unpin(vma);1403err_map:1404	i915_gem_object_unpin_map(obj);1405err_obj:1406	i915_gem_object_put(obj);1407	return err;1408}1409 1410static int live_timeslice_nopreempt(void *arg)1411{1412	struct intel_gt *gt = arg;1413	struct intel_engine_cs *engine;1414	enum intel_engine_id id;1415	struct igt_spinner spin;1416	int err = 0;1417 1418	/*1419	 * We should not timeslice into a request that is marked with1420	 * I915_REQUEST_NOPREEMPT.1421	 */1422	if (!CONFIG_DRM_I915_TIMESLICE_DURATION)1423		return 0;1424 1425	if (igt_spinner_init(&spin, gt))1426		return -ENOMEM;1427 1428	for_each_engine(engine, gt, id) {1429		struct intel_context *ce;1430		struct i915_request *rq;1431		unsigned long timeslice;1432 1433		if (!intel_engine_has_preemption(engine))1434			continue;1435 1436		ce = intel_context_create(engine);1437		if (IS_ERR(ce)) {1438			err = PTR_ERR(ce);1439			break;1440		}1441 1442		st_engine_heartbeat_disable(engine);1443		timeslice = xchg(&engine->props.timeslice_duration_ms, 1);1444 1445		/* Create an unpreemptible spinner */1446 1447		rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);1448		intel_context_put(ce);1449		if (IS_ERR(rq)) {1450			err = PTR_ERR(rq);1451			goto out_heartbeat;1452		}1453 1454		i915_request_get(rq);1455		i915_request_add(rq);1456 1457		if (!igt_wait_for_spinner(&spin, rq)) {1458			i915_request_put(rq);1459			err = -ETIME;1460			goto out_spin;1461		}1462 1463		set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq->fence.flags);1464		i915_request_put(rq);1465 1466		/* Followed by a maximum priority barrier (heartbeat) */1467 1468		ce = intel_context_create(engine);1469		if (IS_ERR(ce)) {1470			err = PTR_ERR(ce);1471			goto out_spin;1472		}1473 1474		rq = intel_context_create_request(ce);1475		intel_context_put(ce);1476		if (IS_ERR(rq)) {1477			err = PTR_ERR(rq);1478			goto out_spin;1479		}1480 1481		rq->sched.attr.priority = I915_PRIORITY_BARRIER;1482		i915_request_get(rq);1483		i915_request_add(rq);1484 1485		/*1486		 * Wait until the barrier is in ELSP, and we know timeslicing1487		 * will have been activated.1488		 */1489		if (wait_for_submit(engine, rq, HZ / 2)) {1490			i915_request_put(rq);1491			err = -ETIME;1492			goto out_spin;1493		}1494 1495		/*1496		 * Since the ELSP[0] request is unpreemptible, it should not1497		 * allow the maximum priority barrier through. Wait long1498		 * enough to see if it is timesliced in by mistake.1499		 */1500		if (i915_request_wait(rq, 0, slice_timeout(engine)) >= 0) {1501			pr_err("%s: I915_PRIORITY_BARRIER request completed, bypassing no-preempt request\n",1502			       engine->name);1503			err = -EINVAL;1504		}1505		i915_request_put(rq);1506 1507out_spin:1508		igt_spinner_end(&spin);1509out_heartbeat:1510		xchg(&engine->props.timeslice_duration_ms, timeslice);1511		st_engine_heartbeat_enable(engine);1512		if (err)1513			break;1514 1515		if (igt_flush_test(gt->i915)) {1516			err = -EIO;1517			break;1518		}1519	}1520 1521	igt_spinner_fini(&spin);1522	return err;1523}1524 1525static int live_busywait_preempt(void *arg)1526{1527	struct intel_gt *gt = arg;1528	struct i915_gem_context *ctx_hi, *ctx_lo;1529	struct intel_engine_cs *engine;1530	struct drm_i915_gem_object *obj;1531	struct i915_vma *vma;1532	enum intel_engine_id id;1533	u32 *map;1534	int err;1535 1536	/*1537	 * Verify that even without HAS_LOGICAL_RING_PREEMPTION, we can1538	 * preempt the busywaits used to synchronise between rings.1539	 */1540 1541	ctx_hi = kernel_context(gt->i915, NULL);1542	if (IS_ERR(ctx_hi))1543		return PTR_ERR(ctx_hi);1544 1545	ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY;1546 1547	ctx_lo = kernel_context(gt->i915, NULL);1548	if (IS_ERR(ctx_lo)) {1549		err = PTR_ERR(ctx_lo);1550		goto err_ctx_hi;1551	}1552 1553	ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY;1554 1555	obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);1556	if (IS_ERR(obj)) {1557		err = PTR_ERR(obj);1558		goto err_ctx_lo;1559	}1560 1561	map = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);1562	if (IS_ERR(map)) {1563		err = PTR_ERR(map);1564		goto err_obj;1565	}1566 1567	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);1568	if (IS_ERR(vma)) {1569		err = PTR_ERR(vma);1570		goto err_map;1571	}1572 1573	err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);1574	if (err)1575		goto err_map;1576 1577	err = i915_vma_sync(vma);1578	if (err)1579		goto err_vma;1580 1581	for_each_engine(engine, gt, id) {1582		struct i915_request *lo, *hi;1583		struct igt_live_test t;1584		u32 *cs;1585 1586		if (!intel_engine_has_preemption(engine))1587			continue;1588 1589		if (!intel_engine_can_store_dword(engine))1590			continue;1591 1592		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {1593			err = -EIO;1594			goto err_vma;1595		}1596 1597		/*1598		 * We create two requests. The low priority request1599		 * busywaits on a semaphore (inside the ringbuffer where1600		 * is should be preemptible) and the high priority requests1601		 * uses a MI_STORE_DWORD_IMM to update the semaphore value1602		 * allowing the first request to complete. If preemption1603		 * fails, we hang instead.1604		 */1605 1606		lo = igt_request_alloc(ctx_lo, engine);1607		if (IS_ERR(lo)) {1608			err = PTR_ERR(lo);1609			goto err_vma;1610		}1611 1612		cs = intel_ring_begin(lo, 8);1613		if (IS_ERR(cs)) {1614			err = PTR_ERR(cs);1615			i915_request_add(lo);1616			goto err_vma;1617		}1618 1619		*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;1620		*cs++ = i915_ggtt_offset(vma);1621		*cs++ = 0;1622		*cs++ = 1;1623 1624		/* XXX Do we need a flush + invalidate here? */1625 1626		*cs++ = MI_SEMAPHORE_WAIT |1627			MI_SEMAPHORE_GLOBAL_GTT |1628			MI_SEMAPHORE_POLL |1629			MI_SEMAPHORE_SAD_EQ_SDD;1630		*cs++ = 0;1631		*cs++ = i915_ggtt_offset(vma);1632		*cs++ = 0;1633 1634		intel_ring_advance(lo, cs);1635 1636		i915_request_get(lo);1637		i915_request_add(lo);1638 1639		if (wait_for(READ_ONCE(*map), 10)) {1640			i915_request_put(lo);1641			err = -ETIMEDOUT;1642			goto err_vma;1643		}1644 1645		/* Low priority request should be busywaiting now */1646		if (i915_request_wait(lo, 0, 1) != -ETIME) {1647			i915_request_put(lo);1648			pr_err("%s: Busywaiting request did not!\n",1649			       engine->name);1650			err = -EIO;1651			goto err_vma;1652		}1653 1654		hi = igt_request_alloc(ctx_hi, engine);1655		if (IS_ERR(hi)) {1656			err = PTR_ERR(hi);1657			i915_request_put(lo);1658			goto err_vma;1659		}1660 1661		cs = intel_ring_begin(hi, 4);1662		if (IS_ERR(cs)) {1663			err = PTR_ERR(cs);1664			i915_request_add(hi);1665			i915_request_put(lo);1666			goto err_vma;1667		}1668 1669		*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;1670		*cs++ = i915_ggtt_offset(vma);1671		*cs++ = 0;1672		*cs++ = 0;1673 1674		intel_ring_advance(hi, cs);1675		i915_request_add(hi);1676 1677		if (i915_request_wait(lo, 0, HZ / 5) < 0) {1678			struct drm_printer p = drm_info_printer(gt->i915->drm.dev);1679 1680			pr_err("%s: Failed to preempt semaphore busywait!\n",1681			       engine->name);1682 1683			intel_engine_dump(engine, &p, "%s\n", engine->name);1684			GEM_TRACE_DUMP();1685 1686			i915_request_put(lo);1687			intel_gt_set_wedged(gt);1688			err = -EIO;1689			goto err_vma;1690		}1691		GEM_BUG_ON(READ_ONCE(*map));1692		i915_request_put(lo);1693 1694		if (igt_live_test_end(&t)) {1695			err = -EIO;1696			goto err_vma;1697		}1698	}1699 1700	err = 0;1701err_vma:1702	i915_vma_unpin(vma);1703err_map:1704	i915_gem_object_unpin_map(obj);1705err_obj:1706	i915_gem_object_put(obj);1707err_ctx_lo:1708	kernel_context_close(ctx_lo);1709err_ctx_hi:1710	kernel_context_close(ctx_hi);1711	return err;1712}1713 1714static struct i915_request *1715spinner_create_request(struct igt_spinner *spin,1716		       struct i915_gem_context *ctx,1717		       struct intel_engine_cs *engine,1718		       u32 arb)1719{1720	struct intel_context *ce;1721	struct i915_request *rq;1722 1723	ce = i915_gem_context_get_engine(ctx, engine->legacy_idx);1724	if (IS_ERR(ce))1725		return ERR_CAST(ce);1726 1727	rq = igt_spinner_create_request(spin, ce, arb);1728	intel_context_put(ce);1729	return rq;1730}1731 1732static int live_preempt(void *arg)1733{1734	struct intel_gt *gt = arg;1735	struct i915_gem_context *ctx_hi, *ctx_lo;1736	struct igt_spinner spin_hi, spin_lo;1737	struct intel_engine_cs *engine;1738	enum intel_engine_id id;1739	int err = -ENOMEM;1740 1741	ctx_hi = kernel_context(gt->i915, NULL);1742	if (!ctx_hi)1743		return -ENOMEM;1744	ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY;1745 1746	ctx_lo = kernel_context(gt->i915, NULL);1747	if (!ctx_lo)1748		goto err_ctx_hi;1749	ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY;1750 1751	if (igt_spinner_init(&spin_hi, gt))1752		goto err_ctx_lo;1753 1754	if (igt_spinner_init(&spin_lo, gt))1755		goto err_spin_hi;1756 1757	for_each_engine(engine, gt, id) {1758		struct igt_live_test t;1759		struct i915_request *rq;1760 1761		if (!intel_engine_has_preemption(engine))1762			continue;1763 1764		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {1765			err = -EIO;1766			goto err_spin_lo;1767		}1768 1769		rq = spinner_create_request(&spin_lo, ctx_lo, engine,1770					    MI_ARB_CHECK);1771		if (IS_ERR(rq)) {1772			err = PTR_ERR(rq);1773			goto err_spin_lo;1774		}1775 1776		i915_request_add(rq);1777		if (!igt_wait_for_spinner(&spin_lo, rq)) {1778			GEM_TRACE("lo spinner failed to start\n");1779			GEM_TRACE_DUMP();1780			intel_gt_set_wedged(gt);1781			err = -EIO;1782			goto err_spin_lo;1783		}1784 1785		rq = spinner_create_request(&spin_hi, ctx_hi, engine,1786					    MI_ARB_CHECK);1787		if (IS_ERR(rq)) {1788			igt_spinner_end(&spin_lo);1789			err = PTR_ERR(rq);1790			goto err_spin_lo;1791		}1792 1793		i915_request_add(rq);1794		if (!igt_wait_for_spinner(&spin_hi, rq)) {1795			GEM_TRACE("hi spinner failed to start\n");1796			GEM_TRACE_DUMP();1797			intel_gt_set_wedged(gt);1798			err = -EIO;1799			goto err_spin_lo;1800		}1801 1802		igt_spinner_end(&spin_hi);1803		igt_spinner_end(&spin_lo);1804 1805		if (igt_live_test_end(&t)) {1806			err = -EIO;1807			goto err_spin_lo;1808		}1809	}1810 1811	err = 0;1812err_spin_lo:1813	igt_spinner_fini(&spin_lo);1814err_spin_hi:1815	igt_spinner_fini(&spin_hi);1816err_ctx_lo:1817	kernel_context_close(ctx_lo);1818err_ctx_hi:1819	kernel_context_close(ctx_hi);1820	return err;1821}1822 1823static int live_late_preempt(void *arg)1824{1825	struct intel_gt *gt = arg;1826	struct i915_gem_context *ctx_hi, *ctx_lo;1827	struct igt_spinner spin_hi, spin_lo;1828	struct intel_engine_cs *engine;1829	struct i915_sched_attr attr = {};1830	enum intel_engine_id id;1831	int err = -ENOMEM;1832 1833	ctx_hi = kernel_context(gt->i915, NULL);1834	if (!ctx_hi)1835		return -ENOMEM;1836 1837	ctx_lo = kernel_context(gt->i915, NULL);1838	if (!ctx_lo)1839		goto err_ctx_hi;1840 1841	if (igt_spinner_init(&spin_hi, gt))1842		goto err_ctx_lo;1843 1844	if (igt_spinner_init(&spin_lo, gt))1845		goto err_spin_hi;1846 1847	/* Make sure ctx_lo stays before ctx_hi until we trigger preemption. */1848	ctx_lo->sched.priority = 1;1849 1850	for_each_engine(engine, gt, id) {1851		struct igt_live_test t;1852		struct i915_request *rq;1853 1854		if (!intel_engine_has_preemption(engine))1855			continue;1856 1857		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {1858			err = -EIO;1859			goto err_spin_lo;1860		}1861 1862		rq = spinner_create_request(&spin_lo, ctx_lo, engine,1863					    MI_ARB_CHECK);1864		if (IS_ERR(rq)) {1865			err = PTR_ERR(rq);1866			goto err_spin_lo;1867		}1868 1869		i915_request_add(rq);1870		if (!igt_wait_for_spinner(&spin_lo, rq)) {1871			pr_err("First context failed to start\n");1872			goto err_wedged;1873		}1874 1875		rq = spinner_create_request(&spin_hi, ctx_hi, engine,1876					    MI_NOOP);1877		if (IS_ERR(rq)) {1878			igt_spinner_end(&spin_lo);1879			err = PTR_ERR(rq);1880			goto err_spin_lo;1881		}1882 1883		i915_request_add(rq);1884		if (igt_wait_for_spinner(&spin_hi, rq)) {1885			pr_err("Second context overtook first?\n");1886			goto err_wedged;1887		}1888 1889		attr.priority = I915_PRIORITY_MAX;1890		engine->sched_engine->schedule(rq, &attr);1891 1892		if (!igt_wait_for_spinner(&spin_hi, rq)) {1893			pr_err("High priority context failed to preempt the low priority context\n");1894			GEM_TRACE_DUMP();1895			goto err_wedged;1896		}1897 1898		igt_spinner_end(&spin_hi);1899		igt_spinner_end(&spin_lo);1900 1901		if (igt_live_test_end(&t)) {1902			err = -EIO;1903			goto err_spin_lo;1904		}1905	}1906 1907	err = 0;1908err_spin_lo:1909	igt_spinner_fini(&spin_lo);1910err_spin_hi:1911	igt_spinner_fini(&spin_hi);1912err_ctx_lo:1913	kernel_context_close(ctx_lo);1914err_ctx_hi:1915	kernel_context_close(ctx_hi);1916	return err;1917 1918err_wedged:1919	igt_spinner_end(&spin_hi);1920	igt_spinner_end(&spin_lo);1921	intel_gt_set_wedged(gt);1922	err = -EIO;1923	goto err_spin_lo;1924}1925 1926struct preempt_client {1927	struct igt_spinner spin;1928	struct i915_gem_context *ctx;1929};1930 1931static int preempt_client_init(struct intel_gt *gt, struct preempt_client *c)1932{1933	c->ctx = kernel_context(gt->i915, NULL);1934	if (!c->ctx)1935		return -ENOMEM;1936 1937	if (igt_spinner_init(&c->spin, gt))1938		goto err_ctx;1939 1940	return 0;1941 1942err_ctx:1943	kernel_context_close(c->ctx);1944	return -ENOMEM;1945}1946 1947static void preempt_client_fini(struct preempt_client *c)1948{1949	igt_spinner_fini(&c->spin);1950	kernel_context_close(c->ctx);1951}1952 1953static int live_nopreempt(void *arg)1954{1955	struct intel_gt *gt = arg;1956	struct intel_engine_cs *engine;1957	struct preempt_client a, b;1958	enum intel_engine_id id;1959	int err = -ENOMEM;1960 1961	/*1962	 * Verify that we can disable preemption for an individual request1963	 * that may be being observed and not want to be interrupted.1964	 */1965 1966	if (preempt_client_init(gt, &a))1967		return -ENOMEM;1968	if (preempt_client_init(gt, &b))1969		goto err_client_a;1970	b.ctx->sched.priority = I915_PRIORITY_MAX;1971 1972	for_each_engine(engine, gt, id) {1973		struct i915_request *rq_a, *rq_b;1974 1975		if (!intel_engine_has_preemption(engine))1976			continue;1977 1978		engine->execlists.preempt_hang.count = 0;1979 1980		rq_a = spinner_create_request(&a.spin,1981					      a.ctx, engine,1982					      MI_ARB_CHECK);1983		if (IS_ERR(rq_a)) {1984			err = PTR_ERR(rq_a);1985			goto err_client_b;1986		}1987 1988		/* Low priority client, but unpreemptable! */1989		__set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq_a->fence.flags);1990 1991		i915_request_add(rq_a);1992		if (!igt_wait_for_spinner(&a.spin, rq_a)) {1993			pr_err("First client failed to start\n");1994			goto err_wedged;1995		}1996 1997		rq_b = spinner_create_request(&b.spin,1998					      b.ctx, engine,1999					      MI_ARB_CHECK);2000		if (IS_ERR(rq_b)) {2001			err = PTR_ERR(rq_b);2002			goto err_client_b;2003		}2004 2005		i915_request_add(rq_b);2006 2007		/* B is much more important than A! (But A is unpreemptable.) */2008		GEM_BUG_ON(rq_prio(rq_b) <= rq_prio(rq_a));2009 2010		/* Wait long enough for preemption and timeslicing */2011		if (igt_wait_for_spinner(&b.spin, rq_b)) {2012			pr_err("Second client started too early!\n");2013			goto err_wedged;2014		}2015 2016		igt_spinner_end(&a.spin);2017 2018		if (!igt_wait_for_spinner(&b.spin, rq_b)) {2019			pr_err("Second client failed to start\n");2020			goto err_wedged;2021		}2022 2023		igt_spinner_end(&b.spin);2024 2025		if (engine->execlists.preempt_hang.count) {2026			pr_err("Preemption recorded x%d; should have been suppressed!\n",2027			       engine->execlists.preempt_hang.count);2028			err = -EINVAL;2029			goto err_wedged;2030		}2031 2032		if (igt_flush_test(gt->i915))2033			goto err_wedged;2034	}2035 2036	err = 0;2037err_client_b:2038	preempt_client_fini(&b);2039err_client_a:2040	preempt_client_fini(&a);2041	return err;2042 2043err_wedged:2044	igt_spinner_end(&b.spin);2045	igt_spinner_end(&a.spin);2046	intel_gt_set_wedged(gt);2047	err = -EIO;2048	goto err_client_b;2049}2050 2051struct live_preempt_cancel {2052	struct intel_engine_cs *engine;2053	struct preempt_client a, b;2054};2055 2056static int __cancel_active0(struct live_preempt_cancel *arg)2057{2058	struct i915_request *rq;2059	struct igt_live_test t;2060	int err;2061 2062	/* Preempt cancel of ELSP0 */2063	GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);2064	if (igt_live_test_begin(&t, arg->engine->i915,2065				__func__, arg->engine->name))2066		return -EIO;2067 2068	rq = spinner_create_request(&arg->a.spin,2069				    arg->a.ctx, arg->engine,2070				    MI_ARB_CHECK);2071	if (IS_ERR(rq))2072		return PTR_ERR(rq);2073 2074	clear_bit(CONTEXT_BANNED, &rq->context->flags);2075	i915_request_get(rq);2076	i915_request_add(rq);2077	if (!igt_wait_for_spinner(&arg->a.spin, rq)) {2078		err = -EIO;2079		goto out;2080	}2081 2082	intel_context_ban(rq->context, rq);2083	err = intel_engine_pulse(arg->engine);2084	if (err)2085		goto out;2086 2087	err = wait_for_reset(arg->engine, rq, HZ / 2);2088	if (err) {2089		pr_err("Cancelled inflight0 request did not reset\n");2090		goto out;2091	}2092 2093out:2094	i915_request_put(rq);2095	if (igt_live_test_end(&t))2096		err = -EIO;2097	return err;2098}2099 2100static int __cancel_active1(struct live_preempt_cancel *arg)2101{2102	struct i915_request *rq[2] = {};2103	struct igt_live_test t;2104	int err;2105 2106	/* Preempt cancel of ELSP1 */2107	GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);2108	if (igt_live_test_begin(&t, arg->engine->i915,2109				__func__, arg->engine->name))2110		return -EIO;2111 2112	rq[0] = spinner_create_request(&arg->a.spin,2113				       arg->a.ctx, arg->engine,2114				       MI_NOOP); /* no preemption */2115	if (IS_ERR(rq[0]))2116		return PTR_ERR(rq[0]);2117 2118	clear_bit(CONTEXT_BANNED, &rq[0]->context->flags);2119	i915_request_get(rq[0]);2120	i915_request_add(rq[0]);2121	if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) {2122		err = -EIO;2123		goto out;2124	}2125 2126	rq[1] = spinner_create_request(&arg->b.spin,2127				       arg->b.ctx, arg->engine,2128				       MI_ARB_CHECK);2129	if (IS_ERR(rq[1])) {2130		err = PTR_ERR(rq[1]);2131		goto out;2132	}2133 2134	clear_bit(CONTEXT_BANNED, &rq[1]->context->flags);2135	i915_request_get(rq[1]);2136	err = i915_request_await_dma_fence(rq[1], &rq[0]->fence);2137	i915_request_add(rq[1]);2138	if (err)2139		goto out;2140 2141	intel_context_ban(rq[1]->context, rq[1]);2142	err = intel_engine_pulse(arg->engine);2143	if (err)2144		goto out;2145 2146	igt_spinner_end(&arg->a.spin);2147	err = wait_for_reset(arg->engine, rq[1], HZ / 2);2148	if (err)2149		goto out;2150 2151	if (rq[0]->fence.error != 0) {2152		pr_err("Normal inflight0 request did not complete\n");2153		err = -EINVAL;2154		goto out;2155	}2156 2157	if (rq[1]->fence.error != -EIO) {2158		pr_err("Cancelled inflight1 request did not report -EIO\n");2159		err = -EINVAL;2160		goto out;2161	}2162 2163out:2164	i915_request_put(rq[1]);2165	i915_request_put(rq[0]);2166	if (igt_live_test_end(&t))2167		err = -EIO;2168	return err;2169}2170 2171static int __cancel_queued(struct live_preempt_cancel *arg)2172{2173	struct i915_request *rq[3] = {};2174	struct igt_live_test t;2175	int err;2176 2177	/* Full ELSP and one in the wings */2178	GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);2179	if (igt_live_test_begin(&t, arg->engine->i915,2180				__func__, arg->engine->name))2181		return -EIO;2182 2183	rq[0] = spinner_create_request(&arg->a.spin,2184				       arg->a.ctx, arg->engine,2185				       MI_ARB_CHECK);2186	if (IS_ERR(rq[0]))2187		return PTR_ERR(rq[0]);2188 2189	clear_bit(CONTEXT_BANNED, &rq[0]->context->flags);2190	i915_request_get(rq[0]);2191	i915_request_add(rq[0]);2192	if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) {2193		err = -EIO;2194		goto out;2195	}2196 2197	rq[1] = igt_request_alloc(arg->b.ctx, arg->engine);2198	if (IS_ERR(rq[1])) {2199		err = PTR_ERR(rq[1]);2200		goto out;2201	}2202 2203	clear_bit(CONTEXT_BANNED, &rq[1]->context->flags);2204	i915_request_get(rq[1]);2205	err = i915_request_await_dma_fence(rq[1], &rq[0]->fence);2206	i915_request_add(rq[1]);2207	if (err)2208		goto out;2209 2210	rq[2] = spinner_create_request(&arg->b.spin,2211				       arg->a.ctx, arg->engine,2212				       MI_ARB_CHECK);2213	if (IS_ERR(rq[2])) {2214		err = PTR_ERR(rq[2]);2215		goto out;2216	}2217 2218	i915_request_get(rq[2]);2219	err = i915_request_await_dma_fence(rq[2], &rq[1]->fence);2220	i915_request_add(rq[2]);2221	if (err)2222		goto out;2223 2224	intel_context_ban(rq[2]->context, rq[2]);2225	err = intel_engine_pulse(arg->engine);2226	if (err)2227		goto out;2228 2229	err = wait_for_reset(arg->engine, rq[2], HZ / 2);2230	if (err)2231		goto out;2232 2233	if (rq[0]->fence.error != -EIO) {2234		pr_err("Cancelled inflight0 request did not report -EIO\n");2235		err = -EINVAL;2236		goto out;2237	}2238 2239	/*2240	 * The behavior between having semaphores and not is different. With2241	 * semaphores the subsequent request is on the hardware and not cancelled2242	 * while without the request is held in the driver and cancelled.2243	 */2244	if (intel_engine_has_semaphores(rq[1]->engine) &&2245	    rq[1]->fence.error != 0) {2246		pr_err("Normal inflight1 request did not complete\n");2247		err = -EINVAL;2248		goto out;2249	}2250 2251	if (rq[2]->fence.error != -EIO) {2252		pr_err("Cancelled queued request did not report -EIO\n");2253		err = -EINVAL;2254		goto out;2255	}2256 2257out:2258	i915_request_put(rq[2]);2259	i915_request_put(rq[1]);2260	i915_request_put(rq[0]);2261	if (igt_live_test_end(&t))2262		err = -EIO;2263	return err;2264}2265 2266static int __cancel_hostile(struct live_preempt_cancel *arg)2267{2268	struct i915_request *rq;2269	int err;2270 2271	/* Preempt cancel non-preemptible spinner in ELSP0 */2272	if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT)2273		return 0;2274 2275	if (!intel_has_reset_engine(arg->engine->gt))2276		return 0;2277 2278	GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);2279	rq = spinner_create_request(&arg->a.spin,2280				    arg->a.ctx, arg->engine,2281				    MI_NOOP); /* preemption disabled */2282	if (IS_ERR(rq))2283		return PTR_ERR(rq);2284 2285	clear_bit(CONTEXT_BANNED, &rq->context->flags);2286	i915_request_get(rq);2287	i915_request_add(rq);2288	if (!igt_wait_for_spinner(&arg->a.spin, rq)) {2289		err = -EIO;2290		goto out;2291	}2292 2293	intel_context_ban(rq->context, rq);2294	err = intel_engine_pulse(arg->engine); /* force reset */2295	if (err)2296		goto out;2297 2298	err = wait_for_reset(arg->engine, rq, HZ / 2);2299	if (err) {2300		pr_err("Cancelled inflight0 request did not reset\n");2301		goto out;2302	}2303 2304out:2305	i915_request_put(rq);2306	if (igt_flush_test(arg->engine->i915))2307		err = -EIO;2308	return err;2309}2310 2311static void force_reset_timeout(struct intel_engine_cs *engine)2312{2313	engine->reset_timeout.probability = 999;2314	atomic_set(&engine->reset_timeout.times, -1);2315}2316 2317static void cancel_reset_timeout(struct intel_engine_cs *engine)2318{2319	memset(&engine->reset_timeout, 0, sizeof(engine->reset_timeout));2320}2321 2322static int __cancel_fail(struct live_preempt_cancel *arg)2323{2324	struct intel_engine_cs *engine = arg->engine;2325	struct i915_request *rq;2326	int err;2327 2328	if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT)2329		return 0;2330 2331	if (!intel_has_reset_engine(engine->gt))2332		return 0;2333 2334	GEM_TRACE("%s(%s)\n", __func__, engine->name);2335	rq = spinner_create_request(&arg->a.spin,2336				    arg->a.ctx, engine,2337				    MI_NOOP); /* preemption disabled */2338	if (IS_ERR(rq))2339		return PTR_ERR(rq);2340 2341	clear_bit(CONTEXT_BANNED, &rq->context->flags);2342	i915_request_get(rq);2343	i915_request_add(rq);2344	if (!igt_wait_for_spinner(&arg->a.spin, rq)) {2345		err = -EIO;2346		goto out;2347	}2348 2349	intel_context_set_banned(rq->context);2350 2351	err = intel_engine_pulse(engine);2352	if (err)2353		goto out;2354 2355	force_reset_timeout(engine);2356 2357	/* force preempt reset [failure] */2358	while (!engine->execlists.pending[0])2359		intel_engine_flush_submission(engine);2360	del_timer_sync(&engine->execlists.preempt);2361	intel_engine_flush_submission(engine);2362 2363	cancel_reset_timeout(engine);2364 2365	/* after failure, require heartbeats to reset device */2366	intel_engine_set_heartbeat(engine, 1);2367	err = wait_for_reset(engine, rq, HZ / 2);2368	intel_engine_set_heartbeat(engine,2369				   engine->defaults.heartbeat_interval_ms);2370	if (err) {2371		pr_err("Cancelled inflight0 request did not reset\n");2372		goto out;2373	}2374 2375out:2376	i915_request_put(rq);2377	if (igt_flush_test(engine->i915))2378		err = -EIO;2379	return err;2380}2381 2382static int live_preempt_cancel(void *arg)2383{2384	struct intel_gt *gt = arg;2385	struct live_preempt_cancel data;2386	enum intel_engine_id id;2387	int err = -ENOMEM;2388 2389	/*2390	 * To cancel an inflight context, we need to first remove it from the2391	 * GPU. That sounds like preemption! Plus a little bit of bookkeeping.2392	 */2393 2394	if (preempt_client_init(gt, &data.a))2395		return -ENOMEM;2396	if (preempt_client_init(gt, &data.b))2397		goto err_client_a;2398 2399	for_each_engine(data.engine, gt, id) {2400		if (!intel_engine_has_preemption(data.engine))2401			continue;2402 2403		err = __cancel_active0(&data);2404		if (err)2405			goto err_wedged;2406 2407		err = __cancel_active1(&data);2408		if (err)2409			goto err_wedged;2410 2411		err = __cancel_queued(&data);2412		if (err)2413			goto err_wedged;2414 2415		err = __cancel_hostile(&data);2416		if (err)2417			goto err_wedged;2418 2419		err = __cancel_fail(&data);2420		if (err)2421			goto err_wedged;2422	}2423 2424	err = 0;2425err_client_b:2426	preempt_client_fini(&data.b);2427err_client_a:2428	preempt_client_fini(&data.a);2429	return err;2430 2431err_wedged:2432	GEM_TRACE_DUMP();2433	igt_spinner_end(&data.b.spin);2434	igt_spinner_end(&data.a.spin);2435	intel_gt_set_wedged(gt);2436	goto err_client_b;2437}2438 2439static int live_suppress_self_preempt(void *arg)2440{2441	struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX };2442	struct intel_gt *gt = arg;2443	struct intel_engine_cs *engine;2444	struct preempt_client a, b;2445	enum intel_engine_id id;2446	int err = -ENOMEM;2447 2448	/*2449	 * Verify that if a preemption request does not cause a change in2450	 * the current execution order, the preempt-to-idle injection is2451	 * skipped and that we do not accidentally apply it after the CS2452	 * completion event.2453	 */2454 2455	if (intel_uc_uses_guc_submission(&gt->uc))2456		return 0; /* presume black blox */2457 2458	if (intel_vgpu_active(gt->i915))2459		return 0; /* GVT forces single port & request submission */2460 2461	if (preempt_client_init(gt, &a))2462		return -ENOMEM;2463	if (preempt_client_init(gt, &b))2464		goto err_client_a;2465 2466	for_each_engine(engine, gt, id) {2467		struct i915_request *rq_a, *rq_b;2468		int depth;2469 2470		if (!intel_engine_has_preemption(engine))2471			continue;2472 2473		if (igt_flush_test(gt->i915))2474			goto err_wedged;2475 2476		st_engine_heartbeat_disable(engine);2477		engine->execlists.preempt_hang.count = 0;2478 2479		rq_a = spinner_create_request(&a.spin,2480					      a.ctx, engine,2481					      MI_NOOP);2482		if (IS_ERR(rq_a)) {2483			err = PTR_ERR(rq_a);2484			st_engine_heartbeat_enable(engine);2485			goto err_client_b;2486		}2487 2488		i915_request_add(rq_a);2489		if (!igt_wait_for_spinner(&a.spin, rq_a)) {2490			pr_err("First client failed to start\n");2491			st_engine_heartbeat_enable(engine);2492			goto err_wedged;2493		}2494 2495		/* Keep postponing the timer to avoid premature slicing */2496		mod_timer(&engine->execlists.timer, jiffies + HZ);2497		for (depth = 0; depth < 8; depth++) {2498			rq_b = spinner_create_request(&b.spin,2499						      b.ctx, engine,2500						      MI_NOOP);2501			if (IS_ERR(rq_b)) {2502				err = PTR_ERR(rq_b);2503				st_engine_heartbeat_enable(engine);2504				goto err_client_b;2505			}2506			i915_request_add(rq_b);2507 2508			GEM_BUG_ON(i915_request_completed(rq_a));2509			engine->sched_engine->schedule(rq_a, &attr);2510			igt_spinner_end(&a.spin);2511 2512			if (!igt_wait_for_spinner(&b.spin, rq_b)) {2513				pr_err("Second client failed to start\n");2514				st_engine_heartbeat_enable(engine);2515				goto err_wedged;2516			}2517 2518			swap(a, b);2519			rq_a = rq_b;2520		}2521		igt_spinner_end(&a.spin);2522 2523		if (engine->execlists.preempt_hang.count) {2524			pr_err("Preemption on %s recorded x%d, depth %d; should have been suppressed!\n",2525			       engine->name,2526			       engine->execlists.preempt_hang.count,2527			       depth);2528			st_engine_heartbeat_enable(engine);2529			err = -EINVAL;2530			goto err_client_b;2531		}2532 2533		st_engine_heartbeat_enable(engine);2534		if (igt_flush_test(gt->i915))2535			goto err_wedged;2536	}2537 2538	err = 0;2539err_client_b:2540	preempt_client_fini(&b);2541err_client_a:2542	preempt_client_fini(&a);2543	return err;2544 2545err_wedged:2546	igt_spinner_end(&b.spin);2547	igt_spinner_end(&a.spin);2548	intel_gt_set_wedged(gt);2549	err = -EIO;2550	goto err_client_b;2551}2552 2553static int live_chain_preempt(void *arg)2554{2555	struct intel_gt *gt = arg;2556	struct intel_engine_cs *engine;2557	struct preempt_client hi, lo;2558	enum intel_engine_id id;2559	int err = -ENOMEM;2560 2561	/*2562	 * Build a chain AB...BA between two contexts (A, B) and request2563	 * preemption of the last request. It should then complete before2564	 * the previously submitted spinner in B.2565	 */2566 2567	if (preempt_client_init(gt, &hi))2568		return -ENOMEM;2569 2570	if (preempt_client_init(gt, &lo))2571		goto err_client_hi;2572 2573	for_each_engine(engine, gt, id) {2574		struct i915_sched_attr attr = { .priority = I915_PRIORITY_MAX };2575		struct igt_live_test t;2576		struct i915_request *rq;2577		int ring_size, count, i;2578 2579		if (!intel_engine_has_preemption(engine))2580			continue;2581 2582		rq = spinner_create_request(&lo.spin,2583					    lo.ctx, engine,2584					    MI_ARB_CHECK);2585		if (IS_ERR(rq))2586			goto err_wedged;2587 2588		i915_request_get(rq);2589		i915_request_add(rq);2590 2591		ring_size = rq->wa_tail - rq->head;2592		if (ring_size < 0)2593			ring_size += rq->ring->size;2594		ring_size = rq->ring->size / ring_size;2595		pr_debug("%s(%s): Using maximum of %d requests\n",2596			 __func__, engine->name, ring_size);2597 2598		igt_spinner_end(&lo.spin);2599		if (i915_request_wait(rq, 0, HZ / 2) < 0) {2600			pr_err("Timed out waiting to flush %s\n", engine->name);2601			i915_request_put(rq);2602			goto err_wedged;2603		}2604		i915_request_put(rq);2605 2606		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {2607			err = -EIO;2608			goto err_wedged;2609		}2610 2611		for_each_prime_number_from(count, 1, ring_size) {2612			rq = spinner_create_request(&hi.spin,2613						    hi.ctx, engine,2614						    MI_ARB_CHECK);2615			if (IS_ERR(rq))2616				goto err_wedged;2617			i915_request_add(rq);2618			if (!igt_wait_for_spinner(&hi.spin, rq))2619				goto err_wedged;2620 2621			rq = spinner_create_request(&lo.spin,2622						    lo.ctx, engine,2623						    MI_ARB_CHECK);2624			if (IS_ERR(rq))2625				goto err_wedged;2626			i915_request_add(rq);2627 2628			for (i = 0; i < count; i++) {2629				rq = igt_request_alloc(lo.ctx, engine);2630				if (IS_ERR(rq))2631					goto err_wedged;2632				i915_request_add(rq);2633			}2634 2635			rq = igt_request_alloc(hi.ctx, engine);2636			if (IS_ERR(rq))2637				goto err_wedged;2638 2639			i915_request_get(rq);2640			i915_request_add(rq);2641			engine->sched_engine->schedule(rq, &attr);2642 2643			igt_spinner_end(&hi.spin);2644			if (i915_request_wait(rq, 0, HZ / 5) < 0) {2645				struct drm_printer p =2646					drm_info_printer(gt->i915->drm.dev);2647 2648				pr_err("Failed to preempt over chain of %d\n",2649				       count);2650				intel_engine_dump(engine, &p,2651						  "%s\n", engine->name);2652				i915_request_put(rq);2653				goto err_wedged;2654			}2655			igt_spinner_end(&lo.spin);2656			i915_request_put(rq);2657 2658			rq = igt_request_alloc(lo.ctx, engine);2659			if (IS_ERR(rq))2660				goto err_wedged;2661 2662			i915_request_get(rq);2663			i915_request_add(rq);2664 2665			if (i915_request_wait(rq, 0, HZ / 5) < 0) {2666				struct drm_printer p =2667					drm_info_printer(gt->i915->drm.dev);2668 2669				pr_err("Failed to flush low priority chain of %d requests\n",2670				       count);2671				intel_engine_dump(engine, &p,2672						  "%s\n", engine->name);2673 2674				i915_request_put(rq);2675				goto err_wedged;2676			}2677			i915_request_put(rq);2678		}2679 2680		if (igt_live_test_end(&t)) {2681			err = -EIO;2682			goto err_wedged;2683		}2684	}2685 2686	err = 0;2687err_client_lo:2688	preempt_client_fini(&lo);2689err_client_hi:2690	preempt_client_fini(&hi);2691	return err;2692 2693err_wedged:2694	igt_spinner_end(&hi.spin);2695	igt_spinner_end(&lo.spin);2696	intel_gt_set_wedged(gt);2697	err = -EIO;2698	goto err_client_lo;2699}2700 2701static int create_gang(struct intel_engine_cs *engine,2702		       struct i915_request **prev)2703{2704	struct drm_i915_gem_object *obj;2705	struct intel_context *ce;2706	struct i915_request *rq;2707	struct i915_vma *vma;2708	u32 *cs;2709	int err;2710 2711	ce = intel_context_create(engine);2712	if (IS_ERR(ce))2713		return PTR_ERR(ce);2714 2715	obj = i915_gem_object_create_internal(engine->i915, 4096);2716	if (IS_ERR(obj)) {2717		err = PTR_ERR(obj);2718		goto err_ce;2719	}2720 2721	vma = i915_vma_instance(obj, ce->vm, NULL);2722	if (IS_ERR(vma)) {2723		err = PTR_ERR(vma);2724		goto err_obj;2725	}2726 2727	err = i915_vma_pin(vma, 0, 0, PIN_USER);2728	if (err)2729		goto err_obj;2730 2731	cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);2732	if (IS_ERR(cs)) {2733		err = PTR_ERR(cs);2734		goto err_obj;2735	}2736 2737	/* Semaphore target: spin until zero */2738	*cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;2739 2740	*cs++ = MI_SEMAPHORE_WAIT |2741		MI_SEMAPHORE_POLL |2742		MI_SEMAPHORE_SAD_EQ_SDD;2743	*cs++ = 0;2744	*cs++ = lower_32_bits(i915_vma_offset(vma));2745	*cs++ = upper_32_bits(i915_vma_offset(vma));2746 2747	if (*prev) {2748		u64 offset = i915_vma_offset((*prev)->batch);2749 2750		/* Terminate the spinner in the next lower priority batch. */2751		*cs++ = MI_STORE_DWORD_IMM_GEN4;2752		*cs++ = lower_32_bits(offset);2753		*cs++ = upper_32_bits(offset);2754		*cs++ = 0;2755	}2756 2757	*cs++ = MI_BATCH_BUFFER_END;2758	i915_gem_object_flush_map(obj);2759	i915_gem_object_unpin_map(obj);2760 2761	rq = intel_context_create_request(ce);2762	if (IS_ERR(rq)) {2763		err = PTR_ERR(rq);2764		goto err_obj;2765	}2766 2767	rq->batch = i915_vma_get(vma);2768	i915_request_get(rq);2769 2770	err = igt_vma_move_to_active_unlocked(vma, rq, 0);2771	if (!err)2772		err = rq->engine->emit_bb_start(rq,2773						i915_vma_offset(vma),2774						PAGE_SIZE, 0);2775	i915_request_add(rq);2776	if (err)2777		goto err_rq;2778 2779	i915_gem_object_put(obj);2780	intel_context_put(ce);2781 2782	rq->mock.link.next = &(*prev)->mock.link;2783	*prev = rq;2784	return 0;2785 2786err_rq:2787	i915_vma_put(rq->batch);2788	i915_request_put(rq);2789err_obj:2790	i915_gem_object_put(obj);2791err_ce:2792	intel_context_put(ce);2793	return err;2794}2795 2796static int __live_preempt_ring(struct intel_engine_cs *engine,2797			       struct igt_spinner *spin,2798			       int queue_sz, int ring_sz)2799{2800	struct intel_context *ce[2] = {};2801	struct i915_request *rq;2802	struct igt_live_test t;2803	int err = 0;2804	int n;2805 2806	if (igt_live_test_begin(&t, engine->i915, __func__, engine->name))2807		return -EIO;2808 2809	for (n = 0; n < ARRAY_SIZE(ce); n++) {2810		struct intel_context *tmp;2811 2812		tmp = intel_context_create(engine);2813		if (IS_ERR(tmp)) {2814			err = PTR_ERR(tmp);2815			goto err_ce;2816		}2817 2818		tmp->ring_size = ring_sz;2819 2820		err = intel_context_pin(tmp);2821		if (err) {2822			intel_context_put(tmp);2823			goto err_ce;2824		}2825 2826		memset32(tmp->ring->vaddr,2827			 0xdeadbeef, /* trigger a hang if executed */2828			 tmp->ring->vma->size / sizeof(u32));2829 2830		ce[n] = tmp;2831	}2832 2833	rq = igt_spinner_create_request(spin, ce[0], MI_ARB_CHECK);2834	if (IS_ERR(rq)) {2835		err = PTR_ERR(rq);2836		goto err_ce;2837	}2838 2839	i915_request_get(rq);2840	rq->sched.attr.priority = I915_PRIORITY_BARRIER;2841	i915_request_add(rq);2842 2843	if (!igt_wait_for_spinner(spin, rq)) {2844		intel_gt_set_wedged(engine->gt);2845		i915_request_put(rq);2846		err = -ETIME;2847		goto err_ce;2848	}2849 2850	/* Fill the ring, until we will cause a wrap */2851	n = 0;2852	while (ce[0]->ring->tail - rq->wa_tail <= queue_sz) {2853		struct i915_request *tmp;2854 2855		tmp = intel_context_create_request(ce[0]);2856		if (IS_ERR(tmp)) {2857			err = PTR_ERR(tmp);2858			i915_request_put(rq);2859			goto err_ce;2860		}2861 2862		i915_request_add(tmp);2863		intel_engine_flush_submission(engine);2864		n++;2865	}2866	intel_engine_flush_submission(engine);2867	pr_debug("%s: Filled %d with %d nop tails {size:%x, tail:%x, emit:%x, rq.tail:%x}\n",2868		 engine->name, queue_sz, n,2869		 ce[0]->ring->size,2870		 ce[0]->ring->tail,2871		 ce[0]->ring->emit,2872		 rq->tail);2873	i915_request_put(rq);2874 2875	/* Create a second request to preempt the first ring */2876	rq = intel_context_create_request(ce[1]);2877	if (IS_ERR(rq)) {2878		err = PTR_ERR(rq);2879		goto err_ce;2880	}2881 2882	rq->sched.attr.priority = I915_PRIORITY_BARRIER;2883	i915_request_get(rq);2884	i915_request_add(rq);2885 2886	err = wait_for_submit(engine, rq, HZ / 2);2887	i915_request_put(rq);2888	if (err) {2889		pr_err("%s: preemption request was not submitted\n",2890		       engine->name);2891		err = -ETIME;2892	}2893 2894	pr_debug("%s: ring[0]:{ tail:%x, emit:%x }, ring[1]:{ tail:%x, emit:%x }\n",2895		 engine->name,2896		 ce[0]->ring->tail, ce[0]->ring->emit,2897		 ce[1]->ring->tail, ce[1]->ring->emit);2898 2899err_ce:2900	intel_engine_flush_submission(engine);2901	igt_spinner_end(spin);2902	for (n = 0; n < ARRAY_SIZE(ce); n++) {2903		if (IS_ERR_OR_NULL(ce[n]))2904			break;2905 2906		intel_context_unpin(ce[n]);2907		intel_context_put(ce[n]);2908	}2909	if (igt_live_test_end(&t))2910		err = -EIO;2911	return err;2912}2913 2914static int live_preempt_ring(void *arg)2915{2916	struct intel_gt *gt = arg;2917	struct intel_engine_cs *engine;2918	struct igt_spinner spin;2919	enum intel_engine_id id;2920	int err = 0;2921 2922	/*2923	 * Check that we rollback large chunks of a ring in order to do a2924	 * preemption event. Similar to live_unlite_ring, but looking at2925	 * ring size rather than the impact of intel_ring_direction().2926	 */2927 2928	if (igt_spinner_init(&spin, gt))2929		return -ENOMEM;2930 2931	for_each_engine(engine, gt, id) {2932		int n;2933 2934		if (!intel_engine_has_preemption(engine))2935			continue;2936 2937		if (!intel_engine_can_store_dword(engine))2938			continue;2939 2940		st_engine_heartbeat_disable(engine);2941 2942		for (n = 0; n <= 3; n++) {2943			err = __live_preempt_ring(engine, &spin,2944						  n * SZ_4K / 4, SZ_4K);2945			if (err)2946				break;2947		}2948 2949		st_engine_heartbeat_enable(engine);2950		if (err)2951			break;2952	}2953 2954	igt_spinner_fini(&spin);2955	return err;2956}2957 2958static int live_preempt_gang(void *arg)2959{2960	struct intel_gt *gt = arg;2961	struct intel_engine_cs *engine;2962	enum intel_engine_id id;2963 2964	/*2965	 * Build as long a chain of preempters as we can, with each2966	 * request higher priority than the last. Once we are ready, we release2967	 * the last batch which then precolates down the chain, each releasing2968	 * the next oldest in turn. The intent is to simply push as hard as we2969	 * can with the number of preemptions, trying to exceed narrow HW2970	 * limits. At a minimum, we insist that we can sort all the user2971	 * high priority levels into execution order.2972	 */2973 2974	for_each_engine(engine, gt, id) {2975		struct i915_request *rq = NULL;2976		struct igt_live_test t;2977		IGT_TIMEOUT(end_time);2978		int prio = 0;2979		int err = 0;2980		u32 *cs;2981 2982		if (!intel_engine_has_preemption(engine))2983			continue;2984 2985		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name))2986			return -EIO;2987 2988		do {2989			struct i915_sched_attr attr = { .priority = prio++ };2990 2991			err = create_gang(engine, &rq);2992			if (err)2993				break;2994 2995			/* Submit each spinner at increasing priority */2996			engine->sched_engine->schedule(rq, &attr);2997		} while (prio <= I915_PRIORITY_MAX &&2998			 !__igt_timeout(end_time, NULL));2999		pr_debug("%s: Preempt chain of %d requests\n",3000			 engine->name, prio);3001 3002		/*3003		 * Such that the last spinner is the highest priority and3004		 * should execute first. When that spinner completes,3005		 * it will terminate the next lowest spinner until there3006		 * are no more spinners and the gang is complete.3007		 */3008		cs = i915_gem_object_pin_map_unlocked(rq->batch->obj, I915_MAP_WC);3009		if (!IS_ERR(cs)) {3010			*cs = 0;3011			i915_gem_object_unpin_map(rq->batch->obj);3012		} else {3013			err = PTR_ERR(cs);3014			intel_gt_set_wedged(gt);3015		}3016 3017		while (rq) { /* wait for each rq from highest to lowest prio */3018			struct i915_request *n = list_next_entry(rq, mock.link);3019 3020			if (err == 0 && i915_request_wait(rq, 0, HZ / 5) < 0) {3021				struct drm_printer p =3022					drm_info_printer(engine->i915->drm.dev);3023 3024				pr_err("Failed to flush chain of %d requests, at %d\n",3025				       prio, rq_prio(rq));3026				intel_engine_dump(engine, &p,3027						  "%s\n", engine->name);3028 3029				err = -ETIME;3030			}3031 3032			i915_vma_put(rq->batch);3033			i915_request_put(rq);3034			rq = n;3035		}3036 3037		if (igt_live_test_end(&t))3038			err = -EIO;3039		if (err)3040			return err;3041	}3042 3043	return 0;3044}3045 3046static struct i915_vma *3047create_gpr_user(struct intel_engine_cs *engine,3048		struct i915_vma *result,3049		unsigned int offset)3050{3051	struct drm_i915_gem_object *obj;3052	struct i915_vma *vma;3053	u32 *cs;3054	int err;3055	int i;3056 3057	obj = i915_gem_object_create_internal(engine->i915, 4096);3058	if (IS_ERR(obj))3059		return ERR_CAST(obj);3060 3061	vma = i915_vma_instance(obj, result->vm, NULL);3062	if (IS_ERR(vma)) {3063		i915_gem_object_put(obj);3064		return vma;3065	}3066 3067	err = i915_vma_pin(vma, 0, 0, PIN_USER);3068	if (err) {3069		i915_vma_put(vma);3070		return ERR_PTR(err);3071	}3072 3073	cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);3074	if (IS_ERR(cs)) {3075		i915_vma_put(vma);3076		return ERR_CAST(cs);3077	}3078 3079	/* All GPR are clear for new contexts. We use GPR(0) as a constant */3080	*cs++ = MI_LOAD_REGISTER_IMM(1);3081	*cs++ = CS_GPR(engine, 0);3082	*cs++ = 1;3083 3084	for (i = 1; i < NUM_GPR; i++) {3085		u64 addr;3086 3087		/*3088		 * Perform: GPR[i]++3089		 *3090		 * As we read and write into the context saved GPR[i], if3091		 * we restart this batch buffer from an earlier point, we3092		 * will repeat the increment and store a value > 1.3093		 */3094		*cs++ = MI_MATH(4);3095		*cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCA, MI_MATH_REG(i));3096		*cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCB, MI_MATH_REG(0));3097		*cs++ = MI_MATH_ADD;3098		*cs++ = MI_MATH_STORE(MI_MATH_REG(i), MI_MATH_REG_ACCU);3099 3100		addr = i915_vma_offset(result) + offset + i * sizeof(*cs);3101		*cs++ = MI_STORE_REGISTER_MEM_GEN8;3102		*cs++ = CS_GPR(engine, 2 * i);3103		*cs++ = lower_32_bits(addr);3104		*cs++ = upper_32_bits(addr);3105 3106		*cs++ = MI_SEMAPHORE_WAIT |3107			MI_SEMAPHORE_POLL |3108			MI_SEMAPHORE_SAD_GTE_SDD;3109		*cs++ = i;3110		*cs++ = lower_32_bits(i915_vma_offset(result));3111		*cs++ = upper_32_bits(i915_vma_offset(result));3112	}3113 3114	*cs++ = MI_BATCH_BUFFER_END;3115	i915_gem_object_flush_map(obj);3116	i915_gem_object_unpin_map(obj);3117 3118	return vma;3119}3120 3121static struct i915_vma *create_global(struct intel_gt *gt, size_t sz)3122{3123	struct drm_i915_gem_object *obj;3124	struct i915_vma *vma;3125	int err;3126 3127	obj = i915_gem_object_create_internal(gt->i915, sz);3128	if (IS_ERR(obj))3129		return ERR_CAST(obj);3130 3131	vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);3132	if (IS_ERR(vma)) {3133		i915_gem_object_put(obj);3134		return vma;3135	}3136 3137	err = i915_ggtt_pin(vma, NULL, 0, 0);3138	if (err) {3139		i915_vma_put(vma);3140		return ERR_PTR(err);3141	}3142 3143	return vma;3144}3145 3146static struct i915_request *3147create_gpr_client(struct intel_engine_cs *engine,3148		  struct i915_vma *global,3149		  unsigned int offset)3150{3151	struct i915_vma *batch, *vma;3152	struct intel_context *ce;3153	struct i915_request *rq;3154	int err;3155 3156	ce = intel_context_create(engine);3157	if (IS_ERR(ce))3158		return ERR_CAST(ce);3159 3160	vma = i915_vma_instance(global->obj, ce->vm, NULL);3161	if (IS_ERR(vma)) {3162		err = PTR_ERR(vma);3163		goto out_ce;3164	}3165 3166	err = i915_vma_pin(vma, 0, 0, PIN_USER);3167	if (err)3168		goto out_ce;3169 3170	batch = create_gpr_user(engine, vma, offset);3171	if (IS_ERR(batch)) {3172		err = PTR_ERR(batch);3173		goto out_vma;3174	}3175 3176	rq = intel_context_create_request(ce);3177	if (IS_ERR(rq)) {3178		err = PTR_ERR(rq);3179		goto out_batch;3180	}3181 3182	err = igt_vma_move_to_active_unlocked(vma, rq, 0);3183 3184	i915_vma_lock(batch);3185	if (!err)3186		err = i915_vma_move_to_active(batch, rq, 0);3187	if (!err)3188		err = rq->engine->emit_bb_start(rq,3189						i915_vma_offset(batch),3190						PAGE_SIZE, 0);3191	i915_vma_unlock(batch);3192	i915_vma_unpin(batch);3193 3194	if (!err)3195		i915_request_get(rq);3196	i915_request_add(rq);3197 3198out_batch:3199	i915_vma_put(batch);3200out_vma:3201	i915_vma_unpin(vma);3202out_ce:3203	intel_context_put(ce);3204	return err ? ERR_PTR(err) : rq;3205}3206 3207static int preempt_user(struct intel_engine_cs *engine,3208			struct i915_vma *global,3209			int id)3210{3211	struct i915_sched_attr attr = {3212		.priority = I915_PRIORITY_MAX3213	};3214	struct i915_request *rq;3215	int err = 0;3216	u32 *cs;3217 3218	rq = intel_engine_create_kernel_request(engine);3219	if (IS_ERR(rq))3220		return PTR_ERR(rq);3221 3222	cs = intel_ring_begin(rq, 4);3223	if (IS_ERR(cs)) {3224		i915_request_add(rq);3225		return PTR_ERR(cs);3226	}3227 3228	*cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;3229	*cs++ = i915_ggtt_offset(global);3230	*cs++ = 0;3231	*cs++ = id;3232 3233	intel_ring_advance(rq, cs);3234 3235	i915_request_get(rq);3236	i915_request_add(rq);3237 3238	engine->sched_engine->schedule(rq, &attr);3239 3240	if (i915_request_wait(rq, 0, HZ / 2) < 0)3241		err = -ETIME;3242	i915_request_put(rq);3243 3244	return err;3245}3246 3247static int live_preempt_user(void *arg)3248{3249	struct intel_gt *gt = arg;3250	struct intel_engine_cs *engine;3251	struct i915_vma *global;3252	enum intel_engine_id id;3253	u32 *result;3254	int err = 0;3255 3256	/*3257	 * In our other tests, we look at preemption in carefully3258	 * controlled conditions in the ringbuffer. Since most of the3259	 * time is spent in user batches, most of our preemptions naturally3260	 * occur there. We want to verify that when we preempt inside a batch3261	 * we continue on from the current instruction and do not roll back3262	 * to the start, or another earlier arbitration point.3263	 *3264	 * To verify this, we create a batch which is a mixture of3265	 * MI_MATH (gpr++) MI_SRM (gpr) and preemption points. Then with3266	 * a few preempting contexts thrown into the mix, we look for any3267	 * repeated instructions (which show up as incorrect values).3268	 */3269 3270	global = create_global(gt, 4096);3271	if (IS_ERR(global))3272		return PTR_ERR(global);3273 3274	result = i915_gem_object_pin_map_unlocked(global->obj, I915_MAP_WC);3275	if (IS_ERR(result)) {3276		i915_vma_unpin_and_release(&global, 0);3277		return PTR_ERR(result);3278	}3279 3280	for_each_engine(engine, gt, id) {3281		struct i915_request *client[3] = {};3282		struct igt_live_test t;3283		int i;3284 3285		if (!intel_engine_has_preemption(engine))3286			continue;3287 3288		if (GRAPHICS_VER(gt->i915) == 8 && engine->class != RENDER_CLASS)3289			continue; /* we need per-context GPR */3290 3291		if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {3292			err = -EIO;3293			break;3294		}3295 3296		memset(result, 0, 4096);3297 3298		for (i = 0; i < ARRAY_SIZE(client); i++) {3299			struct i915_request *rq;3300 3301			rq = create_gpr_client(engine, global,3302					       NUM_GPR * i * sizeof(u32));3303			if (IS_ERR(rq)) {3304				err = PTR_ERR(rq);3305				goto end_test;3306			}3307 3308			client[i] = rq;3309		}3310 3311		/* Continuously preempt the set of 3 running contexts */3312		for (i = 1; i <= NUM_GPR; i++) {3313			err = preempt_user(engine, global, i);3314			if (err)3315				goto end_test;3316		}3317 3318		if (READ_ONCE(result[0]) != NUM_GPR) {3319			pr_err("%s: Failed to release semaphore\n",3320			       engine->name);3321			err = -EIO;3322			goto end_test;3323		}3324 3325		for (i = 0; i < ARRAY_SIZE(client); i++) {3326			int gpr;3327 3328			if (i915_request_wait(client[i], 0, HZ / 2) < 0) {3329				err = -ETIME;3330				goto end_test;3331			}3332 3333			for (gpr = 1; gpr < NUM_GPR; gpr++) {3334				if (result[NUM_GPR * i + gpr] != 1) {3335					pr_err("%s: Invalid result, client %d, gpr %d, result: %d\n",3336					       engine->name,3337					       i, gpr, result[NUM_GPR * i + gpr]);3338					err = -EINVAL;3339					goto end_test;3340				}3341			}3342		}3343 3344end_test:3345		for (i = 0; i < ARRAY_SIZE(client); i++) {3346			if (!client[i])3347				break;3348 3349			i915_request_put(client[i]);3350		}3351 3352		/* Flush the semaphores on error */3353		smp_store_mb(result[0], -1);3354		if (igt_live_test_end(&t))3355			err = -EIO;3356		if (err)3357			break;3358	}3359 3360	i915_vma_unpin_and_release(&global, I915_VMA_RELEASE_MAP);3361	return err;3362}3363 3364static int live_preempt_timeout(void *arg)3365{3366	struct intel_gt *gt = arg;3367	struct i915_gem_context *ctx_hi, *ctx_lo;3368	struct igt_spinner spin_lo;3369	struct intel_engine_cs *engine;3370	enum intel_engine_id id;3371	int err = -ENOMEM;3372 3373	/*3374	 * Check that we force preemption to occur by cancelling the previous3375	 * context if it refuses to yield the GPU.3376	 */3377	if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT)3378		return 0;3379 3380	if (!intel_has_reset_engine(gt))3381		return 0;3382 3383	ctx_hi = kernel_context(gt->i915, NULL);3384	if (!ctx_hi)3385		return -ENOMEM;3386	ctx_hi->sched.priority = I915_CONTEXT_MAX_USER_PRIORITY;3387 3388	ctx_lo = kernel_context(gt->i915, NULL);3389	if (!ctx_lo)3390		goto err_ctx_hi;3391	ctx_lo->sched.priority = I915_CONTEXT_MIN_USER_PRIORITY;3392 3393	if (igt_spinner_init(&spin_lo, gt))3394		goto err_ctx_lo;3395 3396	for_each_engine(engine, gt, id) {3397		unsigned long saved_timeout;3398		struct i915_request *rq;3399 3400		if (!intel_engine_has_preemption(engine))3401			continue;3402 3403		rq = spinner_create_request(&spin_lo, ctx_lo, engine,3404					    MI_NOOP); /* preemption disabled */3405		if (IS_ERR(rq)) {3406			err = PTR_ERR(rq);3407			goto err_spin_lo;3408		}3409 3410		i915_request_add(rq);3411		if (!igt_wait_for_spinner(&spin_lo, rq)) {3412			intel_gt_set_wedged(gt);3413			err = -EIO;3414			goto err_spin_lo;3415		}3416 3417		rq = igt_request_alloc(ctx_hi, engine);3418		if (IS_ERR(rq)) {3419			igt_spinner_end(&spin_lo);3420			err = PTR_ERR(rq);3421			goto err_spin_lo;3422		}3423 3424		/* Flush the previous CS ack before changing timeouts */3425		while (READ_ONCE(engine->execlists.pending[0]))3426			cpu_relax();3427 3428		saved_timeout = engine->props.preempt_timeout_ms;3429		engine->props.preempt_timeout_ms = 1; /* in ms, -> 1 jiffy */3430 3431		i915_request_get(rq);3432		i915_request_add(rq);3433 3434		intel_engine_flush_submission(engine);3435		engine->props.preempt_timeout_ms = saved_timeout;3436 3437		if (i915_request_wait(rq, 0, HZ / 10) < 0) {3438			intel_gt_set_wedged(gt);3439			i915_request_put(rq);3440			err = -ETIME;3441			goto err_spin_lo;3442		}3443 3444		igt_spinner_end(&spin_lo);3445		i915_request_put(rq);3446	}3447 3448	err = 0;3449err_spin_lo:3450	igt_spinner_fini(&spin_lo);3451err_ctx_lo:3452	kernel_context_close(ctx_lo);3453err_ctx_hi:3454	kernel_context_close(ctx_hi);3455	return err;3456}3457 3458static int random_range(struct rnd_state *rnd, int min, int max)3459{3460	return i915_prandom_u32_max_state(max - min, rnd) + min;3461}3462 3463static int random_priority(struct rnd_state *rnd)3464{3465	return random_range(rnd, I915_PRIORITY_MIN, I915_PRIORITY_MAX);3466}3467 3468struct preempt_smoke {3469	struct intel_gt *gt;3470	struct kthread_work work;3471	struct i915_gem_context **contexts;3472	struct intel_engine_cs *engine;3473	struct drm_i915_gem_object *batch;3474	unsigned int ncontext;3475	struct rnd_state prng;3476	unsigned long count;3477	int result;3478};3479 3480static struct i915_gem_context *smoke_context(struct preempt_smoke *smoke)3481{3482	return smoke->contexts[i915_prandom_u32_max_state(smoke->ncontext,3483							  &smoke->prng)];3484}3485 3486static int smoke_submit(struct preempt_smoke *smoke,3487			struct i915_gem_context *ctx, int prio,3488			struct drm_i915_gem_object *batch)3489{3490	struct i915_request *rq;3491	struct i915_vma *vma = NULL;3492	int err = 0;3493 3494	if (batch) {3495		struct i915_address_space *vm;3496 3497		vm = i915_gem_context_get_eb_vm(ctx);3498		vma = i915_vma_instance(batch, vm, NULL);3499		i915_vm_put(vm);3500		if (IS_ERR(vma))3501			return PTR_ERR(vma);3502 3503		err = i915_vma_pin(vma, 0, 0, PIN_USER);3504		if (err)3505			return err;3506	}3507 3508	ctx->sched.priority = prio;3509 3510	rq = igt_request_alloc(ctx, smoke->engine);3511	if (IS_ERR(rq)) {3512		err = PTR_ERR(rq);3513		goto unpin;3514	}3515 3516	if (vma) {3517		err = igt_vma_move_to_active_unlocked(vma, rq, 0);3518		if (!err)3519			err = rq->engine->emit_bb_start(rq,3520							i915_vma_offset(vma),3521							PAGE_SIZE, 0);3522	}3523 3524	i915_request_add(rq);3525 3526unpin:3527	if (vma)3528		i915_vma_unpin(vma);3529 3530	return err;3531}3532 3533static void smoke_crescendo_work(struct kthread_work *work)3534{3535	struct preempt_smoke *smoke = container_of(work, typeof(*smoke), work);3536	IGT_TIMEOUT(end_time);3537	unsigned long count;3538 3539	count = 0;3540	do {3541		struct i915_gem_context *ctx = smoke_context(smoke);3542 3543		smoke->result = smoke_submit(smoke, ctx,3544					     count % I915_PRIORITY_MAX,3545					     smoke->batch);3546 3547		count++;3548	} while (!smoke->result && count < smoke->ncontext &&3549		 !__igt_timeout(end_time, NULL));3550 3551	smoke->count = count;3552}3553 3554static int smoke_crescendo(struct preempt_smoke *smoke, unsigned int flags)3555#define BATCH BIT(0)3556{3557	struct kthread_worker *worker[I915_NUM_ENGINES] = {};3558	struct preempt_smoke *arg;3559	struct intel_engine_cs *engine;3560	enum intel_engine_id id;3561	unsigned long count;3562	int err = 0;3563 3564	arg = kmalloc_array(I915_NUM_ENGINES, sizeof(*arg), GFP_KERNEL);3565	if (!arg)3566		return -ENOMEM;3567 3568	memset(arg, 0, I915_NUM_ENGINES * sizeof(*arg));3569 3570	for_each_engine(engine, smoke->gt, id) {3571		arg[id] = *smoke;3572		arg[id].engine = engine;3573		if (!(flags & BATCH))3574			arg[id].batch = NULL;3575		arg[id].count = 0;3576 3577		worker[id] = kthread_create_worker(0, "igt/smoke:%d", id);3578		if (IS_ERR(worker[id])) {3579			err = PTR_ERR(worker[id]);3580			break;3581		}3582 3583		kthread_init_work(&arg[id].work, smoke_crescendo_work);3584		kthread_queue_work(worker[id], &arg[id].work);3585	}3586 3587	count = 0;3588	for_each_engine(engine, smoke->gt, id) {3589		if (IS_ERR_OR_NULL(worker[id]))3590			continue;3591 3592		kthread_flush_work(&arg[id].work);3593		if (arg[id].result && !err)3594			err = arg[id].result;3595 3596		count += arg[id].count;3597 3598		kthread_destroy_worker(worker[id]);3599	}3600 3601	pr_info("Submitted %lu crescendo:%x requests across %d engines and %d contexts\n",3602		count, flags, smoke->gt->info.num_engines, smoke->ncontext);3603 3604	kfree(arg);3605	return 0;3606}3607 3608static int smoke_random(struct preempt_smoke *smoke, unsigned int flags)3609{3610	enum intel_engine_id id;3611	IGT_TIMEOUT(end_time);3612	unsigned long count;3613 3614	count = 0;3615	do {3616		for_each_engine(smoke->engine, smoke->gt, id) {3617			struct i915_gem_context *ctx = smoke_context(smoke);3618			int err;3619 3620			err = smoke_submit(smoke,3621					   ctx, random_priority(&smoke->prng),3622					   flags & BATCH ? smoke->batch : NULL);3623			if (err)3624				return err;3625 3626			count++;3627		}3628	} while (count < smoke->ncontext && !__igt_timeout(end_time, NULL));3629 3630	pr_info("Submitted %lu random:%x requests across %d engines and %d contexts\n",3631		count, flags, smoke->gt->info.num_engines, smoke->ncontext);3632	return 0;3633}3634 3635static int live_preempt_smoke(void *arg)3636{3637	struct preempt_smoke smoke = {3638		.gt = arg,3639		.prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed),3640		.ncontext = 256,3641	};3642	const unsigned int phase[] = { 0, BATCH };3643	struct igt_live_test t;3644	int err = -ENOMEM;3645	u32 *cs;3646	int n;3647 3648	smoke.contexts = kmalloc_array(smoke.ncontext,3649				       sizeof(*smoke.contexts),3650				       GFP_KERNEL);3651	if (!smoke.contexts)3652		return -ENOMEM;3653 3654	smoke.batch =3655		i915_gem_object_create_internal(smoke.gt->i915, PAGE_SIZE);3656	if (IS_ERR(smoke.batch)) {3657		err = PTR_ERR(smoke.batch);3658		goto err_free;3659	}3660 3661	cs = i915_gem_object_pin_map_unlocked(smoke.batch, I915_MAP_WB);3662	if (IS_ERR(cs)) {3663		err = PTR_ERR(cs);3664		goto err_batch;3665	}3666	for (n = 0; n < PAGE_SIZE / sizeof(*cs) - 1; n++)3667		cs[n] = MI_ARB_CHECK;3668	cs[n] = MI_BATCH_BUFFER_END;3669	i915_gem_object_flush_map(smoke.batch);3670	i915_gem_object_unpin_map(smoke.batch);3671 3672	if (igt_live_test_begin(&t, smoke.gt->i915, __func__, "all")) {3673		err = -EIO;3674		goto err_batch;3675	}3676 3677	for (n = 0; n < smoke.ncontext; n++) {3678		smoke.contexts[n] = kernel_context(smoke.gt->i915, NULL);3679		if (!smoke.contexts[n])3680			goto err_ctx;3681	}3682 3683	for (n = 0; n < ARRAY_SIZE(phase); n++) {3684		err = smoke_crescendo(&smoke, phase[n]);3685		if (err)3686			goto err_ctx;3687 3688		err = smoke_random(&smoke, phase[n]);3689		if (err)3690			goto err_ctx;3691	}3692 3693err_ctx:3694	if (igt_live_test_end(&t))3695		err = -EIO;3696 3697	for (n = 0; n < smoke.ncontext; n++) {3698		if (!smoke.contexts[n])3699			break;3700		kernel_context_close(smoke.contexts[n]);3701	}3702 3703err_batch:3704	i915_gem_object_put(smoke.batch);3705err_free:3706	kfree(smoke.contexts);3707 3708	return err;3709}3710 3711static int nop_virtual_engine(struct intel_gt *gt,3712			      struct intel_engine_cs **siblings,3713			      unsigned int nsibling,3714			      unsigned int nctx,3715			      unsigned int flags)3716#define CHAIN BIT(0)3717{3718	IGT_TIMEOUT(end_time);3719	struct i915_request *request[16] = {};3720	struct intel_context *ve[16];3721	unsigned long n, prime, nc;3722	struct igt_live_test t;3723	ktime_t times[2] = {};3724	int err;3725 3726	GEM_BUG_ON(!nctx || nctx > ARRAY_SIZE(ve));3727 3728	for (n = 0; n < nctx; n++) {3729		ve[n] = intel_engine_create_virtual(siblings, nsibling, 0);3730		if (IS_ERR(ve[n])) {3731			err = PTR_ERR(ve[n]);3732			nctx = n;3733			goto out;3734		}3735 3736		err = intel_context_pin(ve[n]);3737		if (err) {3738			intel_context_put(ve[n]);3739			nctx = n;3740			goto out;3741		}3742	}3743 3744	err = igt_live_test_begin(&t, gt->i915, __func__, ve[0]->engine->name);3745	if (err)3746		goto out;3747 3748	for_each_prime_number_from(prime, 1, 8192) {3749		times[1] = ktime_get_raw();3750 3751		if (flags & CHAIN) {3752			for (nc = 0; nc < nctx; nc++) {3753				for (n = 0; n < prime; n++) {3754					struct i915_request *rq;3755 3756					rq = i915_request_create(ve[nc]);3757					if (IS_ERR(rq)) {3758						err = PTR_ERR(rq);3759						goto out;3760					}3761 3762					if (request[nc])3763						i915_request_put(request[nc]);3764					request[nc] = i915_request_get(rq);3765					i915_request_add(rq);3766				}3767			}3768		} else {3769			for (n = 0; n < prime; n++) {3770				for (nc = 0; nc < nctx; nc++) {3771					struct i915_request *rq;3772 3773					rq = i915_request_create(ve[nc]);3774					if (IS_ERR(rq)) {3775						err = PTR_ERR(rq);3776						goto out;3777					}3778 3779					if (request[nc])3780						i915_request_put(request[nc]);3781					request[nc] = i915_request_get(rq);3782					i915_request_add(rq);3783				}3784			}3785		}3786 3787		for (nc = 0; nc < nctx; nc++) {3788			if (i915_request_wait(request[nc], 0, HZ / 10) < 0) {3789				pr_err("%s(%s): wait for %llx:%lld timed out\n",3790				       __func__, ve[0]->engine->name,3791				       request[nc]->fence.context,3792				       request[nc]->fence.seqno);3793 3794				GEM_TRACE("%s(%s) failed at request %llx:%lld\n",3795					  __func__, ve[0]->engine->name,3796					  request[nc]->fence.context,3797					  request[nc]->fence.seqno);3798				GEM_TRACE_DUMP();3799				intel_gt_set_wedged(gt);3800				break;3801			}3802		}3803 3804		times[1] = ktime_sub(ktime_get_raw(), times[1]);3805		if (prime == 1)3806			times[0] = times[1];3807 3808		for (nc = 0; nc < nctx; nc++) {3809			i915_request_put(request[nc]);3810			request[nc] = NULL;3811		}3812 3813		if (__igt_timeout(end_time, NULL))3814			break;3815	}3816 3817	err = igt_live_test_end(&t);3818	if (err)3819		goto out;3820 3821	pr_info("Requestx%d latencies on %s: 1 = %lluns, %lu = %lluns\n",3822		nctx, ve[0]->engine->name, ktime_to_ns(times[0]),3823		prime, div64_u64(ktime_to_ns(times[1]), prime));3824 3825out:3826	if (igt_flush_test(gt->i915))3827		err = -EIO;3828 3829	for (nc = 0; nc < nctx; nc++) {3830		i915_request_put(request[nc]);3831		intel_context_unpin(ve[nc]);3832		intel_context_put(ve[nc]);3833	}3834	return err;3835}3836 3837static unsigned int3838__select_siblings(struct intel_gt *gt,3839		  unsigned int class,3840		  struct intel_engine_cs **siblings,3841		  bool (*filter)(const struct intel_engine_cs *))3842{3843	unsigned int n = 0;3844	unsigned int inst;3845 3846	for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {3847		if (!gt->engine_class[class][inst])3848			continue;3849 3850		if (filter && !filter(gt->engine_class[class][inst]))3851			continue;3852 3853		siblings[n++] = gt->engine_class[class][inst];3854	}3855 3856	return n;3857}3858 3859static unsigned int3860select_siblings(struct intel_gt *gt,3861		unsigned int class,3862		struct intel_engine_cs **siblings)3863{3864	return __select_siblings(gt, class, siblings, NULL);3865}3866 3867static int live_virtual_engine(void *arg)3868{3869	struct intel_gt *gt = arg;3870	struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];3871	struct intel_engine_cs *engine;3872	enum intel_engine_id id;3873	unsigned int class;3874	int err;3875 3876	if (intel_uc_uses_guc_submission(&gt->uc))3877		return 0;3878 3879	for_each_engine(engine, gt, id) {3880		err = nop_virtual_engine(gt, &engine, 1, 1, 0);3881		if (err) {3882			pr_err("Failed to wrap engine %s: err=%d\n",3883			       engine->name, err);3884			return err;3885		}3886	}3887 3888	for (class = 0; class <= MAX_ENGINE_CLASS; class++) {3889		int nsibling, n;3890 3891		nsibling = select_siblings(gt, class, siblings);3892		if (nsibling < 2)3893			continue;3894 3895		for (n = 1; n <= nsibling + 1; n++) {3896			err = nop_virtual_engine(gt, siblings, nsibling,3897						 n, 0);3898			if (err)3899				return err;3900		}3901 3902		err = nop_virtual_engine(gt, siblings, nsibling, n, CHAIN);3903		if (err)3904			return err;3905	}3906 3907	return 0;3908}3909 3910static int mask_virtual_engine(struct intel_gt *gt,3911			       struct intel_engine_cs **siblings,3912			       unsigned int nsibling)3913{3914	struct i915_request *request[MAX_ENGINE_INSTANCE + 1];3915	struct intel_context *ve;3916	struct igt_live_test t;3917	unsigned int n;3918	int err;3919 3920	/*3921	 * Check that by setting the execution mask on a request, we can3922	 * restrict it to our desired engine within the virtual engine.3923	 */3924 3925	ve = intel_engine_create_virtual(siblings, nsibling, 0);3926	if (IS_ERR(ve)) {3927		err = PTR_ERR(ve);3928		goto out_close;3929	}3930 3931	err = intel_context_pin(ve);3932	if (err)3933		goto out_put;3934 3935	err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name);3936	if (err)3937		goto out_unpin;3938 3939	for (n = 0; n < nsibling; n++) {3940		request[n] = i915_request_create(ve);3941		if (IS_ERR(request[n])) {3942			err = PTR_ERR(request[n]);3943			nsibling = n;3944			goto out;3945		}3946 3947		/* Reverse order as it's more likely to be unnatural */3948		request[n]->execution_mask = siblings[nsibling - n - 1]->mask;3949 3950		i915_request_get(request[n]);3951		i915_request_add(request[n]);3952	}3953 3954	for (n = 0; n < nsibling; n++) {3955		if (i915_request_wait(request[n], 0, HZ / 10) < 0) {3956			pr_err("%s(%s): wait for %llx:%lld timed out\n",3957			       __func__, ve->engine->name,3958			       request[n]->fence.context,3959			       request[n]->fence.seqno);3960 3961			GEM_TRACE("%s(%s) failed at request %llx:%lld\n",3962				  __func__, ve->engine->name,3963				  request[n]->fence.context,3964				  request[n]->fence.seqno);3965			GEM_TRACE_DUMP();3966			intel_gt_set_wedged(gt);3967			err = -EIO;3968			goto out;3969		}3970 3971		if (request[n]->engine != siblings[nsibling - n - 1]) {3972			pr_err("Executed on wrong sibling '%s', expected '%s'\n",3973			       request[n]->engine->name,3974			       siblings[nsibling - n - 1]->name);3975			err = -EINVAL;3976			goto out;3977		}3978	}3979 3980	err = igt_live_test_end(&t);3981out:3982	if (igt_flush_test(gt->i915))3983		err = -EIO;3984 3985	for (n = 0; n < nsibling; n++)3986		i915_request_put(request[n]);3987 3988out_unpin:3989	intel_context_unpin(ve);3990out_put:3991	intel_context_put(ve);3992out_close:3993	return err;3994}3995 3996static int live_virtual_mask(void *arg)3997{3998	struct intel_gt *gt = arg;3999	struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];4000	unsigned int class;4001	int err;4002 4003	if (intel_uc_uses_guc_submission(&gt->uc))4004		return 0;4005 4006	for (class = 0; class <= MAX_ENGINE_CLASS; class++) {4007		unsigned int nsibling;4008 4009		nsibling = select_siblings(gt, class, siblings);4010		if (nsibling < 2)4011			continue;4012 4013		err = mask_virtual_engine(gt, siblings, nsibling);4014		if (err)4015			return err;4016	}4017 4018	return 0;4019}4020 4021static int slicein_virtual_engine(struct intel_gt *gt,4022				  struct intel_engine_cs **siblings,4023				  unsigned int nsibling)4024{4025	const long timeout = slice_timeout(siblings[0]);4026	struct intel_context *ce;4027	struct i915_request *rq;4028	struct igt_spinner spin;4029	unsigned int n;4030	int err = 0;4031 4032	/*4033	 * Virtual requests must take part in timeslicing on the target engines.4034	 */4035 4036	if (igt_spinner_init(&spin, gt))4037		return -ENOMEM;4038 4039	for (n = 0; n < nsibling; n++) {4040		ce = intel_context_create(siblings[n]);4041		if (IS_ERR(ce)) {4042			err = PTR_ERR(ce);4043			goto out;4044		}4045 4046		rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);4047		intel_context_put(ce);4048		if (IS_ERR(rq)) {4049			err = PTR_ERR(rq);4050			goto out;4051		}4052 4053		i915_request_add(rq);4054	}4055 4056	ce = intel_engine_create_virtual(siblings, nsibling, 0);4057	if (IS_ERR(ce)) {4058		err = PTR_ERR(ce);4059		goto out;4060	}4061 4062	rq = intel_context_create_request(ce);4063	intel_context_put(ce);4064	if (IS_ERR(rq)) {4065		err = PTR_ERR(rq);4066		goto out;4067	}4068 4069	i915_request_get(rq);4070	i915_request_add(rq);4071	if (i915_request_wait(rq, 0, timeout) < 0) {4072		GEM_TRACE_ERR("%s(%s) failed to slice in virtual request\n",4073			      __func__, rq->engine->name);4074		GEM_TRACE_DUMP();4075		intel_gt_set_wedged(gt);4076		err = -EIO;4077	}4078	i915_request_put(rq);4079 4080out:4081	igt_spinner_end(&spin);4082	if (igt_flush_test(gt->i915))4083		err = -EIO;4084	igt_spinner_fini(&spin);4085	return err;4086}4087 4088static int sliceout_virtual_engine(struct intel_gt *gt,4089				   struct intel_engine_cs **siblings,4090				   unsigned int nsibling)4091{4092	const long timeout = slice_timeout(siblings[0]);4093	struct intel_context *ce;4094	struct i915_request *rq;4095	struct igt_spinner spin;4096	unsigned int n;4097	int err = 0;4098 4099	/*4100	 * Virtual requests must allow others a fair timeslice.4101	 */4102 4103	if (igt_spinner_init(&spin, gt))4104		return -ENOMEM;4105 4106	/* XXX We do not handle oversubscription and fairness with normal rq */4107	for (n = 0; n < nsibling; n++) {4108		ce = intel_engine_create_virtual(siblings, nsibling, 0);4109		if (IS_ERR(ce)) {4110			err = PTR_ERR(ce);4111			goto out;4112		}4113 4114		rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);4115		intel_context_put(ce);4116		if (IS_ERR(rq)) {4117			err = PTR_ERR(rq);4118			goto out;4119		}4120 4121		i915_request_add(rq);4122	}4123 4124	for (n = 0; !err && n < nsibling; n++) {4125		ce = intel_context_create(siblings[n]);4126		if (IS_ERR(ce)) {4127			err = PTR_ERR(ce);4128			goto out;4129		}4130 4131		rq = intel_context_create_request(ce);4132		intel_context_put(ce);4133		if (IS_ERR(rq)) {4134			err = PTR_ERR(rq);4135			goto out;4136		}4137 4138		i915_request_get(rq);4139		i915_request_add(rq);4140		if (i915_request_wait(rq, 0, timeout) < 0) {4141			GEM_TRACE_ERR("%s(%s) failed to slice out virtual request\n",4142				      __func__, siblings[n]->name);4143			GEM_TRACE_DUMP();4144			intel_gt_set_wedged(gt);4145			err = -EIO;4146		}4147		i915_request_put(rq);4148	}4149 4150out:4151	igt_spinner_end(&spin);4152	if (igt_flush_test(gt->i915))4153		err = -EIO;4154	igt_spinner_fini(&spin);4155	return err;4156}4157 4158static int live_virtual_slice(void *arg)4159{4160	struct intel_gt *gt = arg;4161	struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];4162	unsigned int class;4163	int err;4164 4165	if (intel_uc_uses_guc_submission(&gt->uc))4166		return 0;4167 4168	for (class = 0; class <= MAX_ENGINE_CLASS; class++) {4169		unsigned int nsibling;4170 4171		nsibling = __select_siblings(gt, class, siblings,4172					     intel_engine_has_timeslices);4173		if (nsibling < 2)4174			continue;4175 4176		err = slicein_virtual_engine(gt, siblings, nsibling);4177		if (err)4178			return err;4179 4180		err = sliceout_virtual_engine(gt, siblings, nsibling);4181		if (err)4182			return err;4183	}4184 4185	return 0;4186}4187 4188static int preserved_virtual_engine(struct intel_gt *gt,4189				    struct intel_engine_cs **siblings,4190				    unsigned int nsibling)4191{4192	struct i915_request *last = NULL;4193	struct intel_context *ve;4194	struct i915_vma *scratch;4195	struct igt_live_test t;4196	unsigned int n;4197	int err = 0;4198	u32 *cs;4199 4200	scratch =4201		__vm_create_scratch_for_read_pinned(&siblings[0]->gt->ggtt->vm,4202						    PAGE_SIZE);4203	if (IS_ERR(scratch))4204		return PTR_ERR(scratch);4205 4206	err = i915_vma_sync(scratch);4207	if (err)4208		goto out_scratch;4209 4210	ve = intel_engine_create_virtual(siblings, nsibling, 0);4211	if (IS_ERR(ve)) {4212		err = PTR_ERR(ve);4213		goto out_scratch;4214	}4215 4216	err = intel_context_pin(ve);4217	if (err)4218		goto out_put;4219 4220	err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name);4221	if (err)4222		goto out_unpin;4223 4224	for (n = 0; n < NUM_GPR_DW; n++) {4225		struct intel_engine_cs *engine = siblings[n % nsibling];4226		struct i915_request *rq;4227 4228		rq = i915_request_create(ve);4229		if (IS_ERR(rq)) {4230			err = PTR_ERR(rq);4231			goto out_end;4232		}4233 4234		i915_request_put(last);4235		last = i915_request_get(rq);4236 4237		cs = intel_ring_begin(rq, 8);4238		if (IS_ERR(cs)) {4239			i915_request_add(rq);4240			err = PTR_ERR(cs);4241			goto out_end;4242		}4243 4244		*cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;4245		*cs++ = CS_GPR(engine, n);4246		*cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);4247		*cs++ = 0;4248 4249		*cs++ = MI_LOAD_REGISTER_IMM(1);4250		*cs++ = CS_GPR(engine, (n + 1) % NUM_GPR_DW);4251		*cs++ = n + 1;4252 4253		*cs++ = MI_NOOP;4254		intel_ring_advance(rq, cs);4255 4256		/* Restrict this request to run on a particular engine */4257		rq->execution_mask = engine->mask;4258		i915_request_add(rq);4259	}4260 4261	if (i915_request_wait(last, 0, HZ / 5) < 0) {4262		err = -ETIME;4263		goto out_end;4264	}4265 4266	cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB);4267	if (IS_ERR(cs)) {4268		err = PTR_ERR(cs);4269		goto out_end;4270	}4271 4272	for (n = 0; n < NUM_GPR_DW; n++) {4273		if (cs[n] != n) {4274			pr_err("Incorrect value[%d] found for GPR[%d]\n",4275			       cs[n], n);4276			err = -EINVAL;4277			break;4278		}4279	}4280 4281	i915_gem_object_unpin_map(scratch->obj);4282 4283out_end:4284	if (igt_live_test_end(&t))4285		err = -EIO;4286	i915_request_put(last);4287out_unpin:4288	intel_context_unpin(ve);4289out_put:4290	intel_context_put(ve);4291out_scratch:4292	i915_vma_unpin_and_release(&scratch, 0);4293	return err;4294}4295 4296static int live_virtual_preserved(void *arg)4297{4298	struct intel_gt *gt = arg;4299	struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];4300	unsigned int class;4301 4302	/*4303	 * Check that the context image retains non-privileged (user) registers4304	 * from one engine to the next. For this we check that the CS_GPR4305	 * are preserved.4306	 */4307 4308	if (intel_uc_uses_guc_submission(&gt->uc))4309		return 0;4310 4311	/* As we use CS_GPR we cannot run before they existed on all engines. */4312	if (GRAPHICS_VER(gt->i915) < 9)4313		return 0;4314 4315	for (class = 0; class <= MAX_ENGINE_CLASS; class++) {4316		int nsibling, err;4317 4318		nsibling = select_siblings(gt, class, siblings);4319		if (nsibling < 2)4320			continue;4321 4322		err = preserved_virtual_engine(gt, siblings, nsibling);4323		if (err)4324			return err;4325	}4326 4327	return 0;4328}4329 4330static int reset_virtual_engine(struct intel_gt *gt,4331				struct intel_engine_cs **siblings,4332				unsigned int nsibling)4333{4334	struct intel_engine_cs *engine;4335	struct intel_context *ve;4336	struct igt_spinner spin;4337	struct i915_request *rq;4338	unsigned int n;4339	int err = 0;4340 4341	/*4342	 * In order to support offline error capture for fast preempt reset,4343	 * we need to decouple the guilty request and ensure that it and its4344	 * descendents are not executed while the capture is in progress.4345	 */4346 4347	if (igt_spinner_init(&spin, gt))4348		return -ENOMEM;4349 4350	ve = intel_engine_create_virtual(siblings, nsibling, 0);4351	if (IS_ERR(ve)) {4352		err = PTR_ERR(ve);4353		goto out_spin;4354	}4355 4356	for (n = 0; n < nsibling; n++)4357		st_engine_heartbeat_disable(siblings[n]);4358 4359	rq = igt_spinner_create_request(&spin, ve, MI_ARB_CHECK);4360	if (IS_ERR(rq)) {4361		err = PTR_ERR(rq);4362		goto out_heartbeat;4363	}4364	i915_request_add(rq);4365 4366	if (!igt_wait_for_spinner(&spin, rq)) {4367		intel_gt_set_wedged(gt);4368		err = -ETIME;4369		goto out_heartbeat;4370	}4371 4372	engine = rq->engine;4373	GEM_BUG_ON(engine == ve->engine);4374 4375	/* Take ownership of the reset and tasklet */4376	err = engine_lock_reset_tasklet(engine);4377	if (err)4378		goto out_heartbeat;4379 4380	engine->sched_engine->tasklet.callback(&engine->sched_engine->tasklet);4381	GEM_BUG_ON(execlists_active(&engine->execlists) != rq);4382 4383	/* Fake a preemption event; failed of course */4384	spin_lock_irq(&engine->sched_engine->lock);4385	__unwind_incomplete_requests(engine);4386	spin_unlock_irq(&engine->sched_engine->lock);4387	GEM_BUG_ON(rq->engine != engine);4388 4389	/* Reset the engine while keeping our active request on hold */4390	execlists_hold(engine, rq);4391	GEM_BUG_ON(!i915_request_on_hold(rq));4392 4393	__intel_engine_reset_bh(engine, NULL);4394	GEM_BUG_ON(rq->fence.error != -EIO);4395 4396	/* Release our grasp on the engine, letting CS flow again */4397	engine_unlock_reset_tasklet(engine);4398 4399	/* Check that we do not resubmit the held request */4400	i915_request_get(rq);4401	if (!i915_request_wait(rq, 0, HZ / 5)) {4402		pr_err("%s: on hold request completed!\n",4403		       engine->name);4404		intel_gt_set_wedged(gt);4405		err = -EIO;4406		goto out_rq;4407	}4408	GEM_BUG_ON(!i915_request_on_hold(rq));4409 4410	/* But is resubmitted on release */4411	execlists_unhold(engine, rq);4412	if (i915_request_wait(rq, 0, HZ / 5) < 0) {4413		pr_err("%s: held request did not complete!\n",4414		       engine->name);4415		intel_gt_set_wedged(gt);4416		err = -ETIME;4417	}4418 4419out_rq:4420	i915_request_put(rq);4421out_heartbeat:4422	for (n = 0; n < nsibling; n++)4423		st_engine_heartbeat_enable(siblings[n]);4424 4425	intel_context_put(ve);4426out_spin:4427	igt_spinner_fini(&spin);4428	return err;4429}4430 4431static int live_virtual_reset(void *arg)4432{4433	struct intel_gt *gt = arg;4434	struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];4435	unsigned int class;4436 4437	/*4438	 * Check that we handle a reset event within a virtual engine.4439	 * Only the physical engine is reset, but we have to check the flow4440	 * of the virtual requests around the reset, and make sure it is not4441	 * forgotten.4442	 */4443 4444	if (intel_uc_uses_guc_submission(&gt->uc))4445		return 0;4446 4447	if (!intel_has_reset_engine(gt))4448		return 0;4449 4450	for (class = 0; class <= MAX_ENGINE_CLASS; class++) {4451		int nsibling, err;4452 4453		nsibling = select_siblings(gt, class, siblings);4454		if (nsibling < 2)4455			continue;4456 4457		err = reset_virtual_engine(gt, siblings, nsibling);4458		if (err)4459			return err;4460	}4461 4462	return 0;4463}4464 4465int intel_execlists_live_selftests(struct drm_i915_private *i915)4466{4467	static const struct i915_subtest tests[] = {4468		SUBTEST(live_sanitycheck),4469		SUBTEST(live_unlite_switch),4470		SUBTEST(live_unlite_preempt),4471		SUBTEST(live_unlite_ring),4472		SUBTEST(live_pin_rewind),4473		SUBTEST(live_hold_reset),4474		SUBTEST(live_error_interrupt),4475		SUBTEST(live_timeslice_preempt),4476		SUBTEST(live_timeslice_rewind),4477		SUBTEST(live_timeslice_queue),4478		SUBTEST(live_timeslice_nopreempt),4479		SUBTEST(live_busywait_preempt),4480		SUBTEST(live_preempt),4481		SUBTEST(live_late_preempt),4482		SUBTEST(live_nopreempt),4483		SUBTEST(live_preempt_cancel),4484		SUBTEST(live_suppress_self_preempt),4485		SUBTEST(live_chain_preempt),4486		SUBTEST(live_preempt_ring),4487		SUBTEST(live_preempt_gang),4488		SUBTEST(live_preempt_timeout),4489		SUBTEST(live_preempt_user),4490		SUBTEST(live_preempt_smoke),4491		SUBTEST(live_virtual_engine),4492		SUBTEST(live_virtual_mask),4493		SUBTEST(live_virtual_preserved),4494		SUBTEST(live_virtual_slice),4495		SUBTEST(live_virtual_reset),4496	};4497 4498	if (to_gt(i915)->submission_method != INTEL_SUBMISSION_ELSP)4499		return 0;4500 4501	if (intel_gt_is_wedged(to_gt(i915)))4502		return 0;4503 4504	return intel_gt_live_subtests(tests, to_gt(i915));4505}4506