953 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2023 Intel Corporation4 */5 6#include <linux/bitfield.h>7#include <linux/delay.h>8 9#include <drm/drm_managed.h>10 11#include <kunit/static_stub.h>12#include <kunit/test-bug.h>13 14#include "abi/guc_actions_sriov_abi.h"15#include "abi/guc_relay_actions_abi.h"16#include "abi/guc_relay_communication_abi.h"17 18#include "xe_assert.h"19#include "xe_device.h"20#include "xe_gt.h"21#include "xe_gt_sriov_printk.h"22#include "xe_gt_sriov_pf_service.h"23#include "xe_guc.h"24#include "xe_guc_ct.h"25#include "xe_guc_hxg_helpers.h"26#include "xe_guc_relay.h"27#include "xe_guc_relay_types.h"28#include "xe_sriov.h"29 30/*31 * How long should we wait for the response?32 * XXX this value is subject for the profiling.33 */34#define RELAY_TIMEOUT_MSEC (2500)35 36static void relays_worker_fn(struct work_struct *w);37 38static struct xe_guc *relay_to_guc(struct xe_guc_relay *relay)39{40 return container_of(relay, struct xe_guc, relay);41}42 43static struct xe_guc_ct *relay_to_ct(struct xe_guc_relay *relay)44{45 return &relay_to_guc(relay)->ct;46}47 48static struct xe_gt *relay_to_gt(struct xe_guc_relay *relay)49{50 return guc_to_gt(relay_to_guc(relay));51}52 53static struct xe_device *relay_to_xe(struct xe_guc_relay *relay)54{55 return gt_to_xe(relay_to_gt(relay));56}57 58#define relay_assert(relay, condition) xe_gt_assert(relay_to_gt(relay), condition)59#define relay_notice(relay, msg...) xe_gt_sriov_notice(relay_to_gt(relay), "relay: " msg)60#define relay_debug(relay, msg...) xe_gt_sriov_dbg_verbose(relay_to_gt(relay), "relay: " msg)61 62static int relay_get_totalvfs(struct xe_guc_relay *relay)63{64 struct xe_device *xe = relay_to_xe(relay);65 struct pci_dev *pdev = to_pci_dev(xe->drm.dev);66 67 KUNIT_STATIC_STUB_REDIRECT(relay_get_totalvfs, relay);68 return IS_SRIOV_VF(xe) ? 0 : pci_sriov_get_totalvfs(pdev);69}70 71static bool relay_is_ready(struct xe_guc_relay *relay)72{73 return mempool_initialized(&relay->pool);74}75 76static u32 relay_get_next_rid(struct xe_guc_relay *relay)77{78 u32 rid;79 80 spin_lock(&relay->lock);81 rid = ++relay->last_rid;82 spin_unlock(&relay->lock);83 84 return rid;85}86 87/**88 * struct relay_transaction - internal data used to handle transactions89 *90 * Relation between struct relay_transaction members::91 *92 * <-------------------- GUC_CTB_MAX_DWORDS -------------->93 * <-------- GUC_RELAY_MSG_MAX_LEN --->94 * <--- offset ---> <--- request_len ------->95 * +----------------+-------------------------+----------+--+96 * | | | | |97 * +----------------+-------------------------+----------+--+98 * ^ ^99 * / /100 * request_buf request101 *102 * <-------------------- GUC_CTB_MAX_DWORDS -------------->103 * <-------- GUC_RELAY_MSG_MAX_LEN --->104 * <--- offset ---> <--- response_len --->105 * +----------------+----------------------+-------------+--+106 * | | | | |107 * +----------------+----------------------+-------------+--+108 * ^ ^109 * / /110 * response_buf response111 */112struct relay_transaction {113 /**114 * @incoming: indicates whether this transaction represents an incoming115 * request from the remote VF/PF or this transaction116 * represents outgoing request to the remote VF/PF.117 */118 bool incoming;119 120 /**121 * @remote: PF/VF identifier of the origin (or target) of the relay122 * request message.123 */124 u32 remote;125 126 /** @rid: identifier of the VF/PF relay message. */127 u32 rid;128 129 /**130 * @request: points to the inner VF/PF request message, copied to the131 * #response_buf starting at #offset.132 */133 u32 *request;134 135 /** @request_len: length of the inner VF/PF request message. */136 u32 request_len;137 138 /**139 * @response: points to the placeholder buffer where inner VF/PF140 * response will be located, for outgoing transaction141 * this could be caller's buffer (if provided) otherwise142 * it points to the #response_buf starting at #offset.143 */144 u32 *response;145 146 /**147 * @response_len: length of the inner VF/PF response message (only148 * if #status is 0), initially set to the size of the149 * placeholder buffer where response message will be150 * copied.151 */152 u32 response_len;153 154 /**155 * @offset: offset to the start of the inner VF/PF relay message inside156 * buffers; this offset is equal the length of the outer GuC157 * relay header message.158 */159 u32 offset;160 161 /**162 * @request_buf: buffer with VF/PF request message including outer163 * transport message.164 */165 u32 request_buf[GUC_CTB_MAX_DWORDS];166 167 /**168 * @response_buf: buffer with VF/PF response message including outer169 * transport message.170 */171 u32 response_buf[GUC_CTB_MAX_DWORDS];172 173 /**174 * @reply: status of the reply, 0 means that data pointed by the175 * #response is valid.176 */177 int reply;178 179 /** @done: completion of the outgoing transaction. */180 struct completion done;181 182 /** @link: transaction list link */183 struct list_head link;184};185 186static u32 prepare_pf2guc(u32 *msg, u32 target, u32 rid)187{188 msg[0] = FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |189 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |190 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, XE_GUC_ACTION_PF2GUC_RELAY_TO_VF);191 msg[1] = FIELD_PREP(PF2GUC_RELAY_TO_VF_REQUEST_MSG_1_VFID, target);192 msg[2] = FIELD_PREP(PF2GUC_RELAY_TO_VF_REQUEST_MSG_2_RELAY_ID, rid);193 194 return PF2GUC_RELAY_TO_VF_REQUEST_MSG_MIN_LEN;195}196 197static u32 prepare_vf2guc(u32 *msg, u32 rid)198{199 msg[0] = FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |200 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |201 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, XE_GUC_ACTION_VF2GUC_RELAY_TO_PF);202 msg[1] = FIELD_PREP(VF2GUC_RELAY_TO_PF_REQUEST_MSG_1_RELAY_ID, rid);203 204 return VF2GUC_RELAY_TO_PF_REQUEST_MSG_MIN_LEN;205}206 207static struct relay_transaction *208__relay_get_transaction(struct xe_guc_relay *relay, bool incoming, u32 remote, u32 rid,209 const u32 *action, u32 action_len, u32 *resp, u32 resp_size)210{211 struct relay_transaction *txn;212 213 relay_assert(relay, action_len >= GUC_RELAY_MSG_MIN_LEN);214 relay_assert(relay, action_len <= GUC_RELAY_MSG_MAX_LEN);215 relay_assert(relay, !(!!resp ^ !!resp_size));216 relay_assert(relay, resp_size <= GUC_RELAY_MSG_MAX_LEN);217 relay_assert(relay, resp_size == 0 || resp_size >= GUC_RELAY_MSG_MIN_LEN);218 219 if (unlikely(!relay_is_ready(relay)))220 return ERR_PTR(-ENODEV);221 222 /*223 * For incoming requests we can't use GFP_KERNEL as those are delivered224 * with CTB lock held which is marked as used in the reclaim path.225 * Btw, that's one of the reason why we use mempool here!226 */227 txn = mempool_alloc(&relay->pool, incoming ? GFP_ATOMIC : GFP_KERNEL);228 if (!txn)229 return ERR_PTR(-ENOMEM);230 231 txn->incoming = incoming;232 txn->remote = remote;233 txn->rid = rid;234 txn->offset = remote ?235 prepare_pf2guc(incoming ? txn->response_buf : txn->request_buf, remote, rid) :236 prepare_vf2guc(incoming ? txn->response_buf : txn->request_buf, rid);237 238 relay_assert(relay, txn->offset);239 relay_assert(relay, txn->offset + GUC_RELAY_MSG_MAX_LEN <= ARRAY_SIZE(txn->request_buf));240 relay_assert(relay, txn->offset + GUC_RELAY_MSG_MAX_LEN <= ARRAY_SIZE(txn->response_buf));241 242 txn->request = txn->request_buf + txn->offset;243 memcpy(&txn->request_buf[txn->offset], action, sizeof(u32) * action_len);244 txn->request_len = action_len;245 246 txn->response = resp ?: txn->response_buf + txn->offset;247 txn->response_len = resp_size ?: GUC_RELAY_MSG_MAX_LEN;248 txn->reply = -ENOMSG;249 INIT_LIST_HEAD(&txn->link);250 init_completion(&txn->done);251 252 return txn;253}254 255static struct relay_transaction *256relay_new_transaction(struct xe_guc_relay *relay, u32 target, const u32 *action, u32 len,257 u32 *resp, u32 resp_size)258{259 u32 rid = relay_get_next_rid(relay);260 261 return __relay_get_transaction(relay, false, target, rid, action, len, resp, resp_size);262}263 264static struct relay_transaction *265relay_new_incoming_transaction(struct xe_guc_relay *relay, u32 origin, u32 rid,266 const u32 *action, u32 len)267{268 return __relay_get_transaction(relay, true, origin, rid, action, len, NULL, 0);269}270 271static void relay_release_transaction(struct xe_guc_relay *relay, struct relay_transaction *txn)272{273 relay_assert(relay, list_empty(&txn->link));274 275 txn->offset = 0;276 txn->response = NULL;277 txn->reply = -ESTALE;278 mempool_free(txn, &relay->pool);279}280 281static int relay_send_transaction(struct xe_guc_relay *relay, struct relay_transaction *txn)282{283 u32 len = txn->incoming ? txn->response_len : txn->request_len;284 u32 *buf = txn->incoming ? txn->response_buf : txn->request_buf;285 u32 *msg = buf + txn->offset;286 int ret;287 288 relay_assert(relay, txn->offset);289 relay_assert(relay, txn->offset + len <= GUC_CTB_MAX_DWORDS);290 relay_assert(relay, len >= GUC_RELAY_MSG_MIN_LEN);291 relay_assert(relay, len <= GUC_RELAY_MSG_MAX_LEN);292 293 relay_debug(relay, "sending %s.%u to %u = %*ph\n",294 guc_hxg_type_to_string(FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0])),295 txn->rid, txn->remote, (int)sizeof(u32) * len, msg);296 297 ret = xe_guc_ct_send_block(relay_to_ct(relay), buf, len + txn->offset);298 299 if (unlikely(ret > 0)) {300 relay_notice(relay, "Unexpected data=%d from GuC, wrong ABI?\n", ret);301 ret = -EPROTO;302 }303 if (unlikely(ret < 0)) {304 relay_notice(relay, "Failed to send %s.%x to GuC (%pe) %*ph ...\n",305 guc_hxg_type_to_string(FIELD_GET(GUC_HXG_MSG_0_TYPE, buf[0])),306 FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, buf[0]),307 ERR_PTR(ret), (int)sizeof(u32) * txn->offset, buf);308 relay_notice(relay, "Failed to send %s.%u to %u (%pe) %*ph\n",309 guc_hxg_type_to_string(FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0])),310 txn->rid, txn->remote, ERR_PTR(ret), (int)sizeof(u32) * len, msg);311 }312 313 return ret;314}315 316static void __fini_relay(struct drm_device *drm, void *arg)317{318 struct xe_guc_relay *relay = arg;319 320 mempool_exit(&relay->pool);321}322 323/**324 * xe_guc_relay_init - Initialize a &xe_guc_relay325 * @relay: the &xe_guc_relay to initialize326 *327 * Initialize remaining members of &xe_guc_relay that may depend328 * on the SR-IOV mode.329 *330 * Return: 0 on success or a negative error code on failure.331 */332int xe_guc_relay_init(struct xe_guc_relay *relay)333{334 const int XE_RELAY_MEMPOOL_MIN_NUM = 1;335 struct xe_device *xe = relay_to_xe(relay);336 int err;337 338 relay_assert(relay, !relay_is_ready(relay));339 340 if (!IS_SRIOV(xe))341 return 0;342 343 spin_lock_init(&relay->lock);344 INIT_WORK(&relay->worker, relays_worker_fn);345 INIT_LIST_HEAD(&relay->pending_relays);346 INIT_LIST_HEAD(&relay->incoming_actions);347 348 err = mempool_init_kmalloc_pool(&relay->pool, XE_RELAY_MEMPOOL_MIN_NUM +349 relay_get_totalvfs(relay),350 sizeof(struct relay_transaction));351 if (err)352 return err;353 354 relay_debug(relay, "using mempool with %d elements\n", relay->pool.min_nr);355 356 return drmm_add_action_or_reset(&xe->drm, __fini_relay, relay);357}358 359static u32 to_relay_error(int err)360{361 /* XXX: assume that relay errors match errno codes */362 return err < 0 ? -err : GUC_RELAY_ERROR_UNDISCLOSED;363}364 365static int from_relay_error(u32 error)366{367 /* XXX: assume that relay errors match errno codes */368 return error ? -error : -ENODATA;369}370 371static u32 sanitize_relay_error(u32 error)372{373 /* XXX TBD if generic error codes will be allowed */374 if (!IS_ENABLED(CONFIG_DRM_XE_DEBUG))375 error = GUC_RELAY_ERROR_UNDISCLOSED;376 return error;377}378 379static u32 sanitize_relay_error_hint(u32 hint)380{381 /* XXX TBD if generic error codes will be allowed */382 if (!IS_ENABLED(CONFIG_DRM_XE_DEBUG))383 hint = 0;384 return hint;385}386 387static u32 prepare_error_reply(u32 *msg, u32 error, u32 hint)388{389 msg[0] = FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |390 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_RESPONSE_FAILURE) |391 FIELD_PREP(GUC_HXG_FAILURE_MSG_0_HINT, hint) |392 FIELD_PREP(GUC_HXG_FAILURE_MSG_0_ERROR, error);393 394 XE_WARN_ON(!FIELD_FIT(GUC_HXG_FAILURE_MSG_0_ERROR, error));395 XE_WARN_ON(!FIELD_FIT(GUC_HXG_FAILURE_MSG_0_HINT, hint));396 397 return GUC_HXG_FAILURE_MSG_LEN;398}399 400static void relay_testonly_nop(struct xe_guc_relay *relay)401{402 KUNIT_STATIC_STUB_REDIRECT(relay_testonly_nop, relay);403}404 405static int relay_send_message_and_wait(struct xe_guc_relay *relay,406 struct relay_transaction *txn,407 u32 *buf, u32 buf_size)408{409 unsigned long timeout = msecs_to_jiffies(RELAY_TIMEOUT_MSEC);410 u32 *msg = &txn->request_buf[txn->offset];411 u32 len = txn->request_len;412 u32 type, action, data0;413 int ret;414 long n;415 416 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]);417 action = FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, msg[0]);418 data0 = FIELD_GET(GUC_HXG_REQUEST_MSG_0_DATA0, msg[0]);419 420 relay_debug(relay, "%s.%u to %u action %#x:%u\n",421 guc_hxg_type_to_string(type),422 txn->rid, txn->remote, action, data0);423 424 /* list ordering does not need to match RID ordering */425 spin_lock(&relay->lock);426 list_add_tail(&txn->link, &relay->pending_relays);427 spin_unlock(&relay->lock);428 429resend:430 ret = relay_send_transaction(relay, txn);431 if (unlikely(ret < 0))432 goto unlink;433 434wait:435 n = wait_for_completion_timeout(&txn->done, timeout);436 if (unlikely(n == 0 && txn->reply)) {437 ret = -ETIME;438 goto unlink;439 }440 441 relay_debug(relay, "%u.%u reply %d after %u msec\n",442 txn->remote, txn->rid, txn->reply, jiffies_to_msecs(timeout - n));443 if (unlikely(txn->reply)) {444 reinit_completion(&txn->done);445 if (txn->reply == -EAGAIN)446 goto resend;447 if (txn->reply == -EBUSY) {448 relay_testonly_nop(relay);449 goto wait;450 }451 if (txn->reply > 0)452 ret = from_relay_error(txn->reply);453 else454 ret = txn->reply;455 goto unlink;456 }457 458 relay_debug(relay, "%u.%u response %*ph\n", txn->remote, txn->rid,459 (int)sizeof(u32) * txn->response_len, txn->response);460 relay_assert(relay, txn->response_len >= GUC_RELAY_MSG_MIN_LEN);461 ret = txn->response_len;462 463unlink:464 spin_lock(&relay->lock);465 list_del_init(&txn->link);466 spin_unlock(&relay->lock);467 468 if (unlikely(ret < 0)) {469 relay_notice(relay, "Unsuccessful %s.%u %#x:%u to %u (%pe) %*ph\n",470 guc_hxg_type_to_string(type), txn->rid,471 action, data0, txn->remote, ERR_PTR(ret),472 (int)sizeof(u32) * len, msg);473 }474 475 return ret;476}477 478static int relay_send_to(struct xe_guc_relay *relay, u32 target,479 const u32 *msg, u32 len, u32 *buf, u32 buf_size)480{481 struct relay_transaction *txn;482 int ret;483 484 relay_assert(relay, len >= GUC_RELAY_MSG_MIN_LEN);485 relay_assert(relay, len <= GUC_RELAY_MSG_MAX_LEN);486 relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) == GUC_HXG_ORIGIN_HOST);487 relay_assert(relay, guc_hxg_type_is_action(FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0])));488 489 if (unlikely(!relay_is_ready(relay)))490 return -ENODEV;491 492 txn = relay_new_transaction(relay, target, msg, len, buf, buf_size);493 if (IS_ERR(txn))494 return PTR_ERR(txn);495 496 switch (FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0])) {497 case GUC_HXG_TYPE_REQUEST:498 ret = relay_send_message_and_wait(relay, txn, buf, buf_size);499 break;500 case GUC_HXG_TYPE_FAST_REQUEST:501 relay_assert(relay, !GUC_HXG_TYPE_FAST_REQUEST);502 fallthrough;503 case GUC_HXG_TYPE_EVENT:504 ret = relay_send_transaction(relay, txn);505 break;506 default:507 ret = -EINVAL;508 break;509 }510 511 relay_release_transaction(relay, txn);512 return ret;513}514 515#ifdef CONFIG_PCI_IOV516/**517 * xe_guc_relay_send_to_vf - Send a message to the VF.518 * @relay: the &xe_guc_relay which will send the message519 * @target: target VF number520 * @msg: request message to be sent521 * @len: length of the request message (in dwords, can't be 0)522 * @buf: placeholder for the response message523 * @buf_size: size of the response message placeholder (in dwords)524 *525 * This function can only be used by the driver running in the SR-IOV PF mode.526 *527 * Return: Non-negative response length (in dwords) or528 * a negative error code on failure.529 */530int xe_guc_relay_send_to_vf(struct xe_guc_relay *relay, u32 target,531 const u32 *msg, u32 len, u32 *buf, u32 buf_size)532{533 relay_assert(relay, IS_SRIOV_PF(relay_to_xe(relay)));534 535 return relay_send_to(relay, target, msg, len, buf, buf_size);536}537#endif538 539/**540 * xe_guc_relay_send_to_pf - Send a message to the PF.541 * @relay: the &xe_guc_relay which will send the message542 * @msg: request message to be sent543 * @len: length of the message (in dwords, can't be 0)544 * @buf: placeholder for the response message545 * @buf_size: size of the response message placeholder (in dwords)546 *547 * This function can only be used by driver running in SR-IOV VF mode.548 *549 * Return: Non-negative response length (in dwords) or550 * a negative error code on failure.551 */552int xe_guc_relay_send_to_pf(struct xe_guc_relay *relay,553 const u32 *msg, u32 len, u32 *buf, u32 buf_size)554{555 relay_assert(relay, IS_SRIOV_VF(relay_to_xe(relay)));556 557 return relay_send_to(relay, PFID, msg, len, buf, buf_size);558}559 560static int relay_handle_reply(struct xe_guc_relay *relay, u32 origin,561 u32 rid, int reply, const u32 *msg, u32 len)562{563 struct relay_transaction *pending;564 int err = -ESRCH;565 566 spin_lock(&relay->lock);567 list_for_each_entry(pending, &relay->pending_relays, link) {568 if (pending->remote != origin || pending->rid != rid) {569 relay_debug(relay, "%u.%u still awaits response\n",570 pending->remote, pending->rid);571 continue;572 }573 err = 0; /* found! */574 if (reply == 0) {575 if (len > pending->response_len) {576 reply = -ENOBUFS;577 err = -ENOBUFS;578 } else {579 memcpy(pending->response, msg, 4 * len);580 pending->response_len = len;581 }582 }583 pending->reply = reply;584 complete_all(&pending->done);585 break;586 }587 spin_unlock(&relay->lock);588 589 return err;590}591 592static int relay_handle_failure(struct xe_guc_relay *relay, u32 origin,593 u32 rid, const u32 *msg, u32 len)594{595 int error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, msg[0]);596 u32 hint __maybe_unused = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, msg[0]);597 598 relay_assert(relay, len);599 relay_debug(relay, "%u.%u error %#x (%pe) hint %u debug %*ph\n",600 origin, rid, error, ERR_PTR(-error), hint, 4 * (len - 1), msg + 1);601 602 return relay_handle_reply(relay, origin, rid, error ?: -EREMOTEIO, NULL, 0);603}604 605static int relay_testloop_action_handler(struct xe_guc_relay *relay, u32 origin,606 const u32 *msg, u32 len, u32 *response, u32 size)607{608 static ktime_t last_reply = 0;609 u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]);610 u32 action = FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, msg[0]);611 u32 opcode = FIELD_GET(GUC_HXG_REQUEST_MSG_0_DATA0, msg[0]);612 ktime_t now = ktime_get();613 bool busy;614 int ret;615 616 relay_assert(relay, guc_hxg_type_is_action(type));617 relay_assert(relay, action == GUC_RELAY_ACTION_VFXPF_TESTLOOP);618 619 if (!IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV))620 return -ECONNREFUSED;621 622 if (!last_reply)623 last_reply = now;624 busy = ktime_before(now, ktime_add_ms(last_reply, 2 * RELAY_TIMEOUT_MSEC));625 if (!busy)626 last_reply = now;627 628 switch (opcode) {629 case VFXPF_TESTLOOP_OPCODE_NOP:630 if (type == GUC_HXG_TYPE_EVENT)631 return 0;632 return guc_hxg_msg_encode_success(response, 0);633 case VFXPF_TESTLOOP_OPCODE_BUSY:634 if (type == GUC_HXG_TYPE_EVENT)635 return -EPROTO;636 msleep(RELAY_TIMEOUT_MSEC / 8);637 if (busy)638 return -EINPROGRESS;639 return guc_hxg_msg_encode_success(response, 0);640 case VFXPF_TESTLOOP_OPCODE_RETRY:641 if (type == GUC_HXG_TYPE_EVENT)642 return -EPROTO;643 msleep(RELAY_TIMEOUT_MSEC / 8);644 if (busy)645 return guc_hxg_msg_encode_retry(response, 0);646 return guc_hxg_msg_encode_success(response, 0);647 case VFXPF_TESTLOOP_OPCODE_ECHO:648 if (type == GUC_HXG_TYPE_EVENT)649 return -EPROTO;650 if (size < len)651 return -ENOBUFS;652 ret = guc_hxg_msg_encode_success(response, len);653 memcpy(response + ret, msg + ret, (len - ret) * sizeof(u32));654 return len;655 case VFXPF_TESTLOOP_OPCODE_FAIL:656 return -EHWPOISON;657 default:658 break;659 }660 661 relay_notice(relay, "Unexpected action %#x opcode %#x\n", action, opcode);662 return -EBADRQC;663}664 665static int relay_action_handler(struct xe_guc_relay *relay, u32 origin,666 const u32 *msg, u32 len, u32 *response, u32 size)667{668 struct xe_gt *gt = relay_to_gt(relay);669 u32 type;670 int ret;671 672 relay_assert(relay, len >= GUC_HXG_MSG_MIN_LEN);673 674 if (FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, msg[0]) == GUC_RELAY_ACTION_VFXPF_TESTLOOP)675 return relay_testloop_action_handler(relay, origin, msg, len, response, size);676 677 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]);678 679 if (IS_SRIOV_PF(relay_to_xe(relay)))680 ret = xe_gt_sriov_pf_service_process_request(gt, origin, msg, len, response, size);681 else682 ret = -EOPNOTSUPP;683 684 if (type == GUC_HXG_TYPE_EVENT)685 relay_assert(relay, ret <= 0);686 687 return ret;688}689 690static struct relay_transaction *relay_dequeue_transaction(struct xe_guc_relay *relay)691{692 struct relay_transaction *txn;693 694 spin_lock(&relay->lock);695 txn = list_first_entry_or_null(&relay->incoming_actions, struct relay_transaction, link);696 if (txn)697 list_del_init(&txn->link);698 spin_unlock(&relay->lock);699 700 return txn;701}702 703static void relay_process_incoming_action(struct xe_guc_relay *relay)704{705 struct relay_transaction *txn;706 bool again = false;707 u32 type;708 int ret;709 710 txn = relay_dequeue_transaction(relay);711 if (!txn)712 return;713 714 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, txn->request_buf[txn->offset]);715 716 ret = relay_action_handler(relay, txn->remote,717 txn->request_buf + txn->offset, txn->request_len,718 txn->response_buf + txn->offset,719 ARRAY_SIZE(txn->response_buf) - txn->offset);720 721 if (ret == -EINPROGRESS) {722 again = true;723 ret = guc_hxg_msg_encode_busy(txn->response_buf + txn->offset, 0);724 }725 726 if (ret > 0) {727 txn->response_len = ret;728 ret = relay_send_transaction(relay, txn);729 }730 731 if (ret < 0) {732 u32 error = to_relay_error(ret);733 734 relay_notice(relay, "Failed to handle %s.%u from %u (%pe) %*ph\n",735 guc_hxg_type_to_string(type), txn->rid, txn->remote,736 ERR_PTR(ret), 4 * txn->request_len, txn->request_buf + txn->offset);737 738 txn->response_len = prepare_error_reply(txn->response_buf + txn->offset,739 txn->remote ?740 sanitize_relay_error(error) : error,741 txn->remote ?742 sanitize_relay_error_hint(-ret) : -ret);743 ret = relay_send_transaction(relay, txn);744 again = false;745 }746 747 if (again) {748 spin_lock(&relay->lock);749 list_add(&txn->link, &relay->incoming_actions);750 spin_unlock(&relay->lock);751 return;752 }753 754 if (unlikely(ret < 0))755 relay_notice(relay, "Failed to process action.%u (%pe) %*ph\n",756 txn->rid, ERR_PTR(ret), 4 * txn->request_len,757 txn->request_buf + txn->offset);758 759 relay_release_transaction(relay, txn);760}761 762static bool relay_needs_worker(struct xe_guc_relay *relay)763{764 bool is_empty;765 766 spin_lock(&relay->lock);767 is_empty = list_empty(&relay->incoming_actions);768 spin_unlock(&relay->lock);769 770 return !is_empty;771 772}773 774static void relay_kick_worker(struct xe_guc_relay *relay)775{776 KUNIT_STATIC_STUB_REDIRECT(relay_kick_worker, relay);777 queue_work(relay_to_xe(relay)->sriov.wq, &relay->worker);778}779 780static void relays_worker_fn(struct work_struct *w)781{782 struct xe_guc_relay *relay = container_of(w, struct xe_guc_relay, worker);783 784 relay_process_incoming_action(relay);785 786 if (relay_needs_worker(relay))787 relay_kick_worker(relay);788}789 790static int relay_queue_action_msg(struct xe_guc_relay *relay, u32 origin, u32 rid,791 const u32 *msg, u32 len)792{793 struct relay_transaction *txn;794 795 txn = relay_new_incoming_transaction(relay, origin, rid, msg, len);796 if (IS_ERR(txn))797 return PTR_ERR(txn);798 799 spin_lock(&relay->lock);800 list_add_tail(&txn->link, &relay->incoming_actions);801 spin_unlock(&relay->lock);802 803 relay_kick_worker(relay);804 return 0;805}806 807static int relay_process_msg(struct xe_guc_relay *relay, u32 origin, u32 rid,808 const u32 *msg, u32 len)809{810 u32 type;811 int err;812 813 if (unlikely(len < GUC_HXG_MSG_MIN_LEN))814 return -EPROTO;815 816 if (FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) != GUC_HXG_ORIGIN_HOST)817 return -EPROTO;818 819 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]);820 relay_debug(relay, "received %s.%u from %u = %*ph\n",821 guc_hxg_type_to_string(type), rid, origin, 4 * len, msg);822 823 switch (type) {824 case GUC_HXG_TYPE_REQUEST:825 case GUC_HXG_TYPE_FAST_REQUEST:826 case GUC_HXG_TYPE_EVENT:827 err = relay_queue_action_msg(relay, origin, rid, msg, len);828 break;829 case GUC_HXG_TYPE_RESPONSE_SUCCESS:830 err = relay_handle_reply(relay, origin, rid, 0, msg, len);831 break;832 case GUC_HXG_TYPE_NO_RESPONSE_BUSY:833 err = relay_handle_reply(relay, origin, rid, -EBUSY, NULL, 0);834 break;835 case GUC_HXG_TYPE_NO_RESPONSE_RETRY:836 err = relay_handle_reply(relay, origin, rid, -EAGAIN, NULL, 0);837 break;838 case GUC_HXG_TYPE_RESPONSE_FAILURE:839 err = relay_handle_failure(relay, origin, rid, msg, len);840 break;841 default:842 err = -EBADRQC;843 }844 845 if (unlikely(err))846 relay_notice(relay, "Failed to process %s.%u from %u (%pe) %*ph\n",847 guc_hxg_type_to_string(type), rid, origin,848 ERR_PTR(err), 4 * len, msg);849 850 return err;851}852 853/**854 * xe_guc_relay_process_guc2vf - Handle relay notification message from the GuC.855 * @relay: the &xe_guc_relay which will handle the message856 * @msg: message to be handled857 * @len: length of the message (in dwords)858 *859 * This function will handle relay messages received from the GuC.860 *861 * This function is can only be used if driver is running in SR-IOV mode.862 *863 * Return: 0 on success or a negative error code on failure.864 */865int xe_guc_relay_process_guc2vf(struct xe_guc_relay *relay, const u32 *msg, u32 len)866{867 u32 rid;868 869 relay_assert(relay, len >= GUC_HXG_MSG_MIN_LEN);870 relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) == GUC_HXG_ORIGIN_GUC);871 relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]) == GUC_HXG_TYPE_EVENT);872 relay_assert(relay, FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[0]) ==873 XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF);874 875 if (unlikely(!IS_SRIOV_VF(relay_to_xe(relay)) && !kunit_get_current_test()))876 return -EPERM;877 878 if (unlikely(!relay_is_ready(relay)))879 return -ENODEV;880 881 if (unlikely(len < GUC2VF_RELAY_FROM_PF_EVENT_MSG_MIN_LEN))882 return -EPROTO;883 884 if (unlikely(len > GUC2VF_RELAY_FROM_PF_EVENT_MSG_MAX_LEN))885 return -EMSGSIZE;886 887 if (unlikely(FIELD_GET(GUC_HXG_EVENT_MSG_0_DATA0, msg[0])))888 return -EPFNOSUPPORT;889 890 rid = FIELD_GET(GUC2VF_RELAY_FROM_PF_EVENT_MSG_1_RELAY_ID, msg[1]);891 892 return relay_process_msg(relay, PFID, rid,893 msg + GUC2VF_RELAY_FROM_PF_EVENT_MSG_MIN_LEN,894 len - GUC2VF_RELAY_FROM_PF_EVENT_MSG_MIN_LEN);895}896 897#ifdef CONFIG_PCI_IOV898/**899 * xe_guc_relay_process_guc2pf - Handle relay notification message from the GuC.900 * @relay: the &xe_guc_relay which will handle the message901 * @msg: message to be handled902 * @len: length of the message (in dwords)903 *904 * This function will handle relay messages received from the GuC.905 *906 * This function can only be used if driver is running in SR-IOV PF mode.907 *908 * Return: 0 on success or a negative error code on failure.909 */910int xe_guc_relay_process_guc2pf(struct xe_guc_relay *relay, const u32 *msg, u32 len)911{912 u32 origin, rid;913 int err;914 915 relay_assert(relay, len >= GUC_HXG_EVENT_MSG_MIN_LEN);916 relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) == GUC_HXG_ORIGIN_GUC);917 relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]) == GUC_HXG_TYPE_EVENT);918 relay_assert(relay, FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[0]) ==919 XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF);920 921 if (unlikely(!IS_SRIOV_PF(relay_to_xe(relay)) && !kunit_get_current_test()))922 return -EPERM;923 924 if (unlikely(!relay_is_ready(relay)))925 return -ENODEV;926 927 if (unlikely(len < GUC2PF_RELAY_FROM_VF_EVENT_MSG_MIN_LEN))928 return -EPROTO;929 930 if (unlikely(len > GUC2PF_RELAY_FROM_VF_EVENT_MSG_MAX_LEN))931 return -EMSGSIZE;932 933 if (unlikely(FIELD_GET(GUC_HXG_EVENT_MSG_0_DATA0, msg[0])))934 return -EPFNOSUPPORT;935 936 origin = FIELD_GET(GUC2PF_RELAY_FROM_VF_EVENT_MSG_1_VFID, msg[1]);937 rid = FIELD_GET(GUC2PF_RELAY_FROM_VF_EVENT_MSG_2_RELAY_ID, msg[2]);938 939 if (unlikely(origin > relay_get_totalvfs(relay)))940 return -ENOENT;941 942 err = relay_process_msg(relay, origin, rid,943 msg + GUC2PF_RELAY_FROM_VF_EVENT_MSG_MIN_LEN,944 len - GUC2PF_RELAY_FROM_VF_EVENT_MSG_MIN_LEN);945 946 return err;947}948#endif949 950#if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST)951#include "tests/xe_guc_relay_test.c"952#endif953