2708 lines · c
1/*2 * Copyright 2023 Red Hat Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 */22#include "priv.h"23 24#include <core/pci.h>25#include <subdev/timer.h>26#include <subdev/vfn.h>27#include <engine/fifo/chan.h>28#include <engine/sec2.h>29 30#include <nvfw/fw.h>31 32#include <nvrm/nvtypes.h>33#include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl0000.h>34#include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl0005.h>35#include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl0080.h>36#include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl2080.h>37#include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080event.h>38#include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080gpu.h>39#include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080internal.h>40#include <nvrm/535.113.01/common/sdk/nvidia/inc/nvos.h>41#include <nvrm/535.113.01/common/shared/msgq/inc/msgq/msgq_priv.h>42#include <nvrm/535.113.01/common/uproc/os/common/include/libos_init_args.h>43#include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/gsp/gsp_fw_sr_meta.h>44#include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/gsp/gsp_fw_wpr_meta.h>45#include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/rmRiscvUcode.h>46#include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/rmgspseq.h>47#include <nvrm/535.113.01/nvidia/generated/g_allclasses.h>48#include <nvrm/535.113.01/nvidia/generated/g_os_nvoc.h>49#include <nvrm/535.113.01/nvidia/generated/g_rpc-structures.h>50#include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gsp/gsp_fw_heap.h>51#include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gsp/gsp_init_args.h>52#include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gsp/gsp_static_config.h>53#include <nvrm/535.113.01/nvidia/inc/kernel/gpu/intr/engine_idx.h>54#include <nvrm/535.113.01/nvidia/kernel/inc/vgpu/rpc_global_enums.h>55 56#include <linux/acpi.h>57#include <linux/ctype.h>58#include <linux/parser.h>59 60#define GSP_MSG_MIN_SIZE GSP_PAGE_SIZE61#define GSP_MSG_MAX_SIZE GSP_PAGE_MIN_SIZE * 1662 63struct r535_gsp_msg {64 u8 auth_tag_buffer[16];65 u8 aad_buffer[16];66 u32 checksum;67 u32 sequence;68 u32 elem_count;69 u32 pad;70 u8 data[];71};72 73#define GSP_MSG_HDR_SIZE offsetof(struct r535_gsp_msg, data)74 75static int76r535_rpc_status_to_errno(uint32_t rpc_status)77{78 switch (rpc_status) {79 case 0x55: /* NV_ERR_NOT_READY */80 case 0x66: /* NV_ERR_TIMEOUT_RETRY */81 return -EBUSY;82 case 0x51: /* NV_ERR_NO_MEMORY */83 return -ENOMEM;84 default:85 return -EINVAL;86 }87}88 89static void *90r535_gsp_msgq_wait(struct nvkm_gsp *gsp, u32 repc, u32 *prepc, int *ptime)91{92 struct r535_gsp_msg *mqe;93 u32 size, rptr = *gsp->msgq.rptr;94 int used;95 u8 *msg;96 u32 len;97 98 size = DIV_ROUND_UP(GSP_MSG_HDR_SIZE + repc, GSP_PAGE_SIZE);99 if (WARN_ON(!size || size >= gsp->msgq.cnt))100 return ERR_PTR(-EINVAL);101 102 do {103 u32 wptr = *gsp->msgq.wptr;104 105 used = wptr + gsp->msgq.cnt - rptr;106 if (used >= gsp->msgq.cnt)107 used -= gsp->msgq.cnt;108 if (used >= size)109 break;110 111 usleep_range(1, 2);112 } while (--(*ptime));113 114 if (WARN_ON(!*ptime))115 return ERR_PTR(-ETIMEDOUT);116 117 mqe = (void *)((u8 *)gsp->shm.msgq.ptr + 0x1000 + rptr * 0x1000);118 119 if (prepc) {120 *prepc = (used * GSP_PAGE_SIZE) - sizeof(*mqe);121 return mqe->data;122 }123 124 msg = kvmalloc(repc, GFP_KERNEL);125 if (!msg)126 return ERR_PTR(-ENOMEM);127 128 len = ((gsp->msgq.cnt - rptr) * GSP_PAGE_SIZE) - sizeof(*mqe);129 len = min_t(u32, repc, len);130 memcpy(msg, mqe->data, len);131 132 rptr += DIV_ROUND_UP(len, GSP_PAGE_SIZE);133 if (rptr == gsp->msgq.cnt)134 rptr = 0;135 136 repc -= len;137 138 if (repc) {139 mqe = (void *)((u8 *)gsp->shm.msgq.ptr + 0x1000 + 0 * 0x1000);140 memcpy(msg + len, mqe, repc);141 142 rptr += DIV_ROUND_UP(repc, GSP_PAGE_SIZE);143 }144 145 mb();146 (*gsp->msgq.rptr) = rptr;147 return msg;148}149 150static void *151r535_gsp_msgq_recv(struct nvkm_gsp *gsp, u32 repc, int *ptime)152{153 return r535_gsp_msgq_wait(gsp, repc, NULL, ptime);154}155 156static int157r535_gsp_cmdq_push(struct nvkm_gsp *gsp, void *argv)158{159 struct r535_gsp_msg *cmd = container_of(argv, typeof(*cmd), data);160 struct r535_gsp_msg *cqe;161 u32 argc = cmd->checksum;162 u64 *ptr = (void *)cmd;163 u64 *end;164 u64 csum = 0;165 int free, time = 1000000;166 u32 wptr, size;167 u32 off = 0;168 169 argc = ALIGN(GSP_MSG_HDR_SIZE + argc, GSP_PAGE_SIZE);170 171 end = (u64 *)((char *)ptr + argc);172 cmd->pad = 0;173 cmd->checksum = 0;174 cmd->sequence = gsp->cmdq.seq++;175 cmd->elem_count = DIV_ROUND_UP(argc, 0x1000);176 177 while (ptr < end)178 csum ^= *ptr++;179 180 cmd->checksum = upper_32_bits(csum) ^ lower_32_bits(csum);181 182 wptr = *gsp->cmdq.wptr;183 do {184 do {185 free = *gsp->cmdq.rptr + gsp->cmdq.cnt - wptr - 1;186 if (free >= gsp->cmdq.cnt)187 free -= gsp->cmdq.cnt;188 if (free >= 1)189 break;190 191 usleep_range(1, 2);192 } while(--time);193 194 if (WARN_ON(!time)) {195 kvfree(cmd);196 return -ETIMEDOUT;197 }198 199 cqe = (void *)((u8 *)gsp->shm.cmdq.ptr + 0x1000 + wptr * 0x1000);200 size = min_t(u32, argc, (gsp->cmdq.cnt - wptr) * GSP_PAGE_SIZE);201 memcpy(cqe, (u8 *)cmd + off, size);202 203 wptr += DIV_ROUND_UP(size, 0x1000);204 if (wptr == gsp->cmdq.cnt)205 wptr = 0;206 207 off += size;208 argc -= size;209 } while(argc);210 211 nvkm_trace(&gsp->subdev, "cmdq: wptr %d\n", wptr);212 wmb();213 (*gsp->cmdq.wptr) = wptr;214 mb();215 216 nvkm_falcon_wr32(&gsp->falcon, 0xc00, 0x00000000);217 218 kvfree(cmd);219 return 0;220}221 222static void *223r535_gsp_cmdq_get(struct nvkm_gsp *gsp, u32 argc)224{225 struct r535_gsp_msg *cmd;226 u32 size = GSP_MSG_HDR_SIZE + argc;227 228 size = ALIGN(size, GSP_MSG_MIN_SIZE);229 cmd = kvzalloc(size, GFP_KERNEL);230 if (!cmd)231 return ERR_PTR(-ENOMEM);232 233 cmd->checksum = argc;234 return cmd->data;235}236 237struct nvfw_gsp_rpc {238 u32 header_version;239 u32 signature;240 u32 length;241 u32 function;242 u32 rpc_result;243 u32 rpc_result_private;244 u32 sequence;245 union {246 u32 spare;247 u32 cpuRmGfid;248 };249 u8 data[];250};251 252static void253r535_gsp_msg_done(struct nvkm_gsp *gsp, struct nvfw_gsp_rpc *msg)254{255 kvfree(msg);256}257 258static void259r535_gsp_msg_dump(struct nvkm_gsp *gsp, struct nvfw_gsp_rpc *msg, int lvl)260{261 if (gsp->subdev.debug >= lvl) {262 nvkm_printk__(&gsp->subdev, lvl, info,263 "msg fn:%d len:0x%x/0x%zx res:0x%x resp:0x%x\n",264 msg->function, msg->length, msg->length - sizeof(*msg),265 msg->rpc_result, msg->rpc_result_private);266 print_hex_dump(KERN_INFO, "msg: ", DUMP_PREFIX_OFFSET, 16, 1,267 msg->data, msg->length - sizeof(*msg), true);268 }269}270 271static struct nvfw_gsp_rpc *272r535_gsp_msg_recv(struct nvkm_gsp *gsp, int fn, u32 repc)273{274 struct nvkm_subdev *subdev = &gsp->subdev;275 struct nvfw_gsp_rpc *msg;276 int time = 4000000, i;277 u32 size;278 279retry:280 msg = r535_gsp_msgq_wait(gsp, sizeof(*msg), &size, &time);281 if (IS_ERR_OR_NULL(msg))282 return msg;283 284 msg = r535_gsp_msgq_recv(gsp, msg->length, &time);285 if (IS_ERR_OR_NULL(msg))286 return msg;287 288 if (msg->rpc_result) {289 r535_gsp_msg_dump(gsp, msg, NV_DBG_ERROR);290 r535_gsp_msg_done(gsp, msg);291 return ERR_PTR(-EINVAL);292 }293 294 r535_gsp_msg_dump(gsp, msg, NV_DBG_TRACE);295 296 if (fn && msg->function == fn) {297 if (repc) {298 if (msg->length < sizeof(*msg) + repc) {299 nvkm_error(subdev, "msg len %d < %zd\n",300 msg->length, sizeof(*msg) + repc);301 r535_gsp_msg_dump(gsp, msg, NV_DBG_ERROR);302 r535_gsp_msg_done(gsp, msg);303 return ERR_PTR(-EIO);304 }305 306 return msg;307 }308 309 r535_gsp_msg_done(gsp, msg);310 return NULL;311 }312 313 for (i = 0; i < gsp->msgq.ntfy_nr; i++) {314 struct nvkm_gsp_msgq_ntfy *ntfy = &gsp->msgq.ntfy[i];315 316 if (ntfy->fn == msg->function) {317 if (ntfy->func)318 ntfy->func(ntfy->priv, ntfy->fn, msg->data, msg->length - sizeof(*msg));319 break;320 }321 }322 323 if (i == gsp->msgq.ntfy_nr)324 r535_gsp_msg_dump(gsp, msg, NV_DBG_WARN);325 326 r535_gsp_msg_done(gsp, msg);327 if (fn)328 goto retry;329 330 if (*gsp->msgq.rptr != *gsp->msgq.wptr)331 goto retry;332 333 return NULL;334}335 336static int337r535_gsp_msg_ntfy_add(struct nvkm_gsp *gsp, u32 fn, nvkm_gsp_msg_ntfy_func func, void *priv)338{339 int ret = 0;340 341 mutex_lock(&gsp->msgq.mutex);342 if (WARN_ON(gsp->msgq.ntfy_nr >= ARRAY_SIZE(gsp->msgq.ntfy))) {343 ret = -ENOSPC;344 } else {345 gsp->msgq.ntfy[gsp->msgq.ntfy_nr].fn = fn;346 gsp->msgq.ntfy[gsp->msgq.ntfy_nr].func = func;347 gsp->msgq.ntfy[gsp->msgq.ntfy_nr].priv = priv;348 gsp->msgq.ntfy_nr++;349 }350 mutex_unlock(&gsp->msgq.mutex);351 return ret;352}353 354static int355r535_gsp_rpc_poll(struct nvkm_gsp *gsp, u32 fn)356{357 void *repv;358 359 mutex_lock(&gsp->cmdq.mutex);360 repv = r535_gsp_msg_recv(gsp, fn, 0);361 mutex_unlock(&gsp->cmdq.mutex);362 if (IS_ERR(repv))363 return PTR_ERR(repv);364 365 return 0;366}367 368static void *369r535_gsp_rpc_send(struct nvkm_gsp *gsp, void *argv, bool wait, u32 repc)370{371 struct nvfw_gsp_rpc *rpc = container_of(argv, typeof(*rpc), data);372 struct nvfw_gsp_rpc *msg;373 u32 fn = rpc->function;374 void *repv = NULL;375 int ret;376 377 if (gsp->subdev.debug >= NV_DBG_TRACE) {378 nvkm_trace(&gsp->subdev, "rpc fn:%d len:0x%x/0x%zx\n", rpc->function,379 rpc->length, rpc->length - sizeof(*rpc));380 print_hex_dump(KERN_INFO, "rpc: ", DUMP_PREFIX_OFFSET, 16, 1,381 rpc->data, rpc->length - sizeof(*rpc), true);382 }383 384 ret = r535_gsp_cmdq_push(gsp, rpc);385 if (ret)386 return ERR_PTR(ret);387 388 if (wait) {389 msg = r535_gsp_msg_recv(gsp, fn, repc);390 if (!IS_ERR_OR_NULL(msg))391 repv = msg->data;392 else393 repv = msg;394 }395 396 return repv;397}398 399static void400r535_gsp_event_dtor(struct nvkm_gsp_event *event)401{402 struct nvkm_gsp_device *device = event->device;403 struct nvkm_gsp_client *client = device->object.client;404 struct nvkm_gsp *gsp = client->gsp;405 406 mutex_lock(&gsp->client_id.mutex);407 if (event->func) {408 list_del(&event->head);409 event->func = NULL;410 }411 mutex_unlock(&gsp->client_id.mutex);412 413 nvkm_gsp_rm_free(&event->object);414 event->device = NULL;415}416 417static int418r535_gsp_device_event_get(struct nvkm_gsp_event *event)419{420 struct nvkm_gsp_device *device = event->device;421 NV2080_CTRL_EVENT_SET_NOTIFICATION_PARAMS *ctrl;422 423 ctrl = nvkm_gsp_rm_ctrl_get(&device->subdevice,424 NV2080_CTRL_CMD_EVENT_SET_NOTIFICATION, sizeof(*ctrl));425 if (IS_ERR(ctrl))426 return PTR_ERR(ctrl);427 428 ctrl->event = event->id;429 ctrl->action = NV2080_CTRL_EVENT_SET_NOTIFICATION_ACTION_REPEAT;430 return nvkm_gsp_rm_ctrl_wr(&device->subdevice, ctrl);431}432 433static int434r535_gsp_device_event_ctor(struct nvkm_gsp_device *device, u32 handle, u32 id,435 nvkm_gsp_event_func func, struct nvkm_gsp_event *event)436{437 struct nvkm_gsp_client *client = device->object.client;438 struct nvkm_gsp *gsp = client->gsp;439 NV0005_ALLOC_PARAMETERS *args;440 int ret;441 442 args = nvkm_gsp_rm_alloc_get(&device->subdevice, handle,443 NV01_EVENT_KERNEL_CALLBACK_EX, sizeof(*args),444 &event->object);445 if (IS_ERR(args))446 return PTR_ERR(args);447 448 args->hParentClient = client->object.handle;449 args->hSrcResource = 0;450 args->hClass = NV01_EVENT_KERNEL_CALLBACK_EX;451 args->notifyIndex = NV01_EVENT_CLIENT_RM | id;452 args->data = NULL;453 454 ret = nvkm_gsp_rm_alloc_wr(&event->object, args);455 if (ret)456 return ret;457 458 event->device = device;459 event->id = id;460 461 ret = r535_gsp_device_event_get(event);462 if (ret) {463 nvkm_gsp_event_dtor(event);464 return ret;465 }466 467 mutex_lock(&gsp->client_id.mutex);468 event->func = func;469 list_add(&event->head, &client->events);470 mutex_unlock(&gsp->client_id.mutex);471 return 0;472}473 474static void475r535_gsp_device_dtor(struct nvkm_gsp_device *device)476{477 nvkm_gsp_rm_free(&device->subdevice);478 nvkm_gsp_rm_free(&device->object);479}480 481static int482r535_gsp_subdevice_ctor(struct nvkm_gsp_device *device)483{484 NV2080_ALLOC_PARAMETERS *args;485 486 return nvkm_gsp_rm_alloc(&device->object, 0x5d1d0000, NV20_SUBDEVICE_0, sizeof(*args),487 &device->subdevice);488}489 490static int491r535_gsp_device_ctor(struct nvkm_gsp_client *client, struct nvkm_gsp_device *device)492{493 NV0080_ALLOC_PARAMETERS *args;494 int ret;495 496 args = nvkm_gsp_rm_alloc_get(&client->object, 0xde1d0000, NV01_DEVICE_0, sizeof(*args),497 &device->object);498 if (IS_ERR(args))499 return PTR_ERR(args);500 501 args->hClientShare = client->object.handle;502 503 ret = nvkm_gsp_rm_alloc_wr(&device->object, args);504 if (ret)505 return ret;506 507 ret = r535_gsp_subdevice_ctor(device);508 if (ret)509 nvkm_gsp_rm_free(&device->object);510 511 return ret;512}513 514static void515r535_gsp_client_dtor(struct nvkm_gsp_client *client)516{517 struct nvkm_gsp *gsp = client->gsp;518 519 nvkm_gsp_rm_free(&client->object);520 521 mutex_lock(&gsp->client_id.mutex);522 idr_remove(&gsp->client_id.idr, client->object.handle & 0xffff);523 mutex_unlock(&gsp->client_id.mutex);524 525 client->gsp = NULL;526}527 528static int529r535_gsp_client_ctor(struct nvkm_gsp *gsp, struct nvkm_gsp_client *client)530{531 NV0000_ALLOC_PARAMETERS *args;532 int ret;533 534 mutex_lock(&gsp->client_id.mutex);535 ret = idr_alloc(&gsp->client_id.idr, client, 0, 0xffff + 1, GFP_KERNEL);536 mutex_unlock(&gsp->client_id.mutex);537 if (ret < 0)538 return ret;539 540 client->gsp = gsp;541 client->object.client = client;542 INIT_LIST_HEAD(&client->events);543 544 args = nvkm_gsp_rm_alloc_get(&client->object, 0xc1d00000 | ret, NV01_ROOT, sizeof(*args),545 &client->object);546 if (IS_ERR(args)) {547 r535_gsp_client_dtor(client);548 return ret;549 }550 551 args->hClient = client->object.handle;552 args->processID = ~0;553 554 ret = nvkm_gsp_rm_alloc_wr(&client->object, args);555 if (ret) {556 r535_gsp_client_dtor(client);557 return ret;558 }559 560 return 0;561}562 563static int564r535_gsp_rpc_rm_free(struct nvkm_gsp_object *object)565{566 struct nvkm_gsp_client *client = object->client;567 struct nvkm_gsp *gsp = client->gsp;568 rpc_free_v03_00 *rpc;569 570 nvkm_debug(&gsp->subdev, "cli:0x%08x obj:0x%08x free\n",571 client->object.handle, object->handle);572 573 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_FREE, sizeof(*rpc));574 if (WARN_ON(IS_ERR_OR_NULL(rpc)))575 return -EIO;576 577 rpc->params.hRoot = client->object.handle;578 rpc->params.hObjectParent = 0;579 rpc->params.hObjectOld = object->handle;580 return nvkm_gsp_rpc_wr(gsp, rpc, true);581}582 583static void584r535_gsp_rpc_rm_alloc_done(struct nvkm_gsp_object *object, void *repv)585{586 rpc_gsp_rm_alloc_v03_00 *rpc = container_of(repv, typeof(*rpc), params);587 588 nvkm_gsp_rpc_done(object->client->gsp, rpc);589}590 591static void *592r535_gsp_rpc_rm_alloc_push(struct nvkm_gsp_object *object, void *argv, u32 repc)593{594 rpc_gsp_rm_alloc_v03_00 *rpc = container_of(argv, typeof(*rpc), params);595 struct nvkm_gsp *gsp = object->client->gsp;596 void *ret;597 598 rpc = nvkm_gsp_rpc_push(gsp, rpc, true, sizeof(*rpc) + repc);599 if (IS_ERR_OR_NULL(rpc))600 return rpc;601 602 if (rpc->status) {603 ret = ERR_PTR(r535_rpc_status_to_errno(rpc->status));604 if (PTR_ERR(ret) != -EAGAIN && PTR_ERR(ret) != -EBUSY)605 nvkm_error(&gsp->subdev, "RM_ALLOC: 0x%x\n", rpc->status);606 } else {607 ret = repc ? rpc->params : NULL;608 }609 610 nvkm_gsp_rpc_done(gsp, rpc);611 612 return ret;613}614 615static void *616r535_gsp_rpc_rm_alloc_get(struct nvkm_gsp_object *object, u32 oclass, u32 argc)617{618 struct nvkm_gsp_client *client = object->client;619 struct nvkm_gsp *gsp = client->gsp;620 rpc_gsp_rm_alloc_v03_00 *rpc;621 622 nvkm_debug(&gsp->subdev, "cli:0x%08x obj:0x%08x new obj:0x%08x cls:0x%08x argc:%d\n",623 client->object.handle, object->parent->handle, object->handle, oclass, argc);624 625 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_GSP_RM_ALLOC, sizeof(*rpc) + argc);626 if (IS_ERR(rpc))627 return rpc;628 629 rpc->hClient = client->object.handle;630 rpc->hParent = object->parent->handle;631 rpc->hObject = object->handle;632 rpc->hClass = oclass;633 rpc->status = 0;634 rpc->paramsSize = argc;635 return rpc->params;636}637 638static void639r535_gsp_rpc_rm_ctrl_done(struct nvkm_gsp_object *object, void *repv)640{641 rpc_gsp_rm_control_v03_00 *rpc = container_of(repv, typeof(*rpc), params);642 643 if (!repv)644 return;645 nvkm_gsp_rpc_done(object->client->gsp, rpc);646}647 648static int649r535_gsp_rpc_rm_ctrl_push(struct nvkm_gsp_object *object, void **argv, u32 repc)650{651 rpc_gsp_rm_control_v03_00 *rpc = container_of((*argv), typeof(*rpc), params);652 struct nvkm_gsp *gsp = object->client->gsp;653 int ret = 0;654 655 rpc = nvkm_gsp_rpc_push(gsp, rpc, true, repc);656 if (IS_ERR_OR_NULL(rpc)) {657 *argv = NULL;658 return PTR_ERR(rpc);659 }660 661 if (rpc->status) {662 ret = r535_rpc_status_to_errno(rpc->status);663 if (ret != -EAGAIN && ret != -EBUSY)664 nvkm_error(&gsp->subdev, "cli:0x%08x obj:0x%08x ctrl cmd:0x%08x failed: 0x%08x\n",665 object->client->object.handle, object->handle, rpc->cmd, rpc->status);666 }667 668 if (repc)669 *argv = rpc->params;670 else671 nvkm_gsp_rpc_done(gsp, rpc);672 673 return ret;674}675 676static void *677r535_gsp_rpc_rm_ctrl_get(struct nvkm_gsp_object *object, u32 cmd, u32 argc)678{679 struct nvkm_gsp_client *client = object->client;680 struct nvkm_gsp *gsp = client->gsp;681 rpc_gsp_rm_control_v03_00 *rpc;682 683 nvkm_debug(&gsp->subdev, "cli:0x%08x obj:0x%08x ctrl cmd:0x%08x argc:%d\n",684 client->object.handle, object->handle, cmd, argc);685 686 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL, sizeof(*rpc) + argc);687 if (IS_ERR(rpc))688 return rpc;689 690 rpc->hClient = client->object.handle;691 rpc->hObject = object->handle;692 rpc->cmd = cmd;693 rpc->status = 0;694 rpc->paramsSize = argc;695 return rpc->params;696}697 698static void699r535_gsp_rpc_done(struct nvkm_gsp *gsp, void *repv)700{701 struct nvfw_gsp_rpc *rpc = container_of(repv, typeof(*rpc), data);702 703 r535_gsp_msg_done(gsp, rpc);704}705 706static void *707r535_gsp_rpc_get(struct nvkm_gsp *gsp, u32 fn, u32 argc)708{709 struct nvfw_gsp_rpc *rpc;710 711 rpc = r535_gsp_cmdq_get(gsp, ALIGN(sizeof(*rpc) + argc, sizeof(u64)));712 if (IS_ERR(rpc))713 return ERR_CAST(rpc);714 715 rpc->header_version = 0x03000000;716 rpc->signature = ('C' << 24) | ('P' << 16) | ('R' << 8) | 'V';717 rpc->function = fn;718 rpc->rpc_result = 0xffffffff;719 rpc->rpc_result_private = 0xffffffff;720 rpc->length = sizeof(*rpc) + argc;721 return rpc->data;722}723 724static void *725r535_gsp_rpc_push(struct nvkm_gsp *gsp, void *argv, bool wait, u32 repc)726{727 struct nvfw_gsp_rpc *rpc = container_of(argv, typeof(*rpc), data);728 struct r535_gsp_msg *cmd = container_of((void *)rpc, typeof(*cmd), data);729 const u32 max_msg_size = (16 * 0x1000) - sizeof(struct r535_gsp_msg);730 const u32 max_rpc_size = max_msg_size - sizeof(*rpc);731 u32 rpc_size = rpc->length - sizeof(*rpc);732 void *repv;733 734 mutex_lock(&gsp->cmdq.mutex);735 if (rpc_size > max_rpc_size) {736 const u32 fn = rpc->function;737 738 /* Adjust length, and send initial RPC. */739 rpc->length = sizeof(*rpc) + max_rpc_size;740 cmd->checksum = rpc->length;741 742 repv = r535_gsp_rpc_send(gsp, argv, false, 0);743 if (IS_ERR(repv))744 goto done;745 746 argv += max_rpc_size;747 rpc_size -= max_rpc_size;748 749 /* Remaining chunks sent as CONTINUATION_RECORD RPCs. */750 while (rpc_size) {751 u32 size = min(rpc_size, max_rpc_size);752 void *next;753 754 next = r535_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD, size);755 if (IS_ERR(next)) {756 repv = next;757 goto done;758 }759 760 memcpy(next, argv, size);761 762 repv = r535_gsp_rpc_send(gsp, next, false, 0);763 if (IS_ERR(repv))764 goto done;765 766 argv += size;767 rpc_size -= size;768 }769 770 /* Wait for reply. */771 if (wait) {772 rpc = r535_gsp_msg_recv(gsp, fn, repc);773 if (!IS_ERR_OR_NULL(rpc))774 repv = rpc->data;775 else776 repv = rpc;777 } else {778 repv = NULL;779 }780 } else {781 repv = r535_gsp_rpc_send(gsp, argv, wait, repc);782 }783 784done:785 mutex_unlock(&gsp->cmdq.mutex);786 return repv;787}788 789const struct nvkm_gsp_rm790r535_gsp_rm = {791 .rpc_get = r535_gsp_rpc_get,792 .rpc_push = r535_gsp_rpc_push,793 .rpc_done = r535_gsp_rpc_done,794 795 .rm_ctrl_get = r535_gsp_rpc_rm_ctrl_get,796 .rm_ctrl_push = r535_gsp_rpc_rm_ctrl_push,797 .rm_ctrl_done = r535_gsp_rpc_rm_ctrl_done,798 799 .rm_alloc_get = r535_gsp_rpc_rm_alloc_get,800 .rm_alloc_push = r535_gsp_rpc_rm_alloc_push,801 .rm_alloc_done = r535_gsp_rpc_rm_alloc_done,802 803 .rm_free = r535_gsp_rpc_rm_free,804 805 .client_ctor = r535_gsp_client_ctor,806 .client_dtor = r535_gsp_client_dtor,807 808 .device_ctor = r535_gsp_device_ctor,809 .device_dtor = r535_gsp_device_dtor,810 811 .event_ctor = r535_gsp_device_event_ctor,812 .event_dtor = r535_gsp_event_dtor,813};814 815static void816r535_gsp_msgq_work(struct work_struct *work)817{818 struct nvkm_gsp *gsp = container_of(work, typeof(*gsp), msgq.work);819 820 mutex_lock(&gsp->cmdq.mutex);821 if (*gsp->msgq.rptr != *gsp->msgq.wptr)822 r535_gsp_msg_recv(gsp, 0, 0);823 mutex_unlock(&gsp->cmdq.mutex);824}825 826static irqreturn_t827r535_gsp_intr(struct nvkm_inth *inth)828{829 struct nvkm_gsp *gsp = container_of(inth, typeof(*gsp), subdev.inth);830 struct nvkm_subdev *subdev = &gsp->subdev;831 u32 intr = nvkm_falcon_rd32(&gsp->falcon, 0x0008);832 u32 inte = nvkm_falcon_rd32(&gsp->falcon, gsp->falcon.func->addr2 +833 gsp->falcon.func->riscv_irqmask);834 u32 stat = intr & inte;835 836 if (!stat) {837 nvkm_debug(subdev, "inte %08x %08x\n", intr, inte);838 return IRQ_NONE;839 }840 841 if (stat & 0x00000040) {842 nvkm_falcon_wr32(&gsp->falcon, 0x004, 0x00000040);843 schedule_work(&gsp->msgq.work);844 stat &= ~0x00000040;845 }846 847 if (stat) {848 nvkm_error(subdev, "intr %08x\n", stat);849 nvkm_falcon_wr32(&gsp->falcon, 0x014, stat);850 nvkm_falcon_wr32(&gsp->falcon, 0x004, stat);851 }852 853 nvkm_falcon_intr_retrigger(&gsp->falcon);854 return IRQ_HANDLED;855}856 857static int858r535_gsp_intr_get_table(struct nvkm_gsp *gsp)859{860 NV2080_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE_PARAMS *ctrl;861 int ret = 0;862 863 ctrl = nvkm_gsp_rm_ctrl_get(&gsp->internal.device.subdevice,864 NV2080_CTRL_CMD_INTERNAL_INTR_GET_KERNEL_TABLE, sizeof(*ctrl));865 if (IS_ERR(ctrl))866 return PTR_ERR(ctrl);867 868 ret = nvkm_gsp_rm_ctrl_push(&gsp->internal.device.subdevice, &ctrl, sizeof(*ctrl));869 if (WARN_ON(ret)) {870 nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl);871 return ret;872 }873 874 for (unsigned i = 0; i < ctrl->tableLen; i++) {875 enum nvkm_subdev_type type;876 int inst;877 878 nvkm_debug(&gsp->subdev,879 "%2d: engineIdx %3d pmcIntrMask %08x stall %08x nonStall %08x\n", i,880 ctrl->table[i].engineIdx, ctrl->table[i].pmcIntrMask,881 ctrl->table[i].vectorStall, ctrl->table[i].vectorNonStall);882 883 switch (ctrl->table[i].engineIdx) {884 case MC_ENGINE_IDX_GSP:885 type = NVKM_SUBDEV_GSP;886 inst = 0;887 break;888 case MC_ENGINE_IDX_DISP:889 type = NVKM_ENGINE_DISP;890 inst = 0;891 break;892 case MC_ENGINE_IDX_CE0 ... MC_ENGINE_IDX_CE9:893 type = NVKM_ENGINE_CE;894 inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_CE0;895 break;896 case MC_ENGINE_IDX_GR0:897 type = NVKM_ENGINE_GR;898 inst = 0;899 break;900 case MC_ENGINE_IDX_NVDEC0 ... MC_ENGINE_IDX_NVDEC7:901 type = NVKM_ENGINE_NVDEC;902 inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_NVDEC0;903 break;904 case MC_ENGINE_IDX_MSENC ... MC_ENGINE_IDX_MSENC2:905 type = NVKM_ENGINE_NVENC;906 inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_MSENC;907 break;908 case MC_ENGINE_IDX_NVJPEG0 ... MC_ENGINE_IDX_NVJPEG7:909 type = NVKM_ENGINE_NVJPG;910 inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_NVJPEG0;911 break;912 case MC_ENGINE_IDX_OFA0:913 type = NVKM_ENGINE_OFA;914 inst = 0;915 break;916 default:917 continue;918 }919 920 if (WARN_ON(gsp->intr_nr == ARRAY_SIZE(gsp->intr))) {921 ret = -ENOSPC;922 break;923 }924 925 gsp->intr[gsp->intr_nr].type = type;926 gsp->intr[gsp->intr_nr].inst = inst;927 gsp->intr[gsp->intr_nr].stall = ctrl->table[i].vectorStall;928 gsp->intr[gsp->intr_nr].nonstall = ctrl->table[i].vectorNonStall;929 gsp->intr_nr++;930 }931 932 nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl);933 return ret;934}935 936static int937r535_gsp_rpc_get_gsp_static_info(struct nvkm_gsp *gsp)938{939 GspStaticConfigInfo *rpc;940 int last_usable = -1;941 942 rpc = nvkm_gsp_rpc_rd(gsp, NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO, sizeof(*rpc));943 if (IS_ERR(rpc))944 return PTR_ERR(rpc);945 946 gsp->internal.client.object.client = &gsp->internal.client;947 gsp->internal.client.object.parent = NULL;948 gsp->internal.client.object.handle = rpc->hInternalClient;949 gsp->internal.client.gsp = gsp;950 951 gsp->internal.device.object.client = &gsp->internal.client;952 gsp->internal.device.object.parent = &gsp->internal.client.object;953 gsp->internal.device.object.handle = rpc->hInternalDevice;954 955 gsp->internal.device.subdevice.client = &gsp->internal.client;956 gsp->internal.device.subdevice.parent = &gsp->internal.device.object;957 gsp->internal.device.subdevice.handle = rpc->hInternalSubdevice;958 959 gsp->bar.rm_bar1_pdb = rpc->bar1PdeBase;960 gsp->bar.rm_bar2_pdb = rpc->bar2PdeBase;961 962 for (int i = 0; i < rpc->fbRegionInfoParams.numFBRegions; i++) {963 NV2080_CTRL_CMD_FB_GET_FB_REGION_FB_REGION_INFO *reg =964 &rpc->fbRegionInfoParams.fbRegion[i];965 966 nvkm_debug(&gsp->subdev, "fb region %d: "967 "%016llx-%016llx rsvd:%016llx perf:%08x comp:%d iso:%d prot:%d\n", i,968 reg->base, reg->limit, reg->reserved, reg->performance,969 reg->supportCompressed, reg->supportISO, reg->bProtected);970 971 if (!reg->reserved && !reg->bProtected) {972 if (reg->supportCompressed && reg->supportISO &&973 !WARN_ON_ONCE(gsp->fb.region_nr >= ARRAY_SIZE(gsp->fb.region))) {974 const u64 size = (reg->limit + 1) - reg->base;975 976 gsp->fb.region[gsp->fb.region_nr].addr = reg->base;977 gsp->fb.region[gsp->fb.region_nr].size = size;978 gsp->fb.region_nr++;979 }980 981 last_usable = i;982 }983 }984 985 if (last_usable >= 0) {986 u32 rsvd_base = rpc->fbRegionInfoParams.fbRegion[last_usable].limit + 1;987 988 gsp->fb.rsvd_size = gsp->fb.heap.addr - rsvd_base;989 }990 991 for (int gpc = 0; gpc < ARRAY_SIZE(rpc->tpcInfo); gpc++) {992 if (rpc->gpcInfo.gpcMask & BIT(gpc)) {993 gsp->gr.tpcs += hweight32(rpc->tpcInfo[gpc].tpcMask);994 gsp->gr.gpcs++;995 }996 }997 998 nvkm_gsp_rpc_done(gsp, rpc);999 return 0;1000}1001 1002static void1003nvkm_gsp_mem_dtor(struct nvkm_gsp *gsp, struct nvkm_gsp_mem *mem)1004{1005 if (mem->data) {1006 /*1007 * Poison the buffer to catch any unexpected access from1008 * GSP-RM if the buffer was prematurely freed.1009 */1010 memset(mem->data, 0xFF, mem->size);1011 1012 dma_free_coherent(gsp->subdev.device->dev, mem->size, mem->data, mem->addr);1013 memset(mem, 0, sizeof(*mem));1014 }1015}1016 1017static int1018nvkm_gsp_mem_ctor(struct nvkm_gsp *gsp, size_t size, struct nvkm_gsp_mem *mem)1019{1020 mem->size = size;1021 mem->data = dma_alloc_coherent(gsp->subdev.device->dev, size, &mem->addr, GFP_KERNEL);1022 if (WARN_ON(!mem->data))1023 return -ENOMEM;1024 1025 return 0;1026}1027 1028static int1029r535_gsp_postinit(struct nvkm_gsp *gsp)1030{1031 struct nvkm_device *device = gsp->subdev.device;1032 int ret;1033 1034 ret = r535_gsp_rpc_get_gsp_static_info(gsp);1035 if (WARN_ON(ret))1036 return ret;1037 1038 INIT_WORK(&gsp->msgq.work, r535_gsp_msgq_work);1039 1040 ret = r535_gsp_intr_get_table(gsp);1041 if (WARN_ON(ret))1042 return ret;1043 1044 ret = nvkm_gsp_intr_stall(gsp, gsp->subdev.type, gsp->subdev.inst);1045 if (WARN_ON(ret < 0))1046 return ret;1047 1048 ret = nvkm_inth_add(&device->vfn->intr, ret, NVKM_INTR_PRIO_NORMAL, &gsp->subdev,1049 r535_gsp_intr, &gsp->subdev.inth);1050 if (WARN_ON(ret))1051 return ret;1052 1053 nvkm_inth_allow(&gsp->subdev.inth);1054 nvkm_wr32(device, 0x110004, 0x00000040);1055 1056 /* Release the DMA buffers that were needed only for boot and init */1057 nvkm_gsp_mem_dtor(gsp, &gsp->boot.fw);1058 nvkm_gsp_mem_dtor(gsp, &gsp->libos);1059 1060 return ret;1061}1062 1063static int1064r535_gsp_rpc_unloading_guest_driver(struct nvkm_gsp *gsp, bool suspend)1065{1066 rpc_unloading_guest_driver_v1F_07 *rpc;1067 1068 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER, sizeof(*rpc));1069 if (IS_ERR(rpc))1070 return PTR_ERR(rpc);1071 1072 if (suspend) {1073 rpc->bInPMTransition = 1;1074 rpc->bGc6Entering = 0;1075 rpc->newLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_3;1076 } else {1077 rpc->bInPMTransition = 0;1078 rpc->bGc6Entering = 0;1079 rpc->newLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_0;1080 }1081 1082 return nvkm_gsp_rpc_wr(gsp, rpc, true);1083}1084 1085enum registry_type {1086 REGISTRY_TABLE_ENTRY_TYPE_DWORD = 1, /* 32-bit unsigned integer */1087 REGISTRY_TABLE_ENTRY_TYPE_BINARY = 2, /* Binary blob */1088 REGISTRY_TABLE_ENTRY_TYPE_STRING = 3, /* Null-terminated string */1089};1090 1091/* An arbitrary limit to the length of a registry key */1092#define REGISTRY_MAX_KEY_LENGTH 641093 1094/**1095 * registry_list_entry - linked list member for a registry key/value1096 * @head: list_head struct1097 * @type: dword, binary, or string1098 * @klen: the length of name of the key1099 * @vlen: the length of the value1100 * @key: the key name1101 * @dword: the data, if REGISTRY_TABLE_ENTRY_TYPE_DWORD1102 * @binary: the data, if TYPE_BINARY or TYPE_STRING1103 *1104 * Every registry key/value is represented internally by this struct.1105 *1106 * Type DWORD is a simple 32-bit unsigned integer, and its value is stored in1107 * @dword.1108 *1109 * Types BINARY and STRING are variable-length binary blobs. The only real1110 * difference between BINARY and STRING is that STRING is null-terminated and1111 * is expected to contain only printable characters.1112 *1113 * Note: it is technically possible to have multiple keys with the same name1114 * but different types, but this is not useful since GSP-RM expects keys to1115 * have only one specific type.1116 */1117struct registry_list_entry {1118 struct list_head head;1119 enum registry_type type;1120 size_t klen;1121 char key[REGISTRY_MAX_KEY_LENGTH];1122 size_t vlen;1123 u32 dword; /* TYPE_DWORD */1124 u8 binary[] __counted_by(vlen); /* TYPE_BINARY or TYPE_STRING */1125};1126 1127/**1128 * add_registry -- adds a registry entry1129 * @gsp: gsp pointer1130 * @key: name of the registry key1131 * @type: type of data1132 * @data: pointer to value1133 * @length: size of data, in bytes1134 *1135 * Adds a registry key/value pair to the registry database.1136 *1137 * This function collects the registry information in a linked list. After1138 * all registry keys have been added, build_registry() is used to create the1139 * RPC data structure.1140 *1141 * registry_rpc_size is a running total of the size of all registry keys.1142 * It's used to avoid an O(n) calculation of the size when the RPC is built.1143 *1144 * Returns 0 on success, or negative error code on error.1145 */1146static int add_registry(struct nvkm_gsp *gsp, const char *key,1147 enum registry_type type, const void *data, size_t length)1148{1149 struct registry_list_entry *reg;1150 const size_t nlen = strnlen(key, REGISTRY_MAX_KEY_LENGTH) + 1;1151 size_t alloc_size; /* extra bytes to alloc for binary or string value */1152 1153 if (nlen > REGISTRY_MAX_KEY_LENGTH)1154 return -EINVAL;1155 1156 alloc_size = (type == REGISTRY_TABLE_ENTRY_TYPE_DWORD) ? 0 : length;1157 1158 reg = kmalloc(sizeof(*reg) + alloc_size, GFP_KERNEL);1159 if (!reg)1160 return -ENOMEM;1161 1162 switch (type) {1163 case REGISTRY_TABLE_ENTRY_TYPE_DWORD:1164 reg->dword = *(const u32 *)(data);1165 break;1166 case REGISTRY_TABLE_ENTRY_TYPE_BINARY:1167 case REGISTRY_TABLE_ENTRY_TYPE_STRING:1168 memcpy(reg->binary, data, alloc_size);1169 break;1170 default:1171 nvkm_error(&gsp->subdev, "unrecognized registry type %u for '%s'\n",1172 type, key);1173 kfree(reg);1174 return -EINVAL;1175 }1176 1177 memcpy(reg->key, key, nlen);1178 reg->klen = nlen;1179 reg->vlen = length;1180 reg->type = type;1181 1182 list_add_tail(®->head, &gsp->registry_list);1183 gsp->registry_rpc_size += sizeof(PACKED_REGISTRY_ENTRY) + nlen + alloc_size;1184 1185 return 0;1186}1187 1188static int add_registry_num(struct nvkm_gsp *gsp, const char *key, u32 value)1189{1190 return add_registry(gsp, key, REGISTRY_TABLE_ENTRY_TYPE_DWORD,1191 &value, sizeof(u32));1192}1193 1194static int add_registry_string(struct nvkm_gsp *gsp, const char *key, const char *value)1195{1196 return add_registry(gsp, key, REGISTRY_TABLE_ENTRY_TYPE_STRING,1197 value, strlen(value) + 1);1198}1199 1200/**1201 * build_registry -- create the registry RPC data1202 * @gsp: gsp pointer1203 * @registry: pointer to the RPC payload to fill1204 *1205 * After all registry key/value pairs have been added, call this function to1206 * build the RPC.1207 *1208 * The registry RPC looks like this:1209 *1210 * +-----------------+1211 * |NvU32 size; |1212 * |NvU32 numEntries;|1213 * +-----------------+1214 * +----------------------------------------+1215 * |PACKED_REGISTRY_ENTRY |1216 * +----------------------------------------+1217 * |Null-terminated key (string) for entry 0|1218 * +----------------------------------------+1219 * |Binary/string data value for entry 0 | (only if necessary)1220 * +----------------------------------------+1221 *1222 * +----------------------------------------+1223 * |PACKED_REGISTRY_ENTRY |1224 * +----------------------------------------+1225 * |Null-terminated key (string) for entry 1|1226 * +----------------------------------------+1227 * |Binary/string data value for entry 1 | (only if necessary)1228 * +----------------------------------------+1229 * ... (and so on, one copy for each entry)1230 *1231 *1232 * The 'data' field of an entry is either a 32-bit integer (for type DWORD)1233 * or an offset into the PACKED_REGISTRY_TABLE (for types BINARY and STRING).1234 *1235 * All memory allocated by add_registry() is released.1236 */1237static void build_registry(struct nvkm_gsp *gsp, PACKED_REGISTRY_TABLE *registry)1238{1239 struct registry_list_entry *reg, *n;1240 size_t str_offset;1241 unsigned int i = 0;1242 1243 registry->numEntries = list_count_nodes(&gsp->registry_list);1244 str_offset = struct_size(registry, entries, registry->numEntries);1245 1246 list_for_each_entry_safe(reg, n, &gsp->registry_list, head) {1247 registry->entries[i].type = reg->type;1248 registry->entries[i].length = reg->vlen;1249 1250 /* Append the key name to the table */1251 registry->entries[i].nameOffset = str_offset;1252 memcpy((void *)registry + str_offset, reg->key, reg->klen);1253 str_offset += reg->klen;1254 1255 switch (reg->type) {1256 case REGISTRY_TABLE_ENTRY_TYPE_DWORD:1257 registry->entries[i].data = reg->dword;1258 break;1259 case REGISTRY_TABLE_ENTRY_TYPE_BINARY:1260 case REGISTRY_TABLE_ENTRY_TYPE_STRING:1261 /* If the type is binary or string, also append the value */1262 memcpy((void *)registry + str_offset, reg->binary, reg->vlen);1263 registry->entries[i].data = str_offset;1264 str_offset += reg->vlen;1265 break;1266 default:1267 break;1268 }1269 1270 i++;1271 list_del(®->head);1272 kfree(reg);1273 }1274 1275 /* Double-check that we calculated the sizes correctly */1276 WARN_ON(gsp->registry_rpc_size != str_offset);1277 1278 registry->size = gsp->registry_rpc_size;1279}1280 1281/**1282 * clean_registry -- clean up registry memory in case of error1283 * @gsp: gsp pointer1284 *1285 * Call this function to clean up all memory allocated by add_registry()1286 * in case of error and build_registry() is not called.1287 */1288static void clean_registry(struct nvkm_gsp *gsp)1289{1290 struct registry_list_entry *reg, *n;1291 1292 list_for_each_entry_safe(reg, n, &gsp->registry_list, head) {1293 list_del(®->head);1294 kfree(reg);1295 }1296 1297 gsp->registry_rpc_size = sizeof(PACKED_REGISTRY_TABLE);1298}1299 1300MODULE_PARM_DESC(NVreg_RegistryDwords,1301 "A semicolon-separated list of key=integer pairs of GSP-RM registry keys");1302static char *NVreg_RegistryDwords;1303module_param(NVreg_RegistryDwords, charp, 0400);1304 1305/* dword only */1306struct nv_gsp_registry_entries {1307 const char *name;1308 u32 value;1309};1310 1311/**1312 * r535_registry_entries - required registry entries for GSP-RM1313 *1314 * This array lists registry entries that are required for GSP-RM to1315 * function correctly.1316 *1317 * RMSecBusResetEnable - enables PCI secondary bus reset1318 * RMForcePcieConfigSave - forces GSP-RM to preserve PCI configuration1319 * registers on any PCI reset.1320 */1321static const struct nv_gsp_registry_entries r535_registry_entries[] = {1322 { "RMSecBusResetEnable", 1 },1323 { "RMForcePcieConfigSave", 1 },1324};1325#define NV_GSP_REG_NUM_ENTRIES ARRAY_SIZE(r535_registry_entries)1326 1327/**1328 * strip - strips all characters in 'reject' from 's'1329 * @s: string to strip1330 * @reject: string of characters to remove1331 *1332 * 's' is modified.1333 *1334 * Returns the length of the new string.1335 */1336static size_t strip(char *s, const char *reject)1337{1338 char *p = s, *p2 = s;1339 size_t length = 0;1340 char c;1341 1342 do {1343 while ((c = *p2) && strchr(reject, c))1344 p2++;1345 1346 *p++ = c = *p2++;1347 length++;1348 } while (c);1349 1350 return length;1351}1352 1353/**1354 * r535_gsp_rpc_set_registry - build registry RPC and call GSP-RM1355 * @gsp: gsp pointer1356 *1357 * The GSP-RM registry is a set of key/value pairs that configure some aspects1358 * of GSP-RM. The keys are strings, and the values are 32-bit integers.1359 *1360 * The registry is built from a combination of a static hard-coded list (see1361 * above) and entries passed on the driver's command line.1362 */1363static int1364r535_gsp_rpc_set_registry(struct nvkm_gsp *gsp)1365{1366 PACKED_REGISTRY_TABLE *rpc;1367 unsigned int i;1368 int ret;1369 1370 INIT_LIST_HEAD(&gsp->registry_list);1371 gsp->registry_rpc_size = sizeof(PACKED_REGISTRY_TABLE);1372 1373 for (i = 0; i < NV_GSP_REG_NUM_ENTRIES; i++) {1374 ret = add_registry_num(gsp, r535_registry_entries[i].name,1375 r535_registry_entries[i].value);1376 if (ret)1377 goto fail;1378 }1379 1380 /*1381 * The NVreg_RegistryDwords parameter is a string of key=value1382 * pairs separated by semicolons. We need to extract and trim each1383 * substring, and then parse the substring to extract the key and1384 * value.1385 */1386 if (NVreg_RegistryDwords) {1387 char *p = kstrdup(NVreg_RegistryDwords, GFP_KERNEL);1388 char *start, *next = p, *equal;1389 1390 if (!p) {1391 ret = -ENOMEM;1392 goto fail;1393 }1394 1395 /* Remove any whitespace from the parameter string */1396 strip(p, " \t\n");1397 1398 while ((start = strsep(&next, ";"))) {1399 long value;1400 1401 equal = strchr(start, '=');1402 if (!equal || equal == start || equal[1] == 0) {1403 nvkm_error(&gsp->subdev,1404 "ignoring invalid registry string '%s'\n",1405 start);1406 continue;1407 }1408 1409 /* Truncate the key=value string to just key */1410 *equal = 0;1411 1412 ret = kstrtol(equal + 1, 0, &value);1413 if (!ret) {1414 ret = add_registry_num(gsp, start, value);1415 } else {1416 /* Not a number, so treat it as a string */1417 ret = add_registry_string(gsp, start, equal + 1);1418 }1419 1420 if (ret) {1421 nvkm_error(&gsp->subdev,1422 "ignoring invalid registry key/value '%s=%s'\n",1423 start, equal + 1);1424 continue;1425 }1426 }1427 1428 kfree(p);1429 }1430 1431 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_SET_REGISTRY, gsp->registry_rpc_size);1432 if (IS_ERR(rpc)) {1433 ret = PTR_ERR(rpc);1434 goto fail;1435 }1436 1437 build_registry(gsp, rpc);1438 1439 return nvkm_gsp_rpc_wr(gsp, rpc, false);1440 1441fail:1442 clean_registry(gsp);1443 return ret;1444}1445 1446#if defined(CONFIG_ACPI) && defined(CONFIG_X86)1447static void1448r535_gsp_acpi_caps(acpi_handle handle, CAPS_METHOD_DATA *caps)1449{1450 const guid_t NVOP_DSM_GUID =1451 GUID_INIT(0xA486D8F8, 0x0BDA, 0x471B,1452 0xA7, 0x2B, 0x60, 0x42, 0xA6, 0xB5, 0xBE, 0xE0);1453 u64 NVOP_DSM_REV = 0x00000100;1454 union acpi_object argv4 = {1455 .buffer.type = ACPI_TYPE_BUFFER,1456 .buffer.length = 4,1457 .buffer.pointer = kmalloc(argv4.buffer.length, GFP_KERNEL),1458 }, *obj;1459 1460 caps->status = 0xffff;1461 1462 if (!acpi_check_dsm(handle, &NVOP_DSM_GUID, NVOP_DSM_REV, BIT_ULL(0x1a)))1463 return;1464 1465 obj = acpi_evaluate_dsm(handle, &NVOP_DSM_GUID, NVOP_DSM_REV, 0x1a, &argv4);1466 if (!obj)1467 return;1468 1469 if (WARN_ON(obj->type != ACPI_TYPE_BUFFER) ||1470 WARN_ON(obj->buffer.length != 4))1471 return;1472 1473 caps->status = 0;1474 caps->optimusCaps = *(u32 *)obj->buffer.pointer;1475 1476 ACPI_FREE(obj);1477 1478 kfree(argv4.buffer.pointer);1479}1480 1481static void1482r535_gsp_acpi_jt(acpi_handle handle, JT_METHOD_DATA *jt)1483{1484 const guid_t JT_DSM_GUID =1485 GUID_INIT(0xCBECA351L, 0x067B, 0x4924,1486 0x9C, 0xBD, 0xB4, 0x6B, 0x00, 0xB8, 0x6F, 0x34);1487 u64 JT_DSM_REV = 0x00000103;1488 u32 caps;1489 union acpi_object argv4 = {1490 .buffer.type = ACPI_TYPE_BUFFER,1491 .buffer.length = sizeof(caps),1492 .buffer.pointer = kmalloc(argv4.buffer.length, GFP_KERNEL),1493 }, *obj;1494 1495 jt->status = 0xffff;1496 1497 obj = acpi_evaluate_dsm(handle, &JT_DSM_GUID, JT_DSM_REV, 0x1, &argv4);1498 if (!obj)1499 return;1500 1501 if (WARN_ON(obj->type != ACPI_TYPE_BUFFER) ||1502 WARN_ON(obj->buffer.length != 4))1503 return;1504 1505 jt->status = 0;1506 jt->jtCaps = *(u32 *)obj->buffer.pointer;1507 jt->jtRevId = (jt->jtCaps & 0xfff00000) >> 20;1508 jt->bSBIOSCaps = 0;1509 1510 ACPI_FREE(obj);1511 1512 kfree(argv4.buffer.pointer);1513}1514 1515static void1516r535_gsp_acpi_mux_id(acpi_handle handle, u32 id, MUX_METHOD_DATA_ELEMENT *mode,1517 MUX_METHOD_DATA_ELEMENT *part)1518{1519 union acpi_object mux_arg = { ACPI_TYPE_INTEGER };1520 struct acpi_object_list input = { 1, &mux_arg };1521 acpi_handle iter = NULL, handle_mux = NULL;1522 acpi_status status;1523 unsigned long long value;1524 1525 mode->status = 0xffff;1526 part->status = 0xffff;1527 1528 do {1529 status = acpi_get_next_object(ACPI_TYPE_DEVICE, handle, iter, &iter);1530 if (ACPI_FAILURE(status) || !iter)1531 return;1532 1533 status = acpi_evaluate_integer(iter, "_ADR", NULL, &value);1534 if (ACPI_FAILURE(status) || value != id)1535 continue;1536 1537 handle_mux = iter;1538 } while (!handle_mux);1539 1540 if (!handle_mux)1541 return;1542 1543 /* I -think- 0 means "acquire" according to nvidia's driver source */1544 input.pointer->integer.type = ACPI_TYPE_INTEGER;1545 input.pointer->integer.value = 0;1546 1547 status = acpi_evaluate_integer(handle_mux, "MXDM", &input, &value);1548 if (ACPI_SUCCESS(status)) {1549 mode->acpiId = id;1550 mode->mode = value;1551 mode->status = 0;1552 }1553 1554 status = acpi_evaluate_integer(handle_mux, "MXDS", &input, &value);1555 if (ACPI_SUCCESS(status)) {1556 part->acpiId = id;1557 part->mode = value;1558 part->status = 0;1559 }1560}1561 1562static void1563r535_gsp_acpi_mux(acpi_handle handle, DOD_METHOD_DATA *dod, MUX_METHOD_DATA *mux)1564{1565 mux->tableLen = dod->acpiIdListLen / sizeof(dod->acpiIdList[0]);1566 1567 for (int i = 0; i < mux->tableLen; i++) {1568 r535_gsp_acpi_mux_id(handle, dod->acpiIdList[i], &mux->acpiIdMuxModeTable[i],1569 &mux->acpiIdMuxPartTable[i]);1570 }1571}1572 1573static void1574r535_gsp_acpi_dod(acpi_handle handle, DOD_METHOD_DATA *dod)1575{1576 acpi_status status;1577 struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };1578 union acpi_object *_DOD;1579 1580 dod->status = 0xffff;1581 1582 status = acpi_evaluate_object(handle, "_DOD", NULL, &output);1583 if (ACPI_FAILURE(status))1584 return;1585 1586 _DOD = output.pointer;1587 1588 if (WARN_ON(_DOD->type != ACPI_TYPE_PACKAGE) ||1589 WARN_ON(_DOD->package.count > ARRAY_SIZE(dod->acpiIdList)))1590 return;1591 1592 for (int i = 0; i < _DOD->package.count; i++) {1593 if (WARN_ON(_DOD->package.elements[i].type != ACPI_TYPE_INTEGER))1594 return;1595 1596 dod->acpiIdList[i] = _DOD->package.elements[i].integer.value;1597 dod->acpiIdListLen += sizeof(dod->acpiIdList[0]);1598 }1599 1600 dod->status = 0;1601 kfree(output.pointer);1602}1603#endif1604 1605static void1606r535_gsp_acpi_info(struct nvkm_gsp *gsp, ACPI_METHOD_DATA *acpi)1607{1608#if defined(CONFIG_ACPI) && defined(CONFIG_X86)1609 acpi_handle handle = ACPI_HANDLE(gsp->subdev.device->dev);1610 1611 if (!handle)1612 return;1613 1614 acpi->bValid = 1;1615 1616 r535_gsp_acpi_dod(handle, &acpi->dodMethodData);1617 if (acpi->dodMethodData.status == 0)1618 r535_gsp_acpi_mux(handle, &acpi->dodMethodData, &acpi->muxMethodData);1619 1620 r535_gsp_acpi_jt(handle, &acpi->jtMethodData);1621 r535_gsp_acpi_caps(handle, &acpi->capsMethodData);1622#endif1623}1624 1625static int1626r535_gsp_rpc_set_system_info(struct nvkm_gsp *gsp)1627{1628 struct nvkm_device *device = gsp->subdev.device;1629 struct nvkm_device_pci *pdev = container_of(device, typeof(*pdev), device);1630 GspSystemInfo *info;1631 1632 if (WARN_ON(device->type == NVKM_DEVICE_TEGRA))1633 return -ENOSYS;1634 1635 info = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_GSP_SET_SYSTEM_INFO, sizeof(*info));1636 if (IS_ERR(info))1637 return PTR_ERR(info);1638 1639 info->gpuPhysAddr = device->func->resource_addr(device, 0);1640 info->gpuPhysFbAddr = device->func->resource_addr(device, 1);1641 info->gpuPhysInstAddr = device->func->resource_addr(device, 3);1642 info->nvDomainBusDeviceFunc = pci_dev_id(pdev->pdev);1643 info->maxUserVa = TASK_SIZE;1644 info->pciConfigMirrorBase = 0x088000;1645 info->pciConfigMirrorSize = 0x001000;1646 r535_gsp_acpi_info(gsp, &info->acpiMethodData);1647 1648 return nvkm_gsp_rpc_wr(gsp, info, false);1649}1650 1651static int1652r535_gsp_msg_os_error_log(void *priv, u32 fn, void *repv, u32 repc)1653{1654 struct nvkm_gsp *gsp = priv;1655 struct nvkm_subdev *subdev = &gsp->subdev;1656 rpc_os_error_log_v17_00 *msg = repv;1657 1658 if (WARN_ON(repc < sizeof(*msg)))1659 return -EINVAL;1660 1661 nvkm_error(subdev, "Xid:%d %s\n", msg->exceptType, msg->errString);1662 return 0;1663}1664 1665static int1666r535_gsp_msg_rc_triggered(void *priv, u32 fn, void *repv, u32 repc)1667{1668 rpc_rc_triggered_v17_02 *msg = repv;1669 struct nvkm_gsp *gsp = priv;1670 struct nvkm_subdev *subdev = &gsp->subdev;1671 struct nvkm_chan *chan;1672 unsigned long flags;1673 1674 if (WARN_ON(repc < sizeof(*msg)))1675 return -EINVAL;1676 1677 nvkm_error(subdev, "rc engn:%08x chid:%d type:%d scope:%d part:%d\n",1678 msg->nv2080EngineType, msg->chid, msg->exceptType, msg->scope,1679 msg->partitionAttributionId);1680 1681 chan = nvkm_chan_get_chid(&subdev->device->fifo->engine, msg->chid / 8, &flags);1682 if (!chan) {1683 nvkm_error(subdev, "rc chid:%d not found!\n", msg->chid);1684 return 0;1685 }1686 1687 nvkm_chan_error(chan, false);1688 nvkm_chan_put(&chan, flags);1689 return 0;1690}1691 1692static int1693r535_gsp_msg_mmu_fault_queued(void *priv, u32 fn, void *repv, u32 repc)1694{1695 struct nvkm_gsp *gsp = priv;1696 struct nvkm_subdev *subdev = &gsp->subdev;1697 1698 WARN_ON(repc != 0);1699 1700 nvkm_error(subdev, "mmu fault queued\n");1701 return 0;1702}1703 1704static int1705r535_gsp_msg_post_event(void *priv, u32 fn, void *repv, u32 repc)1706{1707 struct nvkm_gsp *gsp = priv;1708 struct nvkm_gsp_client *client;1709 struct nvkm_subdev *subdev = &gsp->subdev;1710 rpc_post_event_v17_00 *msg = repv;1711 1712 if (WARN_ON(repc < sizeof(*msg)))1713 return -EINVAL;1714 if (WARN_ON(repc != sizeof(*msg) + msg->eventDataSize))1715 return -EINVAL;1716 1717 nvkm_debug(subdev, "event: %08x %08x %d %08x %08x %d %d\n",1718 msg->hClient, msg->hEvent, msg->notifyIndex, msg->data,1719 msg->status, msg->eventDataSize, msg->bNotifyList);1720 1721 mutex_lock(&gsp->client_id.mutex);1722 client = idr_find(&gsp->client_id.idr, msg->hClient & 0xffff);1723 if (client) {1724 struct nvkm_gsp_event *event;1725 bool handled = false;1726 1727 list_for_each_entry(event, &client->events, head) {1728 if (event->object.handle == msg->hEvent) {1729 event->func(event, msg->eventData, msg->eventDataSize);1730 handled = true;1731 }1732 }1733 1734 if (!handled) {1735 nvkm_error(subdev, "event: cid 0x%08x event 0x%08x not found!\n",1736 msg->hClient, msg->hEvent);1737 }1738 } else {1739 nvkm_error(subdev, "event: cid 0x%08x not found!\n", msg->hClient);1740 }1741 mutex_unlock(&gsp->client_id.mutex);1742 return 0;1743}1744 1745/**1746 * r535_gsp_msg_run_cpu_sequencer() -- process I/O commands from the GSP1747 * @priv: gsp pointer1748 * @fn: function number (ignored)1749 * @repv: pointer to libos print RPC1750 * @repc: message size1751 *1752 * The GSP sequencer is a list of I/O commands that the GSP can send to1753 * the driver to perform for various purposes. The most common usage is to1754 * perform a special mid-initialization reset.1755 */1756static int1757r535_gsp_msg_run_cpu_sequencer(void *priv, u32 fn, void *repv, u32 repc)1758{1759 struct nvkm_gsp *gsp = priv;1760 struct nvkm_subdev *subdev = &gsp->subdev;1761 struct nvkm_device *device = subdev->device;1762 rpc_run_cpu_sequencer_v17_00 *seq = repv;1763 int ptr = 0, ret;1764 1765 nvkm_debug(subdev, "seq: %08x %08x\n", seq->bufferSizeDWord, seq->cmdIndex);1766 1767 while (ptr < seq->cmdIndex) {1768 GSP_SEQUENCER_BUFFER_CMD *cmd = (void *)&seq->commandBuffer[ptr];1769 1770 ptr += 1;1771 ptr += GSP_SEQUENCER_PAYLOAD_SIZE_DWORDS(cmd->opCode);1772 1773 switch (cmd->opCode) {1774 case GSP_SEQ_BUF_OPCODE_REG_WRITE: {1775 u32 addr = cmd->payload.regWrite.addr;1776 u32 data = cmd->payload.regWrite.val;1777 1778 nvkm_trace(subdev, "seq wr32 %06x %08x\n", addr, data);1779 nvkm_wr32(device, addr, data);1780 }1781 break;1782 case GSP_SEQ_BUF_OPCODE_REG_MODIFY: {1783 u32 addr = cmd->payload.regModify.addr;1784 u32 mask = cmd->payload.regModify.mask;1785 u32 data = cmd->payload.regModify.val;1786 1787 nvkm_trace(subdev, "seq mask %06x %08x %08x\n", addr, mask, data);1788 nvkm_mask(device, addr, mask, data);1789 }1790 break;1791 case GSP_SEQ_BUF_OPCODE_REG_POLL: {1792 u32 addr = cmd->payload.regPoll.addr;1793 u32 mask = cmd->payload.regPoll.mask;1794 u32 data = cmd->payload.regPoll.val;1795 u32 usec = cmd->payload.regPoll.timeout ?: 4000000;1796 //u32 error = cmd->payload.regPoll.error;1797 1798 nvkm_trace(subdev, "seq poll %06x %08x %08x %d\n", addr, mask, data, usec);1799 nvkm_rd32(device, addr);1800 nvkm_usec(device, usec,1801 if ((nvkm_rd32(device, addr) & mask) == data)1802 break;1803 );1804 }1805 break;1806 case GSP_SEQ_BUF_OPCODE_DELAY_US: {1807 u32 usec = cmd->payload.delayUs.val;1808 1809 nvkm_trace(subdev, "seq usec %d\n", usec);1810 udelay(usec);1811 }1812 break;1813 case GSP_SEQ_BUF_OPCODE_REG_STORE: {1814 u32 addr = cmd->payload.regStore.addr;1815 u32 slot = cmd->payload.regStore.index;1816 1817 seq->regSaveArea[slot] = nvkm_rd32(device, addr);1818 nvkm_trace(subdev, "seq save %08x -> %d: %08x\n", addr, slot,1819 seq->regSaveArea[slot]);1820 }1821 break;1822 case GSP_SEQ_BUF_OPCODE_CORE_RESET:1823 nvkm_trace(subdev, "seq core reset\n");1824 nvkm_falcon_reset(&gsp->falcon);1825 nvkm_falcon_mask(&gsp->falcon, 0x624, 0x00000080, 0x00000080);1826 nvkm_falcon_wr32(&gsp->falcon, 0x10c, 0x00000000);1827 break;1828 case GSP_SEQ_BUF_OPCODE_CORE_START:1829 nvkm_trace(subdev, "seq core start\n");1830 if (nvkm_falcon_rd32(&gsp->falcon, 0x100) & 0x00000040)1831 nvkm_falcon_wr32(&gsp->falcon, 0x130, 0x00000002);1832 else1833 nvkm_falcon_wr32(&gsp->falcon, 0x100, 0x00000002);1834 break;1835 case GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT:1836 nvkm_trace(subdev, "seq core wait halt\n");1837 nvkm_msec(device, 2000,1838 if (nvkm_falcon_rd32(&gsp->falcon, 0x100) & 0x00000010)1839 break;1840 );1841 break;1842 case GSP_SEQ_BUF_OPCODE_CORE_RESUME: {1843 struct nvkm_sec2 *sec2 = device->sec2;1844 u32 mbox0;1845 1846 nvkm_trace(subdev, "seq core resume\n");1847 1848 ret = gsp->func->reset(gsp);1849 if (WARN_ON(ret))1850 return ret;1851 1852 nvkm_falcon_wr32(&gsp->falcon, 0x040, lower_32_bits(gsp->libos.addr));1853 nvkm_falcon_wr32(&gsp->falcon, 0x044, upper_32_bits(gsp->libos.addr));1854 1855 nvkm_falcon_start(&sec2->falcon);1856 1857 if (nvkm_msec(device, 2000,1858 if (nvkm_rd32(device, 0x1180f8) & 0x04000000)1859 break;1860 ) < 0)1861 return -ETIMEDOUT;1862 1863 mbox0 = nvkm_falcon_rd32(&sec2->falcon, 0x040);1864 if (WARN_ON(mbox0)) {1865 nvkm_error(&gsp->subdev, "seq core resume sec2: 0x%x\n", mbox0);1866 return -EIO;1867 }1868 1869 nvkm_falcon_wr32(&gsp->falcon, 0x080, gsp->boot.app_version);1870 1871 if (WARN_ON(!nvkm_falcon_riscv_active(&gsp->falcon)))1872 return -EIO;1873 }1874 break;1875 default:1876 nvkm_error(subdev, "unknown sequencer opcode %08x\n", cmd->opCode);1877 return -EINVAL;1878 }1879 }1880 1881 return 0;1882}1883 1884static int1885r535_gsp_booter_unload(struct nvkm_gsp *gsp, u32 mbox0, u32 mbox1)1886{1887 struct nvkm_subdev *subdev = &gsp->subdev;1888 struct nvkm_device *device = subdev->device;1889 u32 wpr2_hi;1890 int ret;1891 1892 wpr2_hi = nvkm_rd32(device, 0x1fa828);1893 if (!wpr2_hi) {1894 nvkm_debug(subdev, "WPR2 not set - skipping booter unload\n");1895 return 0;1896 }1897 1898 ret = nvkm_falcon_fw_boot(&gsp->booter.unload, &gsp->subdev, true, &mbox0, &mbox1, 0, 0);1899 if (WARN_ON(ret))1900 return ret;1901 1902 wpr2_hi = nvkm_rd32(device, 0x1fa828);1903 if (WARN_ON(wpr2_hi))1904 return -EIO;1905 1906 return 0;1907}1908 1909static int1910r535_gsp_booter_load(struct nvkm_gsp *gsp, u32 mbox0, u32 mbox1)1911{1912 int ret;1913 1914 ret = nvkm_falcon_fw_boot(&gsp->booter.load, &gsp->subdev, true, &mbox0, &mbox1, 0, 0);1915 if (ret)1916 return ret;1917 1918 nvkm_falcon_wr32(&gsp->falcon, 0x080, gsp->boot.app_version);1919 1920 if (WARN_ON(!nvkm_falcon_riscv_active(&gsp->falcon)))1921 return -EIO;1922 1923 return 0;1924}1925 1926static int1927r535_gsp_wpr_meta_init(struct nvkm_gsp *gsp)1928{1929 GspFwWprMeta *meta;1930 int ret;1931 1932 ret = nvkm_gsp_mem_ctor(gsp, 0x1000, &gsp->wpr_meta);1933 if (ret)1934 return ret;1935 1936 meta = gsp->wpr_meta.data;1937 1938 meta->magic = GSP_FW_WPR_META_MAGIC;1939 meta->revision = GSP_FW_WPR_META_REVISION;1940 1941 meta->sysmemAddrOfRadix3Elf = gsp->radix3.lvl0.addr;1942 meta->sizeOfRadix3Elf = gsp->fb.wpr2.elf.size;1943 1944 meta->sysmemAddrOfBootloader = gsp->boot.fw.addr;1945 meta->sizeOfBootloader = gsp->boot.fw.size;1946 meta->bootloaderCodeOffset = gsp->boot.code_offset;1947 meta->bootloaderDataOffset = gsp->boot.data_offset;1948 meta->bootloaderManifestOffset = gsp->boot.manifest_offset;1949 1950 meta->sysmemAddrOfSignature = gsp->sig.addr;1951 meta->sizeOfSignature = gsp->sig.size;1952 1953 meta->gspFwRsvdStart = gsp->fb.heap.addr;1954 meta->nonWprHeapOffset = gsp->fb.heap.addr;1955 meta->nonWprHeapSize = gsp->fb.heap.size;1956 meta->gspFwWprStart = gsp->fb.wpr2.addr;1957 meta->gspFwHeapOffset = gsp->fb.wpr2.heap.addr;1958 meta->gspFwHeapSize = gsp->fb.wpr2.heap.size;1959 meta->gspFwOffset = gsp->fb.wpr2.elf.addr;1960 meta->bootBinOffset = gsp->fb.wpr2.boot.addr;1961 meta->frtsOffset = gsp->fb.wpr2.frts.addr;1962 meta->frtsSize = gsp->fb.wpr2.frts.size;1963 meta->gspFwWprEnd = ALIGN_DOWN(gsp->fb.bios.vga_workspace.addr, 0x20000);1964 meta->fbSize = gsp->fb.size;1965 meta->vgaWorkspaceOffset = gsp->fb.bios.vga_workspace.addr;1966 meta->vgaWorkspaceSize = gsp->fb.bios.vga_workspace.size;1967 meta->bootCount = 0;1968 meta->partitionRpcAddr = 0;1969 meta->partitionRpcRequestOffset = 0;1970 meta->partitionRpcReplyOffset = 0;1971 meta->verified = 0;1972 return 0;1973}1974 1975static int1976r535_gsp_shared_init(struct nvkm_gsp *gsp)1977{1978 struct {1979 msgqTxHeader tx;1980 msgqRxHeader rx;1981 } *cmdq, *msgq;1982 int ret, i;1983 1984 gsp->shm.cmdq.size = 0x40000;1985 gsp->shm.msgq.size = 0x40000;1986 1987 gsp->shm.ptes.nr = (gsp->shm.cmdq.size + gsp->shm.msgq.size) >> GSP_PAGE_SHIFT;1988 gsp->shm.ptes.nr += DIV_ROUND_UP(gsp->shm.ptes.nr * sizeof(u64), GSP_PAGE_SIZE);1989 gsp->shm.ptes.size = ALIGN(gsp->shm.ptes.nr * sizeof(u64), GSP_PAGE_SIZE);1990 1991 ret = nvkm_gsp_mem_ctor(gsp, gsp->shm.ptes.size +1992 gsp->shm.cmdq.size +1993 gsp->shm.msgq.size,1994 &gsp->shm.mem);1995 if (ret)1996 return ret;1997 1998 gsp->shm.ptes.ptr = gsp->shm.mem.data;1999 gsp->shm.cmdq.ptr = (u8 *)gsp->shm.ptes.ptr + gsp->shm.ptes.size;2000 gsp->shm.msgq.ptr = (u8 *)gsp->shm.cmdq.ptr + gsp->shm.cmdq.size;2001 2002 for (i = 0; i < gsp->shm.ptes.nr; i++)2003 gsp->shm.ptes.ptr[i] = gsp->shm.mem.addr + (i << GSP_PAGE_SHIFT);2004 2005 cmdq = gsp->shm.cmdq.ptr;2006 cmdq->tx.version = 0;2007 cmdq->tx.size = gsp->shm.cmdq.size;2008 cmdq->tx.entryOff = GSP_PAGE_SIZE;2009 cmdq->tx.msgSize = GSP_PAGE_SIZE;2010 cmdq->tx.msgCount = (cmdq->tx.size - cmdq->tx.entryOff) / cmdq->tx.msgSize;2011 cmdq->tx.writePtr = 0;2012 cmdq->tx.flags = 1;2013 cmdq->tx.rxHdrOff = offsetof(typeof(*cmdq), rx.readPtr);2014 2015 msgq = gsp->shm.msgq.ptr;2016 2017 gsp->cmdq.cnt = cmdq->tx.msgCount;2018 gsp->cmdq.wptr = &cmdq->tx.writePtr;2019 gsp->cmdq.rptr = &msgq->rx.readPtr;2020 gsp->msgq.cnt = cmdq->tx.msgCount;2021 gsp->msgq.wptr = &msgq->tx.writePtr;2022 gsp->msgq.rptr = &cmdq->rx.readPtr;2023 return 0;2024}2025 2026static int2027r535_gsp_rmargs_init(struct nvkm_gsp *gsp, bool resume)2028{2029 GSP_ARGUMENTS_CACHED *args;2030 int ret;2031 2032 if (!resume) {2033 ret = r535_gsp_shared_init(gsp);2034 if (ret)2035 return ret;2036 2037 ret = nvkm_gsp_mem_ctor(gsp, 0x1000, &gsp->rmargs);2038 if (ret)2039 return ret;2040 }2041 2042 args = gsp->rmargs.data;2043 args->messageQueueInitArguments.sharedMemPhysAddr = gsp->shm.mem.addr;2044 args->messageQueueInitArguments.pageTableEntryCount = gsp->shm.ptes.nr;2045 args->messageQueueInitArguments.cmdQueueOffset =2046 (u8 *)gsp->shm.cmdq.ptr - (u8 *)gsp->shm.mem.data;2047 args->messageQueueInitArguments.statQueueOffset =2048 (u8 *)gsp->shm.msgq.ptr - (u8 *)gsp->shm.mem.data;2049 2050 if (!resume) {2051 args->srInitArguments.oldLevel = 0;2052 args->srInitArguments.flags = 0;2053 args->srInitArguments.bInPMTransition = 0;2054 } else {2055 args->srInitArguments.oldLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_3;2056 args->srInitArguments.flags = 0;2057 args->srInitArguments.bInPMTransition = 1;2058 }2059 2060 return 0;2061}2062 2063static inline u642064r535_gsp_libos_id8(const char *name)2065{2066 u64 id = 0;2067 2068 for (int i = 0; i < sizeof(id) && *name; i++, name++)2069 id = (id << 8) | *name;2070 2071 return id;2072}2073 2074/**2075 * create_pte_array() - creates a PTE array of a physically contiguous buffer2076 * @ptes: pointer to the array2077 * @addr: base address of physically contiguous buffer (GSP_PAGE_SIZE aligned)2078 * @size: size of the buffer2079 *2080 * GSP-RM sometimes expects physically-contiguous buffers to have an array of2081 * "PTEs" for each page in that buffer. Although in theory that allows for2082 * the buffer to be physically discontiguous, GSP-RM does not currently2083 * support that.2084 *2085 * In this case, the PTEs are DMA addresses of each page of the buffer. Since2086 * the buffer is physically contiguous, calculating all the PTEs is simple2087 * math.2088 *2089 * See memdescGetPhysAddrsForGpu()2090 */2091static void create_pte_array(u64 *ptes, dma_addr_t addr, size_t size)2092{2093 unsigned int num_pages = DIV_ROUND_UP_ULL(size, GSP_PAGE_SIZE);2094 unsigned int i;2095 2096 for (i = 0; i < num_pages; i++)2097 ptes[i] = (u64)addr + (i << GSP_PAGE_SHIFT);2098}2099 2100/**2101 * r535_gsp_libos_init() -- create the libos arguments structure2102 * @gsp: gsp pointer2103 *2104 * The logging buffers are byte queues that contain encoded printf-like2105 * messages from GSP-RM. They need to be decoded by a special application2106 * that can parse the buffers.2107 *2108 * The 'loginit' buffer contains logs from early GSP-RM init and2109 * exception dumps. The 'logrm' buffer contains the subsequent logs. Both are2110 * written to directly by GSP-RM and can be any multiple of GSP_PAGE_SIZE.2111 *2112 * The physical address map for the log buffer is stored in the buffer2113 * itself, starting with offset 1. Offset 0 contains the "put" pointer.2114 *2115 * The GSP only understands 4K pages (GSP_PAGE_SIZE), so even if the kernel is2116 * configured for a larger page size (e.g. 64K pages), we need to give2117 * the GSP an array of 4K pages. Fortunately, since the buffer is2118 * physically contiguous, it's simple math to calculate the addresses.2119 *2120 * The buffers must be a multiple of GSP_PAGE_SIZE. GSP-RM also currently2121 * ignores the @kind field for LOGINIT, LOGINTR, and LOGRM, but expects the2122 * buffers to be physically contiguous anyway.2123 *2124 * The memory allocated for the arguments must remain until the GSP sends the2125 * init_done RPC.2126 *2127 * See _kgspInitLibosLoggingStructures (allocates memory for buffers)2128 * See kgspSetupLibosInitArgs_IMPL (creates pLibosInitArgs[] array)2129 */2130static int2131r535_gsp_libos_init(struct nvkm_gsp *gsp)2132{2133 LibosMemoryRegionInitArgument *args;2134 int ret;2135 2136 ret = nvkm_gsp_mem_ctor(gsp, 0x1000, &gsp->libos);2137 if (ret)2138 return ret;2139 2140 args = gsp->libos.data;2141 2142 ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->loginit);2143 if (ret)2144 return ret;2145 2146 args[0].id8 = r535_gsp_libos_id8("LOGINIT");2147 args[0].pa = gsp->loginit.addr;2148 args[0].size = gsp->loginit.size;2149 args[0].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;2150 args[0].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM;2151 create_pte_array(gsp->loginit.data + sizeof(u64), gsp->loginit.addr, gsp->loginit.size);2152 2153 ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->logintr);2154 if (ret)2155 return ret;2156 2157 args[1].id8 = r535_gsp_libos_id8("LOGINTR");2158 args[1].pa = gsp->logintr.addr;2159 args[1].size = gsp->logintr.size;2160 args[1].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;2161 args[1].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM;2162 create_pte_array(gsp->logintr.data + sizeof(u64), gsp->logintr.addr, gsp->logintr.size);2163 2164 ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->logrm);2165 if (ret)2166 return ret;2167 2168 args[2].id8 = r535_gsp_libos_id8("LOGRM");2169 args[2].pa = gsp->logrm.addr;2170 args[2].size = gsp->logrm.size;2171 args[2].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;2172 args[2].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM;2173 create_pte_array(gsp->logrm.data + sizeof(u64), gsp->logrm.addr, gsp->logrm.size);2174 2175 ret = r535_gsp_rmargs_init(gsp, false);2176 if (ret)2177 return ret;2178 2179 args[3].id8 = r535_gsp_libos_id8("RMARGS");2180 args[3].pa = gsp->rmargs.addr;2181 args[3].size = gsp->rmargs.size;2182 args[3].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;2183 args[3].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM;2184 return 0;2185}2186 2187void2188nvkm_gsp_sg_free(struct nvkm_device *device, struct sg_table *sgt)2189{2190 struct scatterlist *sgl;2191 int i;2192 2193 dma_unmap_sgtable(device->dev, sgt, DMA_BIDIRECTIONAL, 0);2194 2195 for_each_sgtable_sg(sgt, sgl, i) {2196 struct page *page = sg_page(sgl);2197 2198 __free_page(page);2199 }2200 2201 sg_free_table(sgt);2202}2203 2204int2205nvkm_gsp_sg(struct nvkm_device *device, u64 size, struct sg_table *sgt)2206{2207 const u64 pages = DIV_ROUND_UP(size, PAGE_SIZE);2208 struct scatterlist *sgl;2209 int ret, i;2210 2211 ret = sg_alloc_table(sgt, pages, GFP_KERNEL);2212 if (ret)2213 return ret;2214 2215 for_each_sgtable_sg(sgt, sgl, i) {2216 struct page *page = alloc_page(GFP_KERNEL);2217 2218 if (!page) {2219 nvkm_gsp_sg_free(device, sgt);2220 return -ENOMEM;2221 }2222 2223 sg_set_page(sgl, page, PAGE_SIZE, 0);2224 }2225 2226 ret = dma_map_sgtable(device->dev, sgt, DMA_BIDIRECTIONAL, 0);2227 if (ret)2228 nvkm_gsp_sg_free(device, sgt);2229 2230 return ret;2231}2232 2233static void2234nvkm_gsp_radix3_dtor(struct nvkm_gsp *gsp, struct nvkm_gsp_radix3 *rx3)2235{2236 nvkm_gsp_sg_free(gsp->subdev.device, &rx3->lvl2);2237 nvkm_gsp_mem_dtor(gsp, &rx3->lvl1);2238 nvkm_gsp_mem_dtor(gsp, &rx3->lvl0);2239}2240 2241/**2242 * nvkm_gsp_radix3_sg - build a radix3 table from a S/G list2243 * @gsp: gsp pointer2244 * @sgt: S/G list to traverse2245 * @size: size of the image, in bytes2246 * @rx3: radix3 array to update2247 *2248 * The GSP uses a three-level page table, called radix3, to map the firmware.2249 * Each 64-bit "pointer" in the table is either the bus address of an entry in2250 * the next table (for levels 0 and 1) or the bus address of the next page in2251 * the GSP firmware image itself.2252 *2253 * Level 0 contains a single entry in one page that points to the first page2254 * of level 1.2255 *2256 * Level 1, since it's also only one page in size, contains up to 512 entries,2257 * one for each page in Level 2.2258 *2259 * Level 2 can be up to 512 pages in size, and each of those entries points to2260 * the next page of the firmware image. Since there can be up to 512*5122261 * pages, that limits the size of the firmware to 512*512*GSP_PAGE_SIZE = 1GB.2262 *2263 * Internally, the GSP has its window into system memory, but the base2264 * physical address of the aperture is not 0. In fact, it varies depending on2265 * the GPU architecture. Since the GPU is a PCI device, this window is2266 * accessed via DMA and is therefore bound by IOMMU translation. The end2267 * result is that GSP-RM must translate the bus addresses in the table to GSP2268 * physical addresses. All this should happen transparently.2269 *2270 * Returns 0 on success, or negative error code2271 *2272 * See kgspCreateRadix3_IMPL2273 */2274static int2275nvkm_gsp_radix3_sg(struct nvkm_gsp *gsp, struct sg_table *sgt, u64 size,2276 struct nvkm_gsp_radix3 *rx3)2277{2278 struct sg_dma_page_iter sg_dma_iter;2279 struct scatterlist *sg;2280 size_t bufsize;2281 u64 *pte;2282 int ret, i, page_idx = 0;2283 2284 ret = nvkm_gsp_mem_ctor(gsp, GSP_PAGE_SIZE, &rx3->lvl0);2285 if (ret)2286 return ret;2287 2288 ret = nvkm_gsp_mem_ctor(gsp, GSP_PAGE_SIZE, &rx3->lvl1);2289 if (ret)2290 goto lvl1_fail;2291 2292 // Allocate level 22293 bufsize = ALIGN((size / GSP_PAGE_SIZE) * sizeof(u64), GSP_PAGE_SIZE);2294 ret = nvkm_gsp_sg(gsp->subdev.device, bufsize, &rx3->lvl2);2295 if (ret)2296 goto lvl2_fail;2297 2298 // Write the bus address of level 1 to level 02299 pte = rx3->lvl0.data;2300 *pte = rx3->lvl1.addr;2301 2302 // Write the bus address of each page in level 2 to level 12303 pte = rx3->lvl1.data;2304 for_each_sgtable_dma_page(&rx3->lvl2, &sg_dma_iter, 0)2305 *pte++ = sg_page_iter_dma_address(&sg_dma_iter);2306 2307 // Finally, write the bus address of each page in sgt to level 22308 for_each_sgtable_sg(&rx3->lvl2, sg, i) {2309 void *sgl_end;2310 2311 pte = sg_virt(sg);2312 sgl_end = (void *)pte + sg->length;2313 2314 for_each_sgtable_dma_page(sgt, &sg_dma_iter, page_idx) {2315 *pte++ = sg_page_iter_dma_address(&sg_dma_iter);2316 page_idx++;2317 2318 // Go to the next scatterlist for level 2 if we've reached the end2319 if ((void *)pte >= sgl_end)2320 break;2321 }2322 }2323 2324 if (ret) {2325lvl2_fail:2326 nvkm_gsp_mem_dtor(gsp, &rx3->lvl1);2327lvl1_fail:2328 nvkm_gsp_mem_dtor(gsp, &rx3->lvl0);2329 }2330 2331 return ret;2332}2333 2334int2335r535_gsp_fini(struct nvkm_gsp *gsp, bool suspend)2336{2337 u32 mbox0 = 0xff, mbox1 = 0xff;2338 int ret;2339 2340 if (!gsp->running)2341 return 0;2342 2343 if (suspend) {2344 GspFwWprMeta *meta = gsp->wpr_meta.data;2345 u64 len = meta->gspFwWprEnd - meta->gspFwWprStart;2346 GspFwSRMeta *sr;2347 2348 ret = nvkm_gsp_sg(gsp->subdev.device, len, &gsp->sr.sgt);2349 if (ret)2350 return ret;2351 2352 ret = nvkm_gsp_radix3_sg(gsp, &gsp->sr.sgt, len, &gsp->sr.radix3);2353 if (ret)2354 return ret;2355 2356 ret = nvkm_gsp_mem_ctor(gsp, sizeof(*sr), &gsp->sr.meta);2357 if (ret)2358 return ret;2359 2360 sr = gsp->sr.meta.data;2361 sr->magic = GSP_FW_SR_META_MAGIC;2362 sr->revision = GSP_FW_SR_META_REVISION;2363 sr->sysmemAddrOfSuspendResumeData = gsp->sr.radix3.lvl0.addr;2364 sr->sizeOfSuspendResumeData = len;2365 2366 mbox0 = lower_32_bits(gsp->sr.meta.addr);2367 mbox1 = upper_32_bits(gsp->sr.meta.addr);2368 }2369 2370 ret = r535_gsp_rpc_unloading_guest_driver(gsp, suspend);2371 if (WARN_ON(ret))2372 return ret;2373 2374 nvkm_msec(gsp->subdev.device, 2000,2375 if (nvkm_falcon_rd32(&gsp->falcon, 0x040) & 0x80000000)2376 break;2377 );2378 2379 nvkm_falcon_reset(&gsp->falcon);2380 2381 ret = nvkm_gsp_fwsec_sb(gsp);2382 WARN_ON(ret);2383 2384 ret = r535_gsp_booter_unload(gsp, mbox0, mbox1);2385 WARN_ON(ret);2386 2387 gsp->running = false;2388 return 0;2389}2390 2391int2392r535_gsp_init(struct nvkm_gsp *gsp)2393{2394 u32 mbox0, mbox1;2395 int ret;2396 2397 if (!gsp->sr.meta.data) {2398 mbox0 = lower_32_bits(gsp->wpr_meta.addr);2399 mbox1 = upper_32_bits(gsp->wpr_meta.addr);2400 } else {2401 r535_gsp_rmargs_init(gsp, true);2402 2403 mbox0 = lower_32_bits(gsp->sr.meta.addr);2404 mbox1 = upper_32_bits(gsp->sr.meta.addr);2405 }2406 2407 /* Execute booter to handle (eventually...) booting GSP-RM. */2408 ret = r535_gsp_booter_load(gsp, mbox0, mbox1);2409 if (WARN_ON(ret))2410 goto done;2411 2412 ret = r535_gsp_rpc_poll(gsp, NV_VGPU_MSG_EVENT_GSP_INIT_DONE);2413 if (ret)2414 goto done;2415 2416 gsp->running = true;2417 2418done:2419 if (gsp->sr.meta.data) {2420 nvkm_gsp_mem_dtor(gsp, &gsp->sr.meta);2421 nvkm_gsp_radix3_dtor(gsp, &gsp->sr.radix3);2422 nvkm_gsp_sg_free(gsp->subdev.device, &gsp->sr.sgt);2423 return ret;2424 }2425 2426 if (ret == 0)2427 ret = r535_gsp_postinit(gsp);2428 2429 return ret;2430}2431 2432static int2433r535_gsp_rm_boot_ctor(struct nvkm_gsp *gsp)2434{2435 const struct firmware *fw = gsp->fws.bl;2436 const struct nvfw_bin_hdr *hdr;2437 RM_RISCV_UCODE_DESC *desc;2438 int ret;2439 2440 hdr = nvfw_bin_hdr(&gsp->subdev, fw->data);2441 desc = (void *)fw->data + hdr->header_offset;2442 2443 ret = nvkm_gsp_mem_ctor(gsp, hdr->data_size, &gsp->boot.fw);2444 if (ret)2445 return ret;2446 2447 memcpy(gsp->boot.fw.data, fw->data + hdr->data_offset, hdr->data_size);2448 2449 gsp->boot.code_offset = desc->monitorCodeOffset;2450 gsp->boot.data_offset = desc->monitorDataOffset;2451 gsp->boot.manifest_offset = desc->manifestOffset;2452 gsp->boot.app_version = desc->appVersion;2453 return 0;2454}2455 2456static const struct nvkm_firmware_func2457r535_gsp_fw = {2458 .type = NVKM_FIRMWARE_IMG_SGT,2459};2460 2461static int2462r535_gsp_elf_section(struct nvkm_gsp *gsp, const char *name, const u8 **pdata, u64 *psize)2463{2464 const u8 *img = gsp->fws.rm->data;2465 const struct elf64_hdr *ehdr = (const struct elf64_hdr *)img;2466 const struct elf64_shdr *shdr = (const struct elf64_shdr *)&img[ehdr->e_shoff];2467 const char *names = &img[shdr[ehdr->e_shstrndx].sh_offset];2468 2469 for (int i = 0; i < ehdr->e_shnum; i++, shdr++) {2470 if (!strcmp(&names[shdr->sh_name], name)) {2471 *pdata = &img[shdr->sh_offset];2472 *psize = shdr->sh_size;2473 return 0;2474 }2475 }2476 2477 nvkm_error(&gsp->subdev, "section '%s' not found\n", name);2478 return -ENOENT;2479}2480 2481static void2482r535_gsp_dtor_fws(struct nvkm_gsp *gsp)2483{2484 nvkm_firmware_put(gsp->fws.bl);2485 gsp->fws.bl = NULL;2486 nvkm_firmware_put(gsp->fws.booter.unload);2487 gsp->fws.booter.unload = NULL;2488 nvkm_firmware_put(gsp->fws.booter.load);2489 gsp->fws.booter.load = NULL;2490 nvkm_firmware_put(gsp->fws.rm);2491 gsp->fws.rm = NULL;2492}2493 2494void2495r535_gsp_dtor(struct nvkm_gsp *gsp)2496{2497 idr_destroy(&gsp->client_id.idr);2498 mutex_destroy(&gsp->client_id.mutex);2499 2500 nvkm_gsp_radix3_dtor(gsp, &gsp->radix3);2501 nvkm_gsp_mem_dtor(gsp, &gsp->sig);2502 nvkm_firmware_dtor(&gsp->fw);2503 2504 nvkm_falcon_fw_dtor(&gsp->booter.unload);2505 nvkm_falcon_fw_dtor(&gsp->booter.load);2506 2507 mutex_destroy(&gsp->msgq.mutex);2508 mutex_destroy(&gsp->cmdq.mutex);2509 2510 r535_gsp_dtor_fws(gsp);2511 2512 nvkm_gsp_mem_dtor(gsp, &gsp->rmargs);2513 nvkm_gsp_mem_dtor(gsp, &gsp->wpr_meta);2514 nvkm_gsp_mem_dtor(gsp, &gsp->shm.mem);2515 nvkm_gsp_mem_dtor(gsp, &gsp->loginit);2516 nvkm_gsp_mem_dtor(gsp, &gsp->logintr);2517 nvkm_gsp_mem_dtor(gsp, &gsp->logrm);2518}2519 2520int2521r535_gsp_oneinit(struct nvkm_gsp *gsp)2522{2523 struct nvkm_device *device = gsp->subdev.device;2524 const u8 *data;2525 u64 size;2526 int ret;2527 2528 mutex_init(&gsp->cmdq.mutex);2529 mutex_init(&gsp->msgq.mutex);2530 2531 ret = gsp->func->booter.ctor(gsp, "booter-load", gsp->fws.booter.load,2532 &device->sec2->falcon, &gsp->booter.load);2533 if (ret)2534 return ret;2535 2536 ret = gsp->func->booter.ctor(gsp, "booter-unload", gsp->fws.booter.unload,2537 &device->sec2->falcon, &gsp->booter.unload);2538 if (ret)2539 return ret;2540 2541 /* Load GSP firmware from ELF image into DMA-accessible memory. */2542 ret = r535_gsp_elf_section(gsp, ".fwimage", &data, &size);2543 if (ret)2544 return ret;2545 2546 ret = nvkm_firmware_ctor(&r535_gsp_fw, "gsp-rm", device, data, size, &gsp->fw);2547 if (ret)2548 return ret;2549 2550 /* Load relevant signature from ELF image. */2551 ret = r535_gsp_elf_section(gsp, gsp->func->sig_section, &data, &size);2552 if (ret)2553 return ret;2554 2555 ret = nvkm_gsp_mem_ctor(gsp, ALIGN(size, 256), &gsp->sig);2556 if (ret)2557 return ret;2558 2559 memcpy(gsp->sig.data, data, size);2560 2561 /* Build radix3 page table for ELF image. */2562 ret = nvkm_gsp_radix3_sg(gsp, &gsp->fw.mem.sgt, gsp->fw.len, &gsp->radix3);2563 if (ret)2564 return ret;2565 2566 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER,2567 r535_gsp_msg_run_cpu_sequencer, gsp);2568 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_POST_EVENT, r535_gsp_msg_post_event, gsp);2569 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_RC_TRIGGERED,2570 r535_gsp_msg_rc_triggered, gsp);2571 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_MMU_FAULT_QUEUED,2572 r535_gsp_msg_mmu_fault_queued, gsp);2573 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_OS_ERROR_LOG, r535_gsp_msg_os_error_log, gsp);2574 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_PERF_BRIDGELESS_INFO_UPDATE, NULL, NULL);2575 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT, NULL, NULL);2576 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_GSP_SEND_USER_SHARED_DATA, NULL, NULL);2577 ret = r535_gsp_rm_boot_ctor(gsp);2578 if (ret)2579 return ret;2580 2581 /* Release FW images - we've copied them to DMA buffers now. */2582 r535_gsp_dtor_fws(gsp);2583 2584 /* Calculate FB layout. */2585 gsp->fb.wpr2.frts.size = 0x100000;2586 gsp->fb.wpr2.frts.addr = ALIGN_DOWN(gsp->fb.bios.addr, 0x20000) - gsp->fb.wpr2.frts.size;2587 2588 gsp->fb.wpr2.boot.size = gsp->boot.fw.size;2589 gsp->fb.wpr2.boot.addr = ALIGN_DOWN(gsp->fb.wpr2.frts.addr - gsp->fb.wpr2.boot.size, 0x1000);2590 2591 gsp->fb.wpr2.elf.size = gsp->fw.len;2592 gsp->fb.wpr2.elf.addr = ALIGN_DOWN(gsp->fb.wpr2.boot.addr - gsp->fb.wpr2.elf.size, 0x10000);2593 2594 {2595 u32 fb_size_gb = DIV_ROUND_UP_ULL(gsp->fb.size, 1 << 30);2596 2597 gsp->fb.wpr2.heap.size =2598 gsp->func->wpr_heap.os_carveout_size +2599 gsp->func->wpr_heap.base_size +2600 ALIGN(GSP_FW_HEAP_PARAM_SIZE_PER_GB_FB * fb_size_gb, 1 << 20) +2601 ALIGN(GSP_FW_HEAP_PARAM_CLIENT_ALLOC_SIZE, 1 << 20);2602 2603 gsp->fb.wpr2.heap.size = max(gsp->fb.wpr2.heap.size, gsp->func->wpr_heap.min_size);2604 }2605 2606 gsp->fb.wpr2.heap.addr = ALIGN_DOWN(gsp->fb.wpr2.elf.addr - gsp->fb.wpr2.heap.size, 0x100000);2607 gsp->fb.wpr2.heap.size = ALIGN_DOWN(gsp->fb.wpr2.elf.addr - gsp->fb.wpr2.heap.addr, 0x100000);2608 2609 gsp->fb.wpr2.addr = ALIGN_DOWN(gsp->fb.wpr2.heap.addr - sizeof(GspFwWprMeta), 0x100000);2610 gsp->fb.wpr2.size = gsp->fb.wpr2.frts.addr + gsp->fb.wpr2.frts.size - gsp->fb.wpr2.addr;2611 2612 gsp->fb.heap.size = 0x100000;2613 gsp->fb.heap.addr = gsp->fb.wpr2.addr - gsp->fb.heap.size;2614 2615 ret = nvkm_gsp_fwsec_frts(gsp);2616 if (WARN_ON(ret))2617 return ret;2618 2619 ret = r535_gsp_libos_init(gsp);2620 if (WARN_ON(ret))2621 return ret;2622 2623 ret = r535_gsp_wpr_meta_init(gsp);2624 if (WARN_ON(ret))2625 return ret;2626 2627 ret = r535_gsp_rpc_set_system_info(gsp);2628 if (WARN_ON(ret))2629 return ret;2630 2631 ret = r535_gsp_rpc_set_registry(gsp);2632 if (WARN_ON(ret))2633 return ret;2634 2635 /* Reset GSP into RISC-V mode. */2636 ret = gsp->func->reset(gsp);2637 if (WARN_ON(ret))2638 return ret;2639 2640 nvkm_falcon_wr32(&gsp->falcon, 0x040, lower_32_bits(gsp->libos.addr));2641 nvkm_falcon_wr32(&gsp->falcon, 0x044, upper_32_bits(gsp->libos.addr));2642 2643 mutex_init(&gsp->client_id.mutex);2644 idr_init(&gsp->client_id.idr);2645 return 0;2646}2647 2648static int2649r535_gsp_load_fw(struct nvkm_gsp *gsp, const char *name, const char *ver,2650 const struct firmware **pfw)2651{2652 char fwname[64];2653 2654 snprintf(fwname, sizeof(fwname), "gsp/%s-%s", name, ver);2655 return nvkm_firmware_get(&gsp->subdev, fwname, 0, pfw);2656}2657 2658int2659r535_gsp_load(struct nvkm_gsp *gsp, int ver, const struct nvkm_gsp_fwif *fwif)2660{2661 struct nvkm_subdev *subdev = &gsp->subdev;2662 int ret;2663 bool enable_gsp = fwif->enable;2664 2665#if IS_ENABLED(CONFIG_DRM_NOUVEAU_GSP_DEFAULT)2666 enable_gsp = true;2667#endif2668 if (!nvkm_boolopt(subdev->device->cfgopt, "NvGspRm", enable_gsp))2669 return -EINVAL;2670 2671 if ((ret = r535_gsp_load_fw(gsp, "gsp", fwif->ver, &gsp->fws.rm)) ||2672 (ret = r535_gsp_load_fw(gsp, "booter_load", fwif->ver, &gsp->fws.booter.load)) ||2673 (ret = r535_gsp_load_fw(gsp, "booter_unload", fwif->ver, &gsp->fws.booter.unload)) ||2674 (ret = r535_gsp_load_fw(gsp, "bootloader", fwif->ver, &gsp->fws.bl))) {2675 r535_gsp_dtor_fws(gsp);2676 return ret;2677 }2678 2679 return 0;2680}2681 2682#define NVKM_GSP_FIRMWARE(chip) \2683MODULE_FIRMWARE("nvidia/"#chip"/gsp/booter_load-535.113.01.bin"); \2684MODULE_FIRMWARE("nvidia/"#chip"/gsp/booter_unload-535.113.01.bin"); \2685MODULE_FIRMWARE("nvidia/"#chip"/gsp/bootloader-535.113.01.bin"); \2686MODULE_FIRMWARE("nvidia/"#chip"/gsp/gsp-535.113.01.bin")2687 2688NVKM_GSP_FIRMWARE(tu102);2689NVKM_GSP_FIRMWARE(tu104);2690NVKM_GSP_FIRMWARE(tu106);2691 2692NVKM_GSP_FIRMWARE(tu116);2693NVKM_GSP_FIRMWARE(tu117);2694 2695NVKM_GSP_FIRMWARE(ga100);2696 2697NVKM_GSP_FIRMWARE(ga102);2698NVKM_GSP_FIRMWARE(ga103);2699NVKM_GSP_FIRMWARE(ga104);2700NVKM_GSP_FIRMWARE(ga106);2701NVKM_GSP_FIRMWARE(ga107);2702 2703NVKM_GSP_FIRMWARE(ad102);2704NVKM_GSP_FIRMWARE(ad103);2705NVKM_GSP_FIRMWARE(ad104);2706NVKM_GSP_FIRMWARE(ad106);2707NVKM_GSP_FIRMWARE(ad107);2708