509 lines · c
1/*2 * Copyright 2023 Red Hat Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 */22#include "gf100.h"23 24#include <core/memory.h>25#include <subdev/gsp.h>26#include <subdev/mmu/vmm.h>27#include <engine/fifo/priv.h>28 29#include <nvif/if900d.h>30 31#include <nvhw/drf.h>32 33#include <nvrm/nvtypes.h>34#include <nvrm/535.113.01/common/sdk/nvidia/inc/alloc/alloc_channel.h>35#include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl0080/ctrl0080fifo.h>36#include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080gpu.h>37#include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080internal.h>38#include <nvrm/535.113.01/nvidia/generated/g_kernel_channel_nvoc.h>39 40#define r535_gr(p) container_of((p), struct r535_gr, base)41 42#define R515_GR_MAX_CTXBUFS 943 44struct r535_gr {45 struct nvkm_gr base;46 47 struct {48 u16 bufferId;49 u32 size;50 u8 page;51 u8 align;52 bool global;53 bool init;54 bool ro;55 } ctxbuf[R515_GR_MAX_CTXBUFS];56 int ctxbuf_nr;57 58 struct nvkm_memory *ctxbuf_mem[R515_GR_MAX_CTXBUFS];59};60 61struct r535_gr_chan {62 struct nvkm_object object;63 struct r535_gr *gr;64 65 struct nvkm_vmm *vmm;66 struct nvkm_chan *chan;67 68 struct nvkm_memory *mem[R515_GR_MAX_CTXBUFS];69 struct nvkm_vma *vma[R515_GR_MAX_CTXBUFS];70};71 72struct r535_gr_obj {73 struct nvkm_object object;74 struct nvkm_gsp_object rm;75};76 77static void *78r535_gr_obj_dtor(struct nvkm_object *object)79{80 struct r535_gr_obj *obj = container_of(object, typeof(*obj), object);81 82 nvkm_gsp_rm_free(&obj->rm);83 return obj;84}85 86static const struct nvkm_object_func87r535_gr_obj = {88 .dtor = r535_gr_obj_dtor,89};90 91static int92r535_gr_obj_ctor(const struct nvkm_oclass *oclass, void *argv, u32 argc,93 struct nvkm_object **pobject)94{95 struct r535_gr_chan *chan = container_of(oclass->parent, typeof(*chan), object);96 struct r535_gr_obj *obj;97 98 if (!(obj = kzalloc(sizeof(*obj), GFP_KERNEL)))99 return -ENOMEM;100 101 nvkm_object_ctor(&r535_gr_obj, oclass, &obj->object);102 *pobject = &obj->object;103 104 return nvkm_gsp_rm_alloc(&chan->chan->rm.object, oclass->handle, oclass->base.oclass, 0,105 &obj->rm);106}107 108static void *109r535_gr_chan_dtor(struct nvkm_object *object)110{111 struct r535_gr_chan *grc = container_of(object, typeof(*grc), object);112 struct r535_gr *gr = grc->gr;113 114 for (int i = 0; i < gr->ctxbuf_nr; i++) {115 nvkm_vmm_put(grc->vmm, &grc->vma[i]);116 nvkm_memory_unref(&grc->mem[i]);117 }118 119 nvkm_vmm_unref(&grc->vmm);120 return grc;121}122 123static const struct nvkm_object_func124r535_gr_chan = {125 .dtor = r535_gr_chan_dtor,126};127 128static int129r535_gr_promote_ctx(struct r535_gr *gr, bool golden, struct nvkm_vmm *vmm,130 struct nvkm_memory **pmem, struct nvkm_vma **pvma,131 struct nvkm_gsp_object *chan)132{133 struct nvkm_subdev *subdev = &gr->base.engine.subdev;134 struct nvkm_device *device = subdev->device;135 NV2080_CTRL_GPU_PROMOTE_CTX_PARAMS *ctrl;136 137 ctrl = nvkm_gsp_rm_ctrl_get(&vmm->rm.device.subdevice,138 NV2080_CTRL_CMD_GPU_PROMOTE_CTX, sizeof(*ctrl));139 if (WARN_ON(IS_ERR(ctrl)))140 return PTR_ERR(ctrl);141 142 ctrl->engineType = 1;143 ctrl->hChanClient = vmm->rm.client.object.handle;144 ctrl->hObject = chan->handle;145 146 for (int i = 0; i < gr->ctxbuf_nr; i++) {147 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ENTRY *entry =148 &ctrl->promoteEntry[ctrl->entryCount];149 const bool alloc = golden || !gr->ctxbuf[i].global;150 int ret;151 152 entry->bufferId = gr->ctxbuf[i].bufferId;153 entry->bInitialize = gr->ctxbuf[i].init && alloc;154 155 if (alloc) {156 ret = nvkm_memory_new(device, gr->ctxbuf[i].init ?157 NVKM_MEM_TARGET_INST : NVKM_MEM_TARGET_INST_SR_LOST,158 gr->ctxbuf[i].size, 1 << gr->ctxbuf[i].page,159 gr->ctxbuf[i].init, &pmem[i]);160 if (WARN_ON(ret))161 return ret;162 163 if (gr->ctxbuf[i].bufferId ==164 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP)165 entry->bNonmapped = 1;166 } else {167 if (gr->ctxbuf[i].bufferId ==168 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP)169 continue;170 171 pmem[i] = nvkm_memory_ref(gr->ctxbuf_mem[i]);172 }173 174 if (!entry->bNonmapped) {175 struct gf100_vmm_map_v0 args = {176 .priv = 1,177 .ro = gr->ctxbuf[i].ro,178 };179 180 mutex_lock(&vmm->mutex.vmm);181 ret = nvkm_vmm_get_locked(vmm, false, true, false, 0, gr->ctxbuf[i].align,182 nvkm_memory_size(pmem[i]), &pvma[i]);183 mutex_unlock(&vmm->mutex.vmm);184 if (ret)185 return ret;186 187 ret = nvkm_memory_map(pmem[i], 0, vmm, pvma[i], &args, sizeof(args));188 if (ret)189 return ret;190 191 entry->gpuVirtAddr = pvma[i]->addr;192 }193 194 if (entry->bInitialize) {195 entry->gpuPhysAddr = nvkm_memory_addr(pmem[i]);196 entry->size = gr->ctxbuf[i].size;197 entry->physAttr = 4;198 }199 200 nvkm_debug(subdev,201 "promote %02d: pa %016llx/%08x sz %016llx va %016llx init:%d nm:%d\n",202 entry->bufferId, entry->gpuPhysAddr, entry->physAttr, entry->size,203 entry->gpuVirtAddr, entry->bInitialize, entry->bNonmapped);204 205 ctrl->entryCount++;206 }207 208 return nvkm_gsp_rm_ctrl_wr(&vmm->rm.device.subdevice, ctrl);209}210 211static int212r535_gr_chan_new(struct nvkm_gr *base, struct nvkm_chan *chan, const struct nvkm_oclass *oclass,213 struct nvkm_object **pobject)214{215 struct r535_gr *gr = r535_gr(base);216 struct r535_gr_chan *grc;217 int ret;218 219 if (!(grc = kzalloc(sizeof(*grc), GFP_KERNEL)))220 return -ENOMEM;221 222 nvkm_object_ctor(&r535_gr_chan, oclass, &grc->object);223 grc->gr = gr;224 grc->vmm = nvkm_vmm_ref(chan->vmm);225 grc->chan = chan;226 *pobject = &grc->object;227 228 ret = r535_gr_promote_ctx(gr, false, grc->vmm, grc->mem, grc->vma, &chan->rm.object);229 if (ret)230 return ret;231 232 return 0;233}234 235static u64236r535_gr_units(struct nvkm_gr *gr)237{238 struct nvkm_gsp *gsp = gr->engine.subdev.device->gsp;239 240 return (gsp->gr.tpcs << 8) | gsp->gr.gpcs;241}242 243static int244r535_gr_oneinit(struct nvkm_gr *base)245{246 NV2080_CTRL_INTERNAL_STATIC_GR_GET_CONTEXT_BUFFERS_INFO_PARAMS *info;247 struct r535_gr *gr = container_of(base, typeof(*gr), base);248 struct nvkm_subdev *subdev = &gr->base.engine.subdev;249 struct nvkm_device *device = subdev->device;250 struct nvkm_gsp *gsp = device->gsp;251 struct nvkm_mmu *mmu = device->mmu;252 struct {253 struct nvkm_memory *inst;254 struct nvkm_vmm *vmm;255 struct nvkm_gsp_object chan;256 struct nvkm_vma *vma[R515_GR_MAX_CTXBUFS];257 } golden = {};258 int ret;259 260 /* Allocate a channel to use for golden context init. */261 ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x12000, 0, true, &golden.inst);262 if (ret)263 goto done;264 265 ret = nvkm_vmm_new(device, 0x1000, 0, NULL, 0, NULL, "grGoldenVmm", &golden.vmm);266 if (ret)267 goto done;268 269 ret = mmu->func->promote_vmm(golden.vmm);270 if (ret)271 goto done;272 273 {274 NV_CHANNELGPFIFO_ALLOCATION_PARAMETERS *args;275 276 args = nvkm_gsp_rm_alloc_get(&golden.vmm->rm.device.object, 0xf1f00000,277 device->fifo->func->chan.user.oclass,278 sizeof(*args), &golden.chan);279 if (IS_ERR(args)) {280 ret = PTR_ERR(args);281 goto done;282 }283 284 args->gpFifoOffset = 0;285 args->gpFifoEntries = 0x1000 / 8;286 args->flags =287 NVDEF(NVOS04, FLAGS, CHANNEL_TYPE, PHYSICAL) |288 NVDEF(NVOS04, FLAGS, VPR, FALSE) |289 NVDEF(NVOS04, FLAGS, CHANNEL_SKIP_MAP_REFCOUNTING, FALSE) |290 NVVAL(NVOS04, FLAGS, GROUP_CHANNEL_RUNQUEUE, 0) |291 NVDEF(NVOS04, FLAGS, PRIVILEGED_CHANNEL, TRUE) |292 NVDEF(NVOS04, FLAGS, DELAY_CHANNEL_SCHEDULING, FALSE) |293 NVDEF(NVOS04, FLAGS, CHANNEL_DENY_PHYSICAL_MODE_CE, FALSE) |294 NVVAL(NVOS04, FLAGS, CHANNEL_USERD_INDEX_VALUE, 0) |295 NVDEF(NVOS04, FLAGS, CHANNEL_USERD_INDEX_FIXED, FALSE) |296 NVVAL(NVOS04, FLAGS, CHANNEL_USERD_INDEX_PAGE_VALUE, 0) |297 NVDEF(NVOS04, FLAGS, CHANNEL_USERD_INDEX_PAGE_FIXED, TRUE) |298 NVDEF(NVOS04, FLAGS, CHANNEL_DENY_AUTH_LEVEL_PRIV, FALSE) |299 NVDEF(NVOS04, FLAGS, CHANNEL_SKIP_SCRUBBER, FALSE) |300 NVDEF(NVOS04, FLAGS, CHANNEL_CLIENT_MAP_FIFO, FALSE) |301 NVDEF(NVOS04, FLAGS, SET_EVICT_LAST_CE_PREFETCH_CHANNEL, FALSE) |302 NVDEF(NVOS04, FLAGS, CHANNEL_VGPU_PLUGIN_CONTEXT, FALSE) |303 NVDEF(NVOS04, FLAGS, CHANNEL_PBDMA_ACQUIRE_TIMEOUT, FALSE) |304 NVDEF(NVOS04, FLAGS, GROUP_CHANNEL_THREAD, DEFAULT) |305 NVDEF(NVOS04, FLAGS, MAP_CHANNEL, FALSE) |306 NVDEF(NVOS04, FLAGS, SKIP_CTXBUFFER_ALLOC, FALSE);307 args->hVASpace = golden.vmm->rm.object.handle;308 args->engineType = 1;309 args->instanceMem.base = nvkm_memory_addr(golden.inst);310 args->instanceMem.size = 0x1000;311 args->instanceMem.addressSpace = 2;312 args->instanceMem.cacheAttrib = 1;313 args->ramfcMem.base = nvkm_memory_addr(golden.inst);314 args->ramfcMem.size = 0x200;315 args->ramfcMem.addressSpace = 2;316 args->ramfcMem.cacheAttrib = 1;317 args->userdMem.base = nvkm_memory_addr(golden.inst) + 0x1000;318 args->userdMem.size = 0x200;319 args->userdMem.addressSpace = 2;320 args->userdMem.cacheAttrib = 1;321 args->mthdbufMem.base = nvkm_memory_addr(golden.inst) + 0x2000;322 args->mthdbufMem.size = 0x5000;323 args->mthdbufMem.addressSpace = 2;324 args->mthdbufMem.cacheAttrib = 1;325 args->internalFlags =326 NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, PRIVILEGE, ADMIN) |327 NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, ERROR_NOTIFIER_TYPE, NONE) |328 NVDEF(NV_KERNELCHANNEL, ALLOC_INTERNALFLAGS, ECC_ERROR_NOTIFIER_TYPE, NONE);329 330 ret = nvkm_gsp_rm_alloc_wr(&golden.chan, args);331 if (ret)332 goto done;333 }334 335 /* Fetch context buffer info from RM and allocate each of them here to use336 * during golden context init (or later as a global context buffer).337 *338 * Also build the information that'll be used to create channel contexts.339 */340 info = nvkm_gsp_rm_ctrl_rd(&gsp->internal.device.subdevice,341 NV2080_CTRL_CMD_INTERNAL_STATIC_KGR_GET_CONTEXT_BUFFERS_INFO,342 sizeof(*info));343 if (WARN_ON(IS_ERR(info))) {344 ret = PTR_ERR(info);345 goto done;346 }347 348 for (int i = 0; i < ARRAY_SIZE(info->engineContextBuffersInfo[0].engine); i++) {349 static const struct {350 u32 id0; /* NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID */351 u32 id1; /* NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID */352 bool global;353 bool init;354 bool ro;355 } map[] = {356#define _A(n,N,G,I,R) { .id0 = NV0080_CTRL_FIFO_GET_ENGINE_CONTEXT_PROPERTIES_ENGINE_ID_##n, \357 .id1 = NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_##N, \358 .global = (G), .init = (I), .ro = (R) }359#define _B(N,G,I,R) _A(GRAPHICS_##N, N, (G), (I), (R))360 /* global init ro */361 _A( GRAPHICS, MAIN, false, true, false),362 _B( PATCH, false, true, false),363 _A( GRAPHICS_BUNDLE_CB, BUFFER_BUNDLE_CB, true, false, false),364 _B( PAGEPOOL, true, false, false),365 _B( ATTRIBUTE_CB, true, false, false),366 _B( RTV_CB_GLOBAL, true, false, false),367 _B( FECS_EVENT, true, true, false),368 _B( PRIV_ACCESS_MAP, true, true, true),369#undef _B370#undef _A371 };372 u32 size = info->engineContextBuffersInfo[0].engine[i].size;373 u8 align, page;374 int id;375 376 for (id = 0; id < ARRAY_SIZE(map); id++) {377 if (map[id].id0 == i)378 break;379 }380 381 nvkm_debug(subdev, "%02x: size:0x%08x %s\n", i,382 size, (id < ARRAY_SIZE(map)) ? "*" : "");383 if (id >= ARRAY_SIZE(map))384 continue;385 386 if (map[id].id1 == NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_MAIN)387 size = ALIGN(size, 0x1000) + 64 * 0x1000; /* per-subctx headers */388 389 if (size >= 1 << 21) page = 21;390 else if (size >= 1 << 16) page = 16;391 else page = 12;392 393 if (map[id].id1 == NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_ATTRIBUTE_CB)394 align = order_base_2(size);395 else396 align = page;397 398 if (WARN_ON(gr->ctxbuf_nr == ARRAY_SIZE(gr->ctxbuf)))399 continue;400 401 gr->ctxbuf[gr->ctxbuf_nr].bufferId = map[id].id1;402 gr->ctxbuf[gr->ctxbuf_nr].size = size;403 gr->ctxbuf[gr->ctxbuf_nr].page = page;404 gr->ctxbuf[gr->ctxbuf_nr].align = align;405 gr->ctxbuf[gr->ctxbuf_nr].global = map[id].global;406 gr->ctxbuf[gr->ctxbuf_nr].init = map[id].init;407 gr->ctxbuf[gr->ctxbuf_nr].ro = map[id].ro;408 gr->ctxbuf_nr++;409 410 if (map[id].id1 == NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_PRIV_ACCESS_MAP) {411 if (WARN_ON(gr->ctxbuf_nr == ARRAY_SIZE(gr->ctxbuf)))412 continue;413 414 gr->ctxbuf[gr->ctxbuf_nr] = gr->ctxbuf[gr->ctxbuf_nr - 1];415 gr->ctxbuf[gr->ctxbuf_nr].bufferId =416 NV2080_CTRL_GPU_PROMOTE_CTX_BUFFER_ID_UNRESTRICTED_PRIV_ACCESS_MAP;417 gr->ctxbuf_nr++;418 }419 }420 421 nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, info);422 423 /* Promote golden context to RM. */424 ret = r535_gr_promote_ctx(gr, true, golden.vmm, gr->ctxbuf_mem, golden.vma, &golden.chan);425 if (ret)426 goto done;427 428 /* Allocate 3D class on channel to trigger golden context init in RM. */429 {430 int i;431 432 for (i = 0; gr->base.func->sclass[i].ctor; i++) {433 if ((gr->base.func->sclass[i].oclass & 0xff) == 0x97) {434 struct nvkm_gsp_object threed;435 436 ret = nvkm_gsp_rm_alloc(&golden.chan, 0x97000000,437 gr->base.func->sclass[i].oclass, 0,438 &threed);439 if (ret)440 goto done;441 442 nvkm_gsp_rm_free(&threed);443 break;444 }445 }446 447 if (WARN_ON(!gr->base.func->sclass[i].ctor)) {448 ret = -EINVAL;449 goto done;450 }451 }452 453done:454 nvkm_gsp_rm_free(&golden.chan);455 for (int i = gr->ctxbuf_nr - 1; i >= 0; i--)456 nvkm_vmm_put(golden.vmm, &golden.vma[i]);457 nvkm_vmm_unref(&golden.vmm);458 nvkm_memory_unref(&golden.inst);459 return ret;460 461}462 463static void *464r535_gr_dtor(struct nvkm_gr *base)465{466 struct r535_gr *gr = r535_gr(base);467 468 while (gr->ctxbuf_nr)469 nvkm_memory_unref(&gr->ctxbuf_mem[--gr->ctxbuf_nr]);470 471 kfree(gr->base.func);472 return gr;473}474 475int476r535_gr_new(const struct gf100_gr_func *hw,477 struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_gr **pgr)478{479 struct nvkm_gr_func *rm;480 struct r535_gr *gr;481 int nclass;482 483 for (nclass = 0; hw->sclass[nclass].oclass; nclass++);484 485 if (!(rm = kzalloc(sizeof(*rm) + (nclass + 1) * sizeof(rm->sclass[0]), GFP_KERNEL)))486 return -ENOMEM;487 488 rm->dtor = r535_gr_dtor;489 rm->oneinit = r535_gr_oneinit;490 rm->units = r535_gr_units;491 rm->chan_new = r535_gr_chan_new;492 493 for (int i = 0; i < nclass; i++) {494 rm->sclass[i].minver = hw->sclass[i].minver;495 rm->sclass[i].maxver = hw->sclass[i].maxver;496 rm->sclass[i].oclass = hw->sclass[i].oclass;497 rm->sclass[i].ctor = r535_gr_obj_ctor;498 }499 500 if (!(gr = kzalloc(sizeof(*gr), GFP_KERNEL))) {501 kfree(rm);502 return -ENOMEM;503 }504 505 *pgr = &gr->base;506 507 return nvkm_gr_ctor(rm, device, type, inst, true, &gr->base);508}509