620 lines · c
1/*2 * Copyright 2021 Red Hat Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 */22#include "priv.h"23#include "cgrp.h"24#include "chan.h"25#include "chid.h"26#include "runl.h"27#include "runq.h"28 29#include <core/gpuobj.h>30#include <subdev/gsp.h>31#include <subdev/top.h>32#include <subdev/vfn.h>33 34#include <nvif/class.h>35 36static u3237ga100_chan_doorbell_handle(struct nvkm_chan *chan)38{39 return (chan->cgrp->runl->doorbell << 16) | chan->id;40}41 42static void43ga100_chan_stop(struct nvkm_chan *chan)44{45 struct nvkm_runl *runl = chan->cgrp->runl;46 47 nvkm_wr32(runl->fifo->engine.subdev.device, runl->chan + (chan->id * 4), 0x00000003);48}49 50static void51ga100_chan_start(struct nvkm_chan *chan)52{53 struct nvkm_runl *runl = chan->cgrp->runl;54 struct nvkm_device *device = runl->fifo->engine.subdev.device;55 const int gfid = 0;56 57 nvkm_wr32(device, runl->chan + (chan->id * 4), 0x00000002);58 nvkm_wr32(device, runl->addr + 0x0090, (gfid << 16) | chan->id); /* INTERNAL_DOORBELL. */59}60 61static void62ga100_chan_unbind(struct nvkm_chan *chan)63{64 struct nvkm_runl *runl = chan->cgrp->runl;65 66 nvkm_wr32(runl->fifo->engine.subdev.device, runl->chan + (chan->id * 4), 0xffffffff);67}68 69static int70ga100_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, bool priv)71{72 const u32 limit2 = ilog2(length / 8);73 74 nvkm_kmap(chan->inst);75 nvkm_wo32(chan->inst, 0x010, 0x0000face);76 nvkm_wo32(chan->inst, 0x030, 0x7ffff902);77 nvkm_wo32(chan->inst, 0x048, lower_32_bits(offset));78 nvkm_wo32(chan->inst, 0x04c, upper_32_bits(offset) | (limit2 << 16));79 nvkm_wo32(chan->inst, 0x084, 0x20400000);80 nvkm_wo32(chan->inst, 0x094, 0x30000000 | devm);81 nvkm_wo32(chan->inst, 0x0e4, priv ? 0x00000020 : 0x00000000);82 nvkm_wo32(chan->inst, 0x0e8, chan->id);83 nvkm_wo32(chan->inst, 0x0f4, 0x00001000 | (priv ? 0x00000100 : 0x00000000));84 nvkm_wo32(chan->inst, 0x0f8, 0x80000000 | chan->cgrp->runl->nonstall.vector);85 nvkm_mo32(chan->inst, 0x218, 0x00000000, 0x00000000);86 nvkm_done(chan->inst);87 return 0;88}89 90static const struct nvkm_chan_func_ramfc91ga100_chan_ramfc = {92 .write = ga100_chan_ramfc_write,93 .devm = 0xfff,94 .priv = true,95};96 97const struct nvkm_chan_func98ga100_chan = {99 .inst = &gf100_chan_inst,100 .userd = &gv100_chan_userd,101 .ramfc = &ga100_chan_ramfc,102 .unbind = ga100_chan_unbind,103 .start = ga100_chan_start,104 .stop = ga100_chan_stop,105 .preempt = gk110_chan_preempt,106 .doorbell_handle = ga100_chan_doorbell_handle,107};108 109static void110ga100_cgrp_preempt(struct nvkm_cgrp *cgrp)111{112 struct nvkm_runl *runl = cgrp->runl;113 114 nvkm_wr32(runl->fifo->engine.subdev.device, runl->addr + 0x098, 0x01000000 | cgrp->id);115}116 117const struct nvkm_cgrp_func118ga100_cgrp = {119 .preempt = ga100_cgrp_preempt,120};121 122static int123ga100_engn_cxid(struct nvkm_engn *engn, bool *cgid)124{125 struct nvkm_runl *runl = engn->runl;126 struct nvkm_device *device = runl->fifo->engine.subdev.device;127 u32 stat = nvkm_rd32(device, runl->addr + 0x200 + engn->id * 0x40);128 129 ENGN_DEBUG(engn, "status %08x", stat);130 *cgid = true;131 132 switch ((stat & 0x0000e000) >> 13) {133 case 0 /* INVALID */: return -ENODEV;134 case 1 /* VALID */:135 case 5 /* SAVE */: return (stat & 0x00000fff);136 case 6 /* LOAD */: return (stat & 0x0fff0000) >> 16;137 case 7 /* SWITCH */:138 if (nvkm_engine_chsw_load(engn->engine))139 return (stat & 0x0fff0000) >> 16;140 return (stat & 0x00000fff);141 default:142 WARN_ON(1);143 break;144 }145 146 return -ENODEV;147}148 149static int150ga100_engn_nonstall(struct nvkm_engn *engn)151{152 struct nvkm_engine *engine = engn->engine;153 154 if (WARN_ON(!engine->func->nonstall))155 return -EINVAL;156 157 return engine->func->nonstall(engine);158}159 160const struct nvkm_engn_func161ga100_engn = {162 .nonstall = ga100_engn_nonstall,163 .cxid = ga100_engn_cxid,164 .ctor = gk104_ectx_ctor,165 .bind = gv100_ectx_bind,166};167 168const struct nvkm_engn_func169ga100_engn_ce = {170 .nonstall = ga100_engn_nonstall,171 .cxid = ga100_engn_cxid,172 .ctor = gv100_ectx_ce_ctor,173 .bind = gv100_ectx_ce_bind,174};175 176static bool177ga100_runq_idle(struct nvkm_runq *runq)178{179 struct nvkm_device *device = runq->fifo->engine.subdev.device;180 181 return !(nvkm_rd32(device, 0x04015c + (runq->id * 0x800)) & 0x0000e000);182}183 184static bool185ga100_runq_intr_1(struct nvkm_runq *runq, struct nvkm_runl *runl)186{187 struct nvkm_device *device = runq->fifo->engine.subdev.device;188 u32 inte = nvkm_rd32(device, 0x040180 + (runq->id * 0x800));189 u32 intr = nvkm_rd32(device, 0x040148 + (runq->id * 0x800));190 u32 stat = intr & inte;191 192 if (!stat) {193 RUNQ_DEBUG(runq, "inte1 %08x %08x", intr, inte);194 return false;195 }196 197 if (stat & 0x80000000) {198 u32 chid = nvkm_rd32(device, 0x040120 + (runq->id * 0x0800)) & runl->chid->mask;199 struct nvkm_chan *chan;200 unsigned long flags;201 202 RUNQ_ERROR(runq, "CTXNOTVALID chid:%d", chid);203 chan = nvkm_runl_chan_get_chid(runl, chid, &flags);204 if (chan) {205 nvkm_chan_error(chan, true);206 nvkm_chan_put(&chan, flags);207 }208 209 nvkm_mask(device, 0x0400ac + (runq->id * 0x800), 0x00030000, 0x00030000);210 stat &= ~0x80000000;211 }212 213 if (stat) {214 RUNQ_ERROR(runq, "intr1 %08x", stat);215 nvkm_wr32(device, 0x0401a0 + (runq->id * 0x800), stat);216 }217 218 nvkm_wr32(device, 0x040148 + (runq->id * 0x800), intr);219 return true;220}221 222static bool223ga100_runq_intr_0(struct nvkm_runq *runq, struct nvkm_runl *runl)224{225 struct nvkm_device *device = runq->fifo->engine.subdev.device;226 u32 inte = nvkm_rd32(device, 0x040170 + (runq->id * 0x800));227 u32 intr = nvkm_rd32(device, 0x040108 + (runq->id * 0x800));228 u32 stat = intr & inte;229 230 if (!stat) {231 RUNQ_DEBUG(runq, "inte0 %08x %08x", intr, inte);232 return false;233 }234 235 /*TODO: expand on this when fixing up gf100's version. */236 if (stat & 0xc6afe000) {237 u32 chid = nvkm_rd32(device, 0x040120 + (runq->id * 0x0800)) & runl->chid->mask;238 struct nvkm_chan *chan;239 unsigned long flags;240 241 RUNQ_ERROR(runq, "intr0 %08x", stat);242 chan = nvkm_runl_chan_get_chid(runl, chid, &flags);243 if (chan) {244 nvkm_chan_error(chan, true);245 nvkm_chan_put(&chan, flags);246 }247 248 stat &= ~0xc6afe000;249 }250 251 if (stat) {252 RUNQ_ERROR(runq, "intr0 %08x", stat);253 nvkm_wr32(device, 0x040190 + (runq->id * 0x800), stat);254 }255 256 nvkm_wr32(device, 0x040108 + (runq->id * 0x800), intr);257 return true;258}259 260static bool261ga100_runq_intr(struct nvkm_runq *runq, struct nvkm_runl *runl)262{263 bool intr0 = ga100_runq_intr_0(runq, runl);264 bool intr1 = ga100_runq_intr_1(runq, runl);265 266 return intr0 || intr1;267}268 269static void270ga100_runq_init(struct nvkm_runq *runq)271{272 struct nvkm_device *device = runq->fifo->engine.subdev.device;273 274 nvkm_wr32(device, 0x040108 + (runq->id * 0x800), 0xffffffff); /* INTR_0 */275 nvkm_wr32(device, 0x040148 + (runq->id * 0x800), 0xffffffff); /* INTR_1 */276 nvkm_wr32(device, 0x040170 + (runq->id * 0x800), 0xffffffff); /* INTR_0_EN_SET_TREE */277 nvkm_wr32(device, 0x040180 + (runq->id * 0x800), 0xffffffff); /* INTR_1_EN_SET_TREE */278}279 280const struct nvkm_runq_func281ga100_runq = {282 .init = ga100_runq_init,283 .intr = ga100_runq_intr,284 .idle = ga100_runq_idle,285};286 287static bool288ga100_runl_preempt_pending(struct nvkm_runl *runl)289{290 return nvkm_rd32(runl->fifo->engine.subdev.device, runl->addr + 0x098) & 0x00100000;291}292 293static void294ga100_runl_preempt(struct nvkm_runl *runl)295{296 nvkm_wr32(runl->fifo->engine.subdev.device, runl->addr + 0x098, 0x00000000);297}298 299static void300ga100_runl_allow(struct nvkm_runl *runl, u32 engm)301{302 nvkm_mask(runl->fifo->engine.subdev.device, runl->addr + 0x094, 0x00000001, 0x00000000);303}304 305static void306ga100_runl_block(struct nvkm_runl *runl, u32 engm)307{308 nvkm_mask(runl->fifo->engine.subdev.device, runl->addr + 0x094, 0x00000001, 0x00000001);309}310 311static bool312ga100_runl_pending(struct nvkm_runl *runl)313{314 struct nvkm_device *device = runl->fifo->engine.subdev.device;315 316 return nvkm_rd32(device, runl->addr + 0x08c) & 0x00008000;317}318 319static void320ga100_runl_commit(struct nvkm_runl *runl, struct nvkm_memory *memory, u32 start, int count)321{322 struct nvkm_device *device = runl->fifo->engine.subdev.device;323 u64 addr = nvkm_memory_addr(memory) + start;324 325 nvkm_wr32(device, runl->addr + 0x080, lower_32_bits(addr));326 nvkm_wr32(device, runl->addr + 0x084, upper_32_bits(addr));327 nvkm_wr32(device, runl->addr + 0x088, count);328}329 330static irqreturn_t331ga100_runl_intr(struct nvkm_inth *inth)332{333 struct nvkm_runl *runl = container_of(inth, typeof(*runl), inth);334 struct nvkm_engn *engn;335 struct nvkm_device *device = runl->fifo->engine.subdev.device;336 u32 inte = nvkm_rd32(device, runl->addr + 0x120);337 u32 intr = nvkm_rd32(device, runl->addr + 0x100);338 u32 stat = intr & inte;339 u32 info;340 341 if (!stat) {342 RUNL_DEBUG(runl, "inte %08x %08x", intr, inte);343 return IRQ_NONE;344 }345 346 if (stat & 0x00000007) {347 nvkm_runl_foreach_engn_cond(engn, runl, stat & BIT(engn->id)) {348 info = nvkm_rd32(device, runl->addr + 0x224 + (engn->id * 0x40));349 350 tu102_fifo_intr_ctxsw_timeout_info(engn, info);351 352 nvkm_wr32(device, runl->addr + 0x100, BIT(engn->id));353 stat &= ~BIT(engn->id);354 }355 }356 357 if (stat & 0x00000300) {358 nvkm_wr32(device, runl->addr + 0x100, stat & 0x00000300);359 stat &= ~0x00000300;360 }361 362 if (stat & 0x00010000) {363 if (runl->runq[0]) {364 if (runl->runq[0]->func->intr(runl->runq[0], runl))365 stat &= ~0x00010000;366 }367 }368 369 if (stat & 0x00020000) {370 if (runl->runq[1]) {371 if (runl->runq[1]->func->intr(runl->runq[1], runl))372 stat &= ~0x00020000;373 }374 }375 376 if (stat) {377 RUNL_ERROR(runl, "intr %08x", stat);378 nvkm_wr32(device, runl->addr + 0x140, stat);379 }380 381 nvkm_wr32(device, runl->addr + 0x180, 0x00000001);382 return IRQ_HANDLED;383}384 385static void386ga100_runl_fini(struct nvkm_runl *runl)387{388 nvkm_mask(runl->fifo->engine.subdev.device, runl->addr + 0x300, 0x80000000, 0x00000000);389 nvkm_inth_block(&runl->inth);390}391 392static void393ga100_runl_init(struct nvkm_runl *runl)394{395 struct nvkm_fifo *fifo = runl->fifo;396 struct nvkm_runq *runq;397 struct nvkm_device *device = fifo->engine.subdev.device;398 int i;399 400 /* Submit NULL runlist and preempt. */401 nvkm_wr32(device, runl->addr + 0x088, 0x00000000);402 runl->func->preempt(runl);403 404 /* Enable doorbell. */405 nvkm_mask(device, runl->addr + 0x300, 0x80000000, 0x80000000);406 407 nvkm_wr32(device, runl->addr + 0x100, 0xffffffff); /* INTR_0 */408 nvkm_wr32(device, runl->addr + 0x140, 0xffffffff); /* INTR_0_EN_CLEAR_TREE(0) */409 nvkm_wr32(device, runl->addr + 0x120, 0x000f1307); /* INTR_0_EN_SET_TREE(0) */410 nvkm_wr32(device, runl->addr + 0x148, 0xffffffff); /* INTR_0_EN_CLEAR_TREE(1) */411 nvkm_wr32(device, runl->addr + 0x128, 0x00000000); /* INTR_0_EN_SET_TREE(1) */412 413 /* Init PBDMA(s). */414 for (i = 0; i < runl->runq_nr; i++) {415 runq = runl->runq[i];416 runq->func->init(runq);417 }418 419 nvkm_inth_allow(&runl->inth);420}421 422const struct nvkm_runl_func423ga100_runl = {424 .init = ga100_runl_init,425 .fini = ga100_runl_fini,426 .size = 16,427 .update = nv50_runl_update,428 .insert_cgrp = gv100_runl_insert_cgrp,429 .insert_chan = gv100_runl_insert_chan,430 .commit = ga100_runl_commit,431 .wait = nv50_runl_wait,432 .pending = ga100_runl_pending,433 .block = ga100_runl_block,434 .allow = ga100_runl_allow,435 .preempt = ga100_runl_preempt,436 .preempt_pending = ga100_runl_preempt_pending,437};438 439static int440ga100_runl_new(struct nvkm_fifo *fifo, int id, u32 addr, struct nvkm_runl **prunl)441{442 struct nvkm_device *device = fifo->engine.subdev.device;443 struct nvkm_top_device *tdev;444 struct nvkm_runl *runl;445 struct nvkm_engn *engn;446 u32 chcfg = nvkm_rd32(device, addr + 0x004);447 u32 chnum = 1 << (chcfg & 0x0000000f);448 u32 chaddr = (chcfg & 0xfffffff0);449 u32 dbcfg = nvkm_rd32(device, addr + 0x008);450 u32 vector = nvkm_rd32(device, addr + 0x160);451 int i, ret;452 453 runl = nvkm_runl_new(fifo, id, addr, chnum);454 if (IS_ERR(runl))455 return PTR_ERR(runl);456 457 *prunl = runl;458 459 for (i = 0; i < 2; i++) {460 u32 pbcfg = nvkm_rd32(device, addr + 0x010 + (i * 0x04));461 if (pbcfg & 0x80000000) {462 runl->runq[runl->runq_nr] =463 nvkm_runq_new(fifo, ((pbcfg & 0x03fffc00) - 0x040000) / 0x800);464 if (!runl->runq[runl->runq_nr]) {465 RUNL_ERROR(runl, "runq %d", runl->runq_nr);466 return -ENOMEM;467 }468 469 runl->runq_nr++;470 }471 }472 473 nvkm_list_foreach(tdev, &device->top->device, head, tdev->runlist == runl->addr) {474 if (tdev->engine < 0) {475 RUNL_DEBUG(runl, "engn !top");476 return -EINVAL;477 }478 479 engn = nvkm_runl_add(runl, tdev->engine, (tdev->type == NVKM_ENGINE_CE) ?480 fifo->func->engn_ce : fifo->func->engn,481 tdev->type, tdev->inst);482 if (!engn)483 return -EINVAL;484 485 if (!engn->engine->func->nonstall) {486 RUNL_DEBUG(runl, "engn %s !nonstall", engn->engine->subdev.name);487 return -EINVAL;488 }489 }490 491 if (list_empty(&runl->engns)) {492 RUNL_DEBUG(runl, "!engns");493 return -EINVAL;494 }495 496 ret = nvkm_inth_add(&device->vfn->intr, vector & 0x00000fff, NVKM_INTR_PRIO_NORMAL,497 &fifo->engine.subdev, ga100_runl_intr, &runl->inth);498 if (ret) {499 RUNL_ERROR(runl, "inth %d", ret);500 return ret;501 }502 503 runl->chan = chaddr;504 runl->doorbell = dbcfg >> 16;505 return 0;506}507 508static irqreturn_t509ga100_fifo_nonstall_intr(struct nvkm_inth *inth)510{511 struct nvkm_runl *runl = container_of(inth, typeof(*runl), nonstall.inth);512 513 nvkm_event_ntfy(&runl->fifo->nonstall.event, runl->id, NVKM_FIFO_NONSTALL_EVENT);514 return IRQ_HANDLED;515}516 517static void518ga100_fifo_nonstall_block(struct nvkm_event *event, int type, int index)519{520 struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), nonstall.event);521 struct nvkm_runl *runl = nvkm_runl_get(fifo, index, 0);522 523 nvkm_inth_block(&runl->nonstall.inth);524}525 526static void527ga100_fifo_nonstall_allow(struct nvkm_event *event, int type, int index)528{529 struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), nonstall.event);530 struct nvkm_runl *runl = nvkm_runl_get(fifo, index, 0);531 532 nvkm_inth_allow(&runl->nonstall.inth);533}534 535const struct nvkm_event_func536ga100_fifo_nonstall = {537 .init = ga100_fifo_nonstall_allow,538 .fini = ga100_fifo_nonstall_block,539};540 541int542ga100_fifo_nonstall_ctor(struct nvkm_fifo *fifo)543{544 struct nvkm_subdev *subdev = &fifo->engine.subdev;545 struct nvkm_vfn *vfn = subdev->device->vfn;546 struct nvkm_runl *runl;547 int ret, nr = 0;548 549 nvkm_runl_foreach(runl, fifo) {550 struct nvkm_engn *engn = list_first_entry(&runl->engns, typeof(*engn), head);551 552 runl->nonstall.vector = engn->func->nonstall(engn);553 554 /* if no nonstall vector just keep going */555 if (runl->nonstall.vector == -1)556 continue;557 if (runl->nonstall.vector < 0) {558 RUNL_ERROR(runl, "nonstall %d", runl->nonstall.vector);559 return runl->nonstall.vector;560 }561 562 ret = nvkm_inth_add(&vfn->intr, runl->nonstall.vector, NVKM_INTR_PRIO_NORMAL,563 subdev, ga100_fifo_nonstall_intr, &runl->nonstall.inth);564 if (ret)565 return ret;566 567 nr = max(nr, runl->id + 1);568 }569 570 return nr;571}572 573int574ga100_fifo_runl_ctor(struct nvkm_fifo *fifo)575{576 struct nvkm_device *device = fifo->engine.subdev.device;577 struct nvkm_top_device *tdev;578 struct nvkm_runl *runl;579 int id = 0, ret;580 581 nvkm_list_foreach(tdev, &device->top->device, head, tdev->runlist >= 0) {582 runl = nvkm_runl_get(fifo, -1, tdev->runlist);583 if (!runl) {584 ret = ga100_runl_new(fifo, id++, tdev->runlist, &runl);585 if (ret) {586 if (runl)587 nvkm_runl_del(runl);588 589 continue;590 }591 }592 }593 594 return 0;595}596 597static const struct nvkm_fifo_func598ga100_fifo = {599 .runl_ctor = ga100_fifo_runl_ctor,600 .mmu_fault = &tu102_fifo_mmu_fault,601 .nonstall_ctor = ga100_fifo_nonstall_ctor,602 .nonstall = &ga100_fifo_nonstall,603 .runl = &ga100_runl,604 .runq = &ga100_runq,605 .engn = &ga100_engn,606 .engn_ce = &ga100_engn_ce,607 .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &ga100_cgrp, .force = true },608 .chan = {{ 0, 0, AMPERE_CHANNEL_GPFIFO_A }, &ga100_chan },609};610 611int612ga100_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst,613 struct nvkm_fifo **pfifo)614{615 if (nvkm_gsp_rm(device->gsp))616 return r535_fifo_new(&ga100_fifo, device, type, inst, pfifo);617 618 return nvkm_fifo_new_(&ga100_fifo, device, type, inst, pfifo);619}620