834 lines · c
1/*2 * Copyright 2012 Red Hat Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 * Authors: Ben Skeggs23 */24#include "priv.h"25#include "cgrp.h"26#include "chan.h"27#include "chid.h"28#include "runl.h"29#include "runq.h"30 31#include <core/gpuobj.h>32#include <subdev/mc.h>33#include <subdev/mmu.h>34#include <subdev/top.h>35 36#include <nvif/class.h>37#include <nvif/if900d.h>38 39void40gk104_chan_stop(struct nvkm_chan *chan)41{42 struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device;43 44 nvkm_mask(device, 0x800004 + (chan->id * 8), 0x00000800, 0x00000800);45}46 47void48gk104_chan_start(struct nvkm_chan *chan)49{50 struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device;51 52 nvkm_mask(device, 0x800004 + (chan->id * 8), 0x00000400, 0x00000400);53}54 55void56gk104_chan_unbind(struct nvkm_chan *chan)57{58 struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device;59 60 nvkm_wr32(device, 0x800000 + (chan->id * 8), 0x00000000);61}62 63void64gk104_chan_bind_inst(struct nvkm_chan *chan)65{66 struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device;67 68 nvkm_wr32(device, 0x800000 + (chan->id * 8), 0x80000000 | chan->inst->addr >> 12);69}70 71void72gk104_chan_bind(struct nvkm_chan *chan)73{74 struct nvkm_runl *runl = chan->cgrp->runl;75 struct nvkm_device *device = runl->fifo->engine.subdev.device;76 77 nvkm_mask(device, 0x800004 + (chan->id * 8), 0x000f0000, runl->id << 16);78 gk104_chan_bind_inst(chan);79}80 81static int82gk104_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, bool priv)83{84 const u64 userd = nvkm_memory_addr(chan->userd.mem) + chan->userd.base;85 const u32 limit2 = ilog2(length / 8);86 87 nvkm_kmap(chan->inst);88 nvkm_wo32(chan->inst, 0x08, lower_32_bits(userd));89 nvkm_wo32(chan->inst, 0x0c, upper_32_bits(userd));90 nvkm_wo32(chan->inst, 0x10, 0x0000face);91 nvkm_wo32(chan->inst, 0x30, 0xfffff902);92 nvkm_wo32(chan->inst, 0x48, lower_32_bits(offset));93 nvkm_wo32(chan->inst, 0x4c, upper_32_bits(offset) | (limit2 << 16));94 nvkm_wo32(chan->inst, 0x84, 0x20400000);95 nvkm_wo32(chan->inst, 0x94, 0x30000000 | devm);96 nvkm_wo32(chan->inst, 0x9c, 0x00000100);97 nvkm_wo32(chan->inst, 0xac, 0x0000001f);98 nvkm_wo32(chan->inst, 0xe4, priv ? 0x00000020 : 0x00000000);99 nvkm_wo32(chan->inst, 0xe8, chan->id);100 nvkm_wo32(chan->inst, 0xb8, 0xf8000000);101 nvkm_wo32(chan->inst, 0xf8, 0x10003080); /* 0x002310 */102 nvkm_wo32(chan->inst, 0xfc, 0x10000010); /* 0x002350 */103 nvkm_done(chan->inst);104 return 0;105}106 107const struct nvkm_chan_func_ramfc108gk104_chan_ramfc = {109 .write = gk104_chan_ramfc_write,110 .devm = 0xfff,111 .priv = true,112};113 114const struct nvkm_chan_func_userd115gk104_chan_userd = {116 .bar = 1,117 .size = 0x200,118 .clear = gf100_chan_userd_clear,119};120 121static const struct nvkm_chan_func122gk104_chan = {123 .inst = &gf100_chan_inst,124 .userd = &gk104_chan_userd,125 .ramfc = &gk104_chan_ramfc,126 .bind = gk104_chan_bind,127 .unbind = gk104_chan_unbind,128 .start = gk104_chan_start,129 .stop = gk104_chan_stop,130 .preempt = gf100_chan_preempt,131};132 133static void134gk104_ectx_bind(struct nvkm_engn *engn, struct nvkm_cctx *cctx, struct nvkm_chan *chan)135{136 u32 ptr0, ptr1 = 0;137 u64 addr = 0ULL;138 139 switch (engn->engine->subdev.type) {140 case NVKM_ENGINE_SW : return;141 case NVKM_ENGINE_GR : ptr0 = 0x0210; break;142 case NVKM_ENGINE_SEC : ptr0 = 0x0220; break;143 case NVKM_ENGINE_MSPDEC: ptr0 = 0x0250; break;144 case NVKM_ENGINE_MSPPP : ptr0 = 0x0260; break;145 case NVKM_ENGINE_MSVLD : ptr0 = 0x0270; break;146 case NVKM_ENGINE_VIC : ptr0 = 0x0280; break;147 case NVKM_ENGINE_MSENC : ptr0 = 0x0290; break;148 case NVKM_ENGINE_NVDEC :149 ptr1 = 0x0270;150 ptr0 = 0x0210;151 break;152 case NVKM_ENGINE_NVENC :153 if (!engn->engine->subdev.inst)154 ptr1 = 0x0290;155 ptr0 = 0x0210;156 break;157 default:158 WARN_ON(1);159 return;160 }161 162 if (cctx) {163 addr = cctx->vctx->vma->addr;164 addr |= 4ULL;165 }166 167 nvkm_kmap(chan->inst);168 nvkm_wo32(chan->inst, ptr0 + 0, lower_32_bits(addr));169 nvkm_wo32(chan->inst, ptr0 + 4, upper_32_bits(addr));170 if (ptr1) {171 nvkm_wo32(chan->inst, ptr1 + 0, lower_32_bits(addr));172 nvkm_wo32(chan->inst, ptr1 + 4, upper_32_bits(addr));173 }174 nvkm_done(chan->inst);175}176 177int178gk104_ectx_ctor(struct nvkm_engn *engn, struct nvkm_vctx *vctx)179{180 struct gf100_vmm_map_v0 args = { .priv = 1 };181 int ret;182 183 ret = nvkm_vmm_get(vctx->vmm, 12, vctx->inst->size, &vctx->vma);184 if (ret)185 return ret;186 187 return nvkm_memory_map(vctx->inst, 0, vctx->vmm, vctx->vma, &args, sizeof(args));188}189 190/*TODO: clean this up */191struct gk104_engn_status {192 bool busy;193 bool faulted;194 bool chsw;195 bool save;196 bool load;197 struct {198 bool tsg;199 u32 id;200 } prev, next, *chan;201};202 203static void204gk104_engn_status(struct nvkm_engn *engn, struct gk104_engn_status *status)205{206 u32 stat = nvkm_rd32(engn->runl->fifo->engine.subdev.device, 0x002640 + (engn->id * 0x08));207 208 status->busy = !!(stat & 0x80000000);209 status->faulted = !!(stat & 0x40000000);210 status->next.tsg = !!(stat & 0x10000000);211 status->next.id = (stat & 0x0fff0000) >> 16;212 status->chsw = !!(stat & 0x00008000);213 status->save = !!(stat & 0x00004000);214 status->load = !!(stat & 0x00002000);215 status->prev.tsg = !!(stat & 0x00001000);216 status->prev.id = (stat & 0x00000fff);217 status->chan = NULL;218 219 if (status->busy && status->chsw) {220 if (status->load && status->save) {221 if (nvkm_engine_chsw_load(engn->engine))222 status->chan = &status->next;223 else224 status->chan = &status->prev;225 } else226 if (status->load) {227 status->chan = &status->next;228 } else {229 status->chan = &status->prev;230 }231 } else232 if (status->load) {233 status->chan = &status->prev;234 }235 236 ENGN_DEBUG(engn, "%08x: busy %d faulted %d chsw %d save %d load %d %sid %d%s-> %sid %d%s",237 stat, status->busy, status->faulted, status->chsw, status->save, status->load,238 status->prev.tsg ? "tsg" : "ch", status->prev.id,239 status->chan == &status->prev ? "*" : " ",240 status->next.tsg ? "tsg" : "ch", status->next.id,241 status->chan == &status->next ? "*" : " ");242}243 244int245gk104_engn_cxid(struct nvkm_engn *engn, bool *cgid)246{247 struct gk104_engn_status status;248 249 gk104_engn_status(engn, &status);250 if (status.chan) {251 *cgid = status.chan->tsg;252 return status.chan->id;253 }254 255 return -ENODEV;256}257 258bool259gk104_engn_chsw(struct nvkm_engn *engn)260{261 struct gk104_engn_status status;262 263 gk104_engn_status(engn, &status);264 if (status.busy && status.chsw)265 return true;266 267 return false;268}269 270const struct nvkm_engn_func271gk104_engn = {272 .chsw = gk104_engn_chsw,273 .cxid = gk104_engn_cxid,274 .mmu_fault_trigger = gf100_engn_mmu_fault_trigger,275 .mmu_fault_triggered = gf100_engn_mmu_fault_triggered,276 .ctor = gk104_ectx_ctor,277 .bind = gk104_ectx_bind,278};279 280const struct nvkm_engn_func281gk104_engn_ce = {282 .chsw = gk104_engn_chsw,283 .cxid = gk104_engn_cxid,284 .mmu_fault_trigger = gf100_engn_mmu_fault_trigger,285 .mmu_fault_triggered = gf100_engn_mmu_fault_triggered,286};287 288bool289gk104_runq_idle(struct nvkm_runq *runq)290{291 struct nvkm_device *device = runq->fifo->engine.subdev.device;292 293 return !(nvkm_rd32(device, 0x003080 + (runq->id * 4)) & 0x0000e000);294}295 296static const struct nvkm_bitfield297gk104_runq_intr_1_names[] = {298 { 0x00000001, "HCE_RE_ILLEGAL_OP" },299 { 0x00000002, "HCE_RE_ALIGNB" },300 { 0x00000004, "HCE_PRIV" },301 { 0x00000008, "HCE_ILLEGAL_MTHD" },302 { 0x00000010, "HCE_ILLEGAL_CLASS" },303 {}304};305 306static bool307gk104_runq_intr_1(struct nvkm_runq *runq)308{309 struct nvkm_subdev *subdev = &runq->fifo->engine.subdev;310 struct nvkm_device *device = subdev->device;311 u32 mask = nvkm_rd32(device, 0x04014c + (runq->id * 0x2000));312 u32 stat = nvkm_rd32(device, 0x040148 + (runq->id * 0x2000)) & mask;313 u32 chid = nvkm_rd32(device, 0x040120 + (runq->id * 0x2000)) & 0xfff;314 char msg[128];315 316 if (stat & 0x80000000) {317 if (runq->func->intr_1_ctxnotvalid &&318 runq->func->intr_1_ctxnotvalid(runq, chid))319 stat &= ~0x80000000;320 }321 322 if (stat) {323 nvkm_snprintbf(msg, sizeof(msg), gk104_runq_intr_1_names, stat);324 nvkm_error(subdev, "PBDMA%d: %08x [%s] ch %d %08x %08x\n",325 runq->id, stat, msg, chid,326 nvkm_rd32(device, 0x040150 + (runq->id * 0x2000)),327 nvkm_rd32(device, 0x040154 + (runq->id * 0x2000)));328 }329 330 nvkm_wr32(device, 0x040148 + (runq->id * 0x2000), stat);331 return true;332}333 334const struct nvkm_bitfield335gk104_runq_intr_0_names[] = {336 { 0x00000001, "MEMREQ" },337 { 0x00000002, "MEMACK_TIMEOUT" },338 { 0x00000004, "MEMACK_EXTRA" },339 { 0x00000008, "MEMDAT_TIMEOUT" },340 { 0x00000010, "MEMDAT_EXTRA" },341 { 0x00000020, "MEMFLUSH" },342 { 0x00000040, "MEMOP" },343 { 0x00000080, "LBCONNECT" },344 { 0x00000100, "LBREQ" },345 { 0x00000200, "LBACK_TIMEOUT" },346 { 0x00000400, "LBACK_EXTRA" },347 { 0x00000800, "LBDAT_TIMEOUT" },348 { 0x00001000, "LBDAT_EXTRA" },349 { 0x00002000, "GPFIFO" },350 { 0x00004000, "GPPTR" },351 { 0x00008000, "GPENTRY" },352 { 0x00010000, "GPCRC" },353 { 0x00020000, "PBPTR" },354 { 0x00040000, "PBENTRY" },355 { 0x00080000, "PBCRC" },356 { 0x00100000, "XBARCONNECT" },357 { 0x00200000, "METHOD" },358 { 0x00400000, "METHODCRC" },359 { 0x00800000, "DEVICE" },360 { 0x02000000, "SEMAPHORE" },361 { 0x04000000, "ACQUIRE" },362 { 0x08000000, "PRI" },363 { 0x20000000, "NO_CTXSW_SEG" },364 { 0x40000000, "PBSEG" },365 { 0x80000000, "SIGNATURE" },366 {}367};368 369bool370gk104_runq_intr(struct nvkm_runq *runq, struct nvkm_runl *null)371{372 bool intr0 = gf100_runq_intr(runq, NULL);373 bool intr1 = gk104_runq_intr_1(runq);374 375 return intr0 || intr1;376}377 378void379gk104_runq_init(struct nvkm_runq *runq)380{381 struct nvkm_device *device = runq->fifo->engine.subdev.device;382 383 gf100_runq_init(runq);384 385 nvkm_wr32(device, 0x040148 + (runq->id * 0x2000), 0xffffffff); /* HCE.INTR */386 nvkm_wr32(device, 0x04014c + (runq->id * 0x2000), 0xffffffff); /* HCE.INTREN */387}388 389static u32390gk104_runq_runm(struct nvkm_runq *runq)391{392 return nvkm_rd32(runq->fifo->engine.subdev.device, 0x002390 + (runq->id * 0x04));393}394 395const struct nvkm_runq_func396gk104_runq = {397 .init = gk104_runq_init,398 .intr = gk104_runq_intr,399 .intr_0_names = gk104_runq_intr_0_names,400 .idle = gk104_runq_idle,401};402 403void404gk104_runl_fault_clear(struct nvkm_runl *runl)405{406 nvkm_wr32(runl->fifo->engine.subdev.device, 0x00262c, BIT(runl->id));407}408 409void410gk104_runl_allow(struct nvkm_runl *runl, u32 engm)411{412 nvkm_mask(runl->fifo->engine.subdev.device, 0x002630, BIT(runl->id), 0x00000000);413}414 415void416gk104_runl_block(struct nvkm_runl *runl, u32 engm)417{418 nvkm_mask(runl->fifo->engine.subdev.device, 0x002630, BIT(runl->id), BIT(runl->id));419}420 421bool422gk104_runl_pending(struct nvkm_runl *runl)423{424 struct nvkm_device *device = runl->fifo->engine.subdev.device;425 426 return nvkm_rd32(device, 0x002284 + (runl->id * 0x08)) & 0x00100000;427}428 429void430gk104_runl_commit(struct nvkm_runl *runl, struct nvkm_memory *memory, u32 start, int count)431{432 struct nvkm_fifo *fifo = runl->fifo;433 struct nvkm_device *device = fifo->engine.subdev.device;434 u64 addr = nvkm_memory_addr(memory) + start;435 int target;436 437 switch (nvkm_memory_target(memory)) {438 case NVKM_MEM_TARGET_VRAM: target = 0; break;439 case NVKM_MEM_TARGET_NCOH: target = 3; break;440 default:441 WARN_ON(1);442 return;443 }444 445 spin_lock_irq(&fifo->lock);446 nvkm_wr32(device, 0x002270, (target << 28) | (addr >> 12));447 nvkm_wr32(device, 0x002274, (runl->id << 20) | count);448 spin_unlock_irq(&fifo->lock);449}450 451void452gk104_runl_insert_chan(struct nvkm_chan *chan, struct nvkm_memory *memory, u64 offset)453{454 nvkm_wo32(memory, offset + 0, chan->id);455 nvkm_wo32(memory, offset + 4, 0x00000000);456}457 458static const struct nvkm_runl_func459gk104_runl = {460 .size = 8,461 .update = nv50_runl_update,462 .insert_chan = gk104_runl_insert_chan,463 .commit = gk104_runl_commit,464 .wait = nv50_runl_wait,465 .pending = gk104_runl_pending,466 .block = gk104_runl_block,467 .allow = gk104_runl_allow,468 .fault_clear = gk104_runl_fault_clear,469 .preempt_pending = gf100_runl_preempt_pending,470};471 472static const struct nvkm_enum473gk104_fifo_mmu_fault_engine[] = {474 { 0x00, "GR", NULL, NVKM_ENGINE_GR },475 { 0x01, "DISPLAY" },476 { 0x02, "CAPTURE" },477 { 0x03, "IFB", NULL, NVKM_ENGINE_IFB },478 { 0x04, "BAR1", NULL, NVKM_SUBDEV_BAR },479 { 0x05, "BAR2", NULL, NVKM_SUBDEV_INSTMEM },480 { 0x06, "SCHED" },481 { 0x07, "HOST0" },482 { 0x08, "HOST1" },483 { 0x09, "HOST2" },484 { 0x0a, "HOST3" },485 { 0x0b, "HOST4" },486 { 0x0c, "HOST5" },487 { 0x0d, "HOST6" },488 { 0x0e, "HOST7" },489 { 0x0f, "HOSTSR" },490 { 0x10, "MSVLD", NULL, NVKM_ENGINE_MSVLD },491 { 0x11, "MSPPP", NULL, NVKM_ENGINE_MSPPP },492 { 0x13, "PERF" },493 { 0x14, "MSPDEC", NULL, NVKM_ENGINE_MSPDEC },494 { 0x15, "CE0", NULL, NVKM_ENGINE_CE, 0 },495 { 0x16, "CE1", NULL, NVKM_ENGINE_CE, 1 },496 { 0x17, "PMU" },497 { 0x18, "PTP" },498 { 0x19, "MSENC", NULL, NVKM_ENGINE_MSENC },499 { 0x1b, "CE2", NULL, NVKM_ENGINE_CE, 2 },500 {}501};502 503const struct nvkm_enum504gk104_fifo_mmu_fault_reason[] = {505 { 0x00, "PDE" },506 { 0x01, "PDE_SIZE" },507 { 0x02, "PTE" },508 { 0x03, "VA_LIMIT_VIOLATION" },509 { 0x04, "UNBOUND_INST_BLOCK" },510 { 0x05, "PRIV_VIOLATION" },511 { 0x06, "RO_VIOLATION" },512 { 0x07, "WO_VIOLATION" },513 { 0x08, "PITCH_MASK_VIOLATION" },514 { 0x09, "WORK_CREATION" },515 { 0x0a, "UNSUPPORTED_APERTURE" },516 { 0x0b, "COMPRESSION_FAILURE" },517 { 0x0c, "UNSUPPORTED_KIND" },518 { 0x0d, "REGION_VIOLATION" },519 { 0x0e, "BOTH_PTES_VALID" },520 { 0x0f, "INFO_TYPE_POISONED" },521 {}522};523 524const struct nvkm_enum525gk104_fifo_mmu_fault_hubclient[] = {526 { 0x00, "VIP" },527 { 0x01, "CE0" },528 { 0x02, "CE1" },529 { 0x03, "DNISO" },530 { 0x04, "FE" },531 { 0x05, "FECS" },532 { 0x06, "HOST" },533 { 0x07, "HOST_CPU" },534 { 0x08, "HOST_CPU_NB" },535 { 0x09, "ISO" },536 { 0x0a, "MMU" },537 { 0x0b, "MSPDEC" },538 { 0x0c, "MSPPP" },539 { 0x0d, "MSVLD" },540 { 0x0e, "NISO" },541 { 0x0f, "P2P" },542 { 0x10, "PD" },543 { 0x11, "PERF" },544 { 0x12, "PMU" },545 { 0x13, "RASTERTWOD" },546 { 0x14, "SCC" },547 { 0x15, "SCC_NB" },548 { 0x16, "SEC" },549 { 0x17, "SSYNC" },550 { 0x18, "GR_CE" },551 { 0x19, "CE2" },552 { 0x1a, "XV" },553 { 0x1b, "MMU_NB" },554 { 0x1c, "MSENC" },555 { 0x1d, "DFALCON" },556 { 0x1e, "SKED" },557 { 0x1f, "AFALCON" },558 {}559};560 561const struct nvkm_enum562gk104_fifo_mmu_fault_gpcclient[] = {563 { 0x00, "L1_0" }, { 0x01, "T1_0" }, { 0x02, "PE_0" },564 { 0x03, "L1_1" }, { 0x04, "T1_1" }, { 0x05, "PE_1" },565 { 0x06, "L1_2" }, { 0x07, "T1_2" }, { 0x08, "PE_2" },566 { 0x09, "L1_3" }, { 0x0a, "T1_3" }, { 0x0b, "PE_3" },567 { 0x0c, "RAST" },568 { 0x0d, "GCC" },569 { 0x0e, "GPCCS" },570 { 0x0f, "PROP_0" },571 { 0x10, "PROP_1" },572 { 0x11, "PROP_2" },573 { 0x12, "PROP_3" },574 { 0x13, "L1_4" }, { 0x14, "T1_4" }, { 0x15, "PE_4" },575 { 0x16, "L1_5" }, { 0x17, "T1_5" }, { 0x18, "PE_5" },576 { 0x19, "L1_6" }, { 0x1a, "T1_6" }, { 0x1b, "PE_6" },577 { 0x1c, "L1_7" }, { 0x1d, "T1_7" }, { 0x1e, "PE_7" },578 { 0x1f, "GPM" },579 { 0x20, "LTP_UTLB_0" },580 { 0x21, "LTP_UTLB_1" },581 { 0x22, "LTP_UTLB_2" },582 { 0x23, "LTP_UTLB_3" },583 { 0x24, "GPC_RGG_UTLB" },584 {}585};586 587const struct nvkm_fifo_func_mmu_fault588gk104_fifo_mmu_fault = {589 .recover = gf100_fifo_mmu_fault_recover,590 .access = gf100_fifo_mmu_fault_access,591 .engine = gk104_fifo_mmu_fault_engine,592 .reason = gk104_fifo_mmu_fault_reason,593 .hubclient = gk104_fifo_mmu_fault_hubclient,594 .gpcclient = gk104_fifo_mmu_fault_gpcclient,595};596 597static const struct nvkm_enum598gk104_fifo_intr_bind_reason[] = {599 { 0x01, "BIND_NOT_UNBOUND" },600 { 0x02, "SNOOP_WITHOUT_BAR1" },601 { 0x03, "UNBIND_WHILE_RUNNING" },602 { 0x05, "INVALID_RUNLIST" },603 { 0x06, "INVALID_CTX_TGT" },604 { 0x0b, "UNBIND_WHILE_PARKED" },605 {}606};607 608void609gk104_fifo_intr_bind(struct nvkm_fifo *fifo)610{611 struct nvkm_subdev *subdev = &fifo->engine.subdev;612 u32 intr = nvkm_rd32(subdev->device, 0x00252c);613 u32 code = intr & 0x000000ff;614 const struct nvkm_enum *en = nvkm_enum_find(gk104_fifo_intr_bind_reason, code);615 616 nvkm_error(subdev, "BIND_ERROR %02x [%s]\n", code, en ? en->name : "");617}618 619void620gk104_fifo_intr_chsw(struct nvkm_fifo *fifo)621{622 struct nvkm_subdev *subdev = &fifo->engine.subdev;623 struct nvkm_device *device = subdev->device;624 u32 stat = nvkm_rd32(device, 0x00256c);625 626 nvkm_error(subdev, "CHSW_ERROR %08x\n", stat);627 nvkm_wr32(device, 0x00256c, stat);628}629 630static void631gk104_fifo_intr_dropped_fault(struct nvkm_fifo *fifo)632{633 struct nvkm_subdev *subdev = &fifo->engine.subdev;634 u32 stat = nvkm_rd32(subdev->device, 0x00259c);635 636 nvkm_error(subdev, "DROPPED_MMU_FAULT %08x\n", stat);637}638 639void640gk104_fifo_intr_runlist(struct nvkm_fifo *fifo)641{642 struct nvkm_device *device = fifo->engine.subdev.device;643 struct nvkm_runl *runl;644 u32 mask = nvkm_rd32(device, 0x002a00);645 646 nvkm_runl_foreach_cond(runl, fifo, mask & BIT(runl->id)) {647 nvkm_wr32(device, 0x002a00, BIT(runl->id));648 }649}650 651irqreturn_t652gk104_fifo_intr(struct nvkm_inth *inth)653{654 struct nvkm_fifo *fifo = container_of(inth, typeof(*fifo), engine.subdev.inth);655 struct nvkm_subdev *subdev = &fifo->engine.subdev;656 struct nvkm_device *device = subdev->device;657 u32 mask = nvkm_rd32(device, 0x002140);658 u32 stat = nvkm_rd32(device, 0x002100) & mask;659 660 if (stat & 0x00000001) {661 gk104_fifo_intr_bind(fifo);662 nvkm_wr32(device, 0x002100, 0x00000001);663 stat &= ~0x00000001;664 }665 666 if (stat & 0x00000010) {667 nvkm_error(subdev, "PIO_ERROR\n");668 nvkm_wr32(device, 0x002100, 0x00000010);669 stat &= ~0x00000010;670 }671 672 if (stat & 0x00000100) {673 gf100_fifo_intr_sched(fifo);674 nvkm_wr32(device, 0x002100, 0x00000100);675 stat &= ~0x00000100;676 }677 678 if (stat & 0x00010000) {679 gk104_fifo_intr_chsw(fifo);680 nvkm_wr32(device, 0x002100, 0x00010000);681 stat &= ~0x00010000;682 }683 684 if (stat & 0x00800000) {685 nvkm_error(subdev, "FB_FLUSH_TIMEOUT\n");686 nvkm_wr32(device, 0x002100, 0x00800000);687 stat &= ~0x00800000;688 }689 690 if (stat & 0x01000000) {691 nvkm_error(subdev, "LB_ERROR\n");692 nvkm_wr32(device, 0x002100, 0x01000000);693 stat &= ~0x01000000;694 }695 696 if (stat & 0x08000000) {697 gk104_fifo_intr_dropped_fault(fifo);698 nvkm_wr32(device, 0x002100, 0x08000000);699 stat &= ~0x08000000;700 }701 702 if (stat & 0x10000000) {703 gf100_fifo_intr_mmu_fault(fifo);704 stat &= ~0x10000000;705 }706 707 if (stat & 0x20000000) {708 if (gf100_fifo_intr_pbdma(fifo))709 stat &= ~0x20000000;710 }711 712 if (stat & 0x40000000) {713 gk104_fifo_intr_runlist(fifo);714 stat &= ~0x40000000;715 }716 717 if (stat & 0x80000000) {718 nvkm_wr32(device, 0x002100, 0x80000000);719 nvkm_event_ntfy(&fifo->nonstall.event, 0, NVKM_FIFO_NONSTALL_EVENT);720 stat &= ~0x80000000;721 }722 723 if (stat) {724 nvkm_error(subdev, "INTR %08x\n", stat);725 spin_lock(&fifo->lock);726 nvkm_mask(device, 0x002140, stat, 0x00000000);727 spin_unlock(&fifo->lock);728 nvkm_wr32(device, 0x002100, stat);729 }730 731 return IRQ_HANDLED;732}733 734void735gk104_fifo_init_pbdmas(struct nvkm_fifo *fifo, u32 mask)736{737 struct nvkm_device *device = fifo->engine.subdev.device;738 739 nvkm_wr32(device, 0x000204, mask);740 nvkm_mask(device, 0x002a04, 0xbfffffff, 0xbfffffff);741}742 743void744gk104_fifo_init(struct nvkm_fifo *fifo)745{746 struct nvkm_device *device = fifo->engine.subdev.device;747 748 if (fifo->func->chan.func->userd->bar == 1)749 nvkm_wr32(device, 0x002254, 0x10000000 | fifo->userd.bar1->addr >> 12);750 751 nvkm_wr32(device, 0x002100, 0xffffffff);752 nvkm_wr32(device, 0x002140, 0x7fffffff);753}754 755int756gk104_fifo_runl_ctor(struct nvkm_fifo *fifo)757{758 struct nvkm_device *device = fifo->engine.subdev.device;759 struct nvkm_top_device *tdev;760 struct nvkm_runl *runl;761 struct nvkm_runq *runq;762 const struct nvkm_engn_func *func;763 764 nvkm_list_foreach(tdev, &device->top->device, head, tdev->runlist >= 0) {765 runl = nvkm_runl_get(fifo, tdev->runlist, tdev->runlist);766 if (!runl) {767 runl = nvkm_runl_new(fifo, tdev->runlist, tdev->runlist, 0);768 if (IS_ERR(runl))769 return PTR_ERR(runl);770 771 nvkm_runq_foreach_cond(runq, fifo, gk104_runq_runm(runq) & BIT(runl->id)) {772 if (WARN_ON(runl->runq_nr == ARRAY_SIZE(runl->runq)))773 return -ENOMEM;774 775 runl->runq[runl->runq_nr++] = runq;776 }777 778 }779 780 if (tdev->engine < 0)781 continue;782 783 switch (tdev->type) {784 case NVKM_ENGINE_CE:785 func = fifo->func->engn_ce;786 break;787 case NVKM_ENGINE_GR:788 nvkm_runl_add(runl, 15, &gf100_engn_sw, NVKM_ENGINE_SW, 0);789 fallthrough;790 default:791 func = fifo->func->engn;792 break;793 }794 795 nvkm_runl_add(runl, tdev->engine, func, tdev->type, tdev->inst);796 }797 798 return 0;799}800 801int802gk104_fifo_chid_nr(struct nvkm_fifo *fifo)803{804 return 4096;805}806 807static const struct nvkm_fifo_func808gk104_fifo = {809 .chid_nr = gk104_fifo_chid_nr,810 .chid_ctor = gf100_fifo_chid_ctor,811 .runq_nr = gf100_fifo_runq_nr,812 .runl_ctor = gk104_fifo_runl_ctor,813 .init = gk104_fifo_init,814 .init_pbdmas = gk104_fifo_init_pbdmas,815 .intr = gk104_fifo_intr,816 .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit,817 .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout,818 .mmu_fault = &gk104_fifo_mmu_fault,819 .nonstall = &gf100_fifo_nonstall,820 .runl = &gk104_runl,821 .runq = &gk104_runq,822 .engn = &gk104_engn,823 .engn_ce = &gk104_engn_ce,824 .cgrp = {{ }, &nv04_cgrp },825 .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk104_chan },826};827 828int829gk104_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst,830 struct nvkm_fifo **pfifo)831{832 return nvkm_fifo_new_(&gk104_fifo, device, type, inst, pfifo);833}834