491 lines · c
1/*2 * Copyright 2012 Red Hat Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 * Authors: Ben Skeggs23 */24#include "chan.h"25#include "chid.h"26#include "cgrp.h"27#include "runl.h"28#include "priv.h"29 30#include <core/ramht.h>31#include <subdev/mmu.h>32#include <engine/dma.h>33 34#include <nvif/if0020.h>35 36const struct nvkm_event_func37nvkm_chan_event = {38};39 40void41nvkm_chan_cctx_bind(struct nvkm_chan *chan, struct nvkm_engn *engn, struct nvkm_cctx *cctx)42{43 struct nvkm_cgrp *cgrp = chan->cgrp;44 struct nvkm_runl *runl = cgrp->runl;45 struct nvkm_engine *engine = engn->engine;46 47 if (!engn->func->bind)48 return;49 50 CHAN_TRACE(chan, "%sbind cctx %d[%s]", cctx ? "" : "un", engn->id, engine->subdev.name);51 52 /* Prevent any channel in channel group from being rescheduled, kick them53 * off host and any engine(s) they're loaded on.54 */55 if (cgrp->hw)56 nvkm_runl_block(runl);57 else58 nvkm_chan_block(chan);59 nvkm_chan_preempt(chan, true);60 61 /* Update context pointer. */62 engn->func->bind(engn, cctx, chan);63 64 /* Resume normal operation. */65 if (cgrp->hw)66 nvkm_runl_allow(runl);67 else68 nvkm_chan_allow(chan);69}70 71void72nvkm_chan_cctx_put(struct nvkm_chan *chan, struct nvkm_cctx **pcctx)73{74 struct nvkm_cctx *cctx = *pcctx;75 76 if (cctx) {77 struct nvkm_engn *engn = cctx->vctx->ectx->engn;78 79 if (refcount_dec_and_mutex_lock(&cctx->refs, &chan->cgrp->mutex)) {80 CHAN_TRACE(chan, "dtor cctx %d[%s]", engn->id, engn->engine->subdev.name);81 nvkm_cgrp_vctx_put(chan->cgrp, &cctx->vctx);82 list_del(&cctx->head);83 kfree(cctx);84 mutex_unlock(&chan->cgrp->mutex);85 }86 87 *pcctx = NULL;88 }89}90 91int92nvkm_chan_cctx_get(struct nvkm_chan *chan, struct nvkm_engn *engn, struct nvkm_cctx **pcctx,93 struct nvkm_client *client)94{95 struct nvkm_cgrp *cgrp = chan->cgrp;96 struct nvkm_vctx *vctx;97 struct nvkm_cctx *cctx;98 int ret;99 100 /* Look for an existing channel context for this engine+VEID. */101 mutex_lock(&cgrp->mutex);102 cctx = nvkm_list_find(cctx, &chan->cctxs, head,103 cctx->vctx->ectx->engn == engn && cctx->vctx->vmm == chan->vmm);104 if (cctx) {105 refcount_inc(&cctx->refs);106 *pcctx = cctx;107 mutex_unlock(&chan->cgrp->mutex);108 return 0;109 }110 111 /* Nope - create a fresh one. But, sub-context first. */112 ret = nvkm_cgrp_vctx_get(cgrp, engn, chan, &vctx, client);113 if (ret) {114 CHAN_ERROR(chan, "vctx %d[%s]: %d", engn->id, engn->engine->subdev.name, ret);115 goto done;116 }117 118 /* Now, create the channel context - to track engine binding. */119 CHAN_TRACE(chan, "ctor cctx %d[%s]", engn->id, engn->engine->subdev.name);120 if (!(cctx = *pcctx = kzalloc(sizeof(*cctx), GFP_KERNEL))) {121 nvkm_cgrp_vctx_put(cgrp, &vctx);122 ret = -ENOMEM;123 goto done;124 }125 126 cctx->vctx = vctx;127 refcount_set(&cctx->refs, 1);128 refcount_set(&cctx->uses, 0);129 list_add_tail(&cctx->head, &chan->cctxs);130done:131 mutex_unlock(&cgrp->mutex);132 return ret;133}134 135int136nvkm_chan_preempt_locked(struct nvkm_chan *chan, bool wait)137{138 struct nvkm_runl *runl = chan->cgrp->runl;139 140 CHAN_TRACE(chan, "preempt");141 chan->func->preempt(chan);142 if (!wait)143 return 0;144 145 return nvkm_runl_preempt_wait(runl);146}147 148int149nvkm_chan_preempt(struct nvkm_chan *chan, bool wait)150{151 int ret;152 153 if (!chan->func->preempt)154 return 0;155 156 mutex_lock(&chan->cgrp->runl->mutex);157 ret = nvkm_chan_preempt_locked(chan, wait);158 mutex_unlock(&chan->cgrp->runl->mutex);159 return ret;160}161 162void163nvkm_chan_remove_locked(struct nvkm_chan *chan)164{165 struct nvkm_cgrp *cgrp = chan->cgrp;166 struct nvkm_runl *runl = cgrp->runl;167 168 if (list_empty(&chan->head))169 return;170 171 CHAN_TRACE(chan, "remove");172 if (!--cgrp->chan_nr) {173 runl->cgrp_nr--;174 list_del(&cgrp->head);175 }176 runl->chan_nr--;177 list_del_init(&chan->head);178 atomic_set(&runl->changed, 1);179}180 181void182nvkm_chan_remove(struct nvkm_chan *chan, bool preempt)183{184 struct nvkm_runl *runl = chan->cgrp->runl;185 186 mutex_lock(&runl->mutex);187 if (preempt && chan->func->preempt)188 nvkm_chan_preempt_locked(chan, true);189 nvkm_chan_remove_locked(chan);190 nvkm_runl_update_locked(runl, true);191 mutex_unlock(&runl->mutex);192}193 194void195nvkm_chan_insert(struct nvkm_chan *chan)196{197 struct nvkm_cgrp *cgrp = chan->cgrp;198 struct nvkm_runl *runl = cgrp->runl;199 200 mutex_lock(&runl->mutex);201 if (WARN_ON(!list_empty(&chan->head))) {202 mutex_unlock(&runl->mutex);203 return;204 }205 206 CHAN_TRACE(chan, "insert");207 list_add_tail(&chan->head, &cgrp->chans);208 runl->chan_nr++;209 if (!cgrp->chan_nr++) {210 list_add_tail(&cgrp->head, &cgrp->runl->cgrps);211 runl->cgrp_nr++;212 }213 atomic_set(&runl->changed, 1);214 nvkm_runl_update_locked(runl, true);215 mutex_unlock(&runl->mutex);216}217 218static void219nvkm_chan_block_locked(struct nvkm_chan *chan)220{221 CHAN_TRACE(chan, "block %d", atomic_read(&chan->blocked));222 if (atomic_inc_return(&chan->blocked) == 1)223 chan->func->stop(chan);224}225 226void227nvkm_chan_error(struct nvkm_chan *chan, bool preempt)228{229 unsigned long flags;230 231 spin_lock_irqsave(&chan->lock, flags);232 if (atomic_inc_return(&chan->errored) == 1) {233 CHAN_ERROR(chan, "errored - disabling channel");234 nvkm_chan_block_locked(chan);235 if (preempt)236 chan->func->preempt(chan);237 nvkm_event_ntfy(&chan->cgrp->runl->chid->event, chan->id, NVKM_CHAN_EVENT_ERRORED);238 }239 spin_unlock_irqrestore(&chan->lock, flags);240}241 242void243nvkm_chan_block(struct nvkm_chan *chan)244{245 spin_lock_irq(&chan->lock);246 nvkm_chan_block_locked(chan);247 spin_unlock_irq(&chan->lock);248}249 250void251nvkm_chan_allow(struct nvkm_chan *chan)252{253 spin_lock_irq(&chan->lock);254 CHAN_TRACE(chan, "allow %d", atomic_read(&chan->blocked));255 if (atomic_dec_and_test(&chan->blocked))256 chan->func->start(chan);257 spin_unlock_irq(&chan->lock);258}259 260void261nvkm_chan_del(struct nvkm_chan **pchan)262{263 struct nvkm_chan *chan = *pchan;264 265 if (!chan)266 return;267 268 if (chan->func->ramfc->clear)269 chan->func->ramfc->clear(chan);270 271 nvkm_ramht_del(&chan->ramht);272 nvkm_gpuobj_del(&chan->pgd);273 nvkm_gpuobj_del(&chan->eng);274 nvkm_gpuobj_del(&chan->cache);275 nvkm_gpuobj_del(&chan->ramfc);276 277 if (chan->cgrp) {278 if (!chan->func->id_put)279 nvkm_chid_put(chan->cgrp->runl->chid, chan->id, &chan->cgrp->lock);280 else281 chan->func->id_put(chan);282 283 nvkm_cgrp_unref(&chan->cgrp);284 }285 286 nvkm_memory_unref(&chan->userd.mem);287 288 if (chan->vmm) {289 nvkm_vmm_part(chan->vmm, chan->inst->memory);290 nvkm_vmm_unref(&chan->vmm);291 }292 293 nvkm_gpuobj_del(&chan->push);294 nvkm_gpuobj_del(&chan->inst);295 kfree(chan);296}297 298void299nvkm_chan_put(struct nvkm_chan **pchan, unsigned long irqflags)300{301 struct nvkm_chan *chan = *pchan;302 303 if (!chan)304 return;305 306 *pchan = NULL;307 spin_unlock_irqrestore(&chan->cgrp->lock, irqflags);308}309 310struct nvkm_chan *311nvkm_chan_get_inst(struct nvkm_engine *engine, u64 inst, unsigned long *pirqflags)312{313 struct nvkm_fifo *fifo = engine->subdev.device->fifo;314 struct nvkm_runl *runl;315 struct nvkm_engn *engn;316 struct nvkm_chan *chan;317 318 nvkm_runl_foreach(runl, fifo) {319 nvkm_runl_foreach_engn(engn, runl) {320 if (engine == &fifo->engine || engn->engine == engine) {321 chan = nvkm_runl_chan_get_inst(runl, inst, pirqflags);322 if (chan || engn->engine == engine)323 return chan;324 }325 }326 }327 328 return NULL;329}330 331struct nvkm_chan *332nvkm_chan_get_chid(struct nvkm_engine *engine, int id, unsigned long *pirqflags)333{334 struct nvkm_fifo *fifo = engine->subdev.device->fifo;335 struct nvkm_runl *runl;336 struct nvkm_engn *engn;337 338 nvkm_runl_foreach(runl, fifo) {339 nvkm_runl_foreach_engn(engn, runl) {340 if (fifo->chid || engn->engine == engine)341 return nvkm_runl_chan_get_chid(runl, id, pirqflags);342 }343 }344 345 return NULL;346}347 348int349nvkm_chan_new_(const struct nvkm_chan_func *func, struct nvkm_runl *runl, int runq,350 struct nvkm_cgrp *cgrp, const char *name, bool priv, u32 devm, struct nvkm_vmm *vmm,351 struct nvkm_dmaobj *dmaobj, u64 offset, u64 length,352 struct nvkm_memory *userd, u64 ouserd, struct nvkm_chan **pchan)353{354 struct nvkm_fifo *fifo = runl->fifo;355 struct nvkm_device *device = fifo->engine.subdev.device;356 struct nvkm_chan *chan;357 int ret;358 359 /* Validate arguments against class requirements. */360 if ((runq && runq >= runl->func->runqs) ||361 (!func->inst->vmm != !vmm) ||362 ((func->userd->bar < 0) == !userd) ||363 (!func->ramfc->ctxdma != !dmaobj) ||364 ((func->ramfc->devm < devm) && devm != BIT(0)) ||365 (!func->ramfc->priv && priv)) {366 RUNL_DEBUG(runl, "args runq:%d:%d vmm:%d:%p userd:%d:%p "367 "push:%d:%p devm:%08x:%08x priv:%d:%d",368 runl->func->runqs, runq, func->inst->vmm, vmm,369 func->userd->bar < 0, userd, func->ramfc->ctxdma, dmaobj,370 func->ramfc->devm, devm, func->ramfc->priv, priv);371 return -EINVAL;372 }373 374 if (!(chan = *pchan = kzalloc(sizeof(*chan), GFP_KERNEL)))375 return -ENOMEM;376 377 chan->func = func;378 strscpy(chan->name, name, sizeof(chan->name));379 chan->runq = runq;380 chan->id = -1;381 spin_lock_init(&chan->lock);382 atomic_set(&chan->blocked, 1);383 atomic_set(&chan->errored, 0);384 INIT_LIST_HEAD(&chan->cctxs);385 INIT_LIST_HEAD(&chan->head);386 387 /* Join channel group.388 *389 * GK110 and newer support channel groups (aka TSGs), where individual channels390 * share a timeslice, and, engine context(s).391 *392 * As such, engine contexts are tracked in nvkm_cgrp and we need them even when393 * channels aren't in an API channel group, and on HW that doesn't support TSGs.394 */395 if (!cgrp) {396 ret = nvkm_cgrp_new(runl, chan->name, vmm, fifo->func->cgrp.force, &chan->cgrp);397 if (ret) {398 RUNL_DEBUG(runl, "cgrp %d", ret);399 return ret;400 }401 402 cgrp = chan->cgrp;403 } else {404 if (cgrp->runl != runl || cgrp->vmm != vmm) {405 RUNL_DEBUG(runl, "cgrp %d %d", cgrp->runl != runl, cgrp->vmm != vmm);406 return -EINVAL;407 }408 409 chan->cgrp = nvkm_cgrp_ref(cgrp);410 }411 412 /* Allocate instance block. */413 ret = nvkm_gpuobj_new(device, func->inst->size, 0x1000, func->inst->zero, NULL,414 &chan->inst);415 if (ret) {416 RUNL_DEBUG(runl, "inst %d", ret);417 return ret;418 }419 420 /* Initialise virtual address-space. */421 if (func->inst->vmm) {422 if (WARN_ON(vmm->mmu != device->mmu))423 return -EINVAL;424 425 ret = nvkm_vmm_join(vmm, chan->inst->memory);426 if (ret) {427 RUNL_DEBUG(runl, "vmm %d", ret);428 return ret;429 }430 431 chan->vmm = nvkm_vmm_ref(vmm);432 }433 434 /* Allocate HW ctxdma for push buffer. */435 if (func->ramfc->ctxdma) {436 ret = nvkm_object_bind(&dmaobj->object, chan->inst, -16, &chan->push);437 if (ret) {438 RUNL_DEBUG(runl, "bind %d", ret);439 return ret;440 }441 }442 443 /* Allocate channel ID. */444 if (!chan->func->id_get) {445 chan->id = nvkm_chid_get(runl->chid, chan);446 if (chan->id >= 0) {447 if (func->userd->bar < 0) {448 if (ouserd + chan->func->userd->size >=449 nvkm_memory_size(userd)) {450 RUNL_DEBUG(runl, "ouserd %llx", ouserd);451 return -EINVAL;452 }453 454 ret = nvkm_memory_kmap(userd, &chan->userd.mem);455 if (ret) {456 RUNL_DEBUG(runl, "userd %d", ret);457 return ret;458 }459 460 chan->userd.base = ouserd;461 } else {462 chan->userd.mem = nvkm_memory_ref(fifo->userd.mem);463 chan->userd.base = chan->id * chan->func->userd->size;464 }465 }466 } else {467 chan->id = chan->func->id_get(chan, userd, ouserd);468 }469 470 if (chan->id < 0) {471 RUNL_ERROR(runl, "!chids");472 return -ENOSPC;473 }474 475 if (cgrp->id < 0)476 cgrp->id = chan->id;477 478 /* Initialise USERD. */479 if (chan->func->userd->clear)480 chan->func->userd->clear(chan);481 482 /* Initialise RAMFC. */483 ret = chan->func->ramfc->write(chan, offset, length, devm, priv);484 if (ret) {485 RUNL_DEBUG(runl, "ramfc %d", ret);486 return ret;487 }488 489 return 0;490}491