399 lines · c
1/*2 * Copyright 2012 Red Hat Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 * Authors: Ben Skeggs23 */24#include "priv.h"25#include "chan.h"26#include "chid.h"27#include "runl.h"28#include "runq.h"29 30#include <core/gpuobj.h>31#include <subdev/bar.h>32#include <subdev/mc.h>33#include <subdev/mmu.h>34 35#include <nvif/cl0080.h>36#include <nvif/unpack.h>37 38bool39nvkm_fifo_ctxsw_in_progress(struct nvkm_engine *engine)40{41 struct nvkm_runl *runl;42 struct nvkm_engn *engn;43 44 nvkm_runl_foreach(runl, engine->subdev.device->fifo) {45 nvkm_runl_foreach_engn(engn, runl) {46 if (engn->engine == engine)47 return engn->func->chsw ? engn->func->chsw(engn) : false;48 }49 }50 51 return false;52}53 54void55nvkm_fifo_pause(struct nvkm_fifo *fifo, unsigned long *flags)56{57 return fifo->func->pause(fifo, flags);58}59 60void61nvkm_fifo_start(struct nvkm_fifo *fifo, unsigned long *flags)62{63 return fifo->func->start(fifo, flags);64}65 66void67nvkm_fifo_fault(struct nvkm_fifo *fifo, struct nvkm_fault_data *info)68{69 return fifo->func->mmu_fault->recover(fifo, info);70}71 72static int73nvkm_fifo_class_new(struct nvkm_device *device, const struct nvkm_oclass *oclass,74 void *argv, u32 argc, struct nvkm_object **pobject)75{76 struct nvkm_fifo *fifo = nvkm_fifo(oclass->engine);77 78 if (oclass->engn == &fifo->func->cgrp.user)79 return nvkm_ucgrp_new(fifo, oclass, argv, argc, pobject);80 81 if (oclass->engn == &fifo->func->chan.user)82 return nvkm_uchan_new(fifo, NULL, oclass, argv, argc, pobject);83 84 WARN_ON(1);85 return -ENOSYS;86}87 88static const struct nvkm_device_oclass89nvkm_fifo_class = {90 .ctor = nvkm_fifo_class_new,91};92 93static int94nvkm_fifo_class_get(struct nvkm_oclass *oclass, int index, const struct nvkm_device_oclass **class)95{96 struct nvkm_fifo *fifo = nvkm_fifo(oclass->engine);97 const struct nvkm_fifo_func_cgrp *cgrp = &fifo->func->cgrp;98 const struct nvkm_fifo_func_chan *chan = &fifo->func->chan;99 int c = 0;100 101 /* *_CHANNEL_GROUP_* */102 if (cgrp->user.oclass) {103 if (c++ == index) {104 oclass->base = cgrp->user;105 oclass->engn = &fifo->func->cgrp.user;106 *class = &nvkm_fifo_class;107 return 0;108 }109 }110 111 /* *_CHANNEL_DMA, *_CHANNEL_GPFIFO_* */112 if (chan->user.oclass) {113 if (c++ == index) {114 oclass->base = chan->user;115 oclass->engn = &fifo->func->chan.user;116 *class = &nvkm_fifo_class;117 return 0;118 }119 }120 121 return c;122}123 124static int125nvkm_fifo_fini(struct nvkm_engine *engine, bool suspend)126{127 struct nvkm_fifo *fifo = nvkm_fifo(engine);128 struct nvkm_runl *runl;129 130 nvkm_inth_block(&fifo->engine.subdev.inth);131 132 nvkm_runl_foreach(runl, fifo)133 nvkm_runl_fini(runl);134 135 return 0;136}137 138static int139nvkm_fifo_init(struct nvkm_engine *engine)140{141 struct nvkm_fifo *fifo = nvkm_fifo(engine);142 struct nvkm_runq *runq;143 struct nvkm_runl *runl;144 u32 mask = 0;145 146 if (fifo->func->init_pbdmas) {147 nvkm_runq_foreach(runq, fifo)148 mask |= BIT(runq->id);149 150 fifo->func->init_pbdmas(fifo, mask);151 152 nvkm_runq_foreach(runq, fifo)153 runq->func->init(runq);154 }155 156 nvkm_runl_foreach(runl, fifo) {157 if (runl->func->init)158 runl->func->init(runl);159 }160 161 if (fifo->func->init)162 fifo->func->init(fifo);163 164 nvkm_inth_allow(&fifo->engine.subdev.inth);165 return 0;166}167 168static int169nvkm_fifo_info(struct nvkm_engine *engine, u64 mthd, u64 *data)170{171 struct nvkm_fifo *fifo = nvkm_fifo(engine);172 struct nvkm_runl *runl;173 struct nvkm_engn *engn;174 int ret;175 176 ret = nvkm_subdev_oneinit(&fifo->engine.subdev);177 if (ret)178 return ret;179 180 switch (mthd) {181 case NV_DEVICE_HOST_CHANNELS: *data = fifo->chid ? fifo->chid->nr : 0; return 0;182 case NV_DEVICE_HOST_RUNLISTS:183 *data = 0;184 nvkm_runl_foreach(runl, fifo)185 *data |= BIT(runl->id);186 return 0;187 case NV_DEVICE_HOST_RUNLIST_ENGINES:188 runl = nvkm_runl_get(fifo, *data, 0);189 if (runl) {190 *data = 0;191 nvkm_runl_foreach_engn(engn, runl) {192#define CASE(n) case NVKM_ENGINE_##n: *data |= NV_DEVICE_HOST_RUNLIST_ENGINES_##n; break193 switch (engn->engine->subdev.type) {194 case NVKM_ENGINE_DMAOBJ:195 break;196 CASE(SW );197 CASE(GR );198 CASE(MPEG );199 CASE(ME );200 CASE(CIPHER);201 CASE(BSP );202 CASE(VP );203 CASE(CE );204 CASE(SEC );205 CASE(MSVLD );206 CASE(MSPDEC);207 CASE(MSPPP );208 CASE(MSENC );209 CASE(VIC );210 CASE(SEC2 );211 CASE(NVDEC );212 CASE(NVENC );213 CASE(NVJPG );214 CASE(OFA );215 default:216 WARN_ON(1);217 break;218 }219#undef CASE220 }221 return 0;222 }223 return -EINVAL;224 case NV_DEVICE_HOST_RUNLIST_CHANNELS:225 if (!fifo->chid) {226 runl = nvkm_runl_get(fifo, *data, 0);227 if (runl) {228 *data = runl->chid->nr;229 return 0;230 }231 }232 return -EINVAL;233 default:234 break;235 }236 237 return -ENOSYS;238}239 240static int241nvkm_fifo_oneinit(struct nvkm_engine *engine)242{243 struct nvkm_subdev *subdev = &engine->subdev;244 struct nvkm_device *device = subdev->device;245 struct nvkm_fifo *fifo = nvkm_fifo(engine);246 struct nvkm_runl *runl;247 struct nvkm_engn *engn;248 int ret, nr, i;249 250 /* Initialise CHID/CGID allocator(s) on GPUs where they aren't per-runlist. */251 if (fifo->func->chid_nr) {252 ret = fifo->func->chid_ctor(fifo, fifo->func->chid_nr(fifo));253 if (ret)254 return ret;255 }256 257 /* Create runqueues for each PBDMA. */258 if (fifo->func->runq_nr) {259 for (nr = fifo->func->runq_nr(fifo), i = 0; i < nr; i++) {260 if (!nvkm_runq_new(fifo, i))261 return -ENOMEM;262 }263 }264 265 /* Create runlists. */266 ret = fifo->func->runl_ctor(fifo);267 if (ret)268 return ret;269 270 nvkm_runl_foreach(runl, fifo) {271 RUNL_DEBUG(runl, "chan:%06x", runl->chan);272 nvkm_runl_foreach_engn(engn, runl) {273 ENGN_DEBUG(engn, "");274 }275 }276 277 /* Register interrupt handler. */278 if (fifo->func->intr) {279 ret = nvkm_inth_add(&device->mc->intr, NVKM_INTR_SUBDEV, NVKM_INTR_PRIO_NORMAL,280 subdev, fifo->func->intr, &subdev->inth);281 if (ret) {282 nvkm_error(subdev, "intr %d\n", ret);283 return ret;284 }285 }286 287 /* Initialise non-stall intr handling. */288 if (fifo->func->nonstall) {289 if (fifo->func->nonstall_ctor) {290 ret = fifo->func->nonstall_ctor(fifo);291 if (ret < 0) {292 nvkm_error(subdev, "nonstall %d\n", ret);293 return ret;294 }295 } else {296 ret = 1;297 }298 299 ret = nvkm_event_init(fifo->func->nonstall, &fifo->engine.subdev, 1, ret,300 &fifo->nonstall.event);301 if (ret)302 return ret;303 }304 305 /* Allocate USERD + BAR1 polling area. */306 if (fifo->func->chan.func->userd->bar == 1) {307 struct nvkm_vmm *bar1 = nvkm_bar_bar1_vmm(device);308 309 ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, fifo->chid->nr *310 fifo->func->chan.func->userd->size, 0, true,311 &fifo->userd.mem);312 if (ret)313 return ret;314 315 ret = nvkm_vmm_get(bar1, 12, nvkm_memory_size(fifo->userd.mem), &fifo->userd.bar1);316 if (ret)317 return ret;318 319 ret = nvkm_memory_map(fifo->userd.mem, 0, bar1, fifo->userd.bar1, NULL, 0);320 if (ret)321 return ret;322 }323 324 return 0;325}326 327static void328nvkm_fifo_preinit(struct nvkm_engine *engine)329{330 nvkm_mc_reset(engine->subdev.device, NVKM_ENGINE_FIFO, 0);331}332 333static void *334nvkm_fifo_dtor(struct nvkm_engine *engine)335{336 struct nvkm_fifo *fifo = nvkm_fifo(engine);337 struct nvkm_runl *runl, *runt;338 struct nvkm_runq *runq, *rtmp;339 340 if (fifo->userd.bar1)341 nvkm_vmm_put(nvkm_bar_bar1_vmm(engine->subdev.device), &fifo->userd.bar1);342 nvkm_memory_unref(&fifo->userd.mem);343 344 list_for_each_entry_safe(runl, runt, &fifo->runls, head)345 nvkm_runl_del(runl);346 list_for_each_entry_safe(runq, rtmp, &fifo->runqs, head)347 nvkm_runq_del(runq);348 349 nvkm_chid_unref(&fifo->cgid);350 nvkm_chid_unref(&fifo->chid);351 352 mutex_destroy(&fifo->userd.mutex);353 354 nvkm_event_fini(&fifo->nonstall.event);355 mutex_destroy(&fifo->mutex);356 357 if (fifo->func->dtor)358 fifo->func->dtor(fifo);359 360 return fifo;361}362 363static const struct nvkm_engine_func364nvkm_fifo = {365 .dtor = nvkm_fifo_dtor,366 .preinit = nvkm_fifo_preinit,367 .oneinit = nvkm_fifo_oneinit,368 .info = nvkm_fifo_info,369 .init = nvkm_fifo_init,370 .fini = nvkm_fifo_fini,371 .base.sclass = nvkm_fifo_class_get,372};373 374int375nvkm_fifo_new_(const struct nvkm_fifo_func *func, struct nvkm_device *device,376 enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo)377{378 struct nvkm_fifo *fifo;379 380 if (!(fifo = *pfifo = kzalloc(sizeof(*fifo), GFP_KERNEL)))381 return -ENOMEM;382 383 fifo->func = func;384 INIT_LIST_HEAD(&fifo->runqs);385 INIT_LIST_HEAD(&fifo->runls);386 /*TODO: Needs to be >CTXSW_TIMEOUT, so RC can recover before this is hit.387 * CTXSW_TIMEOUT HW default seems to differ between GPUs, so just a388 * large number for now until we support changing it.389 */390 fifo->timeout.chan_msec = 10000;391 spin_lock_init(&fifo->lock);392 mutex_init(&fifo->mutex);393 394 INIT_LIST_HEAD(&fifo->userd.list);395 mutex_init(&fifo->userd.mutex);396 397 return nvkm_engine_ctor(&nvkm_fifo, device, type, inst, true, &fifo->engine);398}399