2660 lines · c
1/*2 * Copyright 2012 Red Hat Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 * Authors: Ben Skeggs23 */24#include "gf100.h"25#include "ctxgf100.h"26#include "fuc/os.h"27 28#include <core/client.h>29#include <core/firmware.h>30#include <core/option.h>31#include <subdev/acr.h>32#include <subdev/fb.h>33#include <subdev/mc.h>34#include <subdev/pmu.h>35#include <subdev/therm.h>36#include <subdev/timer.h>37#include <engine/fifo.h>38 39#include <nvif/class.h>40#include <nvif/cl9097.h>41#include <nvif/if900d.h>42#include <nvif/unpack.h>43 44/*******************************************************************************45 * Zero Bandwidth Clear46 ******************************************************************************/47 48static void49gf100_gr_zbc_clear_color(struct gf100_gr *gr, int zbc)50{51 struct nvkm_device *device = gr->base.engine.subdev.device;52 if (gr->zbc_color[zbc].format) {53 nvkm_wr32(device, 0x405804, gr->zbc_color[zbc].ds[0]);54 nvkm_wr32(device, 0x405808, gr->zbc_color[zbc].ds[1]);55 nvkm_wr32(device, 0x40580c, gr->zbc_color[zbc].ds[2]);56 nvkm_wr32(device, 0x405810, gr->zbc_color[zbc].ds[3]);57 }58 nvkm_wr32(device, 0x405814, gr->zbc_color[zbc].format);59 nvkm_wr32(device, 0x405820, zbc);60 nvkm_wr32(device, 0x405824, 0x00000004); /* TRIGGER | WRITE | COLOR */61}62 63static int64gf100_gr_zbc_color_get(struct gf100_gr *gr, int format,65 const u32 ds[4], const u32 l2[4])66{67 struct nvkm_ltc *ltc = gr->base.engine.subdev.device->ltc;68 int zbc = -ENOSPC, i;69 70 for (i = ltc->zbc_color_min; i <= ltc->zbc_color_max; i++) {71 if (gr->zbc_color[i].format) {72 if (gr->zbc_color[i].format != format)73 continue;74 if (memcmp(gr->zbc_color[i].ds, ds, sizeof(75 gr->zbc_color[i].ds)))76 continue;77 if (memcmp(gr->zbc_color[i].l2, l2, sizeof(78 gr->zbc_color[i].l2))) {79 WARN_ON(1);80 return -EINVAL;81 }82 return i;83 } else {84 zbc = (zbc < 0) ? i : zbc;85 }86 }87 88 if (zbc < 0)89 return zbc;90 91 memcpy(gr->zbc_color[zbc].ds, ds, sizeof(gr->zbc_color[zbc].ds));92 memcpy(gr->zbc_color[zbc].l2, l2, sizeof(gr->zbc_color[zbc].l2));93 gr->zbc_color[zbc].format = format;94 nvkm_ltc_zbc_color_get(ltc, zbc, l2);95 gr->func->zbc->clear_color(gr, zbc);96 return zbc;97}98 99static void100gf100_gr_zbc_clear_depth(struct gf100_gr *gr, int zbc)101{102 struct nvkm_device *device = gr->base.engine.subdev.device;103 if (gr->zbc_depth[zbc].format)104 nvkm_wr32(device, 0x405818, gr->zbc_depth[zbc].ds);105 nvkm_wr32(device, 0x40581c, gr->zbc_depth[zbc].format);106 nvkm_wr32(device, 0x405820, zbc);107 nvkm_wr32(device, 0x405824, 0x00000005); /* TRIGGER | WRITE | DEPTH */108}109 110static int111gf100_gr_zbc_depth_get(struct gf100_gr *gr, int format,112 const u32 ds, const u32 l2)113{114 struct nvkm_ltc *ltc = gr->base.engine.subdev.device->ltc;115 int zbc = -ENOSPC, i;116 117 for (i = ltc->zbc_depth_min; i <= ltc->zbc_depth_max; i++) {118 if (gr->zbc_depth[i].format) {119 if (gr->zbc_depth[i].format != format)120 continue;121 if (gr->zbc_depth[i].ds != ds)122 continue;123 if (gr->zbc_depth[i].l2 != l2) {124 WARN_ON(1);125 return -EINVAL;126 }127 return i;128 } else {129 zbc = (zbc < 0) ? i : zbc;130 }131 }132 133 if (zbc < 0)134 return zbc;135 136 gr->zbc_depth[zbc].format = format;137 gr->zbc_depth[zbc].ds = ds;138 gr->zbc_depth[zbc].l2 = l2;139 nvkm_ltc_zbc_depth_get(ltc, zbc, l2);140 gr->func->zbc->clear_depth(gr, zbc);141 return zbc;142}143 144const struct gf100_gr_func_zbc145gf100_gr_zbc = {146 .clear_color = gf100_gr_zbc_clear_color,147 .clear_depth = gf100_gr_zbc_clear_depth,148};149 150/*******************************************************************************151 * Graphics object classes152 ******************************************************************************/153#define gf100_gr_object(p) container_of((p), struct gf100_gr_object, object)154 155struct gf100_gr_object {156 struct nvkm_object object;157 struct gf100_gr_chan *chan;158};159 160static int161gf100_fermi_mthd_zbc_color(struct nvkm_object *object, void *data, u32 size)162{163 struct gf100_gr *gr = gf100_gr(nvkm_gr(object->engine));164 union {165 struct fermi_a_zbc_color_v0 v0;166 } *args = data;167 int ret = -ENOSYS;168 169 if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {170 switch (args->v0.format) {171 case FERMI_A_ZBC_COLOR_V0_FMT_ZERO:172 case FERMI_A_ZBC_COLOR_V0_FMT_UNORM_ONE:173 case FERMI_A_ZBC_COLOR_V0_FMT_RF32_GF32_BF32_AF32:174 case FERMI_A_ZBC_COLOR_V0_FMT_R16_G16_B16_A16:175 case FERMI_A_ZBC_COLOR_V0_FMT_RN16_GN16_BN16_AN16:176 case FERMI_A_ZBC_COLOR_V0_FMT_RS16_GS16_BS16_AS16:177 case FERMI_A_ZBC_COLOR_V0_FMT_RU16_GU16_BU16_AU16:178 case FERMI_A_ZBC_COLOR_V0_FMT_RF16_GF16_BF16_AF16:179 case FERMI_A_ZBC_COLOR_V0_FMT_A8R8G8B8:180 case FERMI_A_ZBC_COLOR_V0_FMT_A8RL8GL8BL8:181 case FERMI_A_ZBC_COLOR_V0_FMT_A2B10G10R10:182 case FERMI_A_ZBC_COLOR_V0_FMT_AU2BU10GU10RU10:183 case FERMI_A_ZBC_COLOR_V0_FMT_A8B8G8R8:184 case FERMI_A_ZBC_COLOR_V0_FMT_A8BL8GL8RL8:185 case FERMI_A_ZBC_COLOR_V0_FMT_AN8BN8GN8RN8:186 case FERMI_A_ZBC_COLOR_V0_FMT_AS8BS8GS8RS8:187 case FERMI_A_ZBC_COLOR_V0_FMT_AU8BU8GU8RU8:188 case FERMI_A_ZBC_COLOR_V0_FMT_A2R10G10B10:189 case FERMI_A_ZBC_COLOR_V0_FMT_BF10GF11RF11:190 ret = gf100_gr_zbc_color_get(gr, args->v0.format,191 args->v0.ds,192 args->v0.l2);193 if (ret >= 0) {194 args->v0.index = ret;195 return 0;196 }197 break;198 default:199 return -EINVAL;200 }201 }202 203 return ret;204}205 206static int207gf100_fermi_mthd_zbc_depth(struct nvkm_object *object, void *data, u32 size)208{209 struct gf100_gr *gr = gf100_gr(nvkm_gr(object->engine));210 union {211 struct fermi_a_zbc_depth_v0 v0;212 } *args = data;213 int ret = -ENOSYS;214 215 if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) {216 switch (args->v0.format) {217 case FERMI_A_ZBC_DEPTH_V0_FMT_FP32:218 ret = gf100_gr_zbc_depth_get(gr, args->v0.format,219 args->v0.ds,220 args->v0.l2);221 return (ret >= 0) ? 0 : -ENOSPC;222 default:223 return -EINVAL;224 }225 }226 227 return ret;228}229 230static int231gf100_fermi_mthd(struct nvkm_object *object, u32 mthd, void *data, u32 size)232{233 nvif_ioctl(object, "fermi mthd %08x\n", mthd);234 switch (mthd) {235 case FERMI_A_ZBC_COLOR:236 return gf100_fermi_mthd_zbc_color(object, data, size);237 case FERMI_A_ZBC_DEPTH:238 return gf100_fermi_mthd_zbc_depth(object, data, size);239 default:240 break;241 }242 return -EINVAL;243}244 245const struct nvkm_object_func246gf100_fermi = {247 .mthd = gf100_fermi_mthd,248};249 250static void251gf100_gr_mthd_set_shader_exceptions(struct nvkm_device *device, u32 data)252{253 nvkm_wr32(device, 0x419e44, data ? 0xffffffff : 0x00000000);254 nvkm_wr32(device, 0x419e4c, data ? 0xffffffff : 0x00000000);255}256 257static bool258gf100_gr_mthd_sw(struct nvkm_device *device, u16 class, u32 mthd, u32 data)259{260 switch (class & 0x00ff) {261 case 0x97:262 case 0xc0:263 switch (mthd) {264 case 0x1528:265 gf100_gr_mthd_set_shader_exceptions(device, data);266 return true;267 default:268 break;269 }270 break;271 default:272 break;273 }274 return false;275}276 277static const struct nvkm_object_func278gf100_gr_object_func = {279};280 281static int282gf100_gr_object_new(const struct nvkm_oclass *oclass, void *data, u32 size,283 struct nvkm_object **pobject)284{285 struct gf100_gr_chan *chan = gf100_gr_chan(oclass->parent);286 struct gf100_gr_object *object;287 288 if (!(object = kzalloc(sizeof(*object), GFP_KERNEL)))289 return -ENOMEM;290 *pobject = &object->object;291 292 nvkm_object_ctor(oclass->base.func ? oclass->base.func :293 &gf100_gr_object_func, oclass, &object->object);294 object->chan = chan;295 return 0;296}297 298static int299gf100_gr_object_get(struct nvkm_gr *base, int index, struct nvkm_sclass *sclass)300{301 struct gf100_gr *gr = gf100_gr(base);302 int c = 0;303 304 while (gr->func->sclass[c].oclass) {305 if (c++ == index) {306 *sclass = gr->func->sclass[index];307 sclass->ctor = gf100_gr_object_new;308 return index;309 }310 }311 312 return c;313}314 315/*******************************************************************************316 * PGRAPH context317 ******************************************************************************/318 319static int320gf100_gr_chan_bind(struct nvkm_object *object, struct nvkm_gpuobj *parent,321 int align, struct nvkm_gpuobj **pgpuobj)322{323 struct gf100_gr_chan *chan = gf100_gr_chan(object);324 struct gf100_gr *gr = chan->gr;325 int ret, i;326 327 ret = nvkm_gpuobj_new(gr->base.engine.subdev.device, gr->size,328 align, false, parent, pgpuobj);329 if (ret)330 return ret;331 332 nvkm_kmap(*pgpuobj);333 for (i = 0; i < gr->size; i += 4)334 nvkm_wo32(*pgpuobj, i, gr->data[i / 4]);335 336 if (!gr->firmware) {337 nvkm_wo32(*pgpuobj, 0x00, chan->mmio_nr / 2);338 nvkm_wo32(*pgpuobj, 0x04, chan->mmio_vma->addr >> 8);339 } else {340 nvkm_wo32(*pgpuobj, 0xf4, 0);341 nvkm_wo32(*pgpuobj, 0xf8, 0);342 nvkm_wo32(*pgpuobj, 0x10, chan->mmio_nr / 2);343 nvkm_wo32(*pgpuobj, 0x14, lower_32_bits(chan->mmio_vma->addr));344 nvkm_wo32(*pgpuobj, 0x18, upper_32_bits(chan->mmio_vma->addr));345 nvkm_wo32(*pgpuobj, 0x1c, 1);346 nvkm_wo32(*pgpuobj, 0x20, 0);347 nvkm_wo32(*pgpuobj, 0x28, 0);348 nvkm_wo32(*pgpuobj, 0x2c, 0);349 }350 nvkm_done(*pgpuobj);351 return 0;352}353 354static void *355gf100_gr_chan_dtor(struct nvkm_object *object)356{357 struct gf100_gr_chan *chan = gf100_gr_chan(object);358 359 nvkm_vmm_put(chan->vmm, &chan->mmio_vma);360 nvkm_memory_unref(&chan->mmio);361 362 nvkm_vmm_put(chan->vmm, &chan->attrib_cb);363 nvkm_vmm_put(chan->vmm, &chan->unknown);364 nvkm_vmm_put(chan->vmm, &chan->bundle_cb);365 nvkm_vmm_put(chan->vmm, &chan->pagepool);366 nvkm_vmm_unref(&chan->vmm);367 return chan;368}369 370static const struct nvkm_object_func371gf100_gr_chan = {372 .dtor = gf100_gr_chan_dtor,373 .bind = gf100_gr_chan_bind,374};375 376static int377gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_chan *fifoch,378 const struct nvkm_oclass *oclass,379 struct nvkm_object **pobject)380{381 struct gf100_gr *gr = gf100_gr(base);382 struct gf100_gr_chan *chan;383 struct gf100_vmm_map_v0 args = { .priv = 1 };384 struct nvkm_device *device = gr->base.engine.subdev.device;385 int ret;386 387 if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL)))388 return -ENOMEM;389 nvkm_object_ctor(&gf100_gr_chan, oclass, &chan->object);390 chan->gr = gr;391 chan->vmm = nvkm_vmm_ref(fifoch->vmm);392 *pobject = &chan->object;393 394 /* Map pagepool. */395 ret = nvkm_vmm_get(chan->vmm, 12, nvkm_memory_size(gr->pagepool), &chan->pagepool);396 if (ret)397 return ret;398 399 ret = nvkm_memory_map(gr->pagepool, 0, chan->vmm, chan->pagepool, &args, sizeof(args));400 if (ret)401 return ret;402 403 /* Map bundle circular buffer. */404 ret = nvkm_vmm_get(chan->vmm, 12, nvkm_memory_size(gr->bundle_cb), &chan->bundle_cb);405 if (ret)406 return ret;407 408 ret = nvkm_memory_map(gr->bundle_cb, 0, chan->vmm, chan->bundle_cb, &args, sizeof(args));409 if (ret)410 return ret;411 412 /* Map attribute circular buffer. */413 ret = nvkm_vmm_get(chan->vmm, 12, nvkm_memory_size(gr->attrib_cb), &chan->attrib_cb);414 if (ret)415 return ret;416 417 if (device->card_type < GP100) {418 ret = nvkm_memory_map(gr->attrib_cb, 0, chan->vmm, chan->attrib_cb, NULL, 0);419 if (ret)420 return ret;421 } else {422 ret = nvkm_memory_map(gr->attrib_cb, 0, chan->vmm, chan->attrib_cb,423 &args, sizeof(args));424 if (ret)425 return ret;426 }427 428 /* Map some context buffer of unknown purpose. */429 if (gr->func->grctx->unknown_size) {430 ret = nvkm_vmm_get(chan->vmm, 12, nvkm_memory_size(gr->unknown), &chan->unknown);431 if (ret)432 return ret;433 434 ret = nvkm_memory_map(gr->unknown, 0, chan->vmm, chan->unknown,435 &args, sizeof(args));436 if (ret)437 return ret;438 }439 440 /* Generate golden context image. */441 mutex_lock(&gr->fecs.mutex);442 if (gr->data == NULL) {443 ret = gf100_grctx_generate(gr, chan, fifoch->inst);444 if (ret) {445 nvkm_error(&base->engine.subdev, "failed to construct context\n");446 return ret;447 }448 }449 mutex_unlock(&gr->fecs.mutex);450 451 /* allocate memory for a "mmio list" buffer that's used by the HUB452 * fuc to modify some per-context register settings on first load453 * of the context.454 */455 ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x100,456 false, &chan->mmio);457 if (ret)458 return ret;459 460 ret = nvkm_vmm_get(fifoch->vmm, 12, 0x1000, &chan->mmio_vma);461 if (ret)462 return ret;463 464 ret = nvkm_memory_map(chan->mmio, 0, fifoch->vmm,465 chan->mmio_vma, &args, sizeof(args));466 if (ret)467 return ret;468 469 /* finally, fill in the mmio list and point the context at it */470 nvkm_kmap(chan->mmio);471 gr->func->grctx->pagepool(chan, chan->pagepool->addr);472 gr->func->grctx->bundle(chan, chan->bundle_cb->addr, gr->func->grctx->bundle_size);473 gr->func->grctx->attrib_cb(chan, chan->attrib_cb->addr, gr->func->grctx->attrib_cb_size(gr));474 gr->func->grctx->attrib(chan);475 if (gr->func->grctx->patch_ltc)476 gr->func->grctx->patch_ltc(chan);477 if (gr->func->grctx->unknown_size)478 gr->func->grctx->unknown(chan, chan->unknown->addr, gr->func->grctx->unknown_size);479 nvkm_done(chan->mmio);480 return 0;481}482 483/*******************************************************************************484 * PGRAPH register lists485 ******************************************************************************/486 487const struct gf100_gr_init488gf100_gr_init_main_0[] = {489 { 0x400080, 1, 0x04, 0x003083c2 },490 { 0x400088, 1, 0x04, 0x00006fe7 },491 { 0x40008c, 1, 0x04, 0x00000000 },492 { 0x400090, 1, 0x04, 0x00000030 },493 { 0x40013c, 1, 0x04, 0x013901f7 },494 { 0x400140, 1, 0x04, 0x00000100 },495 { 0x400144, 1, 0x04, 0x00000000 },496 { 0x400148, 1, 0x04, 0x00000110 },497 { 0x400138, 1, 0x04, 0x00000000 },498 { 0x400130, 2, 0x04, 0x00000000 },499 { 0x400124, 1, 0x04, 0x00000002 },500 {}501};502 503const struct gf100_gr_init504gf100_gr_init_fe_0[] = {505 { 0x40415c, 1, 0x04, 0x00000000 },506 { 0x404170, 1, 0x04, 0x00000000 },507 {}508};509 510const struct gf100_gr_init511gf100_gr_init_pri_0[] = {512 { 0x404488, 2, 0x04, 0x00000000 },513 {}514};515 516const struct gf100_gr_init517gf100_gr_init_rstr2d_0[] = {518 { 0x407808, 1, 0x04, 0x00000000 },519 {}520};521 522const struct gf100_gr_init523gf100_gr_init_pd_0[] = {524 { 0x406024, 1, 0x04, 0x00000000 },525 {}526};527 528const struct gf100_gr_init529gf100_gr_init_ds_0[] = {530 { 0x405844, 1, 0x04, 0x00ffffff },531 { 0x405850, 1, 0x04, 0x00000000 },532 { 0x405908, 1, 0x04, 0x00000000 },533 {}534};535 536const struct gf100_gr_init537gf100_gr_init_scc_0[] = {538 { 0x40803c, 1, 0x04, 0x00000000 },539 {}540};541 542const struct gf100_gr_init543gf100_gr_init_prop_0[] = {544 { 0x4184a0, 1, 0x04, 0x00000000 },545 {}546};547 548const struct gf100_gr_init549gf100_gr_init_gpc_unk_0[] = {550 { 0x418604, 1, 0x04, 0x00000000 },551 { 0x418680, 1, 0x04, 0x00000000 },552 { 0x418714, 1, 0x04, 0x80000000 },553 { 0x418384, 1, 0x04, 0x00000000 },554 {}555};556 557const struct gf100_gr_init558gf100_gr_init_setup_0[] = {559 { 0x418814, 3, 0x04, 0x00000000 },560 {}561};562 563const struct gf100_gr_init564gf100_gr_init_crstr_0[] = {565 { 0x418b04, 1, 0x04, 0x00000000 },566 {}567};568 569const struct gf100_gr_init570gf100_gr_init_setup_1[] = {571 { 0x4188c8, 1, 0x04, 0x80000000 },572 { 0x4188cc, 1, 0x04, 0x00000000 },573 { 0x4188d0, 1, 0x04, 0x00010000 },574 { 0x4188d4, 1, 0x04, 0x00000001 },575 {}576};577 578const struct gf100_gr_init579gf100_gr_init_zcull_0[] = {580 { 0x418910, 1, 0x04, 0x00010001 },581 { 0x418914, 1, 0x04, 0x00000301 },582 { 0x418918, 1, 0x04, 0x00800000 },583 { 0x418980, 1, 0x04, 0x77777770 },584 { 0x418984, 3, 0x04, 0x77777777 },585 {}586};587 588const struct gf100_gr_init589gf100_gr_init_gpm_0[] = {590 { 0x418c04, 1, 0x04, 0x00000000 },591 { 0x418c88, 1, 0x04, 0x00000000 },592 {}593};594 595const struct gf100_gr_init596gf100_gr_init_gpc_unk_1[] = {597 { 0x418d00, 1, 0x04, 0x00000000 },598 { 0x418f08, 1, 0x04, 0x00000000 },599 { 0x418e00, 1, 0x04, 0x00000050 },600 { 0x418e08, 1, 0x04, 0x00000000 },601 {}602};603 604const struct gf100_gr_init605gf100_gr_init_gcc_0[] = {606 { 0x41900c, 1, 0x04, 0x00000000 },607 { 0x419018, 1, 0x04, 0x00000000 },608 {}609};610 611const struct gf100_gr_init612gf100_gr_init_tpccs_0[] = {613 { 0x419d08, 2, 0x04, 0x00000000 },614 { 0x419d10, 1, 0x04, 0x00000014 },615 {}616};617 618const struct gf100_gr_init619gf100_gr_init_tex_0[] = {620 { 0x419ab0, 1, 0x04, 0x00000000 },621 { 0x419ab8, 1, 0x04, 0x000000e7 },622 { 0x419abc, 2, 0x04, 0x00000000 },623 {}624};625 626const struct gf100_gr_init627gf100_gr_init_pe_0[] = {628 { 0x41980c, 3, 0x04, 0x00000000 },629 { 0x419844, 1, 0x04, 0x00000000 },630 { 0x41984c, 1, 0x04, 0x00005bc5 },631 { 0x419850, 4, 0x04, 0x00000000 },632 {}633};634 635const struct gf100_gr_init636gf100_gr_init_l1c_0[] = {637 { 0x419c98, 1, 0x04, 0x00000000 },638 { 0x419ca8, 1, 0x04, 0x80000000 },639 { 0x419cb4, 1, 0x04, 0x00000000 },640 { 0x419cb8, 1, 0x04, 0x00008bf4 },641 { 0x419cbc, 1, 0x04, 0x28137606 },642 { 0x419cc0, 2, 0x04, 0x00000000 },643 {}644};645 646const struct gf100_gr_init647gf100_gr_init_wwdx_0[] = {648 { 0x419bd4, 1, 0x04, 0x00800000 },649 { 0x419bdc, 1, 0x04, 0x00000000 },650 {}651};652 653const struct gf100_gr_init654gf100_gr_init_tpccs_1[] = {655 { 0x419d2c, 1, 0x04, 0x00000000 },656 {}657};658 659const struct gf100_gr_init660gf100_gr_init_mpc_0[] = {661 { 0x419c0c, 1, 0x04, 0x00000000 },662 {}663};664 665static const struct gf100_gr_init666gf100_gr_init_sm_0[] = {667 { 0x419e00, 1, 0x04, 0x00000000 },668 { 0x419ea0, 1, 0x04, 0x00000000 },669 { 0x419ea4, 1, 0x04, 0x00000100 },670 { 0x419ea8, 1, 0x04, 0x00001100 },671 { 0x419eac, 1, 0x04, 0x11100702 },672 { 0x419eb0, 1, 0x04, 0x00000003 },673 { 0x419eb4, 4, 0x04, 0x00000000 },674 { 0x419ec8, 1, 0x04, 0x06060618 },675 { 0x419ed0, 1, 0x04, 0x0eff0e38 },676 { 0x419ed4, 1, 0x04, 0x011104f1 },677 { 0x419edc, 1, 0x04, 0x00000000 },678 { 0x419f00, 1, 0x04, 0x00000000 },679 { 0x419f2c, 1, 0x04, 0x00000000 },680 {}681};682 683const struct gf100_gr_init684gf100_gr_init_be_0[] = {685 { 0x40880c, 1, 0x04, 0x00000000 },686 { 0x408910, 9, 0x04, 0x00000000 },687 { 0x408950, 1, 0x04, 0x00000000 },688 { 0x408954, 1, 0x04, 0x0000ffff },689 { 0x408984, 1, 0x04, 0x00000000 },690 { 0x408988, 1, 0x04, 0x08040201 },691 { 0x40898c, 1, 0x04, 0x80402010 },692 {}693};694 695const struct gf100_gr_init696gf100_gr_init_fe_1[] = {697 { 0x4040f0, 1, 0x04, 0x00000000 },698 {}699};700 701const struct gf100_gr_init702gf100_gr_init_pe_1[] = {703 { 0x419880, 1, 0x04, 0x00000002 },704 {}705};706 707static const struct gf100_gr_pack708gf100_gr_pack_mmio[] = {709 { gf100_gr_init_main_0 },710 { gf100_gr_init_fe_0 },711 { gf100_gr_init_pri_0 },712 { gf100_gr_init_rstr2d_0 },713 { gf100_gr_init_pd_0 },714 { gf100_gr_init_ds_0 },715 { gf100_gr_init_scc_0 },716 { gf100_gr_init_prop_0 },717 { gf100_gr_init_gpc_unk_0 },718 { gf100_gr_init_setup_0 },719 { gf100_gr_init_crstr_0 },720 { gf100_gr_init_setup_1 },721 { gf100_gr_init_zcull_0 },722 { gf100_gr_init_gpm_0 },723 { gf100_gr_init_gpc_unk_1 },724 { gf100_gr_init_gcc_0 },725 { gf100_gr_init_tpccs_0 },726 { gf100_gr_init_tex_0 },727 { gf100_gr_init_pe_0 },728 { gf100_gr_init_l1c_0 },729 { gf100_gr_init_wwdx_0 },730 { gf100_gr_init_tpccs_1 },731 { gf100_gr_init_mpc_0 },732 { gf100_gr_init_sm_0 },733 { gf100_gr_init_be_0 },734 { gf100_gr_init_fe_1 },735 { gf100_gr_init_pe_1 },736 {}737};738 739/*******************************************************************************740 * PGRAPH engine/subdev functions741 ******************************************************************************/742 743static u32744gf100_gr_ctxsw_inst(struct nvkm_gr *gr)745{746 return nvkm_rd32(gr->engine.subdev.device, 0x409b00);747}748 749static int750gf100_gr_fecs_ctrl_ctxsw(struct gf100_gr *gr, u32 mthd)751{752 struct nvkm_device *device = gr->base.engine.subdev.device;753 754 nvkm_wr32(device, 0x409804, 0xffffffff);755 nvkm_wr32(device, 0x409800, 0x00000000);756 nvkm_wr32(device, 0x409500, 0xffffffff);757 nvkm_wr32(device, 0x409504, mthd);758 nvkm_msec(device, 2000,759 u32 stat = nvkm_rd32(device, 0x409804);760 if (stat == 0x00000002)761 return -EIO;762 if (stat == 0x00000001)763 return 0;764 );765 766 return -ETIMEDOUT;767}768 769static int770gf100_gr_fecs_start_ctxsw(struct nvkm_gr *base)771{772 struct gf100_gr *gr = gf100_gr(base);773 int ret = 0;774 775 mutex_lock(&gr->fecs.mutex);776 if (!--gr->fecs.disable) {777 if (WARN_ON(ret = gf100_gr_fecs_ctrl_ctxsw(gr, 0x39)))778 gr->fecs.disable++;779 }780 mutex_unlock(&gr->fecs.mutex);781 return ret;782}783 784static int785gf100_gr_fecs_stop_ctxsw(struct nvkm_gr *base)786{787 struct gf100_gr *gr = gf100_gr(base);788 int ret = 0;789 790 mutex_lock(&gr->fecs.mutex);791 if (!gr->fecs.disable++) {792 if (WARN_ON(ret = gf100_gr_fecs_ctrl_ctxsw(gr, 0x38)))793 gr->fecs.disable--;794 }795 mutex_unlock(&gr->fecs.mutex);796 return ret;797}798 799static int800gf100_gr_fecs_halt_pipeline(struct gf100_gr *gr)801{802 int ret = 0;803 804 if (gr->firmware) {805 mutex_lock(&gr->fecs.mutex);806 ret = gf100_gr_fecs_ctrl_ctxsw(gr, 0x04);807 mutex_unlock(&gr->fecs.mutex);808 }809 810 return ret;811}812 813int814gf100_gr_fecs_wfi_golden_save(struct gf100_gr *gr, u32 inst)815{816 struct nvkm_device *device = gr->base.engine.subdev.device;817 818 nvkm_mask(device, 0x409800, 0x00000003, 0x00000000);819 nvkm_wr32(device, 0x409500, inst);820 nvkm_wr32(device, 0x409504, 0x00000009);821 nvkm_msec(device, 2000,822 u32 stat = nvkm_rd32(device, 0x409800);823 if (stat & 0x00000002)824 return -EIO;825 if (stat & 0x00000001)826 return 0;827 );828 829 return -ETIMEDOUT;830}831 832int833gf100_gr_fecs_bind_pointer(struct gf100_gr *gr, u32 inst)834{835 struct nvkm_device *device = gr->base.engine.subdev.device;836 837 nvkm_mask(device, 0x409800, 0x00000030, 0x00000000);838 nvkm_wr32(device, 0x409500, inst);839 nvkm_wr32(device, 0x409504, 0x00000003);840 nvkm_msec(device, 2000,841 u32 stat = nvkm_rd32(device, 0x409800);842 if (stat & 0x00000020)843 return -EIO;844 if (stat & 0x00000010)845 return 0;846 );847 848 return -ETIMEDOUT;849}850 851static int852gf100_gr_fecs_set_reglist_virtual_address(struct gf100_gr *gr, u64 addr)853{854 struct nvkm_device *device = gr->base.engine.subdev.device;855 856 nvkm_wr32(device, 0x409810, addr >> 8);857 nvkm_wr32(device, 0x409800, 0x00000000);858 nvkm_wr32(device, 0x409500, 0x00000001);859 nvkm_wr32(device, 0x409504, 0x00000032);860 nvkm_msec(device, 2000,861 if (nvkm_rd32(device, 0x409800) == 0x00000001)862 return 0;863 );864 865 return -ETIMEDOUT;866}867 868static int869gf100_gr_fecs_set_reglist_bind_instance(struct gf100_gr *gr, u32 inst)870{871 struct nvkm_device *device = gr->base.engine.subdev.device;872 873 nvkm_wr32(device, 0x409810, inst);874 nvkm_wr32(device, 0x409800, 0x00000000);875 nvkm_wr32(device, 0x409500, 0x00000001);876 nvkm_wr32(device, 0x409504, 0x00000031);877 nvkm_msec(device, 2000,878 if (nvkm_rd32(device, 0x409800) == 0x00000001)879 return 0;880 );881 882 return -ETIMEDOUT;883}884 885static int886gf100_gr_fecs_discover_reglist_image_size(struct gf100_gr *gr, u32 *psize)887{888 struct nvkm_device *device = gr->base.engine.subdev.device;889 890 nvkm_wr32(device, 0x409800, 0x00000000);891 nvkm_wr32(device, 0x409500, 0x00000001);892 nvkm_wr32(device, 0x409504, 0x00000030);893 nvkm_msec(device, 2000,894 if ((*psize = nvkm_rd32(device, 0x409800)))895 return 0;896 );897 898 return -ETIMEDOUT;899}900 901static int902gf100_gr_fecs_elpg_bind(struct gf100_gr *gr)903{904 u32 size;905 int ret;906 907 ret = gf100_gr_fecs_discover_reglist_image_size(gr, &size);908 if (ret)909 return ret;910 911 /*XXX: We need to allocate + map the above into PMU's inst block,912 * which which means we probably need a proper PMU before we913 * even bother.914 */915 916 ret = gf100_gr_fecs_set_reglist_bind_instance(gr, 0);917 if (ret)918 return ret;919 920 return gf100_gr_fecs_set_reglist_virtual_address(gr, 0);921}922 923static int924gf100_gr_fecs_discover_pm_image_size(struct gf100_gr *gr, u32 *psize)925{926 struct nvkm_device *device = gr->base.engine.subdev.device;927 928 nvkm_wr32(device, 0x409800, 0x00000000);929 nvkm_wr32(device, 0x409500, 0x00000000);930 nvkm_wr32(device, 0x409504, 0x00000025);931 nvkm_msec(device, 2000,932 if ((*psize = nvkm_rd32(device, 0x409800)))933 return 0;934 );935 936 return -ETIMEDOUT;937}938 939static int940gf100_gr_fecs_discover_zcull_image_size(struct gf100_gr *gr, u32 *psize)941{942 struct nvkm_device *device = gr->base.engine.subdev.device;943 944 nvkm_wr32(device, 0x409800, 0x00000000);945 nvkm_wr32(device, 0x409500, 0x00000000);946 nvkm_wr32(device, 0x409504, 0x00000016);947 nvkm_msec(device, 2000,948 if ((*psize = nvkm_rd32(device, 0x409800)))949 return 0;950 );951 952 return -ETIMEDOUT;953}954 955static int956gf100_gr_fecs_discover_image_size(struct gf100_gr *gr, u32 *psize)957{958 struct nvkm_device *device = gr->base.engine.subdev.device;959 960 nvkm_wr32(device, 0x409800, 0x00000000);961 nvkm_wr32(device, 0x409500, 0x00000000);962 nvkm_wr32(device, 0x409504, 0x00000010);963 nvkm_msec(device, 2000,964 if ((*psize = nvkm_rd32(device, 0x409800)))965 return 0;966 );967 968 return -ETIMEDOUT;969}970 971static void972gf100_gr_fecs_set_watchdog_timeout(struct gf100_gr *gr, u32 timeout)973{974 struct nvkm_device *device = gr->base.engine.subdev.device;975 976 nvkm_wr32(device, 0x409800, 0x00000000);977 nvkm_wr32(device, 0x409500, timeout);978 nvkm_wr32(device, 0x409504, 0x00000021);979}980 981static bool982gf100_gr_chsw_load(struct nvkm_gr *base)983{984 struct gf100_gr *gr = gf100_gr(base);985 if (!gr->firmware) {986 u32 trace = nvkm_rd32(gr->base.engine.subdev.device, 0x40981c);987 if (trace & 0x00000040)988 return true;989 } else {990 u32 mthd = nvkm_rd32(gr->base.engine.subdev.device, 0x409808);991 if (mthd & 0x00080000)992 return true;993 }994 return false;995}996 997int998gf100_gr_rops(struct gf100_gr *gr)999{1000 struct nvkm_device *device = gr->base.engine.subdev.device;1001 return (nvkm_rd32(device, 0x409604) & 0x001f0000) >> 16;1002}1003 1004void1005gf100_gr_zbc_init(struct gf100_gr *gr)1006{1007 const u32 zero[] = { 0x00000000, 0x00000000, 0x00000000, 0x00000000,1008 0x00000000, 0x00000000, 0x00000000, 0x00000000 };1009 const u32 one[] = { 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000,1010 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff };1011 const u32 f32_0[] = { 0x00000000, 0x00000000, 0x00000000, 0x00000000,1012 0x00000000, 0x00000000, 0x00000000, 0x00000000 };1013 const u32 f32_1[] = { 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000,1014 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000 };1015 struct nvkm_ltc *ltc = gr->base.engine.subdev.device->ltc;1016 int index, c = ltc->zbc_color_min, d = ltc->zbc_depth_min, s = ltc->zbc_depth_min;1017 1018 if (!gr->zbc_color[0].format) {1019 gf100_gr_zbc_color_get(gr, 1, & zero[0], &zero[4]); c++;1020 gf100_gr_zbc_color_get(gr, 2, & one[0], &one[4]); c++;1021 gf100_gr_zbc_color_get(gr, 4, &f32_0[0], &f32_0[4]); c++;1022 gf100_gr_zbc_color_get(gr, 4, &f32_1[0], &f32_1[4]); c++;1023 gf100_gr_zbc_depth_get(gr, 1, 0x00000000, 0x00000000); d++;1024 gf100_gr_zbc_depth_get(gr, 1, 0x3f800000, 0x3f800000); d++;1025 if (gr->func->zbc->stencil_get) {1026 gr->func->zbc->stencil_get(gr, 1, 0x00, 0x00); s++;1027 gr->func->zbc->stencil_get(gr, 1, 0x01, 0x01); s++;1028 gr->func->zbc->stencil_get(gr, 1, 0xff, 0xff); s++;1029 }1030 }1031 1032 for (index = c; index <= ltc->zbc_color_max; index++)1033 gr->func->zbc->clear_color(gr, index);1034 for (index = d; index <= ltc->zbc_depth_max; index++)1035 gr->func->zbc->clear_depth(gr, index);1036 1037 if (gr->func->zbc->clear_stencil) {1038 for (index = s; index <= ltc->zbc_depth_max; index++)1039 gr->func->zbc->clear_stencil(gr, index);1040 }1041}1042 1043/*1044 * Wait until GR goes idle. GR is considered idle if it is disabled by the1045 * MC (0x200) register, or GR is not busy and a context switch is not in1046 * progress.1047 */1048int1049gf100_gr_wait_idle(struct gf100_gr *gr)1050{1051 struct nvkm_subdev *subdev = &gr->base.engine.subdev;1052 struct nvkm_device *device = subdev->device;1053 unsigned long end_jiffies = jiffies + msecs_to_jiffies(2000);1054 bool gr_enabled, ctxsw_active, gr_busy;1055 1056 do {1057 /*1058 * required to make sure FIFO_ENGINE_STATUS (0x2640) is1059 * up-to-date1060 */1061 nvkm_rd32(device, 0x400700);1062 1063 gr_enabled = nvkm_rd32(device, 0x200) & 0x1000;1064 ctxsw_active = nvkm_fifo_ctxsw_in_progress(&gr->base.engine);1065 gr_busy = nvkm_rd32(device, 0x40060c) & 0x1;1066 1067 if (!gr_enabled || (!gr_busy && !ctxsw_active))1068 return 0;1069 } while (time_before(jiffies, end_jiffies));1070 1071 nvkm_error(subdev,1072 "wait for idle timeout (en: %d, ctxsw: %d, busy: %d)\n",1073 gr_enabled, ctxsw_active, gr_busy);1074 return -EAGAIN;1075}1076 1077void1078gf100_gr_mmio(struct gf100_gr *gr, const struct gf100_gr_pack *p)1079{1080 struct nvkm_device *device = gr->base.engine.subdev.device;1081 const struct gf100_gr_pack *pack;1082 const struct gf100_gr_init *init;1083 1084 pack_for_each_init(init, pack, p) {1085 u32 next = init->addr + init->count * init->pitch;1086 u32 addr = init->addr;1087 while (addr < next) {1088 nvkm_wr32(device, addr, init->data);1089 addr += init->pitch;1090 }1091 }1092}1093 1094void1095gf100_gr_icmd(struct gf100_gr *gr, const struct gf100_gr_pack *p)1096{1097 struct nvkm_device *device = gr->base.engine.subdev.device;1098 const struct gf100_gr_pack *pack;1099 const struct gf100_gr_init *init;1100 u64 data = 0;1101 1102 nvkm_wr32(device, 0x400208, 0x80000000);1103 1104 pack_for_each_init(init, pack, p) {1105 u32 next = init->addr + init->count * init->pitch;1106 u32 addr = init->addr;1107 1108 if ((pack == p && init == p->init) || data != init->data) {1109 nvkm_wr32(device, 0x400204, init->data);1110 if (pack->type == 64)1111 nvkm_wr32(device, 0x40020c, upper_32_bits(init->data));1112 data = init->data;1113 }1114 1115 while (addr < next) {1116 nvkm_wr32(device, 0x400200, addr);1117 /**1118 * Wait for GR to go idle after submitting a1119 * GO_IDLE bundle1120 */1121 if ((addr & 0xffff) == 0xe100)1122 gf100_gr_wait_idle(gr);1123 nvkm_msec(device, 2000,1124 if (!(nvkm_rd32(device, 0x400700) & 0x00000004))1125 break;1126 );1127 addr += init->pitch;1128 }1129 }1130 1131 nvkm_wr32(device, 0x400208, 0x00000000);1132}1133 1134void1135gf100_gr_mthd(struct gf100_gr *gr, const struct gf100_gr_pack *p)1136{1137 struct nvkm_device *device = gr->base.engine.subdev.device;1138 const struct gf100_gr_pack *pack;1139 const struct gf100_gr_init *init;1140 u32 data = 0;1141 1142 pack_for_each_init(init, pack, p) {1143 u32 ctrl = 0x80000000 | pack->type;1144 u32 next = init->addr + init->count * init->pitch;1145 u32 addr = init->addr;1146 1147 if ((pack == p && init == p->init) || data != init->data) {1148 nvkm_wr32(device, 0x40448c, init->data);1149 data = init->data;1150 }1151 1152 while (addr < next) {1153 nvkm_wr32(device, 0x404488, ctrl | (addr << 14));1154 addr += init->pitch;1155 }1156 }1157}1158 1159u641160gf100_gr_units(struct nvkm_gr *base)1161{1162 struct gf100_gr *gr = gf100_gr(base);1163 u64 cfg;1164 1165 cfg = (u32)gr->gpc_nr;1166 cfg |= (u32)gr->tpc_total << 8;1167 cfg |= (u64)gr->rop_nr << 32;1168 1169 return cfg;1170}1171 1172static const struct nvkm_bitfield gf100_dispatch_error[] = {1173 { 0x00000001, "INJECTED_BUNDLE_ERROR" },1174 { 0x00000002, "CLASS_SUBCH_MISMATCH" },1175 { 0x00000004, "SUBCHSW_DURING_NOTIFY" },1176 {}1177};1178 1179static const struct nvkm_bitfield gf100_m2mf_error[] = {1180 { 0x00000001, "PUSH_TOO_MUCH_DATA" },1181 { 0x00000002, "PUSH_NOT_ENOUGH_DATA" },1182 {}1183};1184 1185static const struct nvkm_bitfield gf100_unk6_error[] = {1186 { 0x00000001, "TEMP_TOO_SMALL" },1187 {}1188};1189 1190static const struct nvkm_bitfield gf100_ccache_error[] = {1191 { 0x00000001, "INTR" },1192 { 0x00000002, "LDCONST_OOB" },1193 {}1194};1195 1196static const struct nvkm_bitfield gf100_macro_error[] = {1197 { 0x00000001, "TOO_FEW_PARAMS" },1198 { 0x00000002, "TOO_MANY_PARAMS" },1199 { 0x00000004, "ILLEGAL_OPCODE" },1200 { 0x00000008, "DOUBLE_BRANCH" },1201 { 0x00000010, "WATCHDOG" },1202 {}1203};1204 1205static const struct nvkm_bitfield gk104_sked_error[] = {1206 { 0x00000040, "CTA_RESUME" },1207 { 0x00000080, "CONSTANT_BUFFER_SIZE" },1208 { 0x00000200, "LOCAL_MEMORY_SIZE_POS" },1209 { 0x00000400, "LOCAL_MEMORY_SIZE_NEG" },1210 { 0x00000800, "WARP_CSTACK_SIZE" },1211 { 0x00001000, "TOTAL_TEMP_SIZE" },1212 { 0x00002000, "REGISTER_COUNT" },1213 { 0x00040000, "TOTAL_THREADS" },1214 { 0x00100000, "PROGRAM_OFFSET" },1215 { 0x00200000, "SHARED_MEMORY_SIZE" },1216 { 0x00800000, "CTA_THREAD_DIMENSION_ZERO" },1217 { 0x01000000, "MEMORY_WINDOW_OVERLAP" },1218 { 0x02000000, "SHARED_CONFIG_TOO_SMALL" },1219 { 0x04000000, "TOTAL_REGISTER_COUNT" },1220 {}1221};1222 1223static const struct nvkm_bitfield gf100_gpc_rop_error[] = {1224 { 0x00000002, "RT_PITCH_OVERRUN" },1225 { 0x00000010, "RT_WIDTH_OVERRUN" },1226 { 0x00000020, "RT_HEIGHT_OVERRUN" },1227 { 0x00000080, "ZETA_STORAGE_TYPE_MISMATCH" },1228 { 0x00000100, "RT_STORAGE_TYPE_MISMATCH" },1229 { 0x00000400, "RT_LINEAR_MISMATCH" },1230 {}1231};1232 1233static void1234gf100_gr_trap_gpc_rop(struct gf100_gr *gr, int gpc)1235{1236 struct nvkm_subdev *subdev = &gr->base.engine.subdev;1237 struct nvkm_device *device = subdev->device;1238 char error[128];1239 u32 trap[4];1240 1241 trap[0] = nvkm_rd32(device, GPC_UNIT(gpc, 0x0420)) & 0x3fffffff;1242 trap[1] = nvkm_rd32(device, GPC_UNIT(gpc, 0x0434));1243 trap[2] = nvkm_rd32(device, GPC_UNIT(gpc, 0x0438));1244 trap[3] = nvkm_rd32(device, GPC_UNIT(gpc, 0x043c));1245 1246 nvkm_snprintbf(error, sizeof(error), gf100_gpc_rop_error, trap[0]);1247 1248 nvkm_error(subdev, "GPC%d/PROP trap: %08x [%s] x = %u, y = %u, "1249 "format = %x, storage type = %x\n",1250 gpc, trap[0], error, trap[1] & 0xffff, trap[1] >> 16,1251 (trap[2] >> 8) & 0x3f, trap[3] & 0xff);1252 nvkm_wr32(device, GPC_UNIT(gpc, 0x0420), 0xc0000000);1253}1254 1255const struct nvkm_enum gf100_mp_warp_error[] = {1256 { 0x01, "STACK_ERROR" },1257 { 0x02, "API_STACK_ERROR" },1258 { 0x03, "RET_EMPTY_STACK_ERROR" },1259 { 0x04, "PC_WRAP" },1260 { 0x05, "MISALIGNED_PC" },1261 { 0x06, "PC_OVERFLOW" },1262 { 0x07, "MISALIGNED_IMMC_ADDR" },1263 { 0x08, "MISALIGNED_REG" },1264 { 0x09, "ILLEGAL_INSTR_ENCODING" },1265 { 0x0a, "ILLEGAL_SPH_INSTR_COMBO" },1266 { 0x0b, "ILLEGAL_INSTR_PARAM" },1267 { 0x0c, "INVALID_CONST_ADDR" },1268 { 0x0d, "OOR_REG" },1269 { 0x0e, "OOR_ADDR" },1270 { 0x0f, "MISALIGNED_ADDR" },1271 { 0x10, "INVALID_ADDR_SPACE" },1272 { 0x11, "ILLEGAL_INSTR_PARAM2" },1273 { 0x12, "INVALID_CONST_ADDR_LDC" },1274 { 0x13, "GEOMETRY_SM_ERROR" },1275 { 0x14, "DIVERGENT" },1276 { 0x15, "WARP_EXIT" },1277 {}1278};1279 1280const struct nvkm_bitfield gf100_mp_global_error[] = {1281 { 0x00000001, "SM_TO_SM_FAULT" },1282 { 0x00000002, "L1_ERROR" },1283 { 0x00000004, "MULTIPLE_WARP_ERRORS" },1284 { 0x00000008, "PHYSICAL_STACK_OVERFLOW" },1285 { 0x00000010, "BPT_INT" },1286 { 0x00000020, "BPT_PAUSE" },1287 { 0x00000040, "SINGLE_STEP_COMPLETE" },1288 { 0x20000000, "ECC_SEC_ERROR" },1289 { 0x40000000, "ECC_DED_ERROR" },1290 { 0x80000000, "TIMEOUT" },1291 {}1292};1293 1294void1295gf100_gr_trap_mp(struct gf100_gr *gr, int gpc, int tpc)1296{1297 struct nvkm_subdev *subdev = &gr->base.engine.subdev;1298 struct nvkm_device *device = subdev->device;1299 u32 werr = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x648));1300 u32 gerr = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x650));1301 const struct nvkm_enum *warp;1302 char glob[128];1303 1304 nvkm_snprintbf(glob, sizeof(glob), gf100_mp_global_error, gerr);1305 warp = nvkm_enum_find(gf100_mp_warp_error, werr & 0xffff);1306 1307 nvkm_error(subdev, "GPC%i/TPC%i/MP trap: "1308 "global %08x [%s] warp %04x [%s]\n",1309 gpc, tpc, gerr, glob, werr, warp ? warp->name : "");1310 1311 nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x648), 0x00000000);1312 nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x650), gerr);1313}1314 1315static void1316gf100_gr_trap_tpc(struct gf100_gr *gr, int gpc, int tpc)1317{1318 struct nvkm_subdev *subdev = &gr->base.engine.subdev;1319 struct nvkm_device *device = subdev->device;1320 u32 stat = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x0508));1321 1322 if (stat & 0x00000001) {1323 u32 trap = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x0224));1324 nvkm_error(subdev, "GPC%d/TPC%d/TEX: %08x\n", gpc, tpc, trap);1325 nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x0224), 0xc0000000);1326 stat &= ~0x00000001;1327 }1328 1329 if (stat & 0x00000002) {1330 gr->func->trap_mp(gr, gpc, tpc);1331 stat &= ~0x00000002;1332 }1333 1334 if (stat & 0x00000004) {1335 u32 trap = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x0084));1336 nvkm_error(subdev, "GPC%d/TPC%d/POLY: %08x\n", gpc, tpc, trap);1337 nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x0084), 0xc0000000);1338 stat &= ~0x00000004;1339 }1340 1341 if (stat & 0x00000008) {1342 u32 trap = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x048c));1343 nvkm_error(subdev, "GPC%d/TPC%d/L1C: %08x\n", gpc, tpc, trap);1344 nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x048c), 0xc0000000);1345 stat &= ~0x00000008;1346 }1347 1348 if (stat & 0x00000010) {1349 u32 trap = nvkm_rd32(device, TPC_UNIT(gpc, tpc, 0x0430));1350 nvkm_error(subdev, "GPC%d/TPC%d/MPC: %08x\n", gpc, tpc, trap);1351 nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x0430), 0xc0000000);1352 stat &= ~0x00000010;1353 }1354 1355 if (stat) {1356 nvkm_error(subdev, "GPC%d/TPC%d/%08x: unknown\n", gpc, tpc, stat);1357 }1358}1359 1360static void1361gf100_gr_trap_gpc(struct gf100_gr *gr, int gpc)1362{1363 struct nvkm_subdev *subdev = &gr->base.engine.subdev;1364 struct nvkm_device *device = subdev->device;1365 u32 stat = nvkm_rd32(device, GPC_UNIT(gpc, 0x2c90));1366 int tpc;1367 1368 if (stat & 0x00000001) {1369 gf100_gr_trap_gpc_rop(gr, gpc);1370 stat &= ~0x00000001;1371 }1372 1373 if (stat & 0x00000002) {1374 u32 trap = nvkm_rd32(device, GPC_UNIT(gpc, 0x0900));1375 nvkm_error(subdev, "GPC%d/ZCULL: %08x\n", gpc, trap);1376 nvkm_wr32(device, GPC_UNIT(gpc, 0x0900), 0xc0000000);1377 stat &= ~0x00000002;1378 }1379 1380 if (stat & 0x00000004) {1381 u32 trap = nvkm_rd32(device, GPC_UNIT(gpc, 0x1028));1382 nvkm_error(subdev, "GPC%d/CCACHE: %08x\n", gpc, trap);1383 nvkm_wr32(device, GPC_UNIT(gpc, 0x1028), 0xc0000000);1384 stat &= ~0x00000004;1385 }1386 1387 if (stat & 0x00000008) {1388 u32 trap = nvkm_rd32(device, GPC_UNIT(gpc, 0x0824));1389 nvkm_error(subdev, "GPC%d/ESETUP: %08x\n", gpc, trap);1390 nvkm_wr32(device, GPC_UNIT(gpc, 0x0824), 0xc0000000);1391 stat &= ~0x00000009;1392 }1393 1394 for (tpc = 0; tpc < gr->tpc_nr[gpc]; tpc++) {1395 u32 mask = 0x00010000 << tpc;1396 if (stat & mask) {1397 gf100_gr_trap_tpc(gr, gpc, tpc);1398 nvkm_wr32(device, GPC_UNIT(gpc, 0x2c90), mask);1399 stat &= ~mask;1400 }1401 }1402 1403 if (stat) {1404 nvkm_error(subdev, "GPC%d/%08x: unknown\n", gpc, stat);1405 }1406}1407 1408static void1409gf100_gr_trap_intr(struct gf100_gr *gr)1410{1411 struct nvkm_subdev *subdev = &gr->base.engine.subdev;1412 struct nvkm_device *device = subdev->device;1413 char error[128];1414 u32 trap = nvkm_rd32(device, 0x400108);1415 int rop, gpc;1416 1417 if (trap & 0x00000001) {1418 u32 stat = nvkm_rd32(device, 0x404000);1419 1420 nvkm_snprintbf(error, sizeof(error), gf100_dispatch_error,1421 stat & 0x3fffffff);1422 nvkm_error(subdev, "DISPATCH %08x [%s]\n", stat, error);1423 nvkm_wr32(device, 0x404000, 0xc0000000);1424 nvkm_wr32(device, 0x400108, 0x00000001);1425 trap &= ~0x00000001;1426 }1427 1428 if (trap & 0x00000002) {1429 u32 stat = nvkm_rd32(device, 0x404600);1430 1431 nvkm_snprintbf(error, sizeof(error), gf100_m2mf_error,1432 stat & 0x3fffffff);1433 nvkm_error(subdev, "M2MF %08x [%s]\n", stat, error);1434 1435 nvkm_wr32(device, 0x404600, 0xc0000000);1436 nvkm_wr32(device, 0x400108, 0x00000002);1437 trap &= ~0x00000002;1438 }1439 1440 if (trap & 0x00000008) {1441 u32 stat = nvkm_rd32(device, 0x408030);1442 1443 nvkm_snprintbf(error, sizeof(error), gf100_ccache_error,1444 stat & 0x3fffffff);1445 nvkm_error(subdev, "CCACHE %08x [%s]\n", stat, error);1446 nvkm_wr32(device, 0x408030, 0xc0000000);1447 nvkm_wr32(device, 0x400108, 0x00000008);1448 trap &= ~0x00000008;1449 }1450 1451 if (trap & 0x00000010) {1452 u32 stat = nvkm_rd32(device, 0x405840);1453 nvkm_error(subdev, "SHADER %08x, sph: 0x%06x, stage: 0x%02x\n",1454 stat, stat & 0xffffff, (stat >> 24) & 0x3f);1455 nvkm_wr32(device, 0x405840, 0xc0000000);1456 nvkm_wr32(device, 0x400108, 0x00000010);1457 trap &= ~0x00000010;1458 }1459 1460 if (trap & 0x00000040) {1461 u32 stat = nvkm_rd32(device, 0x40601c);1462 1463 nvkm_snprintbf(error, sizeof(error), gf100_unk6_error,1464 stat & 0x3fffffff);1465 nvkm_error(subdev, "UNK6 %08x [%s]\n", stat, error);1466 1467 nvkm_wr32(device, 0x40601c, 0xc0000000);1468 nvkm_wr32(device, 0x400108, 0x00000040);1469 trap &= ~0x00000040;1470 }1471 1472 if (trap & 0x00000080) {1473 u32 stat = nvkm_rd32(device, 0x404490);1474 u32 pc = nvkm_rd32(device, 0x404494);1475 u32 op = nvkm_rd32(device, 0x40449c);1476 1477 nvkm_snprintbf(error, sizeof(error), gf100_macro_error,1478 stat & 0x1fffffff);1479 nvkm_error(subdev, "MACRO %08x [%s], pc: 0x%03x%s, op: 0x%08x\n",1480 stat, error, pc & 0x7ff,1481 (pc & 0x10000000) ? "" : " (invalid)",1482 op);1483 1484 nvkm_wr32(device, 0x404490, 0xc0000000);1485 nvkm_wr32(device, 0x400108, 0x00000080);1486 trap &= ~0x00000080;1487 }1488 1489 if (trap & 0x00000100) {1490 u32 stat = nvkm_rd32(device, 0x407020) & 0x3fffffff;1491 1492 nvkm_snprintbf(error, sizeof(error), gk104_sked_error, stat);1493 nvkm_error(subdev, "SKED: %08x [%s]\n", stat, error);1494 1495 if (stat)1496 nvkm_wr32(device, 0x407020, 0x40000000);1497 nvkm_wr32(device, 0x400108, 0x00000100);1498 trap &= ~0x00000100;1499 }1500 1501 if (trap & 0x01000000) {1502 u32 stat = nvkm_rd32(device, 0x400118);1503 for (gpc = 0; stat && gpc < gr->gpc_nr; gpc++) {1504 u32 mask = 0x00000001 << gpc;1505 if (stat & mask) {1506 gf100_gr_trap_gpc(gr, gpc);1507 nvkm_wr32(device, 0x400118, mask);1508 stat &= ~mask;1509 }1510 }1511 nvkm_wr32(device, 0x400108, 0x01000000);1512 trap &= ~0x01000000;1513 }1514 1515 if (trap & 0x02000000) {1516 for (rop = 0; rop < gr->rop_nr; rop++) {1517 u32 statz = nvkm_rd32(device, ROP_UNIT(rop, 0x070));1518 u32 statc = nvkm_rd32(device, ROP_UNIT(rop, 0x144));1519 nvkm_error(subdev, "ROP%d %08x %08x\n",1520 rop, statz, statc);1521 nvkm_wr32(device, ROP_UNIT(rop, 0x070), 0xc0000000);1522 nvkm_wr32(device, ROP_UNIT(rop, 0x144), 0xc0000000);1523 }1524 nvkm_wr32(device, 0x400108, 0x02000000);1525 trap &= ~0x02000000;1526 }1527 1528 if (trap) {1529 nvkm_error(subdev, "TRAP UNHANDLED %08x\n", trap);1530 nvkm_wr32(device, 0x400108, trap);1531 }1532}1533 1534static void1535gf100_gr_ctxctl_debug_unit(struct gf100_gr *gr, u32 base)1536{1537 struct nvkm_subdev *subdev = &gr->base.engine.subdev;1538 struct nvkm_device *device = subdev->device;1539 nvkm_error(subdev, "%06x - done %08x\n", base,1540 nvkm_rd32(device, base + 0x400));1541 nvkm_error(subdev, "%06x - stat %08x %08x %08x %08x\n", base,1542 nvkm_rd32(device, base + 0x800),1543 nvkm_rd32(device, base + 0x804),1544 nvkm_rd32(device, base + 0x808),1545 nvkm_rd32(device, base + 0x80c));1546 nvkm_error(subdev, "%06x - stat %08x %08x %08x %08x\n", base,1547 nvkm_rd32(device, base + 0x810),1548 nvkm_rd32(device, base + 0x814),1549 nvkm_rd32(device, base + 0x818),1550 nvkm_rd32(device, base + 0x81c));1551}1552 1553void1554gf100_gr_ctxctl_debug(struct gf100_gr *gr)1555{1556 struct nvkm_device *device = gr->base.engine.subdev.device;1557 u32 gpcnr = nvkm_rd32(device, 0x409604) & 0xffff;1558 u32 gpc;1559 1560 gf100_gr_ctxctl_debug_unit(gr, 0x409000);1561 for (gpc = 0; gpc < gpcnr; gpc++)1562 gf100_gr_ctxctl_debug_unit(gr, 0x502000 + (gpc * 0x8000));1563}1564 1565static void1566gf100_gr_ctxctl_isr(struct gf100_gr *gr)1567{1568 struct nvkm_subdev *subdev = &gr->base.engine.subdev;1569 struct nvkm_device *device = subdev->device;1570 u32 stat = nvkm_rd32(device, 0x409c18);1571 1572 if (!gr->firmware && (stat & 0x00000001)) {1573 u32 code = nvkm_rd32(device, 0x409814);1574 if (code == E_BAD_FWMTHD) {1575 u32 class = nvkm_rd32(device, 0x409808);1576 u32 addr = nvkm_rd32(device, 0x40980c);1577 u32 subc = (addr & 0x00070000) >> 16;1578 u32 mthd = (addr & 0x00003ffc);1579 u32 data = nvkm_rd32(device, 0x409810);1580 1581 nvkm_error(subdev, "FECS MTHD subc %d class %04x "1582 "mthd %04x data %08x\n",1583 subc, class, mthd, data);1584 } else {1585 nvkm_error(subdev, "FECS ucode error %d\n", code);1586 }1587 nvkm_wr32(device, 0x409c20, 0x00000001);1588 stat &= ~0x00000001;1589 }1590 1591 if (!gr->firmware && (stat & 0x00080000)) {1592 nvkm_error(subdev, "FECS watchdog timeout\n");1593 gf100_gr_ctxctl_debug(gr);1594 nvkm_wr32(device, 0x409c20, 0x00080000);1595 stat &= ~0x00080000;1596 }1597 1598 if (stat) {1599 nvkm_error(subdev, "FECS %08x\n", stat);1600 gf100_gr_ctxctl_debug(gr);1601 nvkm_wr32(device, 0x409c20, stat);1602 }1603}1604 1605static irqreturn_t1606gf100_gr_intr(struct nvkm_inth *inth)1607{1608 struct gf100_gr *gr = container_of(inth, typeof(*gr), base.engine.subdev.inth);1609 struct nvkm_subdev *subdev = &gr->base.engine.subdev;1610 struct nvkm_device *device = subdev->device;1611 struct nvkm_chan *chan;1612 unsigned long flags;1613 u64 inst = nvkm_rd32(device, 0x409b00) & 0x0fffffff;1614 u32 stat = nvkm_rd32(device, 0x400100);1615 u32 addr = nvkm_rd32(device, 0x400704);1616 u32 mthd = (addr & 0x00003ffc);1617 u32 subc = (addr & 0x00070000) >> 16;1618 u32 data = nvkm_rd32(device, 0x400708);1619 u32 code = nvkm_rd32(device, 0x400110);1620 u32 class;1621 const char *name = "unknown";1622 int chid = -1;1623 1624 chan = nvkm_chan_get_inst(&gr->base.engine, (u64)inst << 12, &flags);1625 if (chan) {1626 name = chan->name;1627 chid = chan->id;1628 }1629 1630 if (device->card_type < NV_E0 || subc < 4)1631 class = nvkm_rd32(device, 0x404200 + (subc * 4));1632 else1633 class = 0x0000;1634 1635 if (stat & 0x00000001) {1636 /*1637 * notifier interrupt, only needed for cyclestats1638 * can be safely ignored1639 */1640 nvkm_wr32(device, 0x400100, 0x00000001);1641 stat &= ~0x00000001;1642 }1643 1644 if (stat & 0x00000010) {1645 if (!gf100_gr_mthd_sw(device, class, mthd, data)) {1646 nvkm_error(subdev, "ILLEGAL_MTHD ch %d [%010llx %s] "1647 "subc %d class %04x mthd %04x data %08x\n",1648 chid, inst << 12, name, subc,1649 class, mthd, data);1650 }1651 nvkm_wr32(device, 0x400100, 0x00000010);1652 stat &= ~0x00000010;1653 }1654 1655 if (stat & 0x00000020) {1656 nvkm_error(subdev, "ILLEGAL_CLASS ch %d [%010llx %s] "1657 "subc %d class %04x mthd %04x data %08x\n",1658 chid, inst << 12, name, subc, class, mthd, data);1659 nvkm_wr32(device, 0x400100, 0x00000020);1660 stat &= ~0x00000020;1661 }1662 1663 if (stat & 0x00100000) {1664 const struct nvkm_enum *en =1665 nvkm_enum_find(nv50_data_error_names, code);1666 nvkm_error(subdev, "DATA_ERROR %08x [%s] ch %d [%010llx %s] "1667 "subc %d class %04x mthd %04x data %08x\n",1668 code, en ? en->name : "", chid, inst << 12,1669 name, subc, class, mthd, data);1670 nvkm_wr32(device, 0x400100, 0x00100000);1671 stat &= ~0x00100000;1672 }1673 1674 if (stat & 0x00200000) {1675 nvkm_error(subdev, "TRAP ch %d [%010llx %s]\n",1676 chid, inst << 12, name);1677 gf100_gr_trap_intr(gr);1678 nvkm_wr32(device, 0x400100, 0x00200000);1679 stat &= ~0x00200000;1680 }1681 1682 if (stat & 0x00080000) {1683 gf100_gr_ctxctl_isr(gr);1684 nvkm_wr32(device, 0x400100, 0x00080000);1685 stat &= ~0x00080000;1686 }1687 1688 if (stat) {1689 nvkm_error(subdev, "intr %08x\n", stat);1690 nvkm_wr32(device, 0x400100, stat);1691 }1692 1693 nvkm_wr32(device, 0x400500, 0x00010001);1694 nvkm_chan_put(&chan, flags);1695 return IRQ_HANDLED;1696}1697 1698static void1699gf100_gr_init_fw(struct nvkm_falcon *falcon,1700 struct nvkm_blob *code, struct nvkm_blob *data)1701{1702 nvkm_falcon_load_dmem(falcon, data->data, 0x0, data->size, 0);1703 nvkm_falcon_load_imem(falcon, code->data, 0x0, code->size, 0, 0, false);1704}1705 1706static void1707gf100_gr_init_csdata(struct gf100_gr *gr,1708 const struct gf100_gr_pack *pack,1709 u32 falcon, u32 starstar, u32 base)1710{1711 struct nvkm_device *device = gr->base.engine.subdev.device;1712 const struct gf100_gr_pack *iter;1713 const struct gf100_gr_init *init;1714 u32 addr = ~0, prev = ~0, xfer = 0;1715 u32 star, temp;1716 1717 nvkm_wr32(device, falcon + 0x01c0, 0x02000000 + starstar);1718 star = nvkm_rd32(device, falcon + 0x01c4);1719 temp = nvkm_rd32(device, falcon + 0x01c4);1720 if (temp > star)1721 star = temp;1722 nvkm_wr32(device, falcon + 0x01c0, 0x01000000 + star);1723 1724 pack_for_each_init(init, iter, pack) {1725 u32 head = init->addr - base;1726 u32 tail = head + init->count * init->pitch;1727 while (head < tail) {1728 if (head != prev + 4 || xfer >= 32) {1729 if (xfer) {1730 u32 data = ((--xfer << 26) | addr);1731 nvkm_wr32(device, falcon + 0x01c4, data);1732 star += 4;1733 }1734 addr = head;1735 xfer = 0;1736 }1737 prev = head;1738 xfer = xfer + 1;1739 head = head + init->pitch;1740 }1741 }1742 1743 nvkm_wr32(device, falcon + 0x01c4, (--xfer << 26) | addr);1744 nvkm_wr32(device, falcon + 0x01c0, 0x01000004 + starstar);1745 nvkm_wr32(device, falcon + 0x01c4, star + 4);1746}1747 1748/* Initialize context from an external (secure or not) firmware */1749static int1750gf100_gr_init_ctxctl_ext(struct gf100_gr *gr)1751{1752 struct nvkm_subdev *subdev = &gr->base.engine.subdev;1753 struct nvkm_device *device = subdev->device;1754 u32 lsf_mask = 0;1755 int ret;1756 1757 /* load fuc microcode */1758 nvkm_mc_unk260(device, 0);1759 1760 /* securely-managed falcons must be reset using secure boot */1761 1762 if (!nvkm_acr_managed_falcon(device, NVKM_ACR_LSF_FECS)) {1763 gf100_gr_init_fw(&gr->fecs.falcon, &gr->fecs.inst,1764 &gr->fecs.data);1765 } else {1766 lsf_mask |= BIT(NVKM_ACR_LSF_FECS);1767 }1768 1769 if (!nvkm_acr_managed_falcon(device, NVKM_ACR_LSF_GPCCS)) {1770 gf100_gr_init_fw(&gr->gpccs.falcon, &gr->gpccs.inst,1771 &gr->gpccs.data);1772 } else {1773 lsf_mask |= BIT(NVKM_ACR_LSF_GPCCS);1774 }1775 1776 if (lsf_mask) {1777 ret = nvkm_acr_bootstrap_falcons(device, lsf_mask);1778 if (ret)1779 return ret;1780 }1781 1782 nvkm_mc_unk260(device, 1);1783 1784 /* start both of them running */1785 nvkm_wr32(device, 0x409800, 0x00000000);1786 nvkm_wr32(device, 0x41a10c, 0x00000000);1787 nvkm_wr32(device, 0x40910c, 0x00000000);1788 1789 nvkm_falcon_start(&gr->gpccs.falcon);1790 nvkm_falcon_start(&gr->fecs.falcon);1791 1792 if (nvkm_msec(device, 2000,1793 if (nvkm_rd32(device, 0x409800) & 0x00000001)1794 break;1795 ) < 0)1796 return -EBUSY;1797 1798 gf100_gr_fecs_set_watchdog_timeout(gr, 0x7fffffff);1799 1800 /* Determine how much memory is required to store main context image. */1801 ret = gf100_gr_fecs_discover_image_size(gr, &gr->size);1802 if (ret)1803 return ret;1804 1805 /* Determine how much memory is required to store ZCULL image. */1806 ret = gf100_gr_fecs_discover_zcull_image_size(gr, &gr->size_zcull);1807 if (ret)1808 return ret;1809 1810 /* Determine how much memory is required to store PerfMon image. */1811 ret = gf100_gr_fecs_discover_pm_image_size(gr, &gr->size_pm);1812 if (ret)1813 return ret;1814 1815 /*XXX: We (likely) require PMU support to even bother with this.1816 *1817 * Also, it seems like not all GPUs support ELPG. Traces I1818 * have here show RM enabling it on Kepler/Turing, but none1819 * of the GPUs between those. NVGPU decides this by PCIID.1820 */1821 if (0) {1822 ret = gf100_gr_fecs_elpg_bind(gr);1823 if (ret)1824 return ret;1825 }1826 1827 return 0;1828}1829 1830static int1831gf100_gr_init_ctxctl_int(struct gf100_gr *gr)1832{1833 const struct gf100_grctx_func *grctx = gr->func->grctx;1834 struct nvkm_subdev *subdev = &gr->base.engine.subdev;1835 struct nvkm_device *device = subdev->device;1836 1837 if (!gr->func->fecs.ucode) {1838 return -ENOSYS;1839 }1840 1841 /* load HUB microcode */1842 nvkm_mc_unk260(device, 0);1843 nvkm_falcon_load_dmem(&gr->fecs.falcon,1844 gr->func->fecs.ucode->data.data, 0x0,1845 gr->func->fecs.ucode->data.size, 0);1846 nvkm_falcon_load_imem(&gr->fecs.falcon,1847 gr->func->fecs.ucode->code.data, 0x0,1848 gr->func->fecs.ucode->code.size, 0, 0, false);1849 1850 /* load GPC microcode */1851 nvkm_falcon_load_dmem(&gr->gpccs.falcon,1852 gr->func->gpccs.ucode->data.data, 0x0,1853 gr->func->gpccs.ucode->data.size, 0);1854 nvkm_falcon_load_imem(&gr->gpccs.falcon,1855 gr->func->gpccs.ucode->code.data, 0x0,1856 gr->func->gpccs.ucode->code.size, 0, 0, false);1857 nvkm_mc_unk260(device, 1);1858 1859 /* load register lists */1860 gf100_gr_init_csdata(gr, grctx->hub, 0x409000, 0x000, 0x000000);1861 gf100_gr_init_csdata(gr, grctx->gpc_0, 0x41a000, 0x000, 0x418000);1862 gf100_gr_init_csdata(gr, grctx->gpc_1, 0x41a000, 0x000, 0x418000);1863 gf100_gr_init_csdata(gr, grctx->tpc, 0x41a000, 0x004, 0x419800);1864 gf100_gr_init_csdata(gr, grctx->ppc, 0x41a000, 0x008, 0x41be00);1865 1866 /* start HUB ucode running, it'll init the GPCs */1867 nvkm_wr32(device, 0x40910c, 0x00000000);1868 nvkm_wr32(device, 0x409100, 0x00000002);1869 if (nvkm_msec(device, 2000,1870 if (nvkm_rd32(device, 0x409800) & 0x80000000)1871 break;1872 ) < 0) {1873 gf100_gr_ctxctl_debug(gr);1874 return -EBUSY;1875 }1876 1877 gr->size = nvkm_rd32(device, 0x409804);1878 return 0;1879}1880 1881int1882gf100_gr_init_ctxctl(struct gf100_gr *gr)1883{1884 int ret;1885 1886 if (gr->firmware)1887 ret = gf100_gr_init_ctxctl_ext(gr);1888 else1889 ret = gf100_gr_init_ctxctl_int(gr);1890 1891 return ret;1892}1893 1894int1895gf100_gr_oneinit_sm_id(struct gf100_gr *gr)1896{1897 int tpc, gpc;1898 1899 for (tpc = 0; tpc < gr->tpc_max; tpc++) {1900 for (gpc = 0; gpc < gr->gpc_nr; gpc++) {1901 if (tpc < gr->tpc_nr[gpc]) {1902 gr->sm[gr->sm_nr].gpc = gpc;1903 gr->sm[gr->sm_nr].tpc = tpc;1904 gr->sm_nr++;1905 }1906 }1907 }1908 1909 return 0;1910}1911 1912void1913gf100_gr_oneinit_tiles(struct gf100_gr *gr)1914{1915 static const u8 primes[] = {1916 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 611917 };1918 int init_frac[GPC_MAX], init_err[GPC_MAX], run_err[GPC_MAX], i, j;1919 u32 mul_factor, comm_denom;1920 u8 gpc_map[GPC_MAX];1921 bool sorted;1922 1923 switch (gr->tpc_total) {1924 case 15: gr->screen_tile_row_offset = 0x06; break;1925 case 14: gr->screen_tile_row_offset = 0x05; break;1926 case 13: gr->screen_tile_row_offset = 0x02; break;1927 case 11: gr->screen_tile_row_offset = 0x07; break;1928 case 10: gr->screen_tile_row_offset = 0x06; break;1929 case 7:1930 case 5: gr->screen_tile_row_offset = 0x01; break;1931 case 3: gr->screen_tile_row_offset = 0x02; break;1932 case 2:1933 case 1: gr->screen_tile_row_offset = 0x01; break;1934 default: gr->screen_tile_row_offset = 0x03;1935 for (i = 0; i < ARRAY_SIZE(primes); i++) {1936 if (gr->tpc_total % primes[i]) {1937 gr->screen_tile_row_offset = primes[i];1938 break;1939 }1940 }1941 break;1942 }1943 1944 /* Sort GPCs by TPC count, highest-to-lowest. */1945 for (i = 0; i < gr->gpc_nr; i++)1946 gpc_map[i] = i;1947 sorted = false;1948 1949 while (!sorted) {1950 for (sorted = true, i = 0; i < gr->gpc_nr - 1; i++) {1951 if (gr->tpc_nr[gpc_map[i + 1]] >1952 gr->tpc_nr[gpc_map[i + 0]]) {1953 u8 swap = gpc_map[i];1954 gpc_map[i + 0] = gpc_map[i + 1];1955 gpc_map[i + 1] = swap;1956 sorted = false;1957 }1958 }1959 }1960 1961 /* Determine tile->GPC mapping */1962 mul_factor = gr->gpc_nr * gr->tpc_max;1963 if (mul_factor & 1)1964 mul_factor = 2;1965 else1966 mul_factor = 1;1967 1968 comm_denom = gr->gpc_nr * gr->tpc_max * mul_factor;1969 1970 for (i = 0; i < gr->gpc_nr; i++) {1971 init_frac[i] = gr->tpc_nr[gpc_map[i]] * gr->gpc_nr * mul_factor;1972 init_err[i] = i * gr->tpc_max * mul_factor - comm_denom/2;1973 run_err[i] = init_frac[i] + init_err[i];1974 }1975 1976 for (i = 0; i < gr->tpc_total;) {1977 for (j = 0; j < gr->gpc_nr; j++) {1978 if ((run_err[j] * 2) >= comm_denom) {1979 gr->tile[i++] = gpc_map[j];1980 run_err[j] += init_frac[j] - comm_denom;1981 } else {1982 run_err[j] += init_frac[j];1983 }1984 }1985 }1986}1987 1988static int1989gf100_gr_oneinit(struct nvkm_gr *base)1990{1991 struct gf100_gr *gr = gf100_gr(base);1992 struct nvkm_subdev *subdev = &gr->base.engine.subdev;1993 struct nvkm_device *device = subdev->device;1994 struct nvkm_intr *intr = &device->mc->intr;1995 enum nvkm_intr_type intr_type = NVKM_INTR_SUBDEV;1996 int ret, i, j;1997 1998 if (gr->func->oneinit_intr)1999 intr = gr->func->oneinit_intr(gr, &intr_type);2000 2001 ret = nvkm_inth_add(intr, intr_type, NVKM_INTR_PRIO_NORMAL, &gr->base.engine.subdev,2002 gf100_gr_intr, &gr->base.engine.subdev.inth);2003 if (ret)2004 return ret;2005 2006 nvkm_pmu_pgob(device->pmu, false);2007 2008 gr->rop_nr = gr->func->rops(gr);2009 gr->gpc_nr = nvkm_rd32(device, 0x409604) & 0x0000001f;2010 for (i = 0; i < gr->gpc_nr; i++) {2011 gr->tpc_nr[i] = nvkm_rd32(device, GPC_UNIT(i, 0x2608));2012 gr->tpc_max = max(gr->tpc_max, gr->tpc_nr[i]);2013 gr->tpc_total += gr->tpc_nr[i];2014 for (j = 0; j < gr->func->ppc_nr; j++) {2015 gr->ppc_tpc_mask[i][j] =2016 nvkm_rd32(device, GPC_UNIT(i, 0x0c30 + (j * 4)));2017 if (gr->ppc_tpc_mask[i][j] == 0)2018 continue;2019 2020 gr->ppc_nr[i]++;2021 2022 gr->ppc_mask[i] |= (1 << j);2023 gr->ppc_tpc_nr[i][j] = hweight8(gr->ppc_tpc_mask[i][j]);2024 if (gr->ppc_tpc_min == 0 ||2025 gr->ppc_tpc_min > gr->ppc_tpc_nr[i][j])2026 gr->ppc_tpc_min = gr->ppc_tpc_nr[i][j];2027 if (gr->ppc_tpc_max < gr->ppc_tpc_nr[i][j])2028 gr->ppc_tpc_max = gr->ppc_tpc_nr[i][j];2029 }2030 2031 gr->ppc_total += gr->ppc_nr[i];2032 }2033 2034 /* Allocate global context buffers. */2035 ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST_SR_LOST,2036 gr->func->grctx->pagepool_size, 0x100, false, &gr->pagepool);2037 if (ret)2038 return ret;2039 2040 ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST_SR_LOST, gr->func->grctx->bundle_size,2041 0x100, false, &gr->bundle_cb);2042 if (ret)2043 return ret;2044 2045 ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST_SR_LOST,2046 gr->func->grctx->attrib_cb_size(gr), 0x1000, false, &gr->attrib_cb);2047 if (ret)2048 return ret;2049 2050 if (gr->func->grctx->unknown_size) {2051 ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, gr->func->grctx->unknown_size,2052 0x100, false, &gr->unknown);2053 if (ret)2054 return ret;2055 }2056 2057 memset(gr->tile, 0xff, sizeof(gr->tile));2058 gr->func->oneinit_tiles(gr);2059 2060 return gr->func->oneinit_sm_id(gr);2061}2062 2063static int2064gf100_gr_init_(struct nvkm_gr *base)2065{2066 struct gf100_gr *gr = gf100_gr(base);2067 struct nvkm_subdev *subdev = &base->engine.subdev;2068 struct nvkm_device *device = subdev->device;2069 bool reset = device->chipset == 0x137 || device->chipset == 0x138;2070 int ret;2071 2072 /* On certain GP107/GP108 boards, we trigger a weird issue where2073 * GR will stop responding to PRI accesses after we've asked the2074 * SEC2 RTOS to boot the GR falcons. This happens with far more2075 * frequency when cold-booting a board (ie. returning from D3).2076 *2077 * The root cause for this is not known and has proven difficult2078 * to isolate, with many avenues being dead-ends.2079 *2080 * A workaround was discovered by Karol, whereby putting GR into2081 * reset for an extended period right before initialisation2082 * prevents the problem from occuring.2083 *2084 * XXX: As RM does not require any such workaround, this is more2085 * of a hack than a true fix.2086 */2087 reset = nvkm_boolopt(device->cfgopt, "NvGrResetWar", reset);2088 if (reset) {2089 nvkm_mask(device, 0x000200, 0x00001000, 0x00000000);2090 nvkm_rd32(device, 0x000200);2091 msleep(50);2092 nvkm_mask(device, 0x000200, 0x00001000, 0x00001000);2093 nvkm_rd32(device, 0x000200);2094 }2095 2096 nvkm_pmu_pgob(gr->base.engine.subdev.device->pmu, false);2097 2098 ret = nvkm_falcon_get(&gr->fecs.falcon, subdev);2099 if (ret)2100 return ret;2101 2102 ret = nvkm_falcon_get(&gr->gpccs.falcon, subdev);2103 if (ret)2104 return ret;2105 2106 ret = gr->func->init(gr);2107 if (ret)2108 return ret;2109 2110 nvkm_inth_allow(&subdev->inth);2111 return 0;2112}2113 2114static int2115gf100_gr_fini(struct nvkm_gr *base, bool suspend)2116{2117 struct gf100_gr *gr = gf100_gr(base);2118 struct nvkm_subdev *subdev = &gr->base.engine.subdev;2119 2120 nvkm_inth_block(&subdev->inth);2121 2122 nvkm_falcon_put(&gr->gpccs.falcon, subdev);2123 nvkm_falcon_put(&gr->fecs.falcon, subdev);2124 return 0;2125}2126 2127static void *2128gf100_gr_dtor(struct nvkm_gr *base)2129{2130 struct gf100_gr *gr = gf100_gr(base);2131 2132 kfree(gr->data);2133 2134 nvkm_memory_unref(&gr->unknown);2135 nvkm_memory_unref(&gr->attrib_cb);2136 nvkm_memory_unref(&gr->bundle_cb);2137 nvkm_memory_unref(&gr->pagepool);2138 2139 nvkm_falcon_dtor(&gr->gpccs.falcon);2140 nvkm_falcon_dtor(&gr->fecs.falcon);2141 2142 nvkm_blob_dtor(&gr->fecs.inst);2143 nvkm_blob_dtor(&gr->fecs.data);2144 nvkm_blob_dtor(&gr->gpccs.inst);2145 nvkm_blob_dtor(&gr->gpccs.data);2146 2147 vfree(gr->bundle64);2148 vfree(gr->bundle_veid);2149 vfree(gr->bundle);2150 vfree(gr->method);2151 vfree(gr->sw_ctx);2152 vfree(gr->sw_nonctx);2153 vfree(gr->sw_nonctx1);2154 vfree(gr->sw_nonctx2);2155 vfree(gr->sw_nonctx3);2156 vfree(gr->sw_nonctx4);2157 2158 return gr;2159}2160 2161static const struct nvkm_falcon_func2162gf100_gr_flcn = {2163 .load_imem = nvkm_falcon_v1_load_imem,2164 .load_dmem = nvkm_falcon_v1_load_dmem,2165 .start = nvkm_falcon_v1_start,2166};2167 2168void2169gf100_gr_init_num_tpc_per_gpc(struct gf100_gr *gr, bool pd, bool ds)2170{2171 struct nvkm_device *device = gr->base.engine.subdev.device;2172 int gpc, i, j;2173 u32 data;2174 2175 for (gpc = 0, i = 0; i < 4; i++) {2176 for (data = 0, j = 0; j < 8 && gpc < gr->gpc_nr; j++, gpc++)2177 data |= gr->tpc_nr[gpc] << (j * 4);2178 if (pd)2179 nvkm_wr32(device, 0x406028 + (i * 4), data);2180 if (ds)2181 nvkm_wr32(device, 0x405870 + (i * 4), data);2182 }2183}2184 2185void2186gf100_gr_init_400054(struct gf100_gr *gr)2187{2188 nvkm_wr32(gr->base.engine.subdev.device, 0x400054, 0x34ce3464);2189}2190 2191void2192gf100_gr_init_exception2(struct gf100_gr *gr)2193{2194 struct nvkm_device *device = gr->base.engine.subdev.device;2195 2196 nvkm_wr32(device, 0x40011c, 0xffffffff);2197 nvkm_wr32(device, 0x400134, 0xffffffff);2198}2199 2200void2201gf100_gr_init_rop_exceptions(struct gf100_gr *gr)2202{2203 struct nvkm_device *device = gr->base.engine.subdev.device;2204 int rop;2205 2206 for (rop = 0; rop < gr->rop_nr; rop++) {2207 nvkm_wr32(device, ROP_UNIT(rop, 0x144), 0x40000000);2208 nvkm_wr32(device, ROP_UNIT(rop, 0x070), 0x40000000);2209 nvkm_wr32(device, ROP_UNIT(rop, 0x204), 0xffffffff);2210 nvkm_wr32(device, ROP_UNIT(rop, 0x208), 0xffffffff);2211 }2212}2213 2214void2215gf100_gr_init_shader_exceptions(struct gf100_gr *gr, int gpc, int tpc)2216{2217 struct nvkm_device *device = gr->base.engine.subdev.device;2218 nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x644), 0x001ffffe);2219 nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x64c), 0x0000000f);2220}2221 2222void2223gf100_gr_init_tex_hww_esr(struct gf100_gr *gr, int gpc, int tpc)2224{2225 struct nvkm_device *device = gr->base.engine.subdev.device;2226 nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x224), 0xc0000000);2227}2228 2229void2230gf100_gr_init_419eb4(struct gf100_gr *gr)2231{2232 struct nvkm_device *device = gr->base.engine.subdev.device;2233 nvkm_mask(device, 0x419eb4, 0x00001000, 0x00001000);2234}2235 2236void2237gf100_gr_init_419cc0(struct gf100_gr *gr)2238{2239 struct nvkm_device *device = gr->base.engine.subdev.device;2240 int gpc, tpc;2241 2242 nvkm_mask(device, 0x419cc0, 0x00000008, 0x00000008);2243 2244 for (gpc = 0; gpc < gr->gpc_nr; gpc++) {2245 for (tpc = 0; tpc < gr->tpc_nr[gpc]; tpc++)2246 nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x48c), 0xc0000000);2247 }2248}2249 2250void2251gf100_gr_init_40601c(struct gf100_gr *gr)2252{2253 nvkm_wr32(gr->base.engine.subdev.device, 0x40601c, 0xc0000000);2254}2255 2256void2257gf100_gr_init_fecs_exceptions(struct gf100_gr *gr)2258{2259 const u32 data = gr->firmware ? 0x000e0000 : 0x000e0001;2260 nvkm_wr32(gr->base.engine.subdev.device, 0x409c24, data);2261}2262 2263void2264gf100_gr_init_gpc_mmu(struct gf100_gr *gr)2265{2266 struct nvkm_device *device = gr->base.engine.subdev.device;2267 struct nvkm_fb *fb = device->fb;2268 2269 nvkm_wr32(device, 0x418880, nvkm_rd32(device, 0x100c80) & 0x00000001);2270 nvkm_wr32(device, 0x4188a4, 0x03000000);2271 nvkm_wr32(device, 0x418888, 0x00000000);2272 nvkm_wr32(device, 0x41888c, 0x00000000);2273 nvkm_wr32(device, 0x418890, 0x00000000);2274 nvkm_wr32(device, 0x418894, 0x00000000);2275 nvkm_wr32(device, 0x4188b4, nvkm_memory_addr(fb->mmu_wr) >> 8);2276 nvkm_wr32(device, 0x4188b8, nvkm_memory_addr(fb->mmu_rd) >> 8);2277}2278 2279void2280gf100_gr_init_num_active_ltcs(struct gf100_gr *gr)2281{2282 struct nvkm_device *device = gr->base.engine.subdev.device;2283 nvkm_wr32(device, GPC_BCAST(0x08ac), nvkm_rd32(device, 0x100800));2284}2285 2286void2287gf100_gr_init_zcull(struct gf100_gr *gr)2288{2289 struct nvkm_device *device = gr->base.engine.subdev.device;2290 const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, gr->tpc_total);2291 const u8 tile_nr = ALIGN(gr->tpc_total, 32);2292 u8 bank[GPC_MAX] = {}, gpc, i, j;2293 u32 data;2294 2295 for (i = 0; i < tile_nr; i += 8) {2296 for (data = 0, j = 0; j < 8 && i + j < gr->tpc_total; j++) {2297 data |= bank[gr->tile[i + j]] << (j * 4);2298 bank[gr->tile[i + j]]++;2299 }2300 nvkm_wr32(device, GPC_BCAST(0x0980 + ((i / 8) * 4)), data);2301 }2302 2303 for (gpc = 0; gpc < gr->gpc_nr; gpc++) {2304 nvkm_wr32(device, GPC_UNIT(gpc, 0x0914),2305 gr->screen_tile_row_offset << 8 | gr->tpc_nr[gpc]);2306 nvkm_wr32(device, GPC_UNIT(gpc, 0x0910), 0x00040000 |2307 gr->tpc_total);2308 nvkm_wr32(device, GPC_UNIT(gpc, 0x0918), magicgpc918);2309 }2310 2311 nvkm_wr32(device, GPC_BCAST(0x1bd4), magicgpc918);2312}2313 2314void2315gf100_gr_init_vsc_stream_master(struct gf100_gr *gr)2316{2317 struct nvkm_device *device = gr->base.engine.subdev.device;2318 nvkm_mask(device, TPC_UNIT(0, 0, 0x05c), 0x00000001, 0x00000001);2319}2320 2321static int2322gf100_gr_reset(struct nvkm_gr *base)2323{2324 struct nvkm_subdev *subdev = &base->engine.subdev;2325 struct nvkm_device *device = subdev->device;2326 struct gf100_gr *gr = gf100_gr(base);2327 2328 nvkm_mask(device, 0x400500, 0x00000001, 0x00000000);2329 2330 WARN_ON(gf100_gr_fecs_halt_pipeline(gr));2331 2332 subdev->func->fini(subdev, false);2333 nvkm_mc_disable(device, subdev->type, subdev->inst);2334 if (gr->func->gpccs.reset)2335 gr->func->gpccs.reset(gr);2336 2337 nvkm_mc_enable(device, subdev->type, subdev->inst);2338 return subdev->func->init(subdev);2339}2340 2341int2342gf100_gr_init(struct gf100_gr *gr)2343{2344 struct nvkm_device *device = gr->base.engine.subdev.device;2345 int gpc, tpc;2346 2347 nvkm_mask(device, 0x400500, 0x00010001, 0x00000000);2348 2349 gr->func->init_gpc_mmu(gr);2350 2351 if (gr->sw_nonctx1) {2352 gf100_gr_mmio(gr, gr->sw_nonctx1);2353 gf100_gr_mmio(gr, gr->sw_nonctx2);2354 gf100_gr_mmio(gr, gr->sw_nonctx3);2355 gf100_gr_mmio(gr, gr->sw_nonctx4);2356 } else2357 if (gr->sw_nonctx) {2358 gf100_gr_mmio(gr, gr->sw_nonctx);2359 } else {2360 gf100_gr_mmio(gr, gr->func->mmio);2361 }2362 2363 gf100_gr_wait_idle(gr);2364 2365 if (gr->func->init_r405a14)2366 gr->func->init_r405a14(gr);2367 2368 if (gr->func->clkgate_pack)2369 nvkm_therm_clkgate_init(device->therm, gr->func->clkgate_pack);2370 2371 if (gr->func->init_bios)2372 gr->func->init_bios(gr);2373 2374 gr->func->init_vsc_stream_master(gr);2375 gr->func->init_zcull(gr);2376 gr->func->init_num_active_ltcs(gr);2377 if (gr->func->init_rop_active_fbps)2378 gr->func->init_rop_active_fbps(gr);2379 if (gr->func->init_bios_2)2380 gr->func->init_bios_2(gr);2381 if (gr->func->init_swdx_pes_mask)2382 gr->func->init_swdx_pes_mask(gr);2383 if (gr->func->init_fs)2384 gr->func->init_fs(gr);2385 2386 nvkm_wr32(device, 0x400500, 0x00010001);2387 2388 nvkm_wr32(device, 0x400100, 0xffffffff);2389 nvkm_wr32(device, 0x40013c, 0xffffffff);2390 nvkm_wr32(device, 0x400124, 0x00000002);2391 2392 gr->func->init_fecs_exceptions(gr);2393 2394 if (gr->func->init_40a790)2395 gr->func->init_40a790(gr);2396 2397 if (gr->func->init_ds_hww_esr_2)2398 gr->func->init_ds_hww_esr_2(gr);2399 2400 nvkm_wr32(device, 0x404000, 0xc0000000);2401 nvkm_wr32(device, 0x404600, 0xc0000000);2402 nvkm_wr32(device, 0x408030, 0xc0000000);2403 2404 if (gr->func->init_40601c)2405 gr->func->init_40601c(gr);2406 2407 nvkm_wr32(device, 0x406018, 0xc0000000);2408 nvkm_wr32(device, 0x404490, 0xc0000000);2409 2410 if (gr->func->init_sked_hww_esr)2411 gr->func->init_sked_hww_esr(gr);2412 2413 nvkm_wr32(device, 0x405840, 0xc0000000);2414 nvkm_wr32(device, 0x405844, 0x00ffffff);2415 2416 if (gr->func->init_419cc0)2417 gr->func->init_419cc0(gr);2418 if (gr->func->init_419eb4)2419 gr->func->init_419eb4(gr);2420 if (gr->func->init_419c9c)2421 gr->func->init_419c9c(gr);2422 2423 if (gr->func->init_ppc_exceptions)2424 gr->func->init_ppc_exceptions(gr);2425 2426 for (gpc = 0; gpc < gr->gpc_nr; gpc++) {2427 nvkm_wr32(device, GPC_UNIT(gpc, 0x0420), 0xc0000000);2428 nvkm_wr32(device, GPC_UNIT(gpc, 0x0900), 0xc0000000);2429 nvkm_wr32(device, GPC_UNIT(gpc, 0x1028), 0xc0000000);2430 nvkm_wr32(device, GPC_UNIT(gpc, 0x0824), 0xc0000000);2431 for (tpc = 0; tpc < gr->tpc_nr[gpc]; tpc++) {2432 nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x508), 0xffffffff);2433 nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x50c), 0xffffffff);2434 if (gr->func->init_tex_hww_esr)2435 gr->func->init_tex_hww_esr(gr, gpc, tpc);2436 nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x084), 0xc0000000);2437 if (gr->func->init_504430)2438 gr->func->init_504430(gr, gpc, tpc);2439 gr->func->init_shader_exceptions(gr, gpc, tpc);2440 }2441 nvkm_wr32(device, GPC_UNIT(gpc, 0x2c90), 0xffffffff);2442 nvkm_wr32(device, GPC_UNIT(gpc, 0x2c94), 0xffffffff);2443 }2444 2445 gr->func->init_rop_exceptions(gr);2446 2447 nvkm_wr32(device, 0x400108, 0xffffffff);2448 nvkm_wr32(device, 0x400138, 0xffffffff);2449 nvkm_wr32(device, 0x400118, 0xffffffff);2450 nvkm_wr32(device, 0x400130, 0xffffffff);2451 if (gr->func->init_exception2)2452 gr->func->init_exception2(gr);2453 2454 if (gr->func->init_400054)2455 gr->func->init_400054(gr);2456 2457 gf100_gr_zbc_init(gr);2458 2459 if (gr->func->init_4188a4)2460 gr->func->init_4188a4(gr);2461 2462 return gf100_gr_init_ctxctl(gr);2463}2464 2465void2466gf100_gr_fecs_reset(struct gf100_gr *gr)2467{2468 struct nvkm_device *device = gr->base.engine.subdev.device;2469 2470 nvkm_wr32(device, 0x409614, 0x00000070);2471 nvkm_usec(device, 10, NVKM_DELAY);2472 nvkm_mask(device, 0x409614, 0x00000700, 0x00000700);2473 nvkm_usec(device, 10, NVKM_DELAY);2474 nvkm_rd32(device, 0x409614);2475}2476 2477#include "fuc/hubgf100.fuc3.h"2478 2479struct gf100_gr_ucode2480gf100_gr_fecs_ucode = {2481 .code.data = gf100_grhub_code,2482 .code.size = sizeof(gf100_grhub_code),2483 .data.data = gf100_grhub_data,2484 .data.size = sizeof(gf100_grhub_data),2485};2486 2487#include "fuc/gpcgf100.fuc3.h"2488 2489struct gf100_gr_ucode2490gf100_gr_gpccs_ucode = {2491 .code.data = gf100_grgpc_code,2492 .code.size = sizeof(gf100_grgpc_code),2493 .data.data = gf100_grgpc_data,2494 .data.size = sizeof(gf100_grgpc_data),2495};2496 2497static int2498gf100_gr_nonstall(struct nvkm_gr *base)2499{2500 struct gf100_gr *gr = gf100_gr(base);2501 2502 if (gr->func->nonstall)2503 return gr->func->nonstall(gr);2504 2505 return -EINVAL;2506}2507 2508static const struct nvkm_gr_func2509gf100_gr_ = {2510 .dtor = gf100_gr_dtor,2511 .oneinit = gf100_gr_oneinit,2512 .init = gf100_gr_init_,2513 .fini = gf100_gr_fini,2514 .nonstall = gf100_gr_nonstall,2515 .reset = gf100_gr_reset,2516 .units = gf100_gr_units,2517 .chan_new = gf100_gr_chan_new,2518 .object_get = gf100_gr_object_get,2519 .chsw_load = gf100_gr_chsw_load,2520 .ctxsw.pause = gf100_gr_fecs_stop_ctxsw,2521 .ctxsw.resume = gf100_gr_fecs_start_ctxsw,2522 .ctxsw.inst = gf100_gr_ctxsw_inst,2523};2524 2525static const struct gf100_gr_func2526gf100_gr = {2527 .oneinit_tiles = gf100_gr_oneinit_tiles,2528 .oneinit_sm_id = gf100_gr_oneinit_sm_id,2529 .init = gf100_gr_init,2530 .init_gpc_mmu = gf100_gr_init_gpc_mmu,2531 .init_vsc_stream_master = gf100_gr_init_vsc_stream_master,2532 .init_zcull = gf100_gr_init_zcull,2533 .init_num_active_ltcs = gf100_gr_init_num_active_ltcs,2534 .init_fecs_exceptions = gf100_gr_init_fecs_exceptions,2535 .init_40601c = gf100_gr_init_40601c,2536 .init_419cc0 = gf100_gr_init_419cc0,2537 .init_419eb4 = gf100_gr_init_419eb4,2538 .init_tex_hww_esr = gf100_gr_init_tex_hww_esr,2539 .init_shader_exceptions = gf100_gr_init_shader_exceptions,2540 .init_rop_exceptions = gf100_gr_init_rop_exceptions,2541 .init_exception2 = gf100_gr_init_exception2,2542 .init_400054 = gf100_gr_init_400054,2543 .trap_mp = gf100_gr_trap_mp,2544 .mmio = gf100_gr_pack_mmio,2545 .fecs.ucode = &gf100_gr_fecs_ucode,2546 .fecs.reset = gf100_gr_fecs_reset,2547 .gpccs.ucode = &gf100_gr_gpccs_ucode,2548 .rops = gf100_gr_rops,2549 .grctx = &gf100_grctx,2550 .zbc = &gf100_gr_zbc,2551 .sclass = {2552 { -1, -1, FERMI_TWOD_A },2553 { -1, -1, FERMI_MEMORY_TO_MEMORY_FORMAT_A },2554 { -1, -1, FERMI_A, &gf100_fermi },2555 { -1, -1, FERMI_COMPUTE_A },2556 {}2557 }2558};2559 2560int2561gf100_gr_nofw(struct gf100_gr *gr, int ver, const struct gf100_gr_fwif *fwif)2562{2563 gr->firmware = false;2564 return 0;2565}2566 2567static int2568gf100_gr_load_fw(struct gf100_gr *gr, const char *name,2569 struct nvkm_blob *blob)2570{2571 struct nvkm_subdev *subdev = &gr->base.engine.subdev;2572 struct nvkm_device *device = subdev->device;2573 const struct firmware *fw;2574 char f[32];2575 int ret;2576 2577 snprintf(f, sizeof(f), "nouveau/nv%02x_%s", device->chipset, name);2578 ret = request_firmware(&fw, f, device->dev);2579 if (ret) {2580 snprintf(f, sizeof(f), "nouveau/%s", name);2581 ret = request_firmware(&fw, f, device->dev);2582 if (ret) {2583 nvkm_error(subdev, "failed to load %s\n", name);2584 return ret;2585 }2586 }2587 2588 blob->size = fw->size;2589 blob->data = kmemdup(fw->data, blob->size, GFP_KERNEL);2590 release_firmware(fw);2591 return (blob->data != NULL) ? 0 : -ENOMEM;2592}2593 2594int2595gf100_gr_load(struct gf100_gr *gr, int ver, const struct gf100_gr_fwif *fwif)2596{2597 struct nvkm_device *device = gr->base.engine.subdev.device;2598 2599 if (!nvkm_boolopt(device->cfgopt, "NvGrUseFW", false))2600 return -EINVAL;2601 2602 if (gf100_gr_load_fw(gr, "fuc409c", &gr->fecs.inst) ||2603 gf100_gr_load_fw(gr, "fuc409d", &gr->fecs.data) ||2604 gf100_gr_load_fw(gr, "fuc41ac", &gr->gpccs.inst) ||2605 gf100_gr_load_fw(gr, "fuc41ad", &gr->gpccs.data))2606 return -ENOENT;2607 2608 gr->firmware = true;2609 return 0;2610}2611 2612static const struct gf100_gr_fwif2613gf100_gr_fwif[] = {2614 { -1, gf100_gr_load, &gf100_gr },2615 { -1, gf100_gr_nofw, &gf100_gr },2616 {}2617};2618 2619int2620gf100_gr_new_(const struct gf100_gr_fwif *fwif, struct nvkm_device *device,2621 enum nvkm_subdev_type type, int inst, struct nvkm_gr **pgr)2622{2623 struct gf100_gr *gr;2624 int ret;2625 2626 if (!(gr = kzalloc(sizeof(*gr), GFP_KERNEL)))2627 return -ENOMEM;2628 *pgr = &gr->base;2629 2630 ret = nvkm_gr_ctor(&gf100_gr_, device, type, inst, true, &gr->base);2631 if (ret)2632 return ret;2633 2634 fwif = nvkm_firmware_load(&gr->base.engine.subdev, fwif, "Gr", gr);2635 if (IS_ERR(fwif))2636 return PTR_ERR(fwif);2637 2638 gr->func = fwif->func;2639 2640 ret = nvkm_falcon_ctor(&gf100_gr_flcn, &gr->base.engine.subdev,2641 "fecs", 0x409000, &gr->fecs.falcon);2642 if (ret)2643 return ret;2644 2645 mutex_init(&gr->fecs.mutex);2646 2647 ret = nvkm_falcon_ctor(&gf100_gr_flcn, &gr->base.engine.subdev,2648 "gpccs", 0x41a000, &gr->gpccs.falcon);2649 if (ret)2650 return ret;2651 2652 return 0;2653}2654 2655int2656gf100_gr_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_gr **pgr)2657{2658 return gf100_gr_new_(gf100_gr_fwif, device, type, inst, pgr);2659}2660