1008 lines · c
1/*2 * Copyright 2013 Red Hat Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 * Authors: Ben Skeggs23 * Roy Spliet <rspliet@eclipso.eu>24 */25#define gt215_ram(p) container_of((p), struct gt215_ram, base)26#include "ram.h"27#include "ramfuc.h"28 29#include <core/memory.h>30#include <core/option.h>31#include <subdev/bios.h>32#include <subdev/bios/M0205.h>33#include <subdev/bios/rammap.h>34#include <subdev/bios/timing.h>35#include <subdev/clk/gt215.h>36#include <subdev/gpio.h>37 38struct gt215_ramfuc {39 struct ramfuc base;40 struct ramfuc_reg r_0x001610;41 struct ramfuc_reg r_0x001700;42 struct ramfuc_reg r_0x002504;43 struct ramfuc_reg r_0x004000;44 struct ramfuc_reg r_0x004004;45 struct ramfuc_reg r_0x004018;46 struct ramfuc_reg r_0x004128;47 struct ramfuc_reg r_0x004168;48 struct ramfuc_reg r_0x100080;49 struct ramfuc_reg r_0x100200;50 struct ramfuc_reg r_0x100210;51 struct ramfuc_reg r_0x100220[9];52 struct ramfuc_reg r_0x100264;53 struct ramfuc_reg r_0x1002d0;54 struct ramfuc_reg r_0x1002d4;55 struct ramfuc_reg r_0x1002dc;56 struct ramfuc_reg r_0x10053c;57 struct ramfuc_reg r_0x1005a0;58 struct ramfuc_reg r_0x1005a4;59 struct ramfuc_reg r_0x100700;60 struct ramfuc_reg r_0x100714;61 struct ramfuc_reg r_0x100718;62 struct ramfuc_reg r_0x10071c;63 struct ramfuc_reg r_0x100720;64 struct ramfuc_reg r_0x100760;65 struct ramfuc_reg r_0x1007a0;66 struct ramfuc_reg r_0x1007e0;67 struct ramfuc_reg r_0x100da0;68 struct ramfuc_reg r_0x10f804;69 struct ramfuc_reg r_0x1110e0;70 struct ramfuc_reg r_0x111100;71 struct ramfuc_reg r_0x111104;72 struct ramfuc_reg r_0x1111e0;73 struct ramfuc_reg r_0x111400;74 struct ramfuc_reg r_0x611200;75 struct ramfuc_reg r_mr[4];76 struct ramfuc_reg r_gpio[4];77};78 79struct gt215_ltrain {80 enum {81 NVA3_TRAIN_UNKNOWN,82 NVA3_TRAIN_UNSUPPORTED,83 NVA3_TRAIN_ONCE,84 NVA3_TRAIN_EXEC,85 NVA3_TRAIN_DONE86 } state;87 u32 r_100720;88 u32 r_1111e0;89 u32 r_111400;90 struct nvkm_memory *memory;91};92 93struct gt215_ram {94 struct nvkm_ram base;95 struct gt215_ramfuc fuc;96 struct gt215_ltrain ltrain;97};98 99static void100gt215_link_train_calc(u32 *vals, struct gt215_ltrain *train)101{102 int i, lo, hi;103 u8 median[8], bins[4] = {0, 0, 0, 0}, bin = 0, qty = 0;104 105 for (i = 0; i < 8; i++) {106 for (lo = 0; lo < 0x40; lo++) {107 if (!(vals[lo] & 0x80000000))108 continue;109 if (vals[lo] & (0x101 << i))110 break;111 }112 113 if (lo == 0x40)114 return;115 116 for (hi = lo + 1; hi < 0x40; hi++) {117 if (!(vals[lo] & 0x80000000))118 continue;119 if (!(vals[hi] & (0x101 << i))) {120 hi--;121 break;122 }123 }124 125 median[i] = ((hi - lo) >> 1) + lo;126 bins[(median[i] & 0xf0) >> 4]++;127 median[i] += 0x30;128 }129 130 /* Find the best value for 0x1111e0 */131 for (i = 0; i < 4; i++) {132 if (bins[i] > qty) {133 bin = i + 3;134 qty = bins[i];135 }136 }137 138 train->r_100720 = 0;139 for (i = 0; i < 8; i++) {140 median[i] = max(median[i], (u8) (bin << 4));141 median[i] = min(median[i], (u8) ((bin << 4) | 0xf));142 143 train->r_100720 |= ((median[i] & 0x0f) << (i << 2));144 }145 146 train->r_1111e0 = 0x02000000 | (bin * 0x101);147 train->r_111400 = 0x0;148}149 150/*151 * Link training for (at least) DDR3152 */153static int154gt215_link_train(struct gt215_ram *ram)155{156 struct gt215_ltrain *train = &ram->ltrain;157 struct gt215_ramfuc *fuc = &ram->fuc;158 struct nvkm_subdev *subdev = &ram->base.fb->subdev;159 struct nvkm_device *device = subdev->device;160 struct nvkm_bios *bios = device->bios;161 struct nvkm_clk *clk = device->clk;162 u32 *result, r1700;163 int ret, i;164 struct nvbios_M0205T M0205T = { 0 };165 u8 ver, hdr, cnt, len, snr, ssz;166 unsigned int clk_current;167 unsigned long flags;168 unsigned long *f = &flags;169 170 if (nvkm_boolopt(device->cfgopt, "NvMemExec", true) != true)171 return -ENOSYS;172 173 /* XXX: Multiple partitions? */174 result = kmalloc_array(64, sizeof(u32), GFP_KERNEL);175 if (!result)176 return -ENOMEM;177 178 train->state = NVA3_TRAIN_EXEC;179 180 /* Clock speeds for training and back */181 nvbios_M0205Tp(bios, &ver, &hdr, &cnt, &len, &snr, &ssz, &M0205T);182 if (M0205T.freq == 0) {183 kfree(result);184 return -ENOENT;185 }186 187 clk_current = nvkm_clk_read(clk, nv_clk_src_mem);188 189 ret = gt215_clk_pre(clk, f);190 if (ret)191 goto out;192 193 /* First: clock up/down */194 ret = ram->base.func->calc(&ram->base, (u32) M0205T.freq * 1000);195 if (ret)196 goto out;197 198 /* Do this *after* calc, eliminates write in script */199 nvkm_wr32(device, 0x111400, 0x00000000);200 /* XXX: Magic writes that improve train reliability? */201 nvkm_mask(device, 0x100674, 0x0000ffff, 0x00000000);202 nvkm_mask(device, 0x1005e4, 0x0000ffff, 0x00000000);203 nvkm_mask(device, 0x100b0c, 0x000000ff, 0x00000000);204 nvkm_wr32(device, 0x100c04, 0x00000400);205 206 /* Now the training script */207 r1700 = ram_rd32(fuc, 0x001700);208 209 ram_mask(fuc, 0x100200, 0x00000800, 0x00000000);210 ram_wr32(fuc, 0x611200, 0x3300);211 ram_wait_vblank(fuc);212 ram_wait(fuc, 0x611200, 0x00000003, 0x00000000, 500000);213 ram_mask(fuc, 0x001610, 0x00000083, 0x00000003);214 ram_mask(fuc, 0x100080, 0x00000020, 0x00000000);215 ram_mask(fuc, 0x10f804, 0x80000000, 0x00000000);216 ram_wr32(fuc, 0x001700, 0x00000000);217 218 ram_train(fuc);219 220 /* Reset */221 ram_mask(fuc, 0x10f804, 0x80000000, 0x80000000);222 ram_wr32(fuc, 0x10053c, 0x0);223 ram_wr32(fuc, 0x100720, train->r_100720);224 ram_wr32(fuc, 0x1111e0, train->r_1111e0);225 ram_wr32(fuc, 0x111400, train->r_111400);226 ram_nuke(fuc, 0x100080);227 ram_mask(fuc, 0x100080, 0x00000020, 0x00000020);228 ram_nsec(fuc, 1000);229 230 ram_wr32(fuc, 0x001700, r1700);231 ram_mask(fuc, 0x001610, 0x00000083, 0x00000080);232 ram_wr32(fuc, 0x611200, 0x3330);233 ram_mask(fuc, 0x100200, 0x00000800, 0x00000800);234 235 ram_exec(fuc, true);236 237 ram->base.func->calc(&ram->base, clk_current);238 ram_exec(fuc, true);239 240 /* Post-processing, avoids flicker */241 nvkm_mask(device, 0x616308, 0x10, 0x10);242 nvkm_mask(device, 0x616b08, 0x10, 0x10);243 244 gt215_clk_post(clk, f);245 246 ram_train_result(ram->base.fb, result, 64);247 for (i = 0; i < 64; i++)248 nvkm_debug(subdev, "Train: %08x", result[i]);249 gt215_link_train_calc(result, train);250 251 nvkm_debug(subdev, "Train: %08x %08x %08x", train->r_100720,252 train->r_1111e0, train->r_111400);253 254 kfree(result);255 256 train->state = NVA3_TRAIN_DONE;257 258 return ret;259 260out:261 if(ret == -EBUSY)262 f = NULL;263 264 train->state = NVA3_TRAIN_UNSUPPORTED;265 266 gt215_clk_post(clk, f);267 kfree(result);268 return ret;269}270 271static int272gt215_link_train_init(struct gt215_ram *ram)273{274 static const u32 pattern[16] = {275 0xaaaaaaaa, 0xcccccccc, 0xdddddddd, 0xeeeeeeee,276 0x00000000, 0x11111111, 0x44444444, 0xdddddddd,277 0x33333333, 0x55555555, 0x77777777, 0x66666666,278 0x99999999, 0x88888888, 0xeeeeeeee, 0xbbbbbbbb,279 };280 struct gt215_ltrain *train = &ram->ltrain;281 struct nvkm_device *device = ram->base.fb->subdev.device;282 struct nvkm_bios *bios = device->bios;283 struct nvbios_M0205E M0205E;284 u8 ver, hdr, cnt, len;285 u32 r001700;286 u64 addr;287 int ret, i = 0;288 289 train->state = NVA3_TRAIN_UNSUPPORTED;290 291 /* We support type "5"292 * XXX: training pattern table appears to be unused for this routine */293 if (!nvbios_M0205Ep(bios, i, &ver, &hdr, &cnt, &len, &M0205E))294 return -ENOENT;295 296 if (M0205E.type != 5)297 return 0;298 299 train->state = NVA3_TRAIN_ONCE;300 301 ret = nvkm_ram_get(device, NVKM_RAM_MM_NORMAL, 0x01, 16, 0x8000,302 true, true, &ram->ltrain.memory);303 if (ret)304 return ret;305 306 addr = nvkm_memory_addr(ram->ltrain.memory);307 308 nvkm_wr32(device, 0x100538, 0x10000000 | (addr >> 16));309 nvkm_wr32(device, 0x1005a8, 0x0000ffff);310 nvkm_mask(device, 0x10f800, 0x00000001, 0x00000001);311 312 for (i = 0; i < 0x30; i++) {313 nvkm_wr32(device, 0x10f8c0, (i << 8) | i);314 nvkm_wr32(device, 0x10f900, pattern[i % 16]);315 }316 317 for (i = 0; i < 0x30; i++) {318 nvkm_wr32(device, 0x10f8e0, (i << 8) | i);319 nvkm_wr32(device, 0x10f920, pattern[i % 16]);320 }321 322 /* And upload the pattern */323 r001700 = nvkm_rd32(device, 0x1700);324 nvkm_wr32(device, 0x1700, addr >> 16);325 for (i = 0; i < 16; i++)326 nvkm_wr32(device, 0x700000 + (i << 2), pattern[i]);327 for (i = 0; i < 16; i++)328 nvkm_wr32(device, 0x700100 + (i << 2), pattern[i]);329 nvkm_wr32(device, 0x1700, r001700);330 331 train->r_100720 = nvkm_rd32(device, 0x100720);332 train->r_1111e0 = nvkm_rd32(device, 0x1111e0);333 train->r_111400 = nvkm_rd32(device, 0x111400);334 return 0;335}336 337static void338gt215_link_train_fini(struct gt215_ram *ram)339{340 nvkm_memory_unref(&ram->ltrain.memory);341}342 343/*344 * RAM reclocking345 */346#define T(t) cfg->timing_10_##t347static int348gt215_ram_timing_calc(struct gt215_ram *ram, u32 *timing)349{350 struct nvbios_ramcfg *cfg = &ram->base.target.bios;351 struct nvkm_subdev *subdev = &ram->base.fb->subdev;352 struct nvkm_device *device = subdev->device;353 int tUNK_base, tUNK_40_0, prevCL;354 u32 cur2, cur3, cur7, cur8;355 356 cur2 = nvkm_rd32(device, 0x100228);357 cur3 = nvkm_rd32(device, 0x10022c);358 cur7 = nvkm_rd32(device, 0x10023c);359 cur8 = nvkm_rd32(device, 0x100240);360 361 362 switch ((!T(CWL)) * ram->base.type) {363 case NVKM_RAM_TYPE_DDR2:364 T(CWL) = T(CL) - 1;365 break;366 case NVKM_RAM_TYPE_GDDR3:367 T(CWL) = ((cur2 & 0xff000000) >> 24) + 1;368 break;369 }370 371 prevCL = (cur3 & 0x000000ff) + 1;372 tUNK_base = ((cur7 & 0x00ff0000) >> 16) - prevCL;373 374 timing[0] = (T(RP) << 24 | T(RAS) << 16 | T(RFC) << 8 | T(RC));375 timing[1] = (T(WR) + 1 + T(CWL)) << 24 |376 max_t(u8,T(18), 1) << 16 |377 (T(WTR) + 1 + T(CWL)) << 8 |378 (5 + T(CL) - T(CWL));379 timing[2] = (T(CWL) - 1) << 24 |380 (T(RRD) << 16) |381 (T(RCDWR) << 8) |382 T(RCDRD);383 timing[3] = (cur3 & 0x00ff0000) |384 (0x30 + T(CL)) << 24 |385 (0xb + T(CL)) << 8 |386 (T(CL) - 1);387 timing[4] = T(20) << 24 |388 T(21) << 16 |389 T(13) << 8 |390 T(13);391 timing[5] = T(RFC) << 24 |392 max_t(u8,T(RCDRD), T(RCDWR)) << 16 |393 max_t(u8, (T(CWL) + 6), (T(CL) + 2)) << 8 |394 T(RP);395 timing[6] = (0x5a + T(CL)) << 16 |396 max_t(u8, 1, (6 - T(CL) + T(CWL))) << 8 |397 (0x50 + T(CL) - T(CWL));398 timing[7] = (cur7 & 0xff000000) |399 ((tUNK_base + T(CL)) << 16) |400 0x202;401 timing[8] = cur8 & 0xffffff00;402 403 switch (ram->base.type) {404 case NVKM_RAM_TYPE_DDR2:405 case NVKM_RAM_TYPE_GDDR3:406 tUNK_40_0 = prevCL - (cur8 & 0xff);407 if (tUNK_40_0 > 0)408 timing[8] |= T(CL);409 break;410 default:411 break;412 }413 414 nvkm_debug(subdev, "Entry: 220: %08x %08x %08x %08x\n",415 timing[0], timing[1], timing[2], timing[3]);416 nvkm_debug(subdev, " 230: %08x %08x %08x %08x\n",417 timing[4], timing[5], timing[6], timing[7]);418 nvkm_debug(subdev, " 240: %08x\n", timing[8]);419 return 0;420}421#undef T422 423static void424nvkm_sddr2_dll_reset(struct gt215_ramfuc *fuc)425{426 ram_mask(fuc, mr[0], 0x100, 0x100);427 ram_nsec(fuc, 1000);428 ram_mask(fuc, mr[0], 0x100, 0x000);429 ram_nsec(fuc, 1000);430}431 432static void433nvkm_sddr3_dll_disable(struct gt215_ramfuc *fuc, u32 *mr)434{435 u32 mr1_old = ram_rd32(fuc, mr[1]);436 437 if (!(mr1_old & 0x1)) {438 ram_wr32(fuc, 0x1002d4, 0x00000001);439 ram_wr32(fuc, mr[1], mr[1]);440 ram_nsec(fuc, 1000);441 }442}443 444static void445nvkm_gddr3_dll_disable(struct gt215_ramfuc *fuc, u32 *mr)446{447 u32 mr1_old = ram_rd32(fuc, mr[1]);448 449 if (!(mr1_old & 0x40)) {450 ram_wr32(fuc, mr[1], mr[1]);451 ram_nsec(fuc, 1000);452 }453}454 455static void456gt215_ram_lock_pll(struct gt215_ramfuc *fuc, struct gt215_clk_info *mclk)457{458 ram_wr32(fuc, 0x004004, mclk->pll);459 ram_mask(fuc, 0x004000, 0x00000001, 0x00000001);460 ram_mask(fuc, 0x004000, 0x00000010, 0x00000000);461 ram_wait(fuc, 0x004000, 0x00020000, 0x00020000, 64000);462 ram_mask(fuc, 0x004000, 0x00000010, 0x00000010);463}464 465static void466gt215_ram_gpio(struct gt215_ramfuc *fuc, u8 tag, u32 val)467{468 struct nvkm_gpio *gpio = fuc->base.fb->subdev.device->gpio;469 struct dcb_gpio_func func;470 u32 reg, sh, gpio_val;471 int ret;472 473 if (nvkm_gpio_get(gpio, 0, tag, DCB_GPIO_UNUSED) != val) {474 ret = nvkm_gpio_find(gpio, 0, tag, DCB_GPIO_UNUSED, &func);475 if (ret)476 return;477 478 reg = func.line >> 3;479 sh = (func.line & 0x7) << 2;480 gpio_val = ram_rd32(fuc, gpio[reg]);481 if (gpio_val & (8 << sh))482 val = !val;483 if (!(func.log[1] & 1))484 val = !val;485 486 ram_mask(fuc, gpio[reg], (0x3 << sh), ((val | 0x2) << sh));487 ram_nsec(fuc, 20000);488 }489}490 491static int492gt215_ram_calc(struct nvkm_ram *base, u32 freq)493{494 struct gt215_ram *ram = gt215_ram(base);495 struct gt215_ramfuc *fuc = &ram->fuc;496 struct gt215_ltrain *train = &ram->ltrain;497 struct nvkm_subdev *subdev = &ram->base.fb->subdev;498 struct nvkm_device *device = subdev->device;499 struct nvkm_bios *bios = device->bios;500 struct gt215_clk_info mclk;501 struct nvkm_gpio *gpio = device->gpio;502 struct nvkm_ram_data *next;503 u8 ver, hdr, cnt, len, strap;504 u32 data;505 u32 r004018, r100760, r100da0, r111100, ctrl;506 u32 unk714, unk718, unk71c;507 int ret, i;508 u32 timing[9];509 bool pll2pll;510 511 next = &ram->base.target;512 next->freq = freq;513 ram->base.next = next;514 515 if (ram->ltrain.state == NVA3_TRAIN_ONCE)516 gt215_link_train(ram);517 518 /* lookup memory config data relevant to the target frequency */519 data = nvbios_rammapEm(bios, freq / 1000, &ver, &hdr, &cnt, &len,520 &next->bios);521 if (!data || ver != 0x10 || hdr < 0x05) {522 nvkm_error(subdev, "invalid/missing rammap entry\n");523 return -EINVAL;524 }525 526 /* locate specific data set for the attached memory */527 strap = nvbios_ramcfg_index(subdev);528 if (strap >= cnt) {529 nvkm_error(subdev, "invalid ramcfg strap\n");530 return -EINVAL;531 }532 533 data = nvbios_rammapSp(bios, data, ver, hdr, cnt, len, strap,534 &ver, &hdr, &next->bios);535 if (!data || ver != 0x10 || hdr < 0x09) {536 nvkm_error(subdev, "invalid/missing ramcfg entry\n");537 return -EINVAL;538 }539 540 /* lookup memory timings, if bios says they're present */541 if (next->bios.ramcfg_timing != 0xff) {542 data = nvbios_timingEp(bios, next->bios.ramcfg_timing,543 &ver, &hdr, &cnt, &len,544 &next->bios);545 if (!data || ver != 0x10 || hdr < 0x17) {546 nvkm_error(subdev, "invalid/missing timing entry\n");547 return -EINVAL;548 }549 }550 551 ret = gt215_pll_info(device->clk, 0x12, 0x4000, freq, &mclk);552 if (ret < 0) {553 nvkm_error(subdev, "failed mclk calculation\n");554 return ret;555 }556 557 gt215_ram_timing_calc(ram, timing);558 559 ret = ram_init(fuc, ram->base.fb);560 if (ret)561 return ret;562 563 /* Determine ram-specific MR values */564 ram->base.mr[0] = ram_rd32(fuc, mr[0]);565 ram->base.mr[1] = ram_rd32(fuc, mr[1]);566 ram->base.mr[2] = ram_rd32(fuc, mr[2]);567 568 switch (ram->base.type) {569 case NVKM_RAM_TYPE_DDR2:570 ret = nvkm_sddr2_calc(&ram->base);571 break;572 case NVKM_RAM_TYPE_DDR3:573 ret = nvkm_sddr3_calc(&ram->base);574 break;575 case NVKM_RAM_TYPE_GDDR3:576 ret = nvkm_gddr3_calc(&ram->base);577 break;578 default:579 ret = -ENOSYS;580 break;581 }582 583 if (ret)584 return ret;585 586 /* XXX: 750MHz seems rather arbitrary */587 if (freq <= 750000) {588 r004018 = 0x10000000;589 r100760 = 0x22222222;590 r100da0 = 0x00000010;591 } else {592 r004018 = 0x00000000;593 r100760 = 0x00000000;594 r100da0 = 0x00000000;595 }596 597 if (!next->bios.ramcfg_DLLoff)598 r004018 |= 0x00004000;599 600 /* pll2pll requires to switch to a safe clock first */601 ctrl = ram_rd32(fuc, 0x004000);602 pll2pll = (!(ctrl & 0x00000008)) && mclk.pll;603 604 /* Pre, NVIDIA does this outside the script */605 if (next->bios.ramcfg_10_02_10) {606 ram_mask(fuc, 0x111104, 0x00000600, 0x00000000);607 } else {608 ram_mask(fuc, 0x111100, 0x40000000, 0x40000000);609 ram_mask(fuc, 0x111104, 0x00000180, 0x00000000);610 }611 /* Always disable this bit during reclock */612 ram_mask(fuc, 0x100200, 0x00000800, 0x00000000);613 614 /* If switching from non-pll to pll, lock before disabling FB */615 if (mclk.pll && !pll2pll) {616 ram_mask(fuc, 0x004128, 0x003f3141, mclk.clk | 0x00000101);617 gt215_ram_lock_pll(fuc, &mclk);618 }619 620 /* Start with disabling some CRTCs and PFIFO? */621 ram_wait_vblank(fuc);622 ram_wr32(fuc, 0x611200, 0x3300);623 ram_mask(fuc, 0x002504, 0x1, 0x1);624 ram_nsec(fuc, 10000);625 ram_wait(fuc, 0x002504, 0x10, 0x10, 20000); /* XXX: or longer? */626 ram_block(fuc);627 ram_nsec(fuc, 2000);628 629 if (!next->bios.ramcfg_10_02_10) {630 if (ram->base.type == NVKM_RAM_TYPE_GDDR3)631 ram_mask(fuc, 0x111100, 0x04020000, 0x00020000);632 else633 ram_mask(fuc, 0x111100, 0x04020000, 0x04020000);634 }635 636 /* If we're disabling the DLL, do it now */637 switch (next->bios.ramcfg_DLLoff * ram->base.type) {638 case NVKM_RAM_TYPE_DDR3:639 nvkm_sddr3_dll_disable(fuc, ram->base.mr);640 break;641 case NVKM_RAM_TYPE_GDDR3:642 nvkm_gddr3_dll_disable(fuc, ram->base.mr);643 break;644 }645 646 if (next->bios.timing_10_ODT)647 gt215_ram_gpio(fuc, 0x2e, 1);648 649 /* Brace RAM for impact */650 ram_wr32(fuc, 0x1002d4, 0x00000001);651 ram_wr32(fuc, 0x1002d0, 0x00000001);652 ram_wr32(fuc, 0x1002d0, 0x00000001);653 ram_wr32(fuc, 0x100210, 0x00000000);654 ram_wr32(fuc, 0x1002dc, 0x00000001);655 ram_nsec(fuc, 2000);656 657 if (device->chipset == 0xa3 && freq <= 500000)658 ram_mask(fuc, 0x100700, 0x00000006, 0x00000006);659 660 /* Alter FBVDD/Q, apparently must be done with PLL disabled, thus661 * set it to bypass */662 if (nvkm_gpio_get(gpio, 0, 0x18, DCB_GPIO_UNUSED) ==663 next->bios.ramcfg_FBVDDQ) {664 data = ram_rd32(fuc, 0x004000) & 0x9;665 666 if (data == 0x1)667 ram_mask(fuc, 0x004000, 0x8, 0x8);668 if (data & 0x1)669 ram_mask(fuc, 0x004000, 0x1, 0x0);670 671 gt215_ram_gpio(fuc, 0x18, !next->bios.ramcfg_FBVDDQ);672 673 if (data & 0x1)674 ram_mask(fuc, 0x004000, 0x1, 0x1);675 }676 677 /* Fiddle with clocks */678 /* There's 4 scenario's679 * pll->pll: first switch to a 324MHz clock, set up new PLL, switch680 * clk->pll: Set up new PLL, switch681 * pll->clk: Set up clock, switch682 * clk->clk: Overwrite ctrl and other bits, switch */683 684 /* Switch to regular clock - 324MHz */685 if (pll2pll) {686 ram_mask(fuc, 0x004000, 0x00000004, 0x00000004);687 ram_mask(fuc, 0x004168, 0x003f3141, 0x00083101);688 ram_mask(fuc, 0x004000, 0x00000008, 0x00000008);689 ram_mask(fuc, 0x1110e0, 0x00088000, 0x00088000);690 ram_wr32(fuc, 0x004018, 0x00001000);691 gt215_ram_lock_pll(fuc, &mclk);692 }693 694 if (mclk.pll) {695 ram_mask(fuc, 0x004000, 0x00000105, 0x00000105);696 ram_wr32(fuc, 0x004018, 0x00001000 | r004018);697 ram_wr32(fuc, 0x100da0, r100da0);698 } else {699 ram_mask(fuc, 0x004168, 0x003f3141, mclk.clk | 0x00000101);700 ram_mask(fuc, 0x004000, 0x00000108, 0x00000008);701 ram_mask(fuc, 0x1110e0, 0x00088000, 0x00088000);702 ram_wr32(fuc, 0x004018, 0x00009000 | r004018);703 ram_wr32(fuc, 0x100da0, r100da0);704 }705 ram_nsec(fuc, 20000);706 707 if (next->bios.rammap_10_04_08) {708 ram_wr32(fuc, 0x1005a0, next->bios.ramcfg_10_06 << 16 |709 next->bios.ramcfg_10_05 << 8 |710 next->bios.ramcfg_10_05);711 ram_wr32(fuc, 0x1005a4, next->bios.ramcfg_10_08 << 8 |712 next->bios.ramcfg_10_07);713 ram_wr32(fuc, 0x10f804, next->bios.ramcfg_10_09_f0 << 20 |714 next->bios.ramcfg_10_03_0f << 16 |715 next->bios.ramcfg_10_09_0f |716 0x80000000);717 ram_mask(fuc, 0x10053c, 0x00001000, 0x00000000);718 } else {719 if (train->state == NVA3_TRAIN_DONE) {720 ram_wr32(fuc, 0x100080, 0x1020);721 ram_mask(fuc, 0x111400, 0xffffffff, train->r_111400);722 ram_mask(fuc, 0x1111e0, 0xffffffff, train->r_1111e0);723 ram_mask(fuc, 0x100720, 0xffffffff, train->r_100720);724 }725 ram_mask(fuc, 0x10053c, 0x00001000, 0x00001000);726 ram_mask(fuc, 0x10f804, 0x80000000, 0x00000000);727 ram_mask(fuc, 0x100760, 0x22222222, r100760);728 ram_mask(fuc, 0x1007a0, 0x22222222, r100760);729 ram_mask(fuc, 0x1007e0, 0x22222222, r100760);730 }731 732 if (device->chipset == 0xa3 && freq > 500000) {733 ram_mask(fuc, 0x100700, 0x00000006, 0x00000000);734 }735 736 /* Final switch */737 if (mclk.pll) {738 ram_mask(fuc, 0x1110e0, 0x00088000, 0x00011000);739 ram_mask(fuc, 0x004000, 0x00000008, 0x00000000);740 }741 742 ram_wr32(fuc, 0x1002dc, 0x00000000);743 ram_wr32(fuc, 0x1002d4, 0x00000001);744 ram_wr32(fuc, 0x100210, 0x80000000);745 ram_nsec(fuc, 2000);746 747 /* Set RAM MR parameters and timings */748 for (i = 2; i >= 0; i--) {749 if (ram_rd32(fuc, mr[i]) != ram->base.mr[i]) {750 ram_wr32(fuc, mr[i], ram->base.mr[i]);751 ram_nsec(fuc, 1000);752 }753 }754 755 ram_wr32(fuc, 0x100220[3], timing[3]);756 ram_wr32(fuc, 0x100220[1], timing[1]);757 ram_wr32(fuc, 0x100220[6], timing[6]);758 ram_wr32(fuc, 0x100220[7], timing[7]);759 ram_wr32(fuc, 0x100220[2], timing[2]);760 ram_wr32(fuc, 0x100220[4], timing[4]);761 ram_wr32(fuc, 0x100220[5], timing[5]);762 ram_wr32(fuc, 0x100220[0], timing[0]);763 ram_wr32(fuc, 0x100220[8], timing[8]);764 765 /* Misc */766 ram_mask(fuc, 0x100200, 0x00001000, !next->bios.ramcfg_10_02_08 << 12);767 768 /* XXX: A lot of "chipset"/"ram type" specific stuff...? */769 unk714 = ram_rd32(fuc, 0x100714) & ~0xf0000130;770 unk718 = ram_rd32(fuc, 0x100718) & ~0x00000100;771 unk71c = ram_rd32(fuc, 0x10071c) & ~0x00000100;772 r111100 = ram_rd32(fuc, 0x111100) & ~0x3a800000;773 774 /* NVA8 seems to skip various bits related to ramcfg_10_02_04 */775 if (device->chipset == 0xa8) {776 r111100 |= 0x08000000;777 if (!next->bios.ramcfg_10_02_04)778 unk714 |= 0x00000010;779 } else {780 if (next->bios.ramcfg_10_02_04) {781 switch (ram->base.type) {782 case NVKM_RAM_TYPE_DDR2:783 case NVKM_RAM_TYPE_DDR3:784 r111100 &= ~0x00000020;785 if (next->bios.ramcfg_10_02_10)786 r111100 |= 0x08000004;787 else788 r111100 |= 0x00000024;789 break;790 default:791 break;792 }793 } else {794 switch (ram->base.type) {795 case NVKM_RAM_TYPE_DDR2:796 case NVKM_RAM_TYPE_DDR3:797 r111100 &= ~0x00000024;798 r111100 |= 0x12800000;799 800 if (next->bios.ramcfg_10_02_10)801 r111100 |= 0x08000000;802 unk714 |= 0x00000010;803 break;804 case NVKM_RAM_TYPE_GDDR3:805 r111100 |= 0x30000000;806 unk714 |= 0x00000020;807 break;808 default:809 break;810 }811 }812 }813 814 unk714 |= (next->bios.ramcfg_10_04_01) << 8;815 816 if (next->bios.ramcfg_10_02_20)817 unk714 |= 0xf0000000;818 if (next->bios.ramcfg_10_02_02)819 unk718 |= 0x00000100;820 if (next->bios.ramcfg_10_02_01)821 unk71c |= 0x00000100;822 if (next->bios.timing_10_24 != 0xff) {823 unk718 &= ~0xf0000000;824 unk718 |= next->bios.timing_10_24 << 28;825 }826 if (next->bios.ramcfg_10_02_10)827 r111100 &= ~0x04020000;828 829 ram_mask(fuc, 0x100714, 0xffffffff, unk714);830 ram_mask(fuc, 0x10071c, 0xffffffff, unk71c);831 ram_mask(fuc, 0x100718, 0xffffffff, unk718);832 ram_mask(fuc, 0x111100, 0xffffffff, r111100);833 834 if (!next->bios.timing_10_ODT)835 gt215_ram_gpio(fuc, 0x2e, 0);836 837 /* Reset DLL */838 if (!next->bios.ramcfg_DLLoff)839 nvkm_sddr2_dll_reset(fuc);840 841 if (ram->base.type == NVKM_RAM_TYPE_GDDR3) {842 ram_nsec(fuc, 31000);843 } else {844 ram_nsec(fuc, 14000);845 }846 847 if (ram->base.type == NVKM_RAM_TYPE_DDR3) {848 ram_wr32(fuc, 0x100264, 0x1);849 ram_nsec(fuc, 2000);850 }851 852 ram_nuke(fuc, 0x100700);853 ram_mask(fuc, 0x100700, 0x01000000, 0x01000000);854 ram_mask(fuc, 0x100700, 0x01000000, 0x00000000);855 856 /* Re-enable FB */857 ram_unblock(fuc);858 ram_wr32(fuc, 0x611200, 0x3330);859 860 /* Post fiddlings */861 if (next->bios.rammap_10_04_02)862 ram_mask(fuc, 0x100200, 0x00000800, 0x00000800);863 if (next->bios.ramcfg_10_02_10) {864 ram_mask(fuc, 0x111104, 0x00000180, 0x00000180);865 ram_mask(fuc, 0x111100, 0x40000000, 0x00000000);866 } else {867 ram_mask(fuc, 0x111104, 0x00000600, 0x00000600);868 }869 870 if (mclk.pll) {871 ram_mask(fuc, 0x004168, 0x00000001, 0x00000000);872 ram_mask(fuc, 0x004168, 0x00000100, 0x00000000);873 } else {874 ram_mask(fuc, 0x004000, 0x00000001, 0x00000000);875 ram_mask(fuc, 0x004128, 0x00000001, 0x00000000);876 ram_mask(fuc, 0x004128, 0x00000100, 0x00000000);877 }878 879 return 0;880}881 882static int883gt215_ram_prog(struct nvkm_ram *base)884{885 struct gt215_ram *ram = gt215_ram(base);886 struct gt215_ramfuc *fuc = &ram->fuc;887 struct nvkm_device *device = ram->base.fb->subdev.device;888 bool exec = nvkm_boolopt(device->cfgopt, "NvMemExec", true);889 890 if (exec) {891 nvkm_mask(device, 0x001534, 0x2, 0x2);892 893 ram_exec(fuc, true);894 895 /* Post-processing, avoids flicker */896 nvkm_mask(device, 0x002504, 0x1, 0x0);897 nvkm_mask(device, 0x001534, 0x2, 0x0);898 899 nvkm_mask(device, 0x616308, 0x10, 0x10);900 nvkm_mask(device, 0x616b08, 0x10, 0x10);901 } else {902 ram_exec(fuc, false);903 }904 return 0;905}906 907static void908gt215_ram_tidy(struct nvkm_ram *base)909{910 struct gt215_ram *ram = gt215_ram(base);911 ram_exec(&ram->fuc, false);912}913 914static int915gt215_ram_init(struct nvkm_ram *base)916{917 struct gt215_ram *ram = gt215_ram(base);918 gt215_link_train_init(ram);919 return 0;920}921 922static void *923gt215_ram_dtor(struct nvkm_ram *base)924{925 struct gt215_ram *ram = gt215_ram(base);926 gt215_link_train_fini(ram);927 return ram;928}929 930static const struct nvkm_ram_func931gt215_ram_func = {932 .dtor = gt215_ram_dtor,933 .init = gt215_ram_init,934 .calc = gt215_ram_calc,935 .prog = gt215_ram_prog,936 .tidy = gt215_ram_tidy,937};938 939int940gt215_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)941{942 struct gt215_ram *ram;943 int ret, i;944 945 if (!(ram = kzalloc(sizeof(*ram), GFP_KERNEL)))946 return -ENOMEM;947 *pram = &ram->base;948 949 ret = nv50_ram_ctor(>215_ram_func, fb, &ram->base);950 if (ret)951 return ret;952 953 ram->fuc.r_0x001610 = ramfuc_reg(0x001610);954 ram->fuc.r_0x001700 = ramfuc_reg(0x001700);955 ram->fuc.r_0x002504 = ramfuc_reg(0x002504);956 ram->fuc.r_0x004000 = ramfuc_reg(0x004000);957 ram->fuc.r_0x004004 = ramfuc_reg(0x004004);958 ram->fuc.r_0x004018 = ramfuc_reg(0x004018);959 ram->fuc.r_0x004128 = ramfuc_reg(0x004128);960 ram->fuc.r_0x004168 = ramfuc_reg(0x004168);961 ram->fuc.r_0x100080 = ramfuc_reg(0x100080);962 ram->fuc.r_0x100200 = ramfuc_reg(0x100200);963 ram->fuc.r_0x100210 = ramfuc_reg(0x100210);964 for (i = 0; i < 9; i++)965 ram->fuc.r_0x100220[i] = ramfuc_reg(0x100220 + (i * 4));966 ram->fuc.r_0x100264 = ramfuc_reg(0x100264);967 ram->fuc.r_0x1002d0 = ramfuc_reg(0x1002d0);968 ram->fuc.r_0x1002d4 = ramfuc_reg(0x1002d4);969 ram->fuc.r_0x1002dc = ramfuc_reg(0x1002dc);970 ram->fuc.r_0x10053c = ramfuc_reg(0x10053c);971 ram->fuc.r_0x1005a0 = ramfuc_reg(0x1005a0);972 ram->fuc.r_0x1005a4 = ramfuc_reg(0x1005a4);973 ram->fuc.r_0x100700 = ramfuc_reg(0x100700);974 ram->fuc.r_0x100714 = ramfuc_reg(0x100714);975 ram->fuc.r_0x100718 = ramfuc_reg(0x100718);976 ram->fuc.r_0x10071c = ramfuc_reg(0x10071c);977 ram->fuc.r_0x100720 = ramfuc_reg(0x100720);978 ram->fuc.r_0x100760 = ramfuc_stride(0x100760, 4, ram->base.part_mask);979 ram->fuc.r_0x1007a0 = ramfuc_stride(0x1007a0, 4, ram->base.part_mask);980 ram->fuc.r_0x1007e0 = ramfuc_stride(0x1007e0, 4, ram->base.part_mask);981 ram->fuc.r_0x100da0 = ramfuc_stride(0x100da0, 4, ram->base.part_mask);982 ram->fuc.r_0x10f804 = ramfuc_reg(0x10f804);983 ram->fuc.r_0x1110e0 = ramfuc_stride(0x1110e0, 4, ram->base.part_mask);984 ram->fuc.r_0x111100 = ramfuc_reg(0x111100);985 ram->fuc.r_0x111104 = ramfuc_reg(0x111104);986 ram->fuc.r_0x1111e0 = ramfuc_reg(0x1111e0);987 ram->fuc.r_0x111400 = ramfuc_reg(0x111400);988 ram->fuc.r_0x611200 = ramfuc_reg(0x611200);989 990 if (ram->base.ranks > 1) {991 ram->fuc.r_mr[0] = ramfuc_reg2(0x1002c0, 0x1002c8);992 ram->fuc.r_mr[1] = ramfuc_reg2(0x1002c4, 0x1002cc);993 ram->fuc.r_mr[2] = ramfuc_reg2(0x1002e0, 0x1002e8);994 ram->fuc.r_mr[3] = ramfuc_reg2(0x1002e4, 0x1002ec);995 } else {996 ram->fuc.r_mr[0] = ramfuc_reg(0x1002c0);997 ram->fuc.r_mr[1] = ramfuc_reg(0x1002c4);998 ram->fuc.r_mr[2] = ramfuc_reg(0x1002e0);999 ram->fuc.r_mr[3] = ramfuc_reg(0x1002e4);1000 }1001 ram->fuc.r_gpio[0] = ramfuc_reg(0x00e104);1002 ram->fuc.r_gpio[1] = ramfuc_reg(0x00e108);1003 ram->fuc.r_gpio[2] = ramfuc_reg(0x00e120);1004 ram->fuc.r_gpio[3] = ramfuc_reg(0x00e124);1005 1006 return 0;1007}1008