250 lines · c
1/*2 * Copyright (C) 2007 Ben Skeggs.3 * All Rights Reserved.4 *5 * Permission is hereby granted, free of charge, to any person obtaining6 * a copy of this software and associated documentation files (the7 * "Software"), to deal in the Software without restriction, including8 * without limitation the rights to use, copy, modify, merge, publish,9 * distribute, sublicense, and/or sell copies of the Software, and to10 * permit persons to whom the Software is furnished to do so, subject to11 * the following conditions:12 *13 * The above copyright notice and this permission notice (including the14 * next paragraph) shall be included in all copies or substantial15 * portions of the Software.16 *17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.24 *25 */26 27#include "nouveau_drv.h"28#include "nouveau_dma.h"29#include "nouveau_vmm.h"30 31#include <nvif/user.h>32 33/* Fetch and adjust GPU GET pointer34 *35 * Returns:36 * value >= 0, the adjusted GET pointer37 * -EINVAL if GET pointer currently outside main push buffer38 * -EBUSY if timeout exceeded39 */40static inline int41READ_GET(struct nouveau_channel *chan, uint64_t *prev_get, int *timeout)42{43 uint64_t val;44 45 val = nvif_rd32(chan->userd, chan->user_get);46 if (chan->user_get_hi)47 val |= (uint64_t)nvif_rd32(chan->userd, chan->user_get_hi) << 32;48 49 /* reset counter as long as GET is still advancing, this is50 * to avoid misdetecting a GPU lockup if the GPU happens to51 * just be processing an operation that takes a long time52 */53 if (val != *prev_get) {54 *prev_get = val;55 *timeout = 0;56 }57 58 if ((++*timeout & 0xff) == 0) {59 udelay(1);60 if (*timeout > 100000)61 return -EBUSY;62 }63 64 if (val < chan->push.addr ||65 val > chan->push.addr + (chan->dma.max << 2))66 return -EINVAL;67 68 return (val - chan->push.addr) >> 2;69}70 71void72nv50_dma_push(struct nouveau_channel *chan, u64 offset, u32 length,73 bool no_prefetch)74{75 struct nvif_user *user = &chan->cli->drm->client.device.user;76 struct nouveau_bo *pb = chan->push.buffer;77 int ip = (chan->dma.ib_put * 2) + chan->dma.ib_base;78 79 BUG_ON(chan->dma.ib_free < 1);80 WARN_ON(length > NV50_DMA_PUSH_MAX_LENGTH);81 82 nouveau_bo_wr32(pb, ip++, lower_32_bits(offset));83 nouveau_bo_wr32(pb, ip++, upper_32_bits(offset) | length << 8 |84 (no_prefetch ? (1 << 31) : 0));85 86 chan->dma.ib_put = (chan->dma.ib_put + 1) & chan->dma.ib_max;87 88 mb();89 /* Flush writes. */90 nouveau_bo_rd32(pb, 0);91 92 nvif_wr32(chan->userd, 0x8c, chan->dma.ib_put);93 if (user->func && user->func->doorbell)94 user->func->doorbell(user, chan->token);95 chan->dma.ib_free--;96}97 98static int99nv50_dma_push_wait(struct nouveau_channel *chan, int count)100{101 uint32_t cnt = 0, prev_get = 0;102 103 while (chan->dma.ib_free < count) {104 uint32_t get = nvif_rd32(chan->userd, 0x88);105 if (get != prev_get) {106 prev_get = get;107 cnt = 0;108 }109 110 if ((++cnt & 0xff) == 0) {111 udelay(1);112 if (cnt > 100000)113 return -EBUSY;114 }115 116 chan->dma.ib_free = get - chan->dma.ib_put;117 if (chan->dma.ib_free <= 0)118 chan->dma.ib_free += chan->dma.ib_max;119 }120 121 return 0;122}123 124static int125nv50_dma_wait(struct nouveau_channel *chan, int slots, int count)126{127 uint64_t prev_get = 0;128 int ret, cnt = 0;129 130 ret = nv50_dma_push_wait(chan, slots + 1);131 if (unlikely(ret))132 return ret;133 134 while (chan->dma.free < count) {135 int get = READ_GET(chan, &prev_get, &cnt);136 if (unlikely(get < 0)) {137 if (get == -EINVAL)138 continue;139 140 return get;141 }142 143 if (get <= chan->dma.cur) {144 chan->dma.free = chan->dma.max - chan->dma.cur;145 if (chan->dma.free >= count)146 break;147 148 FIRE_RING(chan);149 do {150 get = READ_GET(chan, &prev_get, &cnt);151 if (unlikely(get < 0)) {152 if (get == -EINVAL)153 continue;154 return get;155 }156 } while (get == 0);157 chan->dma.cur = 0;158 chan->dma.put = 0;159 }160 161 chan->dma.free = get - chan->dma.cur - 1;162 }163 164 return 0;165}166 167int168nouveau_dma_wait(struct nouveau_channel *chan, int slots, int size)169{170 uint64_t prev_get = 0;171 int cnt = 0, get;172 173 if (chan->dma.ib_max)174 return nv50_dma_wait(chan, slots, size);175 176 while (chan->dma.free < size) {177 get = READ_GET(chan, &prev_get, &cnt);178 if (unlikely(get == -EBUSY))179 return -EBUSY;180 181 /* loop until we have a usable GET pointer. the value182 * we read from the GPU may be outside the main ring if183 * PFIFO is processing a buffer called from the main ring,184 * discard these values until something sensible is seen.185 *186 * the other case we discard GET is while the GPU is fetching187 * from the SKIPS area, so the code below doesn't have to deal188 * with some fun corner cases.189 */190 if (unlikely(get == -EINVAL) || get < NOUVEAU_DMA_SKIPS)191 continue;192 193 if (get <= chan->dma.cur) {194 /* engine is fetching behind us, or is completely195 * idle (GET == PUT) so we have free space up until196 * the end of the push buffer197 *198 * we can only hit that path once per call due to199 * looping back to the beginning of the push buffer,200 * we'll hit the fetching-ahead-of-us path from that201 * point on.202 *203 * the *one* exception to that rule is if we read204 * GET==PUT, in which case the below conditional will205 * always succeed and break us out of the wait loop.206 */207 chan->dma.free = chan->dma.max - chan->dma.cur;208 if (chan->dma.free >= size)209 break;210 211 /* not enough space left at the end of the push buffer,212 * instruct the GPU to jump back to the start right213 * after processing the currently pending commands.214 */215 OUT_RING(chan, chan->push.addr | 0x20000000);216 217 /* wait for GET to depart from the skips area.218 * prevents writing GET==PUT and causing a race219 * condition that causes us to think the GPU is220 * idle when it's not.221 */222 do {223 get = READ_GET(chan, &prev_get, &cnt);224 if (unlikely(get == -EBUSY))225 return -EBUSY;226 if (unlikely(get == -EINVAL))227 continue;228 } while (get <= NOUVEAU_DMA_SKIPS);229 WRITE_PUT(NOUVEAU_DMA_SKIPS);230 231 /* we're now submitting commands at the start of232 * the push buffer.233 */234 chan->dma.cur =235 chan->dma.put = NOUVEAU_DMA_SKIPS;236 }237 238 /* engine fetching ahead of us, we have space up until the239 * current GET pointer. the "- 1" is to ensure there's240 * space left to emit a jump back to the beginning of the241 * push buffer if we require it. we can never get GET == PUT242 * here, so this is safe.243 */244 chan->dma.free = get - chan->dma.cur - 1;245 }246 247 return 0;248}249 250