366 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Tegra host1x Command DMA4 *5 * Copyright (c) 2010-2013, NVIDIA Corporation.6 */7 8#include <linux/slab.h>9#include <linux/scatterlist.h>10#include <linux/dma-mapping.h>11 12#include "../cdma.h"13#include "../channel.h"14#include "../dev.h"15#include "../debug.h"16 17/*18 * Put the restart at the end of pushbuffer memory19 */20static void push_buffer_init(struct push_buffer *pb)21{22 *(u32 *)(pb->mapped + pb->size) = host1x_opcode_restart(0);23}24 25/*26 * Increment timedout buffer's syncpt via CPU.27 */28static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr,29 u32 syncpt_incrs, u32 syncval, u32 nr_slots)30{31 unsigned int i;32 33 for (i = 0; i < syncpt_incrs; i++)34 host1x_syncpt_incr(cdma->timeout.syncpt);35 36 /* after CPU incr, ensure shadow is up to date */37 host1x_syncpt_load(cdma->timeout.syncpt);38}39 40/*41 * Start channel DMA42 */43static void cdma_start(struct host1x_cdma *cdma)44{45 struct host1x_channel *ch = cdma_to_channel(cdma);46 u64 start, end;47 48 if (cdma->running)49 return;50 51 cdma->last_pos = cdma->push_buffer.pos;52 start = cdma->push_buffer.dma;53 end = cdma->push_buffer.size + 4;54 55 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,56 HOST1X_CHANNEL_DMACTRL);57 58 /* set base, put and end pointer */59 host1x_ch_writel(ch, lower_32_bits(start), HOST1X_CHANNEL_DMASTART);60#if HOST1X_HW >= 661 host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI);62#endif63 host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);64#if HOST1X_HW >= 665 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMAPUT_HI);66#endif67 host1x_ch_writel(ch, lower_32_bits(end), HOST1X_CHANNEL_DMAEND);68#if HOST1X_HW >= 669 host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI);70#endif71 72 /* reset GET */73 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |74 HOST1X_CHANNEL_DMACTRL_DMAGETRST |75 HOST1X_CHANNEL_DMACTRL_DMAINITGET,76 HOST1X_CHANNEL_DMACTRL);77 78 /* start the command DMA */79 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);80 81 cdma->running = true;82}83 84/*85 * Similar to cdma_start(), but rather than starting from an idle86 * state (where DMA GET is set to DMA PUT), on a timeout we restore87 * DMA GET from an explicit value (so DMA may again be pending).88 */89static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr)90{91 struct host1x *host1x = cdma_to_host1x(cdma);92 struct host1x_channel *ch = cdma_to_channel(cdma);93 u64 start, end;94 95 if (cdma->running)96 return;97 98 cdma->last_pos = cdma->push_buffer.pos;99 100 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,101 HOST1X_CHANNEL_DMACTRL);102 103 start = cdma->push_buffer.dma;104 end = cdma->push_buffer.size + 4;105 106 /* set base, end pointer (all of memory) */107 host1x_ch_writel(ch, lower_32_bits(start), HOST1X_CHANNEL_DMASTART);108#if HOST1X_HW >= 6109 host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI);110#endif111 host1x_ch_writel(ch, lower_32_bits(end), HOST1X_CHANNEL_DMAEND);112#if HOST1X_HW >= 6113 host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI);114#endif115 116 /* set GET, by loading the value in PUT (then reset GET) */117 host1x_ch_writel(ch, getptr, HOST1X_CHANNEL_DMAPUT);118 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |119 HOST1X_CHANNEL_DMACTRL_DMAGETRST |120 HOST1X_CHANNEL_DMACTRL_DMAINITGET,121 HOST1X_CHANNEL_DMACTRL);122 123 dev_dbg(host1x->dev,124 "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", __func__,125 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),126 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),127 cdma->last_pos);128 129 /* deassert GET reset and set PUT */130 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,131 HOST1X_CHANNEL_DMACTRL);132 host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);133 134 /* start the command DMA */135 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);136 137 cdma->running = true;138}139 140/*141 * Kick channel DMA into action by writing its PUT offset (if it has changed)142 */143static void cdma_flush(struct host1x_cdma *cdma)144{145 struct host1x_channel *ch = cdma_to_channel(cdma);146 147 if (cdma->push_buffer.pos != cdma->last_pos) {148 host1x_ch_writel(ch, cdma->push_buffer.pos,149 HOST1X_CHANNEL_DMAPUT);150 cdma->last_pos = cdma->push_buffer.pos;151 }152}153 154static void cdma_stop(struct host1x_cdma *cdma)155{156 struct host1x_channel *ch = cdma_to_channel(cdma);157 158 mutex_lock(&cdma->lock);159 160 if (cdma->running) {161 host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY);162 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,163 HOST1X_CHANNEL_DMACTRL);164 cdma->running = false;165 }166 167 mutex_unlock(&cdma->lock);168}169 170static void cdma_hw_cmdproc_stop(struct host1x *host, struct host1x_channel *ch,171 bool stop)172{173#if HOST1X_HW >= 6174 host1x_ch_writel(ch, stop ? 0x1 : 0x0, HOST1X_CHANNEL_CMDPROC_STOP);175#else176 u32 cmdproc_stop = host1x_sync_readl(host, HOST1X_SYNC_CMDPROC_STOP);177 if (stop)178 cmdproc_stop |= BIT(ch->id);179 else180 cmdproc_stop &= ~BIT(ch->id);181 host1x_sync_writel(host, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);182#endif183}184 185static void cdma_hw_teardown(struct host1x *host, struct host1x_channel *ch)186{187#if HOST1X_HW >= 6188 host1x_ch_writel(ch, 0x1, HOST1X_CHANNEL_TEARDOWN);189#else190 host1x_sync_writel(host, BIT(ch->id), HOST1X_SYNC_CH_TEARDOWN);191#endif192}193 194/*195 * Stops both channel's command processor and CDMA immediately.196 * Also, tears down the channel and resets corresponding module.197 */198static void cdma_freeze(struct host1x_cdma *cdma)199{200 struct host1x *host = cdma_to_host1x(cdma);201 struct host1x_channel *ch = cdma_to_channel(cdma);202 203 if (cdma->torndown && !cdma->running) {204 dev_warn(host->dev, "Already torn down\n");205 return;206 }207 208 dev_dbg(host->dev, "freezing channel (id %d)\n", ch->id);209 210 cdma_hw_cmdproc_stop(host, ch, true);211 212 dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n",213 __func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),214 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),215 cdma->last_pos);216 217 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,218 HOST1X_CHANNEL_DMACTRL);219 220 cdma_hw_teardown(host, ch);221 222 cdma->running = false;223 cdma->torndown = true;224}225 226static void cdma_resume(struct host1x_cdma *cdma, u32 getptr)227{228 struct host1x *host1x = cdma_to_host1x(cdma);229 struct host1x_channel *ch = cdma_to_channel(cdma);230 231 dev_dbg(host1x->dev,232 "resuming channel (id %u, DMAGET restart = 0x%x)\n",233 ch->id, getptr);234 235 cdma_hw_cmdproc_stop(host1x, ch, false);236 237 cdma->torndown = false;238 cdma_timeout_restart(cdma, getptr);239}240 241static void timeout_release_mlock(struct host1x_cdma *cdma)242{243#if HOST1X_HW >= 8244 /* Tegra186 and Tegra194 require a more complicated MLOCK release245 * sequence. Furthermore, those chips by default don't enforce MLOCKs,246 * so it turns out that if we don't /actually/ need MLOCKs, we can just247 * ignore them.248 *249 * As such, for now just implement this on Tegra234 where things are250 * stricter but also easy to implement.251 */252 struct host1x_channel *ch = cdma_to_channel(cdma);253 struct host1x *host1x = cdma_to_host1x(cdma);254 u32 offset;255 256 switch (ch->client->class) {257 case HOST1X_CLASS_VIC:258 offset = HOST1X_COMMON_VIC_MLOCK;259 break;260 case HOST1X_CLASS_NVDEC:261 offset = HOST1X_COMMON_NVDEC_MLOCK;262 break;263 default:264 WARN(1, "%s was not updated for class %u", __func__, ch->client->class);265 return;266 }267 268 host1x_common_writel(host1x, 0x0, offset);269#endif270}271 272/*273 * If this timeout fires, it indicates the current sync_queue entry has274 * exceeded its TTL and the userctx should be timed out and remaining275 * submits already issued cleaned up (future submits return an error).276 */277static void cdma_timeout_handler(struct work_struct *work)278{279 u32 syncpt_val;280 struct host1x_cdma *cdma;281 struct host1x *host1x;282 struct host1x_channel *ch;283 284 cdma = container_of(to_delayed_work(work), struct host1x_cdma,285 timeout.wq);286 host1x = cdma_to_host1x(cdma);287 ch = cdma_to_channel(cdma);288 289 host1x_debug_dump(cdma_to_host1x(cdma));290 291 mutex_lock(&cdma->lock);292 293 if (!cdma->timeout.client) {294 dev_dbg(host1x->dev,295 "cdma_timeout: expired, but has no clientid\n");296 mutex_unlock(&cdma->lock);297 return;298 }299 300 /* stop processing to get a clean snapshot */301 cdma_hw_cmdproc_stop(host1x, ch, true);302 303 syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);304 305 /* has buffer actually completed? */306 if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) {307 dev_dbg(host1x->dev,308 "cdma_timeout: expired, but buffer had completed\n");309 /* restore */310 cdma_hw_cmdproc_stop(host1x, ch, false);311 mutex_unlock(&cdma->lock);312 return;313 }314 315 dev_warn(host1x->dev, "%s: timeout: %u (%s), HW thresh %d, done %d\n",316 __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name,317 syncpt_val, cdma->timeout.syncpt_val);318 319 /* stop HW, resetting channel/module */320 host1x_hw_cdma_freeze(host1x, cdma);321 322 /* release any held MLOCK */323 timeout_release_mlock(cdma);324 325 host1x_cdma_update_sync_queue(cdma, ch->dev);326 mutex_unlock(&cdma->lock);327}328 329/*330 * Init timeout resources331 */332static int cdma_timeout_init(struct host1x_cdma *cdma)333{334 INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler);335 cdma->timeout.initialized = true;336 337 return 0;338}339 340/*341 * Clean up timeout resources342 */343static void cdma_timeout_destroy(struct host1x_cdma *cdma)344{345 if (cdma->timeout.initialized)346 cancel_delayed_work(&cdma->timeout.wq);347 348 cdma->timeout.initialized = false;349}350 351static const struct host1x_cdma_ops host1x_cdma_ops = {352 .start = cdma_start,353 .stop = cdma_stop,354 .flush = cdma_flush,355 356 .timeout_init = cdma_timeout_init,357 .timeout_destroy = cdma_timeout_destroy,358 .freeze = cdma_freeze,359 .resume = cdma_resume,360 .timeout_cpu_incr = cdma_timeout_cpu_incr,361};362 363static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = {364 .init = push_buffer_init,365};366