693 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Tegra host1x Command DMA4 *5 * Copyright (c) 2010-2013, NVIDIA Corporation.6 */7 8 9#include <asm/cacheflush.h>10#include <linux/device.h>11#include <linux/dma-mapping.h>12#include <linux/host1x.h>13#include <linux/interrupt.h>14#include <linux/kernel.h>15#include <linux/kfifo.h>16#include <linux/slab.h>17#include <trace/events/host1x.h>18 19#include "cdma.h"20#include "channel.h"21#include "dev.h"22#include "debug.h"23#include "job.h"24 25/*26 * push_buffer27 *28 * The push buffer is a circular array of words to be fetched by command DMA.29 * Note that it works slightly differently to the sync queue; fence == pos30 * means that the push buffer is full, not empty.31 */32 33/*34 * Typically the commands written into the push buffer are a pair of words. We35 * use slots to represent each of these pairs and to simplify things. Note the36 * strange number of slots allocated here. 512 slots will fit exactly within a37 * single memory page. We also need one additional word at the end of the push38 * buffer for the RESTART opcode that will instruct the CDMA to jump back to39 * the beginning of the push buffer. With 512 slots, this means that we'll use40 * 2 memory pages and waste 4092 bytes of the second page that will never be41 * used.42 */43#define HOST1X_PUSHBUFFER_SLOTS 51144 45/*46 * Clean up push buffer resources47 */48static void host1x_pushbuffer_destroy(struct push_buffer *pb)49{50 struct host1x_cdma *cdma = pb_to_cdma(pb);51 struct host1x *host1x = cdma_to_host1x(cdma);52 53 if (!pb->mapped)54 return;55 56 if (host1x->domain) {57 iommu_unmap(host1x->domain, pb->dma, pb->alloc_size);58 free_iova(&host1x->iova, iova_pfn(&host1x->iova, pb->dma));59 }60 61 dma_free_wc(host1x->dev, pb->alloc_size, pb->mapped, pb->phys);62 63 pb->mapped = NULL;64 pb->phys = 0;65}66 67/*68 * Init push buffer resources69 */70static int host1x_pushbuffer_init(struct push_buffer *pb)71{72 struct host1x_cdma *cdma = pb_to_cdma(pb);73 struct host1x *host1x = cdma_to_host1x(cdma);74 struct iova *alloc;75 u32 size;76 int err;77 78 pb->mapped = NULL;79 pb->phys = 0;80 pb->size = HOST1X_PUSHBUFFER_SLOTS * 8;81 82 size = pb->size + 4;83 84 /* initialize buffer pointers */85 pb->fence = pb->size - 8;86 pb->pos = 0;87 88 if (host1x->domain) {89 unsigned long shift;90 91 size = iova_align(&host1x->iova, size);92 93 pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,94 GFP_KERNEL);95 if (!pb->mapped)96 return -ENOMEM;97 98 shift = iova_shift(&host1x->iova);99 alloc = alloc_iova(&host1x->iova, size >> shift,100 host1x->iova_end >> shift, true);101 if (!alloc) {102 err = -ENOMEM;103 goto iommu_free_mem;104 }105 106 pb->dma = iova_dma_addr(&host1x->iova, alloc);107 err = iommu_map(host1x->domain, pb->dma, pb->phys, size,108 IOMMU_READ, GFP_KERNEL);109 if (err)110 goto iommu_free_iova;111 } else {112 pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,113 GFP_KERNEL);114 if (!pb->mapped)115 return -ENOMEM;116 117 pb->dma = pb->phys;118 }119 120 pb->alloc_size = size;121 122 host1x_hw_pushbuffer_init(host1x, pb);123 124 return 0;125 126iommu_free_iova:127 __free_iova(&host1x->iova, alloc);128iommu_free_mem:129 dma_free_wc(host1x->dev, size, pb->mapped, pb->phys);130 131 return err;132}133 134/*135 * Push two words to the push buffer136 * Caller must ensure push buffer is not full137 */138static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2)139{140 u32 *p = (u32 *)((void *)pb->mapped + pb->pos);141 142 WARN_ON(pb->pos == pb->fence);143 *(p++) = op1;144 *(p++) = op2;145 pb->pos += 8;146 147 if (pb->pos >= pb->size)148 pb->pos -= pb->size;149}150 151/*152 * Pop a number of two word slots from the push buffer153 * Caller must ensure push buffer is not empty154 */155static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots)156{157 /* Advance the next write position */158 pb->fence += slots * 8;159 160 if (pb->fence >= pb->size)161 pb->fence -= pb->size;162}163 164/*165 * Return the number of two word slots free in the push buffer166 */167static u32 host1x_pushbuffer_space(struct push_buffer *pb)168{169 unsigned int fence = pb->fence;170 171 if (pb->fence < pb->pos)172 fence += pb->size;173 174 return (fence - pb->pos) / 8;175}176 177/*178 * Sleep (if necessary) until the requested event happens179 * - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty.180 * - Returns 1181 * - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer182 * - Return the amount of space (> 0)183 * Must be called with the cdma lock held.184 */185unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma,186 enum cdma_event event)187{188 for (;;) {189 struct push_buffer *pb = &cdma->push_buffer;190 unsigned int space;191 192 switch (event) {193 case CDMA_EVENT_SYNC_QUEUE_EMPTY:194 space = list_empty(&cdma->sync_queue) ? 1 : 0;195 break;196 197 case CDMA_EVENT_PUSH_BUFFER_SPACE:198 space = host1x_pushbuffer_space(pb);199 break;200 201 default:202 WARN_ON(1);203 return -EINVAL;204 }205 206 if (space)207 return space;208 209 trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),210 event);211 212 /* If somebody has managed to already start waiting, yield */213 if (cdma->event != CDMA_EVENT_NONE) {214 mutex_unlock(&cdma->lock);215 schedule();216 mutex_lock(&cdma->lock);217 continue;218 }219 220 cdma->event = event;221 222 mutex_unlock(&cdma->lock);223 wait_for_completion(&cdma->complete);224 mutex_lock(&cdma->lock);225 }226 227 return 0;228}229 230/*231 * Sleep (if necessary) until the push buffer has enough free space.232 *233 * Must be called with the cdma lock held.234 */235static int host1x_cdma_wait_pushbuffer_space(struct host1x *host1x,236 struct host1x_cdma *cdma,237 unsigned int needed)238{239 while (true) {240 struct push_buffer *pb = &cdma->push_buffer;241 unsigned int space;242 243 space = host1x_pushbuffer_space(pb);244 if (space >= needed)245 break;246 247 trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),248 CDMA_EVENT_PUSH_BUFFER_SPACE);249 250 host1x_hw_cdma_flush(host1x, cdma);251 252 /* If somebody has managed to already start waiting, yield */253 if (cdma->event != CDMA_EVENT_NONE) {254 mutex_unlock(&cdma->lock);255 schedule();256 mutex_lock(&cdma->lock);257 continue;258 }259 260 cdma->event = CDMA_EVENT_PUSH_BUFFER_SPACE;261 262 mutex_unlock(&cdma->lock);263 wait_for_completion(&cdma->complete);264 mutex_lock(&cdma->lock);265 }266 267 return 0;268}269/*270 * Start timer that tracks the time spent by the job.271 * Must be called with the cdma lock held.272 */273static void cdma_start_timer_locked(struct host1x_cdma *cdma,274 struct host1x_job *job)275{276 if (cdma->timeout.client) {277 /* timer already started */278 return;279 }280 281 cdma->timeout.client = job->client;282 cdma->timeout.syncpt = job->syncpt;283 cdma->timeout.syncpt_val = job->syncpt_end;284 cdma->timeout.start_ktime = ktime_get();285 286 schedule_delayed_work(&cdma->timeout.wq,287 msecs_to_jiffies(job->timeout));288}289 290/*291 * Stop timer when a buffer submission completes.292 * Must be called with the cdma lock held.293 */294static void stop_cdma_timer_locked(struct host1x_cdma *cdma)295{296 cancel_delayed_work(&cdma->timeout.wq);297 cdma->timeout.client = NULL;298}299 300/*301 * For all sync queue entries that have already finished according to the302 * current sync point registers:303 * - unpin & unref their mems304 * - pop their push buffer slots305 * - remove them from the sync queue306 * This is normally called from the host code's worker thread, but can be307 * called manually if necessary.308 * Must be called with the cdma lock held.309 */310static void update_cdma_locked(struct host1x_cdma *cdma)311{312 bool signal = false;313 struct host1x_job *job, *n;314 315 /*316 * Walk the sync queue, reading the sync point registers as necessary,317 * to consume as many sync queue entries as possible without blocking318 */319 list_for_each_entry_safe(job, n, &cdma->sync_queue, list) {320 struct host1x_syncpt *sp = job->syncpt;321 322 /* Check whether this syncpt has completed, and bail if not */323 if (!host1x_syncpt_is_expired(sp, job->syncpt_end) &&324 !job->cancelled) {325 /* Start timer on next pending syncpt */326 if (job->timeout)327 cdma_start_timer_locked(cdma, job);328 329 break;330 }331 332 /* Cancel timeout, when a buffer completes */333 if (cdma->timeout.client)334 stop_cdma_timer_locked(cdma);335 336 /* Unpin the memory */337 host1x_job_unpin(job);338 339 /* Pop push buffer slots */340 if (job->num_slots) {341 struct push_buffer *pb = &cdma->push_buffer;342 343 host1x_pushbuffer_pop(pb, job->num_slots);344 345 if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE)346 signal = true;347 }348 349 list_del(&job->list);350 host1x_job_put(job);351 }352 353 if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY &&354 list_empty(&cdma->sync_queue))355 signal = true;356 357 if (signal) {358 cdma->event = CDMA_EVENT_NONE;359 complete(&cdma->complete);360 }361}362 363void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma,364 struct device *dev)365{366 struct host1x *host1x = cdma_to_host1x(cdma);367 u32 restart_addr, syncpt_incrs, syncpt_val;368 struct host1x_job *job, *next_job = NULL;369 370 syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);371 372 dev_dbg(dev, "%s: starting cleanup (thresh %d)\n",373 __func__, syncpt_val);374 375 /*376 * Move the sync_queue read pointer to the first entry that hasn't377 * completed based on the current HW syncpt value. It's likely there378 * won't be any (i.e. we're still at the head), but covers the case379 * where a syncpt incr happens just prior/during the teardown.380 */381 382 dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n",383 __func__);384 385 list_for_each_entry(job, &cdma->sync_queue, list) {386 if (syncpt_val < job->syncpt_end) {387 388 if (!list_is_last(&job->list, &cdma->sync_queue))389 next_job = list_next_entry(job, list);390 391 goto syncpt_incr;392 }393 394 host1x_job_dump(dev, job);395 }396 397 /* all jobs have been completed */398 job = NULL;399 400syncpt_incr:401 402 /*403 * Increment with CPU the remaining syncpts of a partially executed job.404 *405 * CDMA will continue execution starting with the next job or will get406 * into idle state.407 */408 if (next_job)409 restart_addr = next_job->first_get;410 else411 restart_addr = cdma->last_pos;412 413 if (!job)414 goto resume;415 416 /* do CPU increments for the remaining syncpts */417 if (job->syncpt_recovery) {418 dev_dbg(dev, "%s: perform CPU incr on pending buffers\n",419 __func__);420 421 /* won't need a timeout when replayed */422 job->timeout = 0;423 424 syncpt_incrs = job->syncpt_end - syncpt_val;425 dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs);426 427 host1x_job_dump(dev, job);428 429 /* safe to use CPU to incr syncpts */430 host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get,431 syncpt_incrs, job->syncpt_end,432 job->num_slots);433 434 dev_dbg(dev, "%s: finished sync_queue modification\n",435 __func__);436 } else {437 struct host1x_job *failed_job = job;438 439 host1x_job_dump(dev, job);440 441 host1x_syncpt_set_locked(job->syncpt);442 failed_job->cancelled = true;443 444 list_for_each_entry_continue(job, &cdma->sync_queue, list) {445 unsigned int i;446 447 if (job->syncpt != failed_job->syncpt)448 continue;449 450 for (i = 0; i < job->num_slots; i++) {451 unsigned int slot = (job->first_get/8 + i) %452 HOST1X_PUSHBUFFER_SLOTS;453 u32 *mapped = cdma->push_buffer.mapped;454 455 /*456 * Overwrite opcodes with 0 word writes457 * to offset 0xbad. This does nothing but458 * has a easily detected signature in debug459 * traces.460 *461 * On systems with MLOCK enforcement enabled,462 * the above 0 word writes would fall foul of463 * the enforcement. As such, in the first slot464 * put a RESTART_W opcode to the beginning465 * of the next job. We don't use this for older466 * chips since those only support the RESTART467 * opcode with inconvenient alignment requirements.468 */469 if (i == 0 && host1x->info->has_wide_gather) {470 unsigned int next_job = (job->first_get/8 + job->num_slots)471 % HOST1X_PUSHBUFFER_SLOTS;472 mapped[2*slot+0] = (0xd << 28) | (next_job * 2);473 mapped[2*slot+1] = 0x0;474 } else {475 mapped[2*slot+0] = 0x1bad0000;476 mapped[2*slot+1] = 0x1bad0000;477 }478 }479 480 job->cancelled = true;481 }482 483 wmb();484 485 update_cdma_locked(cdma);486 }487 488resume:489 /* roll back DMAGET and start up channel again */490 host1x_hw_cdma_resume(host1x, cdma, restart_addr);491}492 493static void cdma_update_work(struct work_struct *work)494{495 struct host1x_cdma *cdma = container_of(work, struct host1x_cdma, update_work);496 497 mutex_lock(&cdma->lock);498 update_cdma_locked(cdma);499 mutex_unlock(&cdma->lock);500}501 502/*503 * Create a cdma504 */505int host1x_cdma_init(struct host1x_cdma *cdma)506{507 int err;508 509 mutex_init(&cdma->lock);510 init_completion(&cdma->complete);511 INIT_WORK(&cdma->update_work, cdma_update_work);512 513 INIT_LIST_HEAD(&cdma->sync_queue);514 515 cdma->event = CDMA_EVENT_NONE;516 cdma->running = false;517 cdma->torndown = false;518 519 err = host1x_pushbuffer_init(&cdma->push_buffer);520 if (err)521 return err;522 523 return 0;524}525 526/*527 * Destroy a cdma528 */529int host1x_cdma_deinit(struct host1x_cdma *cdma)530{531 struct push_buffer *pb = &cdma->push_buffer;532 struct host1x *host1x = cdma_to_host1x(cdma);533 534 if (cdma->running) {535 pr_warn("%s: CDMA still running\n", __func__);536 return -EBUSY;537 }538 539 host1x_pushbuffer_destroy(pb);540 host1x_hw_cdma_timeout_destroy(host1x, cdma);541 542 return 0;543}544 545/*546 * Begin a cdma submit547 */548int host1x_cdma_begin(struct host1x_cdma *cdma, struct host1x_job *job)549{550 struct host1x *host1x = cdma_to_host1x(cdma);551 552 mutex_lock(&cdma->lock);553 554 /*555 * Check if syncpoint was locked due to previous job timeout.556 * This needs to be done within the cdma lock to avoid a race557 * with the timeout handler.558 */559 if (job->syncpt->locked) {560 mutex_unlock(&cdma->lock);561 return -EPERM;562 }563 564 if (job->timeout) {565 /* init state on first submit with timeout value */566 if (!cdma->timeout.initialized) {567 int err;568 569 err = host1x_hw_cdma_timeout_init(host1x, cdma);570 if (err) {571 mutex_unlock(&cdma->lock);572 return err;573 }574 }575 }576 577 if (!cdma->running)578 host1x_hw_cdma_start(host1x, cdma);579 580 cdma->slots_free = 0;581 cdma->slots_used = 0;582 cdma->first_get = cdma->push_buffer.pos;583 584 trace_host1x_cdma_begin(dev_name(job->channel->dev));585 return 0;586}587 588/*589 * Push two words into a push buffer slot590 * Blocks as necessary if the push buffer is full.591 */592void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2)593{594 struct host1x *host1x = cdma_to_host1x(cdma);595 struct push_buffer *pb = &cdma->push_buffer;596 u32 slots_free = cdma->slots_free;597 598 if (host1x_debug_trace_cmdbuf)599 trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev),600 op1, op2);601 602 if (slots_free == 0) {603 host1x_hw_cdma_flush(host1x, cdma);604 slots_free = host1x_cdma_wait_locked(cdma,605 CDMA_EVENT_PUSH_BUFFER_SPACE);606 }607 608 cdma->slots_free = slots_free - 1;609 cdma->slots_used++;610 host1x_pushbuffer_push(pb, op1, op2);611}612 613/*614 * Push four words into two consecutive push buffer slots. Note that extra615 * care needs to be taken not to split the two slots across the end of the616 * push buffer. Otherwise the RESTART opcode at the end of the push buffer617 * that ensures processing will restart at the beginning will break up the618 * four words.619 *620 * Blocks as necessary if the push buffer is full.621 */622void host1x_cdma_push_wide(struct host1x_cdma *cdma, u32 op1, u32 op2,623 u32 op3, u32 op4)624{625 struct host1x_channel *channel = cdma_to_channel(cdma);626 struct host1x *host1x = cdma_to_host1x(cdma);627 struct push_buffer *pb = &cdma->push_buffer;628 unsigned int space, needed = 2, extra = 0;629 630 if (host1x_debug_trace_cmdbuf)631 trace_host1x_cdma_push_wide(dev_name(channel->dev), op1, op2,632 op3, op4);633 634 /* compute number of extra slots needed for padding */635 if (pb->pos + 16 > pb->size) {636 extra = (pb->size - pb->pos) / 8;637 needed += extra;638 }639 640 host1x_cdma_wait_pushbuffer_space(host1x, cdma, needed);641 space = host1x_pushbuffer_space(pb);642 643 cdma->slots_free = space - needed;644 cdma->slots_used += needed;645 646 if (extra > 0) {647 /*648 * If there isn't enough space at the tail of the pushbuffer,649 * insert a RESTART(0) here to go back to the beginning.650 * The code above adjusted the indexes appropriately.651 */652 host1x_pushbuffer_push(pb, (0x5 << 28), 0xdead0000);653 }654 655 host1x_pushbuffer_push(pb, op1, op2);656 host1x_pushbuffer_push(pb, op3, op4);657}658 659/*660 * End a cdma submit661 * Kick off DMA, add job to the sync queue, and a number of slots to be freed662 * from the pushbuffer. The handles for a submit must all be pinned at the same663 * time, but they can be unpinned in smaller chunks.664 */665void host1x_cdma_end(struct host1x_cdma *cdma,666 struct host1x_job *job)667{668 struct host1x *host1x = cdma_to_host1x(cdma);669 bool idle = list_empty(&cdma->sync_queue);670 671 host1x_hw_cdma_flush(host1x, cdma);672 673 job->first_get = cdma->first_get;674 job->num_slots = cdma->slots_used;675 host1x_job_get(job);676 list_add_tail(&job->list, &cdma->sync_queue);677 678 /* start timer on idle -> active transitions */679 if (job->timeout && idle)680 cdma_start_timer_locked(cdma, job);681 682 trace_host1x_cdma_end(dev_name(job->channel->dev));683 mutex_unlock(&cdma->lock);684}685 686/*687 * Update cdma state according to current sync point values688 */689void host1x_cdma_update(struct host1x_cdma *cdma)690{691 schedule_work(&cdma->update_work);692}693