591 lines · c
1// SPDX-License-Identifier: GPL-2.0+2/* Copyright (C) 2015-2018 Broadcom */3 4#include <linux/delay.h>5#include <linux/mutex.h>6#include <linux/spinlock_types.h>7#include <linux/workqueue.h>8 9#include <drm/drm_encoder.h>10#include <drm/drm_gem.h>11#include <drm/drm_gem_shmem_helper.h>12#include <drm/gpu_scheduler.h>13 14#include "v3d_performance_counters.h"15 16#include "uapi/drm/v3d_drm.h"17 18struct clk;19struct platform_device;20struct reset_control;21 22#define GMP_GRANULARITY (128 * 1024)23 24#define V3D_MMU_PAGE_SHIFT 1225 26#define V3D_MAX_QUEUES (V3D_CPU + 1)27 28static inline char *v3d_queue_to_string(enum v3d_queue queue)29{30 switch (queue) {31 case V3D_BIN: return "bin";32 case V3D_RENDER: return "render";33 case V3D_TFU: return "tfu";34 case V3D_CSD: return "csd";35 case V3D_CACHE_CLEAN: return "cache_clean";36 case V3D_CPU: return "cpu";37 }38 return "UNKNOWN";39}40 41struct v3d_stats {42 u64 start_ns;43 u64 enabled_ns;44 u64 jobs_completed;45 46 /*47 * This seqcount is used to protect the access to the GPU stats48 * variables. It must be used as, while we are reading the stats,49 * IRQs can happen and the stats can be updated.50 */51 seqcount_t lock;52};53 54struct v3d_queue_state {55 struct drm_gpu_scheduler sched;56 57 u64 fence_context;58 u64 emit_seqno;59 60 /* Stores the GPU stats for this queue in the global context. */61 struct v3d_stats stats;62};63 64/* Performance monitor object. The perform lifetime is controlled by userspace65 * using perfmon related ioctls. A perfmon can be attached to a submit_cl66 * request, and when this is the case, HW perf counters will be activated just67 * before the submit_cl is submitted to the GPU and disabled when the job is68 * done. This way, only events related to a specific job will be counted.69 */70struct v3d_perfmon {71 /* Tracks the number of users of the perfmon, when this counter reaches72 * zero the perfmon is destroyed.73 */74 refcount_t refcnt;75 76 /* Protects perfmon stop, as it can be invoked from multiple places. */77 struct mutex lock;78 79 /* Number of counters activated in this perfmon instance80 * (should be less than DRM_V3D_MAX_PERF_COUNTERS).81 */82 u8 ncounters;83 84 /* Events counted by the HW perf counters. */85 u8 counters[DRM_V3D_MAX_PERF_COUNTERS];86 87 /* Storage for counter values. Counters are incremented by the88 * HW perf counter values every time the perfmon is attached89 * to a GPU job. This way, perfmon users don't have to90 * retrieve the results after each job if they want to track91 * events covering several submissions. Note that counter92 * values can't be reset, but you can fake a reset by93 * destroying the perfmon and creating a new one.94 */95 u64 values[] __counted_by(ncounters);96};97 98struct v3d_dev {99 struct drm_device drm;100 101 /* Short representation (e.g. 33, 41) of the V3D tech version */102 int ver;103 104 /* Short representation (e.g. 5, 6) of the V3D tech revision */105 int rev;106 107 bool single_irq_line;108 109 struct v3d_perfmon_info perfmon_info;110 111 void __iomem *hub_regs;112 void __iomem *core_regs[3];113 void __iomem *bridge_regs;114 void __iomem *gca_regs;115 struct clk *clk;116 struct reset_control *reset;117 118 /* Virtual and DMA addresses of the single shared page table. */119 volatile u32 *pt;120 dma_addr_t pt_paddr;121 122 /* Virtual and DMA addresses of the MMU's scratch page. When123 * a read or write is invalid in the MMU, it will be124 * redirected here.125 */126 void *mmu_scratch;127 dma_addr_t mmu_scratch_paddr;128 /* virtual address bits from V3D to the MMU. */129 int va_width;130 131 /* Number of V3D cores. */132 u32 cores;133 134 /* Allocator managing the address space. All units are in135 * number of pages.136 */137 struct drm_mm mm;138 spinlock_t mm_lock;139 140 struct work_struct overflow_mem_work;141 142 struct v3d_bin_job *bin_job;143 struct v3d_render_job *render_job;144 struct v3d_tfu_job *tfu_job;145 struct v3d_csd_job *csd_job;146 struct v3d_cpu_job *cpu_job;147 148 struct v3d_queue_state queue[V3D_MAX_QUEUES];149 150 /* Spinlock used to synchronize the overflow memory151 * management against bin job submission.152 */153 spinlock_t job_lock;154 155 /* Used to track the active perfmon if any. */156 struct v3d_perfmon *active_perfmon;157 158 /* Protects bo_stats */159 struct mutex bo_lock;160 161 /* Lock taken when resetting the GPU, to keep multiple162 * processes from trying to park the scheduler threads and163 * reset at once.164 */165 struct mutex reset_lock;166 167 /* Lock taken when creating and pushing the GPU scheduler168 * jobs, to keep the sched-fence seqnos in order.169 */170 struct mutex sched_lock;171 172 /* Lock taken during a cache clean and when initiating an L2173 * flush, to keep L2 flushes from interfering with the174 * synchronous L2 cleans.175 */176 struct mutex cache_clean_lock;177 178 struct {179 u32 num_allocated;180 u32 pages_allocated;181 } bo_stats;182};183 184static inline struct v3d_dev *185to_v3d_dev(struct drm_device *dev)186{187 return container_of(dev, struct v3d_dev, drm);188}189 190static inline bool191v3d_has_csd(struct v3d_dev *v3d)192{193 return v3d->ver >= 41;194}195 196#define v3d_to_pdev(v3d) to_platform_device((v3d)->drm.dev)197 198/* The per-fd struct, which tracks the MMU mappings. */199struct v3d_file_priv {200 struct v3d_dev *v3d;201 202 struct {203 struct idr idr;204 struct mutex lock;205 } perfmon;206 207 struct drm_sched_entity sched_entity[V3D_MAX_QUEUES];208 209 /* Stores the GPU stats for a specific queue for this fd. */210 struct v3d_stats stats[V3D_MAX_QUEUES];211};212 213struct v3d_bo {214 struct drm_gem_shmem_object base;215 216 struct drm_mm_node node;217 218 /* List entry for the BO's position in219 * v3d_render_job->unref_list220 */221 struct list_head unref_head;222 223 void *vaddr;224};225 226static inline struct v3d_bo *227to_v3d_bo(struct drm_gem_object *bo)228{229 return (struct v3d_bo *)bo;230}231 232struct v3d_fence {233 struct dma_fence base;234 struct drm_device *dev;235 /* v3d seqno for signaled() test */236 u64 seqno;237 enum v3d_queue queue;238};239 240static inline struct v3d_fence *241to_v3d_fence(struct dma_fence *fence)242{243 return (struct v3d_fence *)fence;244}245 246#define V3D_READ(offset) readl(v3d->hub_regs + offset)247#define V3D_WRITE(offset, val) writel(val, v3d->hub_regs + offset)248 249#define V3D_BRIDGE_READ(offset) readl(v3d->bridge_regs + offset)250#define V3D_BRIDGE_WRITE(offset, val) writel(val, v3d->bridge_regs + offset)251 252#define V3D_GCA_READ(offset) readl(v3d->gca_regs + offset)253#define V3D_GCA_WRITE(offset, val) writel(val, v3d->gca_regs + offset)254 255#define V3D_CORE_READ(core, offset) readl(v3d->core_regs[core] + offset)256#define V3D_CORE_WRITE(core, offset, val) writel(val, v3d->core_regs[core] + offset)257 258struct v3d_job {259 struct drm_sched_job base;260 261 struct kref refcount;262 263 struct v3d_dev *v3d;264 265 /* This is the array of BOs that were looked up at the start266 * of submission.267 */268 struct drm_gem_object **bo;269 u32 bo_count;270 271 /* v3d fence to be signaled by IRQ handler when the job is complete. */272 struct dma_fence *irq_fence;273 274 /* scheduler fence for when the job is considered complete and275 * the BO reservations can be released.276 */277 struct dma_fence *done_fence;278 279 /* Pointer to a performance monitor object if the user requested it,280 * NULL otherwise.281 */282 struct v3d_perfmon *perfmon;283 284 /* File descriptor of the process that submitted the job that could be used285 * for collecting stats by process of GPU usage.286 */287 struct drm_file *file;288 289 /* Callback for the freeing of the job on refcount going to 0. */290 void (*free)(struct kref *ref);291};292 293struct v3d_bin_job {294 struct v3d_job base;295 296 /* GPU virtual addresses of the start/end of the CL job. */297 u32 start, end;298 299 u32 timedout_ctca, timedout_ctra;300 301 /* Corresponding render job, for attaching our overflow memory. */302 struct v3d_render_job *render;303 304 /* Submitted tile memory allocation start/size, tile state. */305 u32 qma, qms, qts;306};307 308struct v3d_render_job {309 struct v3d_job base;310 311 /* GPU virtual addresses of the start/end of the CL job. */312 u32 start, end;313 314 u32 timedout_ctca, timedout_ctra;315 316 /* List of overflow BOs used in the job that need to be317 * released once the job is complete.318 */319 struct list_head unref_list;320};321 322struct v3d_tfu_job {323 struct v3d_job base;324 325 struct drm_v3d_submit_tfu args;326};327 328struct v3d_csd_job {329 struct v3d_job base;330 331 u32 timedout_batches;332 333 struct drm_v3d_submit_csd args;334};335 336enum v3d_cpu_job_type {337 V3D_CPU_JOB_TYPE_INDIRECT_CSD = 1,338 V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY,339 V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY,340 V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY,341 V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY,342 V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY,343};344 345struct v3d_timestamp_query {346 /* Offset of this query in the timestamp BO for its value. */347 u32 offset;348 349 /* Syncobj that indicates the timestamp availability */350 struct drm_syncobj *syncobj;351};352 353struct v3d_performance_query {354 /* Performance monitor IDs for this query */355 u32 *kperfmon_ids;356 357 /* Syncobj that indicates the query availability */358 struct drm_syncobj *syncobj;359};360 361struct v3d_indirect_csd_info {362 /* Indirect CSD */363 struct v3d_csd_job *job;364 365 /* Clean cache job associated to the Indirect CSD job */366 struct v3d_job *clean_job;367 368 /* Offset within the BO where the workgroup counts are stored */369 u32 offset;370 371 /* Workgroups size */372 u32 wg_size;373 374 /* Indices of the uniforms with the workgroup dispatch counts375 * in the uniform stream.376 */377 u32 wg_uniform_offsets[3];378 379 /* Indirect BO */380 struct drm_gem_object *indirect;381 382 /* Context of the Indirect CSD job */383 struct ww_acquire_ctx acquire_ctx;384};385 386struct v3d_timestamp_query_info {387 struct v3d_timestamp_query *queries;388 389 u32 count;390};391 392struct v3d_performance_query_info {393 struct v3d_performance_query *queries;394 395 /* Number of performance queries */396 u32 count;397 398 /* Number of performance monitors related to that query pool */399 u32 nperfmons;400 401 /* Number of performance counters related to that query pool */402 u32 ncounters;403};404 405struct v3d_copy_query_results_info {406 /* Define if should write to buffer using 64 or 32 bits */407 bool do_64bit;408 409 /* Define if it can write to buffer even if the query is not available */410 bool do_partial;411 412 /* Define if it should write availability bit to buffer */413 bool availability_bit;414 415 /* Offset of the copy buffer in the BO */416 u32 offset;417 418 /* Stride of the copy buffer in the BO */419 u32 stride;420};421 422struct v3d_cpu_job {423 struct v3d_job base;424 425 enum v3d_cpu_job_type job_type;426 427 struct v3d_indirect_csd_info indirect_csd;428 429 struct v3d_timestamp_query_info timestamp_query;430 431 struct v3d_copy_query_results_info copy;432 433 struct v3d_performance_query_info performance_query;434};435 436typedef void (*v3d_cpu_job_fn)(struct v3d_cpu_job *);437 438struct v3d_submit_outsync {439 struct drm_syncobj *syncobj;440};441 442struct v3d_submit_ext {443 u32 flags;444 u32 wait_stage;445 446 u32 in_sync_count;447 u64 in_syncs;448 449 u32 out_sync_count;450 struct v3d_submit_outsync *out_syncs;451};452 453/**454 * __wait_for - magic wait macro455 *456 * Macro to help avoid open coding check/wait/timeout patterns. Note that it's457 * important that we check the condition again after having timed out, since the458 * timeout could be due to preemption or similar and we've never had a chance to459 * check the condition before the timeout.460 */461#define __wait_for(OP, COND, US, Wmin, Wmax) ({ \462 const ktime_t end__ = ktime_add_ns(ktime_get_raw(), 1000ll * (US)); \463 long wait__ = (Wmin); /* recommended min for usleep is 10 us */ \464 int ret__; \465 might_sleep(); \466 for (;;) { \467 const bool expired__ = ktime_after(ktime_get_raw(), end__); \468 OP; \469 /* Guarantee COND check prior to timeout */ \470 barrier(); \471 if (COND) { \472 ret__ = 0; \473 break; \474 } \475 if (expired__) { \476 ret__ = -ETIMEDOUT; \477 break; \478 } \479 usleep_range(wait__, wait__ * 2); \480 if (wait__ < (Wmax)) \481 wait__ <<= 1; \482 } \483 ret__; \484})485 486#define _wait_for(COND, US, Wmin, Wmax) __wait_for(, (COND), (US), (Wmin), \487 (Wmax))488#define wait_for(COND, MS) _wait_for((COND), (MS) * 1000, 10, 1000)489 490static inline unsigned long nsecs_to_jiffies_timeout(const u64 n)491{492 /* nsecs_to_jiffies64() does not guard against overflow */493 if ((NSEC_PER_SEC % HZ) != 0 &&494 div_u64(n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ)495 return MAX_JIFFY_OFFSET;496 497 return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1);498}499 500/* v3d_bo.c */501struct drm_gem_object *v3d_create_object(struct drm_device *dev, size_t size);502void v3d_free_object(struct drm_gem_object *gem_obj);503struct v3d_bo *v3d_bo_create(struct drm_device *dev, struct drm_file *file_priv,504 size_t size);505void v3d_get_bo_vaddr(struct v3d_bo *bo);506void v3d_put_bo_vaddr(struct v3d_bo *bo);507int v3d_create_bo_ioctl(struct drm_device *dev, void *data,508 struct drm_file *file_priv);509int v3d_mmap_bo_ioctl(struct drm_device *dev, void *data,510 struct drm_file *file_priv);511int v3d_get_bo_offset_ioctl(struct drm_device *dev, void *data,512 struct drm_file *file_priv);513int v3d_wait_bo_ioctl(struct drm_device *dev, void *data,514 struct drm_file *file_priv);515struct drm_gem_object *v3d_prime_import_sg_table(struct drm_device *dev,516 struct dma_buf_attachment *attach,517 struct sg_table *sgt);518 519/* v3d_debugfs.c */520void v3d_debugfs_init(struct drm_minor *minor);521 522/* v3d_drv.c */523void v3d_get_stats(const struct v3d_stats *stats, u64 timestamp,524 u64 *active_runtime, u64 *jobs_completed);525 526/* v3d_fence.c */527extern const struct dma_fence_ops v3d_fence_ops;528struct dma_fence *v3d_fence_create(struct v3d_dev *v3d, enum v3d_queue queue);529 530/* v3d_gem.c */531int v3d_gem_init(struct drm_device *dev);532void v3d_gem_destroy(struct drm_device *dev);533void v3d_reset(struct v3d_dev *v3d);534void v3d_invalidate_caches(struct v3d_dev *v3d);535void v3d_clean_caches(struct v3d_dev *v3d);536 537/* v3d_submit.c */538void v3d_job_cleanup(struct v3d_job *job);539void v3d_job_put(struct v3d_job *job);540int v3d_submit_cl_ioctl(struct drm_device *dev, void *data,541 struct drm_file *file_priv);542int v3d_submit_tfu_ioctl(struct drm_device *dev, void *data,543 struct drm_file *file_priv);544int v3d_submit_csd_ioctl(struct drm_device *dev, void *data,545 struct drm_file *file_priv);546int v3d_submit_cpu_ioctl(struct drm_device *dev, void *data,547 struct drm_file *file_priv);548 549/* v3d_irq.c */550int v3d_irq_init(struct v3d_dev *v3d);551void v3d_irq_enable(struct v3d_dev *v3d);552void v3d_irq_disable(struct v3d_dev *v3d);553void v3d_irq_reset(struct v3d_dev *v3d);554 555/* v3d_mmu.c */556int v3d_mmu_set_page_table(struct v3d_dev *v3d);557void v3d_mmu_insert_ptes(struct v3d_bo *bo);558void v3d_mmu_remove_ptes(struct v3d_bo *bo);559 560/* v3d_sched.c */561void v3d_timestamp_query_info_free(struct v3d_timestamp_query_info *query_info,562 unsigned int count);563void v3d_performance_query_info_free(struct v3d_performance_query_info *query_info,564 unsigned int count);565void v3d_job_update_stats(struct v3d_job *job, enum v3d_queue queue);566int v3d_sched_init(struct v3d_dev *v3d);567void v3d_sched_fini(struct v3d_dev *v3d);568 569/* v3d_perfmon.c */570void v3d_perfmon_init(struct v3d_dev *v3d);571void v3d_perfmon_get(struct v3d_perfmon *perfmon);572void v3d_perfmon_put(struct v3d_perfmon *perfmon);573void v3d_perfmon_start(struct v3d_dev *v3d, struct v3d_perfmon *perfmon);574void v3d_perfmon_stop(struct v3d_dev *v3d, struct v3d_perfmon *perfmon,575 bool capture);576struct v3d_perfmon *v3d_perfmon_find(struct v3d_file_priv *v3d_priv, int id);577void v3d_perfmon_open_file(struct v3d_file_priv *v3d_priv);578void v3d_perfmon_close_file(struct v3d_file_priv *v3d_priv);579int v3d_perfmon_create_ioctl(struct drm_device *dev, void *data,580 struct drm_file *file_priv);581int v3d_perfmon_destroy_ioctl(struct drm_device *dev, void *data,582 struct drm_file *file_priv);583int v3d_perfmon_get_values_ioctl(struct drm_device *dev, void *data,584 struct drm_file *file_priv);585int v3d_perfmon_get_counter_ioctl(struct drm_device *dev, void *data,586 struct drm_file *file_priv);587 588/* v3d_sysfs.c */589int v3d_sysfs_init(struct device *dev);590void v3d_sysfs_destroy(struct device *dev);591