363 lines · c
1/* SPDX-License-Identifier: GPL-2.0 OR MIT */2/*3 * Copyright 2014-2022 Advanced Micro Devices, Inc.4 *5 * Permission is hereby granted, free of charge, to any person obtaining a6 * copy of this software and associated documentation files (the "Software"),7 * to deal in the Software without restriction, including without limitation8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,9 * and/or sell copies of the Software, and to permit persons to whom the10 * Software is furnished to do so, subject to the following conditions:11 *12 * The above copyright notice and this permission notice shall be included in13 * all copies or substantial portions of the Software.14 *15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR21 * OTHER DEALINGS IN THE SOFTWARE.22 *23 */24 25#ifndef KFD_DEVICE_QUEUE_MANAGER_H_26#define KFD_DEVICE_QUEUE_MANAGER_H_27 28#include <linux/rwsem.h>29#include <linux/list.h>30#include <linux/mutex.h>31#include <linux/sched/mm.h>32#include "kfd_priv.h"33#include "kfd_mqd_manager.h"34 35 36#define VMID_NUM 1637 38#define KFD_MES_PROCESS_QUANTUM 10000039#define KFD_MES_GANG_QUANTUM 1000040#define USE_DEFAULT_GRACE_PERIOD 0xffffffff41 42struct device_process_node {43 struct qcm_process_device *qpd;44 struct list_head list;45};46 47union SQ_CMD_BITS {48 struct {49 uint32_t cmd:3;50 uint32_t:1;51 uint32_t mode:3;52 uint32_t check_vmid:1;53 uint32_t trap_id:3;54 uint32_t:5;55 uint32_t wave_id:4;56 uint32_t simd_id:2;57 uint32_t:2;58 uint32_t queue_id:3;59 uint32_t:1;60 uint32_t vm_id:4;61 } bitfields, bits;62 uint32_t u32All;63 signed int i32All;64 float f32All;65};66 67union GRBM_GFX_INDEX_BITS {68 struct {69 uint32_t instance_index:8;70 uint32_t sh_index:8;71 uint32_t se_index:8;72 uint32_t:5;73 uint32_t sh_broadcast_writes:1;74 uint32_t instance_broadcast_writes:1;75 uint32_t se_broadcast_writes:1;76 } bitfields, bits;77 uint32_t u32All;78 signed int i32All;79 float f32All;80};81 82/**83 * struct device_queue_manager_ops84 *85 * @create_queue: Queue creation routine.86 *87 * @destroy_queue: Queue destruction routine.88 *89 * @update_queue: Queue update routine.90 *91 * @exeute_queues: Dispatches the queues list to the H/W.92 *93 * @register_process: This routine associates a specific process with device.94 *95 * @unregister_process: destroys the associations between process to device.96 *97 * @initialize: Initializes the pipelines and memory module for that device.98 *99 * @start: Initializes the resources/modules the device needs for queues100 * execution. This function is called on device initialization and after the101 * system woke up after suspension.102 *103 * @stop: This routine stops execution of all the active queue running on the104 * H/W and basically this function called on system suspend.105 *106 * @uninitialize: Destroys all the device queue manager resources allocated in107 * initialize routine.108 *109 * @halt: This routine unmaps queues from runlist and set halt status to true110 * so no more queues will be mapped to runlist until unhalt.111 *112 * @unhalt: This routine unset halt status to flase and maps queues back to113 * runlist.114 *115 * @create_kernel_queue: Creates kernel queue. Used for debug queue.116 *117 * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue.118 *119 * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the120 * memory apertures.121 *122 * @process_termination: Clears all process queues belongs to that device.123 *124 * @evict_process_queues: Evict all active queues of a process125 *126 * @restore_process_queues: Restore all evicted queues of a process127 *128 * @get_wave_state: Retrieves context save state and optionally copies the129 * control stack, if kept in the MQD, to the given userspace address.130 *131 * @reset_queues: reset queues which consume RAS poison132 * @get_queue_checkpoint_info: Retrieves queue size information for CRIU checkpoint.133 *134 * @checkpoint_mqd: checkpoint queue MQD contents for CRIU.135 */136 137struct device_queue_manager_ops {138 int (*create_queue)(struct device_queue_manager *dqm,139 struct queue *q,140 struct qcm_process_device *qpd,141 const struct kfd_criu_queue_priv_data *qd,142 const void *restore_mqd,143 const void *restore_ctl_stack);144 145 int (*destroy_queue)(struct device_queue_manager *dqm,146 struct qcm_process_device *qpd,147 struct queue *q);148 149 int (*update_queue)(struct device_queue_manager *dqm,150 struct queue *q, struct mqd_update_info *minfo);151 152 int (*register_process)(struct device_queue_manager *dqm,153 struct qcm_process_device *qpd);154 155 int (*unregister_process)(struct device_queue_manager *dqm,156 struct qcm_process_device *qpd);157 158 int (*initialize)(struct device_queue_manager *dqm);159 int (*start)(struct device_queue_manager *dqm);160 int (*stop)(struct device_queue_manager *dqm);161 void (*uninitialize)(struct device_queue_manager *dqm);162 int (*halt)(struct device_queue_manager *dqm);163 int (*unhalt)(struct device_queue_manager *dqm);164 int (*create_kernel_queue)(struct device_queue_manager *dqm,165 struct kernel_queue *kq,166 struct qcm_process_device *qpd);167 168 void (*destroy_kernel_queue)(struct device_queue_manager *dqm,169 struct kernel_queue *kq,170 struct qcm_process_device *qpd);171 172 bool (*set_cache_memory_policy)(struct device_queue_manager *dqm,173 struct qcm_process_device *qpd,174 enum cache_policy default_policy,175 enum cache_policy alternate_policy,176 void __user *alternate_aperture_base,177 uint64_t alternate_aperture_size);178 179 int (*process_termination)(struct device_queue_manager *dqm,180 struct qcm_process_device *qpd);181 182 int (*evict_process_queues)(struct device_queue_manager *dqm,183 struct qcm_process_device *qpd);184 int (*restore_process_queues)(struct device_queue_manager *dqm,185 struct qcm_process_device *qpd);186 187 int (*get_wave_state)(struct device_queue_manager *dqm,188 struct queue *q,189 void __user *ctl_stack,190 u32 *ctl_stack_used_size,191 u32 *save_area_used_size);192 193 int (*reset_queues)(struct device_queue_manager *dqm,194 uint16_t pasid);195 void (*get_queue_checkpoint_info)(struct device_queue_manager *dqm,196 const struct queue *q, u32 *mqd_size,197 u32 *ctl_stack_size);198 199 int (*checkpoint_mqd)(struct device_queue_manager *dqm,200 const struct queue *q,201 void *mqd,202 void *ctl_stack);203};204 205struct device_queue_manager_asic_ops {206 int (*update_qpd)(struct device_queue_manager *dqm,207 struct qcm_process_device *qpd);208 bool (*set_cache_memory_policy)(struct device_queue_manager *dqm,209 struct qcm_process_device *qpd,210 enum cache_policy default_policy,211 enum cache_policy alternate_policy,212 void __user *alternate_aperture_base,213 uint64_t alternate_aperture_size);214 void (*init_sdma_vm)(struct device_queue_manager *dqm,215 struct queue *q,216 struct qcm_process_device *qpd);217 struct mqd_manager * (*mqd_manager_init)(enum KFD_MQD_TYPE type,218 struct kfd_node *dev);219};220 221struct dqm_detect_hang_info {222 int pipe_id;223 int queue_id;224 int xcc_id;225 uint64_t queue_address;226};227 228/**229 * struct device_queue_manager230 *231 * This struct is a base class for the kfd queues scheduler in the232 * device level. The device base class should expose the basic operations233 * for queue creation and queue destruction. This base class hides the234 * scheduling mode of the driver and the specific implementation of the235 * concrete device. This class is the only class in the queues scheduler236 * that configures the H/W.237 *238 */239 240struct device_queue_manager {241 struct device_queue_manager_ops ops;242 struct device_queue_manager_asic_ops asic_ops;243 244 struct mqd_manager *mqd_mgrs[KFD_MQD_TYPE_MAX];245 struct packet_manager packet_mgr;246 struct kfd_node *dev;247 struct mutex lock_hidden; /* use dqm_lock/unlock(dqm) */248 struct list_head queues;249 unsigned int saved_flags;250 unsigned int processes_count;251 unsigned int active_queue_count;252 unsigned int active_cp_queue_count;253 unsigned int gws_queue_count;254 unsigned int total_queue_count;255 unsigned int next_pipe_to_allocate;256 unsigned int *allocated_queues;257 DECLARE_BITMAP(sdma_bitmap, KFD_MAX_SDMA_QUEUES);258 DECLARE_BITMAP(xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES);259 /* the pasid mapping for each kfd vmid */260 uint16_t vmid_pasid[VMID_NUM];261 uint64_t pipelines_addr;262 uint64_t fence_gpu_addr;263 uint64_t *fence_addr;264 struct kfd_mem_obj *fence_mem;265 bool active_runlist;266 int sched_policy;267 uint32_t trap_debug_vmid;268 269 /* hw exception */270 bool is_hws_hang;271 bool is_resetting;272 struct work_struct hw_exception_work;273 struct kfd_mem_obj hiq_sdma_mqd;274 bool sched_running;275 bool sched_halt;276 277 /* used for GFX 9.4.3 only */278 uint32_t current_logical_xcc_start;279 280 uint32_t wait_times;281 282 wait_queue_head_t destroy_wait;283 284 /* for per-queue reset support */285 struct dqm_detect_hang_info *detect_hang_info;286 size_t detect_hang_info_size;287 int detect_hang_count;288};289 290void device_queue_manager_init_cik(291 struct device_queue_manager_asic_ops *asic_ops);292void device_queue_manager_init_vi(293 struct device_queue_manager_asic_ops *asic_ops);294void device_queue_manager_init_v9(295 struct device_queue_manager_asic_ops *asic_ops);296void device_queue_manager_init_v10(297 struct device_queue_manager_asic_ops *asic_ops);298void device_queue_manager_init_v11(299 struct device_queue_manager_asic_ops *asic_ops);300void device_queue_manager_init_v12(301 struct device_queue_manager_asic_ops *asic_ops);302void program_sh_mem_settings(struct device_queue_manager *dqm,303 struct qcm_process_device *qpd);304unsigned int get_cp_queues_num(struct device_queue_manager *dqm);305unsigned int get_queues_per_pipe(struct device_queue_manager *dqm);306unsigned int get_pipes_per_mec(struct device_queue_manager *dqm);307unsigned int get_num_sdma_queues(struct device_queue_manager *dqm);308unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm);309int reserve_debug_trap_vmid(struct device_queue_manager *dqm,310 struct qcm_process_device *qpd);311int release_debug_trap_vmid(struct device_queue_manager *dqm,312 struct qcm_process_device *qpd);313int suspend_queues(struct kfd_process *p,314 uint32_t num_queues,315 uint32_t grace_period,316 uint64_t exception_clear_mask,317 uint32_t *usr_queue_id_array);318int resume_queues(struct kfd_process *p,319 uint32_t num_queues,320 uint32_t *usr_queue_id_array);321void set_queue_snapshot_entry(struct queue *q,322 uint64_t exception_clear_mask,323 struct kfd_queue_snapshot_entry *qss_entry);324int debug_lock_and_unmap(struct device_queue_manager *dqm);325int debug_map_and_unlock(struct device_queue_manager *dqm);326int debug_refresh_runlist(struct device_queue_manager *dqm);327bool kfd_dqm_is_queue_in_process(struct device_queue_manager *dqm,328 struct qcm_process_device *qpd,329 int doorbell_off, u32 *queue_format);330 331static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)332{333 return (pdd->lds_base >> 16) & 0xFF;334}335 336static inline unsigned int337get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd)338{339 return (pdd->lds_base >> 60) & 0x0E;340}341 342/* The DQM lock can be taken in MMU notifiers. Make sure no reclaim-FS343 * happens while holding this lock anywhere to prevent deadlocks when344 * an MMU notifier runs in reclaim-FS context.345 */346static inline void dqm_lock(struct device_queue_manager *dqm)347{348 mutex_lock(&dqm->lock_hidden);349 dqm->saved_flags = memalloc_noreclaim_save();350}351static inline void dqm_unlock(struct device_queue_manager *dqm)352{353 memalloc_noreclaim_restore(dqm->saved_flags);354 mutex_unlock(&dqm->lock_hidden);355}356 357static inline int read_sdma_queue_counter(uint64_t __user *q_rptr, uint64_t *val)358{359 /* SDMA activity counter is stored at queue's RPTR + 0x8 location. */360 return get_user(*val, q_rptr + 1);361}362#endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */363