310 lines · c
1// SPDX-License-Identifier: GPL-2.0-only OR MIT2/*3 * Copyright (C) 2020-2024 Intel Corporation4 */5 6#include <drm/drm_file.h>7 8#include "ivpu_drv.h"9#include "ivpu_gem.h"10#include "ivpu_jsm_msg.h"11#include "ivpu_ms.h"12#include "ivpu_pm.h"13 14#define MS_INFO_BUFFER_SIZE SZ_16K15#define MS_NUM_BUFFERS 216#define MS_READ_PERIOD_MULTIPLIER 217#define MS_MIN_SAMPLE_PERIOD_NS 100000018 19static struct ivpu_ms_instance *20get_instance_by_mask(struct ivpu_file_priv *file_priv, u64 metric_mask)21{22 struct ivpu_ms_instance *ms;23 24 lockdep_assert_held(&file_priv->ms_lock);25 26 list_for_each_entry(ms, &file_priv->ms_instance_list, ms_instance_node)27 if (ms->mask == metric_mask)28 return ms;29 30 return NULL;31}32 33int ivpu_ms_start_ioctl(struct drm_device *dev, void *data, struct drm_file *file)34{35 struct ivpu_file_priv *file_priv = file->driver_priv;36 struct drm_ivpu_metric_streamer_start *args = data;37 struct ivpu_device *vdev = file_priv->vdev;38 struct ivpu_ms_instance *ms;39 u64 single_buff_size;40 u32 sample_size;41 int ret;42 43 if (!args->metric_group_mask || !args->read_period_samples ||44 args->sampling_period_ns < MS_MIN_SAMPLE_PERIOD_NS)45 return -EINVAL;46 47 mutex_lock(&file_priv->ms_lock);48 49 if (get_instance_by_mask(file_priv, args->metric_group_mask)) {50 ivpu_err(vdev, "Instance already exists (mask %#llx)\n", args->metric_group_mask);51 ret = -EALREADY;52 goto unlock;53 }54 55 ms = kzalloc(sizeof(*ms), GFP_KERNEL);56 if (!ms) {57 ret = -ENOMEM;58 goto unlock;59 }60 61 ms->mask = args->metric_group_mask;62 63 ret = ivpu_jsm_metric_streamer_info(vdev, ms->mask, 0, 0, &sample_size, NULL);64 if (ret)65 goto err_free_ms;66 67 single_buff_size = sample_size *68 ((u64)args->read_period_samples * MS_READ_PERIOD_MULTIPLIER);69 ms->bo = ivpu_bo_create_global(vdev, PAGE_ALIGN(single_buff_size * MS_NUM_BUFFERS),70 DRM_IVPU_BO_CACHED | DRM_IVPU_BO_MAPPABLE);71 if (!ms->bo) {72 ivpu_err(vdev, "Failed to allocate MS buffer (size %llu)\n", single_buff_size);73 ret = -ENOMEM;74 goto err_free_ms;75 }76 77 ms->buff_size = ivpu_bo_size(ms->bo) / MS_NUM_BUFFERS;78 ms->active_buff_vpu_addr = ms->bo->vpu_addr;79 ms->inactive_buff_vpu_addr = ms->bo->vpu_addr + ms->buff_size;80 ms->active_buff_ptr = ivpu_bo_vaddr(ms->bo);81 ms->inactive_buff_ptr = ivpu_bo_vaddr(ms->bo) + ms->buff_size;82 83 ret = ivpu_jsm_metric_streamer_start(vdev, ms->mask, args->sampling_period_ns,84 ms->active_buff_vpu_addr, ms->buff_size);85 if (ret)86 goto err_free_bo;87 88 args->sample_size = sample_size;89 args->max_data_size = ivpu_bo_size(ms->bo);90 list_add_tail(&ms->ms_instance_node, &file_priv->ms_instance_list);91 goto unlock;92 93err_free_bo:94 ivpu_bo_free(ms->bo);95err_free_ms:96 kfree(ms);97unlock:98 mutex_unlock(&file_priv->ms_lock);99 return ret;100}101 102static int103copy_leftover_bytes(struct ivpu_ms_instance *ms,104 void __user *user_ptr, u64 user_size, u64 *user_bytes_copied)105{106 u64 copy_bytes;107 108 if (ms->leftover_bytes) {109 copy_bytes = min(user_size - *user_bytes_copied, ms->leftover_bytes);110 if (copy_to_user(user_ptr + *user_bytes_copied, ms->leftover_addr, copy_bytes))111 return -EFAULT;112 113 ms->leftover_bytes -= copy_bytes;114 ms->leftover_addr += copy_bytes;115 *user_bytes_copied += copy_bytes;116 }117 118 return 0;119}120 121static int122copy_samples_to_user(struct ivpu_device *vdev, struct ivpu_ms_instance *ms,123 void __user *user_ptr, u64 user_size, u64 *user_bytes_copied)124{125 u64 bytes_written;126 int ret;127 128 *user_bytes_copied = 0;129 130 ret = copy_leftover_bytes(ms, user_ptr, user_size, user_bytes_copied);131 if (ret)132 return ret;133 134 if (*user_bytes_copied == user_size)135 return 0;136 137 ret = ivpu_jsm_metric_streamer_update(vdev, ms->mask, ms->inactive_buff_vpu_addr,138 ms->buff_size, &bytes_written);139 if (ret)140 return ret;141 142 swap(ms->active_buff_vpu_addr, ms->inactive_buff_vpu_addr);143 swap(ms->active_buff_ptr, ms->inactive_buff_ptr);144 145 ms->leftover_bytes = bytes_written;146 ms->leftover_addr = ms->inactive_buff_ptr;147 148 return copy_leftover_bytes(ms, user_ptr, user_size, user_bytes_copied);149}150 151int ivpu_ms_get_data_ioctl(struct drm_device *dev, void *data, struct drm_file *file)152{153 struct drm_ivpu_metric_streamer_get_data *args = data;154 struct ivpu_file_priv *file_priv = file->driver_priv;155 struct ivpu_device *vdev = file_priv->vdev;156 struct ivpu_ms_instance *ms;157 u64 bytes_written;158 int ret;159 160 if (!args->metric_group_mask)161 return -EINVAL;162 163 mutex_lock(&file_priv->ms_lock);164 165 ms = get_instance_by_mask(file_priv, args->metric_group_mask);166 if (!ms) {167 ivpu_err(vdev, "Instance doesn't exist for mask: %#llx\n", args->metric_group_mask);168 ret = -EINVAL;169 goto unlock;170 }171 172 if (!args->buffer_size) {173 ret = ivpu_jsm_metric_streamer_update(vdev, ms->mask, 0, 0, &bytes_written);174 if (ret)175 goto unlock;176 args->data_size = bytes_written + ms->leftover_bytes;177 goto unlock;178 }179 180 if (!args->buffer_ptr) {181 ret = -EINVAL;182 goto unlock;183 }184 185 ret = copy_samples_to_user(vdev, ms, u64_to_user_ptr(args->buffer_ptr),186 args->buffer_size, &args->data_size);187unlock:188 mutex_unlock(&file_priv->ms_lock);189 190 return ret;191}192 193static void free_instance(struct ivpu_file_priv *file_priv, struct ivpu_ms_instance *ms)194{195 lockdep_assert_held(&file_priv->ms_lock);196 197 list_del(&ms->ms_instance_node);198 ivpu_jsm_metric_streamer_stop(file_priv->vdev, ms->mask);199 ivpu_bo_free(ms->bo);200 kfree(ms);201}202 203int ivpu_ms_stop_ioctl(struct drm_device *dev, void *data, struct drm_file *file)204{205 struct ivpu_file_priv *file_priv = file->driver_priv;206 struct drm_ivpu_metric_streamer_stop *args = data;207 struct ivpu_ms_instance *ms;208 209 if (!args->metric_group_mask)210 return -EINVAL;211 212 mutex_lock(&file_priv->ms_lock);213 214 ms = get_instance_by_mask(file_priv, args->metric_group_mask);215 if (ms)216 free_instance(file_priv, ms);217 218 mutex_unlock(&file_priv->ms_lock);219 220 return ms ? 0 : -EINVAL;221}222 223static inline struct ivpu_bo *get_ms_info_bo(struct ivpu_file_priv *file_priv)224{225 lockdep_assert_held(&file_priv->ms_lock);226 227 if (file_priv->ms_info_bo)228 return file_priv->ms_info_bo;229 230 file_priv->ms_info_bo = ivpu_bo_create_global(file_priv->vdev, MS_INFO_BUFFER_SIZE,231 DRM_IVPU_BO_CACHED | DRM_IVPU_BO_MAPPABLE);232 return file_priv->ms_info_bo;233}234 235int ivpu_ms_get_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file)236{237 struct drm_ivpu_metric_streamer_get_data *args = data;238 struct ivpu_file_priv *file_priv = file->driver_priv;239 struct ivpu_device *vdev = file_priv->vdev;240 struct ivpu_bo *bo;241 u64 info_size;242 int ret;243 244 if (!args->metric_group_mask)245 return -EINVAL;246 247 if (!args->buffer_size)248 return ivpu_jsm_metric_streamer_info(vdev, args->metric_group_mask,249 0, 0, NULL, &args->data_size);250 if (!args->buffer_ptr)251 return -EINVAL;252 253 mutex_lock(&file_priv->ms_lock);254 255 bo = get_ms_info_bo(file_priv);256 if (!bo) {257 ret = -ENOMEM;258 goto unlock;259 }260 261 ret = ivpu_jsm_metric_streamer_info(vdev, args->metric_group_mask, bo->vpu_addr,262 ivpu_bo_size(bo), NULL, &info_size);263 if (ret)264 goto unlock;265 266 if (args->buffer_size < info_size) {267 ret = -ENOSPC;268 goto unlock;269 }270 271 if (copy_to_user(u64_to_user_ptr(args->buffer_ptr), ivpu_bo_vaddr(bo), info_size))272 ret = -EFAULT;273 274 args->data_size = info_size;275unlock:276 mutex_unlock(&file_priv->ms_lock);277 278 return ret;279}280 281void ivpu_ms_cleanup(struct ivpu_file_priv *file_priv)282{283 struct ivpu_ms_instance *ms, *tmp;284 285 mutex_lock(&file_priv->ms_lock);286 287 if (file_priv->ms_info_bo) {288 ivpu_bo_free(file_priv->ms_info_bo);289 file_priv->ms_info_bo = NULL;290 }291 292 list_for_each_entry_safe(ms, tmp, &file_priv->ms_instance_list, ms_instance_node)293 free_instance(file_priv, ms);294 295 mutex_unlock(&file_priv->ms_lock);296}297 298void ivpu_ms_cleanup_all(struct ivpu_device *vdev)299{300 struct ivpu_file_priv *file_priv;301 unsigned long ctx_id;302 303 mutex_lock(&vdev->context_list_lock);304 305 xa_for_each(&vdev->context_xa, ctx_id, file_priv)306 ivpu_ms_cleanup(file_priv);307 308 mutex_unlock(&vdev->context_list_lock);309}310