551 lines · c
1// SPDX-License-Identifier: GPL-2.0-only OR MIT2/* Copyright (c) 2023 Imagination Technologies Ltd. */3 4#include "pvr_free_list.h"5#include "pvr_hwrt.h"6#include "pvr_gem.h"7#include "pvr_rogue_cr_defs_client.h"8#include "pvr_rogue_fwif.h"9 10#include <drm/drm_gem.h>11#include <linux/bitops.h>12#include <linux/math.h>13#include <linux/slab.h>14#include <linux/xarray.h>15#include <uapi/drm/pvr_drm.h>16 17static_assert(ROGUE_FWIF_NUM_RTDATAS == 2);18static_assert(ROGUE_FWIF_NUM_GEOMDATAS == 1);19static_assert(ROGUE_FWIF_NUM_RTDATA_FREELISTS == 2);20 21/*22 * struct pvr_rt_mtile_info - Render target macrotile information23 */24struct pvr_rt_mtile_info {25 u32 mtile_x[3];26 u32 mtile_y[3];27 u32 tile_max_x;28 u32 tile_max_y;29 u32 tile_size_x;30 u32 tile_size_y;31 u32 num_tiles_x;32 u32 num_tiles_y;33};34 35/* Size of Shadow Render Target Cache entry */36#define SRTC_ENTRY_SIZE sizeof(u32)37/* Size of Renders Accumulation Array entry */38#define RAA_ENTRY_SIZE sizeof(u32)39 40static int41hwrt_init_kernel_structure(struct pvr_file *pvr_file,42 struct drm_pvr_ioctl_create_hwrt_dataset_args *args,43 struct pvr_hwrt_dataset *hwrt)44{45 struct pvr_device *pvr_dev = pvr_file->pvr_dev;46 int err;47 int i;48 49 hwrt->pvr_dev = pvr_dev;50 hwrt->max_rts = args->layers;51 52 /* Get pointers to the free lists */53 for (i = 0; i < ARRAY_SIZE(hwrt->free_lists); i++) {54 hwrt->free_lists[i] = pvr_free_list_lookup(pvr_file, args->free_list_handles[i]);55 if (!hwrt->free_lists[i]) {56 err = -EINVAL;57 goto err_put_free_lists;58 }59 }60 61 if (hwrt->free_lists[ROGUE_FW_LOCAL_FREELIST]->current_pages <62 pvr_get_free_list_min_pages(pvr_dev)) {63 err = -EINVAL;64 goto err_put_free_lists;65 }66 67 return 0;68 69err_put_free_lists:70 for (i = 0; i < ARRAY_SIZE(hwrt->free_lists); i++) {71 pvr_free_list_put(hwrt->free_lists[i]);72 hwrt->free_lists[i] = NULL;73 }74 75 return err;76}77 78static void79hwrt_fini_kernel_structure(struct pvr_hwrt_dataset *hwrt)80{81 int i;82 83 for (i = 0; i < ARRAY_SIZE(hwrt->free_lists); i++) {84 pvr_free_list_put(hwrt->free_lists[i]);85 hwrt->free_lists[i] = NULL;86 }87}88 89static void90hwrt_fini_common_fw_structure(struct pvr_hwrt_dataset *hwrt)91{92 pvr_fw_object_destroy(hwrt->common_fw_obj);93}94 95static int96get_cr_isp_mtile_size_val(struct pvr_device *pvr_dev, u32 samples,97 struct pvr_rt_mtile_info *info, u32 *value_out)98{99 u32 x = info->mtile_x[0];100 u32 y = info->mtile_y[0];101 u32 samples_per_pixel;102 int err;103 104 err = PVR_FEATURE_VALUE(pvr_dev, isp_samples_per_pixel, &samples_per_pixel);105 if (err)106 return err;107 108 if (samples_per_pixel == 1) {109 if (samples >= 4)110 x <<= 1;111 if (samples >= 2)112 y <<= 1;113 } else if (samples_per_pixel == 2) {114 if (samples >= 8)115 x <<= 1;116 if (samples >= 4)117 y <<= 1;118 } else if (samples_per_pixel == 4) {119 if (samples >= 8)120 y <<= 1;121 } else {122 WARN(true, "Unsupported ISP samples per pixel value");123 return -EINVAL;124 }125 126 *value_out = ((x << ROGUE_CR_ISP_MTILE_SIZE_X_SHIFT) & ~ROGUE_CR_ISP_MTILE_SIZE_X_CLRMSK) |127 ((y << ROGUE_CR_ISP_MTILE_SIZE_Y_SHIFT) & ~ROGUE_CR_ISP_MTILE_SIZE_Y_CLRMSK);128 129 return 0;130}131 132static int133get_cr_multisamplectl_val(u32 samples, bool y_flip, u64 *value_out)134{135 static const struct {136 u8 x[8];137 u8 y[8];138 } sample_positions[4] = {139 /* 1 sample */140 {141 .x = { 8 },142 .y = { 8 },143 },144 /* 2 samples */145 {146 .x = { 12, 4 },147 .y = { 12, 4 },148 },149 /* 4 samples */150 {151 .x = { 6, 14, 2, 10 },152 .y = { 2, 6, 10, 14 },153 },154 /* 8 samples */155 {156 .x = { 9, 7, 13, 5, 3, 1, 11, 15 },157 .y = { 5, 11, 9, 3, 13, 7, 15, 1 },158 },159 };160 const int idx = fls(samples) - 1;161 u64 value = 0;162 163 if (idx < 0 || idx > 3)164 return -EINVAL;165 166 for (u32 i = 0; i < 8; i++) {167 value |= ((u64)sample_positions[idx].x[i]) << (i * 8);168 if (y_flip)169 value |= (((u64)(16 - sample_positions[idx].y[i]) & 0xf)) << (i * 8 + 4);170 else171 value |= ((u64)sample_positions[idx].y[i]) << (i * 8 + 4);172 }173 174 *value_out = value;175 176 return 0;177}178 179static int180get_cr_te_aa_val(struct pvr_device *pvr_dev, u32 samples, u32 *value_out)181{182 u32 samples_per_pixel;183 u32 value = 0;184 int err = 0;185 186 err = PVR_FEATURE_VALUE(pvr_dev, isp_samples_per_pixel, &samples_per_pixel);187 if (err)188 return err;189 190 switch (samples_per_pixel) {191 case 1:192 if (samples >= 2)193 value |= ROGUE_CR_TE_AA_Y_EN;194 if (samples >= 4)195 value |= ROGUE_CR_TE_AA_X_EN;196 break;197 case 2:198 if (samples >= 2)199 value |= ROGUE_CR_TE_AA_X2_EN;200 if (samples >= 4)201 value |= ROGUE_CR_TE_AA_Y_EN;202 if (samples >= 8)203 value |= ROGUE_CR_TE_AA_X_EN;204 break;205 case 4:206 if (samples >= 2)207 value |= ROGUE_CR_TE_AA_X2_EN;208 if (samples >= 4)209 value |= ROGUE_CR_TE_AA_Y2_EN;210 if (samples >= 8)211 value |= ROGUE_CR_TE_AA_Y_EN;212 break;213 default:214 WARN(true, "Unsupported ISP samples per pixel value");215 return -EINVAL;216 }217 218 *value_out = value;219 220 return 0;221}222 223static void224hwrtdata_common_init(void *cpu_ptr, void *priv)225{226 struct pvr_hwrt_dataset *hwrt = priv;227 228 memcpy(cpu_ptr, &hwrt->common, sizeof(hwrt->common));229}230 231static int232hwrt_init_common_fw_structure(struct pvr_file *pvr_file,233 struct drm_pvr_ioctl_create_hwrt_dataset_args *args,234 struct pvr_hwrt_dataset *hwrt)235{236 struct drm_pvr_create_hwrt_geom_data_args *geom_data_args = &args->geom_data_args;237 struct pvr_device *pvr_dev = pvr_file->pvr_dev;238 struct pvr_rt_mtile_info info;239 int err;240 241 err = PVR_FEATURE_VALUE(pvr_dev, tile_size_x, &info.tile_size_x);242 if (WARN_ON(err))243 return err;244 245 err = PVR_FEATURE_VALUE(pvr_dev, tile_size_y, &info.tile_size_y);246 if (WARN_ON(err))247 return err;248 249 info.num_tiles_x = DIV_ROUND_UP(args->width, info.tile_size_x);250 info.num_tiles_y = DIV_ROUND_UP(args->height, info.tile_size_y);251 252 if (PVR_HAS_FEATURE(pvr_dev, simple_parameter_format_version)) {253 u32 parameter_format;254 255 err = PVR_FEATURE_VALUE(pvr_dev, simple_parameter_format_version,256 ¶meter_format);257 if (WARN_ON(err))258 return err;259 260 WARN_ON(parameter_format != 2);261 262 /*263 * Set up 16 macrotiles with a multiple of 2x2 tiles per macrotile, which is264 * aligned to a tile group.265 */266 info.mtile_x[0] = DIV_ROUND_UP(info.num_tiles_x, 8) * 2;267 info.mtile_y[0] = DIV_ROUND_UP(info.num_tiles_y, 8) * 2;268 info.mtile_x[1] = 0;269 info.mtile_y[1] = 0;270 info.mtile_x[2] = 0;271 info.mtile_y[2] = 0;272 info.tile_max_x = round_up(info.num_tiles_x, 2) - 1;273 info.tile_max_y = round_up(info.num_tiles_y, 2) - 1;274 } else {275 /* Set up 16 macrotiles with a multiple of 4x4 tiles per macrotile. */276 info.mtile_x[0] = round_up(DIV_ROUND_UP(info.num_tiles_x, 4), 4);277 info.mtile_y[0] = round_up(DIV_ROUND_UP(info.num_tiles_y, 4), 4);278 info.mtile_x[1] = info.mtile_x[0] * 2;279 info.mtile_y[1] = info.mtile_y[0] * 2;280 info.mtile_x[2] = info.mtile_x[0] * 3;281 info.mtile_y[2] = info.mtile_y[0] * 3;282 info.tile_max_x = info.num_tiles_x - 1;283 info.tile_max_y = info.num_tiles_y - 1;284 }285 286 hwrt->common.geom_caches_need_zeroing = false;287 288 hwrt->common.isp_merge_lower_x = args->isp_merge_lower_x;289 hwrt->common.isp_merge_lower_y = args->isp_merge_lower_y;290 hwrt->common.isp_merge_upper_x = args->isp_merge_upper_x;291 hwrt->common.isp_merge_upper_y = args->isp_merge_upper_y;292 hwrt->common.isp_merge_scale_x = args->isp_merge_scale_x;293 hwrt->common.isp_merge_scale_y = args->isp_merge_scale_y;294 295 err = get_cr_multisamplectl_val(args->samples, false,296 &hwrt->common.multi_sample_ctl);297 if (err)298 return err;299 300 err = get_cr_multisamplectl_val(args->samples, true,301 &hwrt->common.flipped_multi_sample_ctl);302 if (err)303 return err;304 305 hwrt->common.mtile_stride = info.mtile_x[0] * info.mtile_y[0];306 307 err = get_cr_te_aa_val(pvr_dev, args->samples, &hwrt->common.teaa);308 if (err)309 return err;310 311 hwrt->common.screen_pixel_max =312 (((args->width - 1) << ROGUE_CR_PPP_SCREEN_PIXXMAX_SHIFT) &313 ~ROGUE_CR_PPP_SCREEN_PIXXMAX_CLRMSK) |314 (((args->height - 1) << ROGUE_CR_PPP_SCREEN_PIXYMAX_SHIFT) &315 ~ROGUE_CR_PPP_SCREEN_PIXYMAX_CLRMSK);316 317 hwrt->common.te_screen =318 ((info.tile_max_x << ROGUE_CR_TE_SCREEN_XMAX_SHIFT) &319 ~ROGUE_CR_TE_SCREEN_XMAX_CLRMSK) |320 ((info.tile_max_y << ROGUE_CR_TE_SCREEN_YMAX_SHIFT) &321 ~ROGUE_CR_TE_SCREEN_YMAX_CLRMSK);322 hwrt->common.te_mtile1 =323 ((info.mtile_x[0] << ROGUE_CR_TE_MTILE1_X1_SHIFT) & ~ROGUE_CR_TE_MTILE1_X1_CLRMSK) |324 ((info.mtile_x[1] << ROGUE_CR_TE_MTILE1_X2_SHIFT) & ~ROGUE_CR_TE_MTILE1_X2_CLRMSK) |325 ((info.mtile_x[2] << ROGUE_CR_TE_MTILE1_X3_SHIFT) & ~ROGUE_CR_TE_MTILE1_X3_CLRMSK);326 hwrt->common.te_mtile2 =327 ((info.mtile_y[0] << ROGUE_CR_TE_MTILE2_Y1_SHIFT) & ~ROGUE_CR_TE_MTILE2_Y1_CLRMSK) |328 ((info.mtile_y[1] << ROGUE_CR_TE_MTILE2_Y2_SHIFT) & ~ROGUE_CR_TE_MTILE2_Y2_CLRMSK) |329 ((info.mtile_y[2] << ROGUE_CR_TE_MTILE2_Y3_SHIFT) & ~ROGUE_CR_TE_MTILE2_Y3_CLRMSK);330 331 err = get_cr_isp_mtile_size_val(pvr_dev, args->samples, &info,332 &hwrt->common.isp_mtile_size);333 if (err)334 return err;335 336 hwrt->common.tpc_stride = geom_data_args->tpc_stride;337 hwrt->common.tpc_size = geom_data_args->tpc_size;338 339 hwrt->common.rgn_header_size = args->region_header_size;340 341 err = pvr_fw_object_create(pvr_dev, sizeof(struct rogue_fwif_hwrtdata_common),342 PVR_BO_FW_FLAGS_DEVICE_UNCACHED, hwrtdata_common_init, hwrt,343 &hwrt->common_fw_obj);344 345 return err;346}347 348static void349hwrt_fw_data_init(void *cpu_ptr, void *priv)350{351 struct pvr_hwrt_data *hwrt_data = priv;352 353 memcpy(cpu_ptr, &hwrt_data->data, sizeof(hwrt_data->data));354}355 356static int357hwrt_data_init_fw_structure(struct pvr_file *pvr_file,358 struct pvr_hwrt_dataset *hwrt,359 struct drm_pvr_ioctl_create_hwrt_dataset_args *args,360 struct drm_pvr_create_hwrt_rt_data_args *rt_data_args,361 struct pvr_hwrt_data *hwrt_data)362{363 struct drm_pvr_create_hwrt_geom_data_args *geom_data_args = &args->geom_data_args;364 struct pvr_device *pvr_dev = pvr_file->pvr_dev;365 struct rogue_fwif_rta_ctl *rta_ctl;366 int free_list_i;367 int err;368 369 pvr_fw_object_get_fw_addr(hwrt->common_fw_obj,370 &hwrt_data->data.hwrt_data_common_fw_addr);371 372 for (free_list_i = 0; free_list_i < ARRAY_SIZE(hwrt->free_lists); free_list_i++) {373 pvr_fw_object_get_fw_addr(hwrt->free_lists[free_list_i]->fw_obj,374 &hwrt_data->data.freelists_fw_addr[free_list_i]);375 }376 377 hwrt_data->data.tail_ptrs_dev_addr = geom_data_args->tpc_dev_addr;378 hwrt_data->data.vheap_table_dev_addr = geom_data_args->vheap_table_dev_addr;379 hwrt_data->data.rtc_dev_addr = geom_data_args->rtc_dev_addr;380 381 hwrt_data->data.pm_mlist_dev_addr = rt_data_args->pm_mlist_dev_addr;382 hwrt_data->data.macrotile_array_dev_addr = rt_data_args->macrotile_array_dev_addr;383 hwrt_data->data.rgn_header_dev_addr = rt_data_args->region_header_dev_addr;384 385 rta_ctl = &hwrt_data->data.rta_ctl;386 387 rta_ctl->render_target_index = 0;388 rta_ctl->active_render_targets = 0;389 rta_ctl->valid_render_targets_fw_addr = 0;390 rta_ctl->rta_num_partial_renders_fw_addr = 0;391 rta_ctl->max_rts = args->layers;392 393 if (args->layers > 1) {394 err = pvr_fw_object_create(pvr_dev, args->layers * SRTC_ENTRY_SIZE,395 PVR_BO_FW_FLAGS_DEVICE_UNCACHED,396 NULL, NULL, &hwrt_data->srtc_obj);397 if (err)398 return err;399 pvr_fw_object_get_fw_addr(hwrt_data->srtc_obj,400 &rta_ctl->valid_render_targets_fw_addr);401 402 err = pvr_fw_object_create(pvr_dev, args->layers * RAA_ENTRY_SIZE,403 PVR_BO_FW_FLAGS_DEVICE_UNCACHED,404 NULL, NULL, &hwrt_data->raa_obj);405 if (err)406 goto err_put_shadow_rt_cache;407 pvr_fw_object_get_fw_addr(hwrt_data->raa_obj,408 &rta_ctl->rta_num_partial_renders_fw_addr);409 }410 411 err = pvr_fw_object_create(pvr_dev, sizeof(struct rogue_fwif_hwrtdata),412 PVR_BO_FW_FLAGS_DEVICE_UNCACHED,413 hwrt_fw_data_init, hwrt_data, &hwrt_data->fw_obj);414 if (err)415 goto err_put_raa_obj;416 417 pvr_free_list_add_hwrt(hwrt->free_lists[0], hwrt_data);418 419 return 0;420 421err_put_raa_obj:422 if (args->layers > 1)423 pvr_fw_object_destroy(hwrt_data->raa_obj);424 425err_put_shadow_rt_cache:426 if (args->layers > 1)427 pvr_fw_object_destroy(hwrt_data->srtc_obj);428 429 return err;430}431 432static void433hwrt_data_fini_fw_structure(struct pvr_hwrt_dataset *hwrt, int hwrt_nr)434{435 struct pvr_hwrt_data *hwrt_data = &hwrt->data[hwrt_nr];436 437 pvr_free_list_remove_hwrt(hwrt->free_lists[0], hwrt_data);438 439 if (hwrt->max_rts > 1) {440 pvr_fw_object_destroy(hwrt_data->raa_obj);441 pvr_fw_object_destroy(hwrt_data->srtc_obj);442 }443 444 pvr_fw_object_destroy(hwrt_data->fw_obj);445}446 447/**448 * pvr_hwrt_dataset_create() - Create a new HWRT dataset449 * @pvr_file: Pointer to pvr_file structure.450 * @args: Creation arguments from userspace.451 *452 * Return:453 * * Pointer to new HWRT, or454 * * ERR_PTR(-%ENOMEM) on out of memory.455 */456struct pvr_hwrt_dataset *457pvr_hwrt_dataset_create(struct pvr_file *pvr_file,458 struct drm_pvr_ioctl_create_hwrt_dataset_args *args)459{460 struct pvr_hwrt_dataset *hwrt;461 int err, i = 0;462 463 /* Create and fill out the kernel structure */464 hwrt = kzalloc(sizeof(*hwrt), GFP_KERNEL);465 466 if (!hwrt)467 return ERR_PTR(-ENOMEM);468 469 err = hwrt_init_kernel_structure(pvr_file, args, hwrt);470 if (err < 0)471 goto err_free;472 473 err = hwrt_init_common_fw_structure(pvr_file, args, hwrt);474 if (err < 0)475 goto err_fini_kernel_structure;476 477 for (; i < ARRAY_SIZE(hwrt->data); i++) {478 err = hwrt_data_init_fw_structure(pvr_file, hwrt, args,479 &args->rt_data_args[i],480 &hwrt->data[i]);481 if (err < 0)482 goto err_fini_data_structures;483 484 hwrt->data[i].hwrt_dataset = hwrt;485 }486 487 kref_init(&hwrt->ref_count);488 return hwrt;489 490err_fini_data_structures:491 while (--i >= 0)492 hwrt_data_fini_fw_structure(hwrt, i);493 494err_fini_kernel_structure:495 hwrt_fini_kernel_structure(hwrt);496 497err_free:498 kfree(hwrt);499 500 return ERR_PTR(err);501}502 503static void504pvr_hwrt_dataset_release(struct kref *ref_count)505{506 struct pvr_hwrt_dataset *hwrt =507 container_of(ref_count, struct pvr_hwrt_dataset, ref_count);508 509 for (int i = ARRAY_SIZE(hwrt->data) - 1; i >= 0; i--) {510 WARN_ON(pvr_fw_structure_cleanup(hwrt->pvr_dev, ROGUE_FWIF_CLEANUP_HWRTDATA,511 hwrt->data[i].fw_obj, 0));512 hwrt_data_fini_fw_structure(hwrt, i);513 }514 515 hwrt_fini_common_fw_structure(hwrt);516 hwrt_fini_kernel_structure(hwrt);517 518 kfree(hwrt);519}520 521/**522 * pvr_destroy_hwrt_datasets_for_file: Destroy any HWRT datasets associated523 * with the given file.524 * @pvr_file: Pointer to pvr_file structure.525 *526 * Removes all HWRT datasets associated with @pvr_file from the device527 * hwrt_dataset list and drops initial references. HWRT datasets will then be528 * destroyed once all outstanding references are dropped.529 */530void pvr_destroy_hwrt_datasets_for_file(struct pvr_file *pvr_file)531{532 struct pvr_hwrt_dataset *hwrt;533 unsigned long handle;534 535 xa_for_each(&pvr_file->hwrt_handles, handle, hwrt) {536 (void)hwrt;537 pvr_hwrt_dataset_put(xa_erase(&pvr_file->hwrt_handles, handle));538 }539}540 541/**542 * pvr_hwrt_dataset_put() - Release reference on HWRT dataset543 * @hwrt: Pointer to HWRT dataset to release reference on544 */545void546pvr_hwrt_dataset_put(struct pvr_hwrt_dataset *hwrt)547{548 if (hwrt)549 kref_put(&hwrt->ref_count, pvr_hwrt_dataset_release);550}551