719 lines · c
1// SPDX-License-Identifier: GPL-2.02 3/*4 * Copyright 2021 HabanaLabs, Ltd.5 * All Rights Reserved.6 */7 8#include <linux/vmalloc.h>9#include <uapi/drm/habanalabs_accel.h>10#include "habanalabs.h"11 12/**13 * hl_format_as_binary - helper function, format an integer as binary14 * using supplied scratch buffer15 * @buf: the buffer to use16 * @buf_len: buffer capacity17 * @n: number to format18 *19 * Returns pointer to buffer20 */21char *hl_format_as_binary(char *buf, size_t buf_len, u32 n)22{23 int i;24 u32 bit;25 bool leading0 = true;26 char *wrptr = buf;27 28 if (buf_len > 0 && buf_len < 3) {29 *wrptr = '\0';30 return buf;31 }32 33 wrptr[0] = '0';34 wrptr[1] = 'b';35 wrptr += 2;36 /* Remove 3 characters from length for '0b' and '\0' termination */37 buf_len -= 3;38 39 for (i = 0; i < sizeof(n) * BITS_PER_BYTE && buf_len; ++i, n <<= 1) {40 /* Writing bit calculation in one line would cause a false41 * positive static code analysis error, so splitting.42 */43 bit = n & (1 << (sizeof(n) * BITS_PER_BYTE - 1));44 bit = !!bit;45 leading0 &= !bit;46 if (!leading0) {47 *wrptr = '0' + bit;48 ++wrptr;49 }50 }51 52 *wrptr = '\0';53 54 return buf;55}56 57/**58 * resize_to_fit - helper function, resize buffer to fit given amount of data59 * @buf: destination buffer double pointer60 * @size: pointer to the size container61 * @desired_size: size the buffer must contain62 *63 * Returns 0 on success or error code on failure.64 * On success, the size of buffer is at least desired_size. Buffer is allocated65 * via vmalloc and must be freed with vfree.66 */67static int resize_to_fit(char **buf, size_t *size, size_t desired_size)68{69 char *resized_buf;70 size_t new_size;71 72 if (*size >= desired_size)73 return 0;74 75 /* Not enough space to print all, have to resize */76 new_size = max_t(size_t, PAGE_SIZE, round_up(desired_size, PAGE_SIZE));77 resized_buf = vmalloc(new_size);78 if (!resized_buf)79 return -ENOMEM;80 memcpy(resized_buf, *buf, *size);81 vfree(*buf);82 *buf = resized_buf;83 *size = new_size;84 85 return 1;86}87 88/**89 * hl_snprintf_resize() - print formatted data to buffer, resize as needed90 * @buf: buffer double pointer, to be written to and resized, must be either91 * NULL or allocated with vmalloc.92 * @size: current size of the buffer93 * @offset: current offset to write to94 * @format: format of the data95 *96 * This function will write formatted data into the buffer. If buffer is not97 * large enough, it will be resized using vmalloc. Size may be modified if the98 * buffer was resized, offset will be advanced by the number of bytes written99 * not including the terminating character100 *101 * Returns 0 on success or error code on failure102 *103 * Note that the buffer has to be manually released using vfree.104 */105int hl_snprintf_resize(char **buf, size_t *size, size_t *offset,106 const char *format, ...)107{108 va_list args;109 size_t length;110 int rc;111 112 if (*buf == NULL && (*size != 0 || *offset != 0))113 return -EINVAL;114 115 va_start(args, format);116 length = vsnprintf(*buf + *offset, *size - *offset, format, args);117 va_end(args);118 119 rc = resize_to_fit(buf, size, *offset + length + 1);120 if (rc < 0)121 return rc;122 else if (rc > 0) {123 /* Resize was needed, write again */124 va_start(args, format);125 length = vsnprintf(*buf + *offset, *size - *offset, format,126 args);127 va_end(args);128 }129 130 *offset += length;131 132 return 0;133}134 135/**136 * hl_sync_engine_to_string - convert engine type enum to string literal137 * @engine_type: engine type (TPC/MME/DMA)138 *139 * Return the resolved string literal140 */141const char *hl_sync_engine_to_string(enum hl_sync_engine_type engine_type)142{143 switch (engine_type) {144 case ENGINE_DMA:145 return "DMA";146 case ENGINE_MME:147 return "MME";148 case ENGINE_TPC:149 return "TPC";150 }151 return "Invalid Engine Type";152}153 154/**155 * hl_print_resize_sync_engine - helper function, format engine name and ID156 * using hl_snprintf_resize157 * @buf: destination buffer double pointer to be used with hl_snprintf_resize158 * @size: pointer to the size container159 * @offset: pointer to the offset container160 * @engine_type: engine type (TPC/MME/DMA)161 * @engine_id: engine numerical id162 *163 * Returns 0 on success or error code on failure164 */165static int hl_print_resize_sync_engine(char **buf, size_t *size, size_t *offset,166 enum hl_sync_engine_type engine_type,167 u32 engine_id)168{169 return hl_snprintf_resize(buf, size, offset, "%s%u",170 hl_sync_engine_to_string(engine_type), engine_id);171}172 173/**174 * hl_state_dump_get_sync_name - transform sync object id to name if available175 * @hdev: pointer to the device176 * @sync_id: sync object id177 *178 * Returns a name literal or NULL if not resolved.179 * Note: returning NULL shall not be considered as a failure, as not all180 * sync objects are named.181 */182const char *hl_state_dump_get_sync_name(struct hl_device *hdev, u32 sync_id)183{184 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;185 struct hl_hw_obj_name_entry *entry;186 187 hash_for_each_possible(sds->so_id_to_str_tb, entry,188 node, sync_id)189 if (sync_id == entry->id)190 return entry->name;191 192 return NULL;193}194 195/**196 * hl_state_dump_get_monitor_name - transform monitor object dump to monitor197 * name if available198 * @hdev: pointer to the device199 * @mon: monitor state dump200 *201 * Returns a name literal or NULL if not resolved.202 * Note: returning NULL shall not be considered as a failure, as not all203 * monitors are named.204 */205const char *hl_state_dump_get_monitor_name(struct hl_device *hdev,206 struct hl_mon_state_dump *mon)207{208 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;209 struct hl_hw_obj_name_entry *entry;210 211 hash_for_each_possible(sds->monitor_id_to_str_tb,212 entry, node, mon->id)213 if (mon->id == entry->id)214 return entry->name;215 216 return NULL;217}218 219/**220 * hl_state_dump_free_sync_to_engine_map - free sync object to engine map221 * @map: sync object to engine map222 *223 * Note: generic free implementation, the allocation is implemented per ASIC.224 */225void hl_state_dump_free_sync_to_engine_map(struct hl_sync_to_engine_map *map)226{227 struct hl_sync_to_engine_map_entry *entry;228 struct hlist_node *tmp_node;229 int i;230 231 hash_for_each_safe(map->tb, i, tmp_node, entry, node) {232 hash_del(&entry->node);233 kfree(entry);234 }235}236 237/**238 * hl_state_dump_get_sync_to_engine - transform sync_id to239 * hl_sync_to_engine_map_entry if available for current id240 * @map: sync object to engine map241 * @sync_id: sync object id242 *243 * Returns the translation entry if found or NULL if not.244 * Note, returned NULL shall not be considered as a failure as the map245 * does not cover all possible, it is a best effort sync ids.246 */247static struct hl_sync_to_engine_map_entry *248hl_state_dump_get_sync_to_engine(struct hl_sync_to_engine_map *map, u32 sync_id)249{250 struct hl_sync_to_engine_map_entry *entry;251 252 hash_for_each_possible(map->tb, entry, node, sync_id)253 if (entry->sync_id == sync_id)254 return entry;255 return NULL;256}257 258/**259 * hl_state_dump_read_sync_objects - read sync objects array260 * @hdev: pointer to the device261 * @index: sync manager block index starting with E_N262 *263 * Returns array of size SP_SYNC_OBJ_AMOUNT on success or NULL on failure264 */265static u32 *hl_state_dump_read_sync_objects(struct hl_device *hdev, u32 index)266{267 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;268 u32 *sync_objects;269 s64 base_addr; /* Base addr can be negative */270 int i;271 272 base_addr = sds->props[SP_SYNC_OBJ_BASE_ADDR] +273 sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index;274 275 sync_objects = vmalloc(sds->props[SP_SYNC_OBJ_AMOUNT] * sizeof(u32));276 if (!sync_objects)277 return NULL;278 279 for (i = 0; i < sds->props[SP_SYNC_OBJ_AMOUNT]; ++i)280 sync_objects[i] = RREG32(base_addr + i * sizeof(u32));281 282 return sync_objects;283}284 285/**286 * hl_state_dump_free_sync_objects - free sync objects array allocated by287 * hl_state_dump_read_sync_objects288 * @sync_objects: sync objects array289 */290static void hl_state_dump_free_sync_objects(u32 *sync_objects)291{292 vfree(sync_objects);293}294 295 296/**297 * hl_state_dump_print_syncs_single_block - print active sync objects on a298 * single block299 * @hdev: pointer to the device300 * @index: sync manager block index starting with E_N301 * @buf: destination buffer double pointer to be used with hl_snprintf_resize302 * @size: pointer to the size container303 * @offset: pointer to the offset container304 * @map: sync engines names map305 *306 * Returns 0 on success or error code on failure307 */308static int309hl_state_dump_print_syncs_single_block(struct hl_device *hdev, u32 index,310 char **buf, size_t *size, size_t *offset,311 struct hl_sync_to_engine_map *map)312{313 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;314 const char *sync_name;315 u32 *sync_objects = NULL;316 int rc = 0, i;317 318 if (sds->sync_namager_names) {319 rc = hl_snprintf_resize(320 buf, size, offset, "%s\n",321 sds->sync_namager_names[index]);322 if (rc)323 goto out;324 }325 326 sync_objects = hl_state_dump_read_sync_objects(hdev, index);327 if (!sync_objects) {328 rc = -ENOMEM;329 goto out;330 }331 332 for (i = 0; i < sds->props[SP_SYNC_OBJ_AMOUNT]; ++i) {333 struct hl_sync_to_engine_map_entry *entry;334 u64 sync_object_addr;335 336 if (!sync_objects[i])337 continue;338 339 sync_object_addr = sds->props[SP_SYNC_OBJ_BASE_ADDR] +340 sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index +341 i * sizeof(u32);342 343 rc = hl_snprintf_resize(buf, size, offset, "sync id: %u", i);344 if (rc)345 goto free_sync_objects;346 sync_name = hl_state_dump_get_sync_name(hdev, i);347 if (sync_name) {348 rc = hl_snprintf_resize(buf, size, offset, " %s",349 sync_name);350 if (rc)351 goto free_sync_objects;352 }353 rc = hl_snprintf_resize(buf, size, offset, ", value: %u",354 sync_objects[i]);355 if (rc)356 goto free_sync_objects;357 358 /* Append engine string */359 entry = hl_state_dump_get_sync_to_engine(map,360 (u32)sync_object_addr);361 if (entry) {362 rc = hl_snprintf_resize(buf, size, offset,363 ", Engine: ");364 if (rc)365 goto free_sync_objects;366 rc = hl_print_resize_sync_engine(buf, size, offset,367 entry->engine_type,368 entry->engine_id);369 if (rc)370 goto free_sync_objects;371 }372 373 rc = hl_snprintf_resize(buf, size, offset, "\n");374 if (rc)375 goto free_sync_objects;376 }377 378free_sync_objects:379 hl_state_dump_free_sync_objects(sync_objects);380out:381 return rc;382}383 384/**385 * hl_state_dump_print_syncs - print active sync objects386 * @hdev: pointer to the device387 * @buf: destination buffer double pointer to be used with hl_snprintf_resize388 * @size: pointer to the size container389 * @offset: pointer to the offset container390 *391 * Returns 0 on success or error code on failure392 */393static int hl_state_dump_print_syncs(struct hl_device *hdev,394 char **buf, size_t *size,395 size_t *offset)396 397{398 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;399 struct hl_sync_to_engine_map *map;400 u32 index;401 int rc = 0;402 403 map = kzalloc(sizeof(*map), GFP_KERNEL);404 if (!map)405 return -ENOMEM;406 407 rc = sds->funcs.gen_sync_to_engine_map(hdev, map);408 if (rc)409 goto free_map_mem;410 411 rc = hl_snprintf_resize(buf, size, offset, "Non zero sync objects:\n");412 if (rc)413 goto out;414 415 if (sds->sync_namager_names) {416 for (index = 0; sds->sync_namager_names[index]; ++index) {417 rc = hl_state_dump_print_syncs_single_block(418 hdev, index, buf, size, offset, map);419 if (rc)420 goto out;421 }422 } else {423 for (index = 0; index < sds->props[SP_NUM_CORES]; ++index) {424 rc = hl_state_dump_print_syncs_single_block(425 hdev, index, buf, size, offset, map);426 if (rc)427 goto out;428 }429 }430 431out:432 hl_state_dump_free_sync_to_engine_map(map);433free_map_mem:434 kfree(map);435 436 return rc;437}438 439/**440 * hl_state_dump_alloc_read_sm_block_monitors - read monitors for a specific441 * block442 * @hdev: pointer to the device443 * @index: sync manager block index starting with E_N444 *445 * Returns an array of monitor data of size SP_MONITORS_AMOUNT or NULL446 * on error447 */448static struct hl_mon_state_dump *449hl_state_dump_alloc_read_sm_block_monitors(struct hl_device *hdev, u32 index)450{451 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;452 struct hl_mon_state_dump *monitors;453 s64 base_addr; /* Base addr can be negative */454 int i;455 456 monitors = vmalloc(sds->props[SP_MONITORS_AMOUNT] *457 sizeof(struct hl_mon_state_dump));458 if (!monitors)459 return NULL;460 461 base_addr = sds->props[SP_NEXT_SYNC_OBJ_ADDR] * index;462 463 for (i = 0; i < sds->props[SP_MONITORS_AMOUNT]; ++i) {464 monitors[i].id = i;465 monitors[i].wr_addr_low =466 RREG32(base_addr + sds->props[SP_MON_OBJ_WR_ADDR_LOW] +467 i * sizeof(u32));468 469 monitors[i].wr_addr_high =470 RREG32(base_addr + sds->props[SP_MON_OBJ_WR_ADDR_HIGH] +471 i * sizeof(u32));472 473 monitors[i].wr_data =474 RREG32(base_addr + sds->props[SP_MON_OBJ_WR_DATA] +475 i * sizeof(u32));476 477 monitors[i].arm_data =478 RREG32(base_addr + sds->props[SP_MON_OBJ_ARM_DATA] +479 i * sizeof(u32));480 481 monitors[i].status =482 RREG32(base_addr + sds->props[SP_MON_OBJ_STATUS] +483 i * sizeof(u32));484 }485 486 return monitors;487}488 489/**490 * hl_state_dump_free_monitors - free the monitors structure491 * @monitors: monitors array created with492 * hl_state_dump_alloc_read_sm_block_monitors493 */494static void hl_state_dump_free_monitors(struct hl_mon_state_dump *monitors)495{496 vfree(monitors);497}498 499/**500 * hl_state_dump_print_monitors_single_block - print active monitors on a501 * single block502 * @hdev: pointer to the device503 * @index: sync manager block index starting with E_N504 * @buf: destination buffer double pointer to be used with hl_snprintf_resize505 * @size: pointer to the size container506 * @offset: pointer to the offset container507 *508 * Returns 0 on success or error code on failure509 */510static int hl_state_dump_print_monitors_single_block(struct hl_device *hdev,511 u32 index,512 char **buf, size_t *size,513 size_t *offset)514{515 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;516 struct hl_mon_state_dump *monitors = NULL;517 int rc = 0, i;518 519 if (sds->sync_namager_names) {520 rc = hl_snprintf_resize(521 buf, size, offset, "%s\n",522 sds->sync_namager_names[index]);523 if (rc)524 goto out;525 }526 527 monitors = hl_state_dump_alloc_read_sm_block_monitors(hdev, index);528 if (!monitors) {529 rc = -ENOMEM;530 goto out;531 }532 533 for (i = 0; i < sds->props[SP_MONITORS_AMOUNT]; ++i) {534 if (!(sds->funcs.monitor_valid(&monitors[i])))535 continue;536 537 /* Monitor is valid, dump it */538 rc = sds->funcs.print_single_monitor(buf, size, offset, hdev,539 &monitors[i]);540 if (rc)541 goto free_monitors;542 543 hl_snprintf_resize(buf, size, offset, "\n");544 }545 546free_monitors:547 hl_state_dump_free_monitors(monitors);548out:549 return rc;550}551 552/**553 * hl_state_dump_print_monitors - print active monitors554 * @hdev: pointer to the device555 * @buf: destination buffer double pointer to be used with hl_snprintf_resize556 * @size: pointer to the size container557 * @offset: pointer to the offset container558 *559 * Returns 0 on success or error code on failure560 */561static int hl_state_dump_print_monitors(struct hl_device *hdev,562 char **buf, size_t *size,563 size_t *offset)564{565 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;566 u32 index;567 int rc = 0;568 569 rc = hl_snprintf_resize(buf, size, offset,570 "Valid (armed) monitor objects:\n");571 if (rc)572 goto out;573 574 if (sds->sync_namager_names) {575 for (index = 0; sds->sync_namager_names[index]; ++index) {576 rc = hl_state_dump_print_monitors_single_block(577 hdev, index, buf, size, offset);578 if (rc)579 goto out;580 }581 } else {582 for (index = 0; index < sds->props[SP_NUM_CORES]; ++index) {583 rc = hl_state_dump_print_monitors_single_block(584 hdev, index, buf, size, offset);585 if (rc)586 goto out;587 }588 }589 590out:591 return rc;592}593 594/**595 * hl_state_dump_print_engine_fences - print active fences for a specific596 * engine597 * @hdev: pointer to the device598 * @engine_type: engine type to use599 * @buf: destination buffer double pointer to be used with hl_snprintf_resize600 * @size: pointer to the size container601 * @offset: pointer to the offset container602 */603static int604hl_state_dump_print_engine_fences(struct hl_device *hdev,605 enum hl_sync_engine_type engine_type,606 char **buf, size_t *size, size_t *offset)607{608 struct hl_state_dump_specs *sds = &hdev->state_dump_specs;609 int rc = 0, i, n_fences;610 u64 base_addr, next_fence;611 612 switch (engine_type) {613 case ENGINE_TPC:614 n_fences = sds->props[SP_NUM_OF_TPC_ENGINES];615 base_addr = sds->props[SP_TPC0_CMDQ];616 next_fence = sds->props[SP_NEXT_TPC];617 break;618 case ENGINE_MME:619 n_fences = sds->props[SP_NUM_OF_MME_ENGINES];620 base_addr = sds->props[SP_MME_CMDQ];621 next_fence = sds->props[SP_NEXT_MME];622 break;623 case ENGINE_DMA:624 n_fences = sds->props[SP_NUM_OF_DMA_ENGINES];625 base_addr = sds->props[SP_DMA_CMDQ];626 next_fence = sds->props[SP_DMA_QUEUES_OFFSET];627 break;628 default:629 return -EINVAL;630 }631 for (i = 0; i < n_fences; ++i) {632 rc = sds->funcs.print_fences_single_engine(633 hdev,634 base_addr + next_fence * i +635 sds->props[SP_FENCE0_CNT_OFFSET],636 base_addr + next_fence * i +637 sds->props[SP_CP_STS_OFFSET],638 engine_type, i, buf, size, offset);639 if (rc)640 goto out;641 }642out:643 return rc;644}645 646/**647 * hl_state_dump_print_fences - print active fences648 * @hdev: pointer to the device649 * @buf: destination buffer double pointer to be used with hl_snprintf_resize650 * @size: pointer to the size container651 * @offset: pointer to the offset container652 */653static int hl_state_dump_print_fences(struct hl_device *hdev, char **buf,654 size_t *size, size_t *offset)655{656 int rc = 0;657 658 rc = hl_snprintf_resize(buf, size, offset, "Valid (armed) fences:\n");659 if (rc)660 goto out;661 662 rc = hl_state_dump_print_engine_fences(hdev, ENGINE_TPC, buf, size, offset);663 if (rc)664 goto out;665 666 rc = hl_state_dump_print_engine_fences(hdev, ENGINE_MME, buf, size, offset);667 if (rc)668 goto out;669 670 rc = hl_state_dump_print_engine_fences(hdev, ENGINE_DMA, buf, size, offset);671 if (rc)672 goto out;673 674out:675 return rc;676}677 678/**679 * hl_state_dump() - dump system state680 * @hdev: pointer to device structure681 */682int hl_state_dump(struct hl_device *hdev)683{684 char *buf = NULL;685 size_t offset = 0, size = 0;686 int rc;687 688 rc = hl_snprintf_resize(&buf, &size, &offset,689 "Timestamp taken on: %llu\n\n",690 ktime_to_ns(ktime_get()));691 if (rc)692 goto err;693 694 rc = hl_state_dump_print_syncs(hdev, &buf, &size, &offset);695 if (rc)696 goto err;697 698 hl_snprintf_resize(&buf, &size, &offset, "\n");699 700 rc = hl_state_dump_print_monitors(hdev, &buf, &size, &offset);701 if (rc)702 goto err;703 704 hl_snprintf_resize(&buf, &size, &offset, "\n");705 706 rc = hl_state_dump_print_fences(hdev, &buf, &size, &offset);707 if (rc)708 goto err;709 710 hl_snprintf_resize(&buf, &size, &offset, "\n");711 712 hl_debugfs_set_state_dump(hdev, buf, size);713 714 return 0;715err:716 vfree(buf);717 return rc;718}719