brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.02 3/*4 * Copyright 2016-2022 HabanaLabs, Ltd.5 * All Rights Reserved.6 */7 8#include "habanalabs.h"9#include <linux/habanalabs/hl_boot_if.h>10 11#include <linux/pci.h>12#include <linux/firmware.h>13#include <linux/crc32.h>14#include <linux/slab.h>15#include <linux/ctype.h>16#include <linux/vmalloc.h>17 18#include <trace/events/habanalabs.h>19 20#define FW_FILE_MAX_SIZE		0x1400000 /* maximum size of 20MB */21 22static char *comms_cmd_str_arr[COMMS_INVLD_LAST] = {23	[COMMS_NOOP] = __stringify(COMMS_NOOP),24	[COMMS_CLR_STS] = __stringify(COMMS_CLR_STS),25	[COMMS_RST_STATE] = __stringify(COMMS_RST_STATE),26	[COMMS_PREP_DESC] = __stringify(COMMS_PREP_DESC),27	[COMMS_DATA_RDY] = __stringify(COMMS_DATA_RDY),28	[COMMS_EXEC] = __stringify(COMMS_EXEC),29	[COMMS_RST_DEV] = __stringify(COMMS_RST_DEV),30	[COMMS_GOTO_WFE] = __stringify(COMMS_GOTO_WFE),31	[COMMS_SKIP_BMC] = __stringify(COMMS_SKIP_BMC),32	[COMMS_PREP_DESC_ELBI] = __stringify(COMMS_PREP_DESC_ELBI),33};34 35static char *comms_sts_str_arr[COMMS_STS_INVLD_LAST] = {36	[COMMS_STS_NOOP] = __stringify(COMMS_STS_NOOP),37	[COMMS_STS_ACK] = __stringify(COMMS_STS_ACK),38	[COMMS_STS_OK] = __stringify(COMMS_STS_OK),39	[COMMS_STS_ERR] = __stringify(COMMS_STS_ERR),40	[COMMS_STS_VALID_ERR] = __stringify(COMMS_STS_VALID_ERR),41	[COMMS_STS_TIMEOUT_ERR] = __stringify(COMMS_STS_TIMEOUT_ERR),42};43 44/**45 * hl_fw_version_cmp() - compares the FW version to a specific version46 *47 * @hdev: pointer to hl_device structure48 * @major: major number of a reference version49 * @minor: minor number of a reference version50 * @subminor: sub-minor number of a reference version51 *52 * Return 1 if FW version greater than the reference version, -1 if it's53 *         smaller and 0 if versions are identical.54 */55int hl_fw_version_cmp(struct hl_device *hdev, u32 major, u32 minor, u32 subminor)56{57	if (hdev->fw_sw_major_ver != major)58		return (hdev->fw_sw_major_ver > major) ? 1 : -1;59 60	if (hdev->fw_sw_minor_ver != minor)61		return (hdev->fw_sw_minor_ver > minor) ? 1 : -1;62 63	if (hdev->fw_sw_sub_minor_ver != subminor)64		return (hdev->fw_sw_sub_minor_ver > subminor) ? 1 : -1;65 66	return 0;67}68 69static char *extract_fw_ver_from_str(const char *fw_str)70{71	char *str, *fw_ver, *whitespace;72	u32 ver_offset;73 74	fw_ver = kmalloc(VERSION_MAX_LEN, GFP_KERNEL);75	if (!fw_ver)76		return NULL;77 78	str = strnstr(fw_str, "fw-", VERSION_MAX_LEN);79	if (!str)80		goto free_fw_ver;81 82	/* Skip the fw- part */83	str += 3;84	ver_offset = str - fw_str;85 86	/* Copy until the next whitespace */87	whitespace = strnstr(str, " ", VERSION_MAX_LEN - ver_offset);88	if (!whitespace)89		goto free_fw_ver;90 91	strscpy(fw_ver, str, whitespace - str + 1);92 93	return fw_ver;94 95free_fw_ver:96	kfree(fw_ver);97	return NULL;98}99 100/**101 * extract_u32_until_given_char() - given a string of the format "<u32><char>*", extract the u32.102 * @str: the given string103 * @ver_num: the pointer to the extracted u32 to be returned to the caller.104 * @given_char: the given char at the end of the u32 in the string105 *106 * Return: Upon success, return a pointer to the given_char in the string. Upon failure, return NULL107 */108static char *extract_u32_until_given_char(char *str, u32 *ver_num, char given_char)109{110	char num_str[8] = {}, *ch;111 112	ch = strchrnul(str, given_char);113	if (*ch == '\0' || ch == str || ch - str >= sizeof(num_str))114		return NULL;115 116	memcpy(num_str, str, ch - str);117	if (kstrtou32(num_str, 10, ver_num))118		return NULL;119	return ch;120}121 122/**123 * hl_get_sw_major_minor_subminor() - extract the FW's SW version major, minor, sub-minor124 *				      from the version string125 * @hdev: pointer to the hl_device126 * @fw_str: the FW's version string127 *128 * The extracted version is set in the hdev fields: fw_sw_{major/minor/sub_minor}_ver.129 *130 * fw_str is expected to have one of two possible formats, examples:131 * 1) 'Preboot version hl-gaudi2-1.9.0-fw-42.0.1-sec-3'132 * 2) 'Preboot version hl-gaudi2-1.9.0-rc-fw-42.0.1-sec-3'133 * In those examples, the SW major,minor,subminor are correspondingly: 1,9,0.134 *135 * Return: 0 for success or a negative error code for failure.136 */137static int hl_get_sw_major_minor_subminor(struct hl_device *hdev, const char *fw_str)138{139	char *end, *start;140 141	end = strnstr(fw_str, "-rc-", VERSION_MAX_LEN);142	if (end == fw_str)143		return -EINVAL;144 145	if (!end)146		end = strnstr(fw_str, "-fw-", VERSION_MAX_LEN);147 148	if (end == fw_str)149		return -EINVAL;150 151	if (!end)152		return -EINVAL;153 154	for (start = end - 1; start != fw_str; start--) {155		if (*start == '-')156			break;157	}158 159	if (start == fw_str)160		return -EINVAL;161 162	/* start/end point each to the starting and ending hyphen of the sw version e.g. -1.9.0- */163	start++;164	start = extract_u32_until_given_char(start, &hdev->fw_sw_major_ver, '.');165	if (!start)166		goto err_zero_ver;167 168	start++;169	start = extract_u32_until_given_char(start, &hdev->fw_sw_minor_ver, '.');170	if (!start)171		goto err_zero_ver;172 173	start++;174	start = extract_u32_until_given_char(start, &hdev->fw_sw_sub_minor_ver, '-');175	if (!start)176		goto err_zero_ver;177 178	return 0;179 180err_zero_ver:181	hdev->fw_sw_major_ver = 0;182	hdev->fw_sw_minor_ver = 0;183	hdev->fw_sw_sub_minor_ver = 0;184	return -EINVAL;185}186 187/**188 * hl_get_preboot_major_minor() - extract the FW's version major, minor from the version string.189 * @hdev: pointer to the hl_device190 * @preboot_ver: the FW's version string191 *192 * preboot_ver is expected to be the format of <major>.<minor>.<sub minor>*, e.g: 42.0.1-sec-3193 * The extracted version is set in the hdev fields: fw_inner_{major/minor}_ver.194 *195 * Return: 0 on success, negative error code for failure.196 */197static int hl_get_preboot_major_minor(struct hl_device *hdev, char *preboot_ver)198{199	preboot_ver = extract_u32_until_given_char(preboot_ver, &hdev->fw_inner_major_ver, '.');200	if (!preboot_ver) {201		dev_err(hdev->dev, "Error parsing preboot major version\n");202		goto err_zero_ver;203	}204 205	preboot_ver++;206 207	preboot_ver = extract_u32_until_given_char(preboot_ver, &hdev->fw_inner_minor_ver, '.');208	if (!preboot_ver) {209		dev_err(hdev->dev, "Error parsing preboot minor version\n");210		goto err_zero_ver;211	}212	return 0;213 214err_zero_ver:215	hdev->fw_inner_major_ver = 0;216	hdev->fw_inner_minor_ver = 0;217	return -EINVAL;218}219 220static int hl_request_fw(struct hl_device *hdev,221				const struct firmware **firmware_p,222				const char *fw_name)223{224	size_t fw_size;225	int rc;226 227	rc = request_firmware(firmware_p, fw_name, hdev->dev);228	if (rc) {229		dev_err(hdev->dev, "Firmware file %s is not found! (error %d)\n",230				fw_name, rc);231		goto out;232	}233 234	fw_size = (*firmware_p)->size;235	if ((fw_size % 4) != 0) {236		dev_err(hdev->dev, "Illegal %s firmware size %zu\n",237				fw_name, fw_size);238		rc = -EINVAL;239		goto release_fw;240	}241 242	dev_dbg(hdev->dev, "%s firmware size == %zu\n", fw_name, fw_size);243 244	if (fw_size > FW_FILE_MAX_SIZE) {245		dev_err(hdev->dev,246			"FW file size %zu exceeds maximum of %u bytes\n",247			fw_size, FW_FILE_MAX_SIZE);248		rc = -EINVAL;249		goto release_fw;250	}251 252	return 0;253 254release_fw:255	release_firmware(*firmware_p);256out:257	return rc;258}259 260/**261 * hl_release_firmware() - release FW262 *263 * @fw: fw descriptor264 *265 * note: this inline function added to serve as a comprehensive mirror for the266 *       hl_request_fw function.267 */268static inline void hl_release_firmware(const struct firmware *fw)269{270	release_firmware(fw);271}272 273/**274 * hl_fw_copy_fw_to_device() - copy FW to device275 *276 * @hdev: pointer to hl_device structure.277 * @fw: fw descriptor278 * @dst: IO memory mapped address space to copy firmware to279 * @src_offset: offset in src FW to copy from280 * @size: amount of bytes to copy (0 to copy the whole binary)281 *282 * actual copy of FW binary data to device, shared by static and dynamic loaders283 */284static int hl_fw_copy_fw_to_device(struct hl_device *hdev,285				const struct firmware *fw, void __iomem *dst,286				u32 src_offset, u32 size)287{288	const void *fw_data;289 290	/* size 0 indicates to copy the whole file */291	if (!size)292		size = fw->size;293 294	if (src_offset + size > fw->size) {295		dev_err(hdev->dev,296			"size to copy(%u) and offset(%u) are invalid\n",297			size, src_offset);298		return -EINVAL;299	}300 301	fw_data = (const void *) fw->data;302 303	memcpy_toio(dst, fw_data + src_offset, size);304	return 0;305}306 307/**308 * hl_fw_copy_msg_to_device() - copy message to device309 *310 * @hdev: pointer to hl_device structure.311 * @msg: message312 * @dst: IO memory mapped address space to copy firmware to313 * @src_offset: offset in src message to copy from314 * @size: amount of bytes to copy (0 to copy the whole binary)315 *316 * actual copy of message data to device.317 */318static int hl_fw_copy_msg_to_device(struct hl_device *hdev,319		struct lkd_msg_comms *msg, void __iomem *dst,320		u32 src_offset, u32 size)321{322	void *msg_data;323 324	/* size 0 indicates to copy the whole file */325	if (!size)326		size = sizeof(struct lkd_msg_comms);327 328	if (src_offset + size > sizeof(struct lkd_msg_comms)) {329		dev_err(hdev->dev,330			"size to copy(%u) and offset(%u) are invalid\n",331			size, src_offset);332		return -EINVAL;333	}334 335	msg_data = (void *) msg;336 337	memcpy_toio(dst, msg_data + src_offset, size);338 339	return 0;340}341 342/**343 * hl_fw_load_fw_to_device() - Load F/W code to device's memory.344 *345 * @hdev: pointer to hl_device structure.346 * @fw_name: the firmware image name347 * @dst: IO memory mapped address space to copy firmware to348 * @src_offset: offset in src FW to copy from349 * @size: amount of bytes to copy (0 to copy the whole binary)350 *351 * Copy fw code from firmware file to device memory.352 *353 * Return: 0 on success, non-zero for failure.354 */355int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,356				void __iomem *dst, u32 src_offset, u32 size)357{358	const struct firmware *fw;359	int rc;360 361	rc = hl_request_fw(hdev, &fw, fw_name);362	if (rc)363		return rc;364 365	rc = hl_fw_copy_fw_to_device(hdev, fw, dst, src_offset, size);366 367	hl_release_firmware(fw);368	return rc;369}370 371int hl_fw_send_pci_access_msg(struct hl_device *hdev, u32 opcode, u64 value)372{373	struct cpucp_packet pkt = {};374	int rc;375 376	pkt.ctl = cpu_to_le32(opcode << CPUCP_PKT_CTL_OPCODE_SHIFT);377	pkt.value = cpu_to_le64(value);378 379	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);380	if (rc)381		dev_err(hdev->dev, "Failed to disable FW's PCI access\n");382 383	return rc;384}385 386/**387 * hl_fw_send_cpu_message() - send CPU message to the device.388 *389 * @hdev: pointer to hl_device structure.390 * @hw_queue_id: HW queue ID391 * @msg: raw data of the message/packet392 * @size: size of @msg in bytes393 * @timeout_us: timeout in usec to wait for CPU reply on the message394 * @result: return code reported by FW395 *396 * send message to the device CPU.397 *398 * Return: 0 on success, non-zero for failure.399 *     -ENOMEM: memory allocation failure400 *     -EAGAIN: CPU is disabled (try again when enabled)401 *     -ETIMEDOUT: timeout waiting for FW response402 *     -EIO: protocol error403 */404int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,405				u16 size, u32 timeout_us, u64 *result)406{407	struct hl_hw_queue *queue = &hdev->kernel_queues[hw_queue_id];408	struct asic_fixed_properties *prop = &hdev->asic_prop;409	u32 tmp, expected_ack_val, pi, opcode;410	struct cpucp_packet *pkt;411	dma_addr_t pkt_dma_addr;412	struct hl_bd *sent_bd;413	int rc = 0, fw_rc;414 415	pkt = hl_cpu_accessible_dma_pool_alloc(hdev, size, &pkt_dma_addr);416	if (!pkt) {417		dev_err(hdev->dev, "Failed to allocate DMA memory for packet to CPU\n");418		return -ENOMEM;419	}420 421	memcpy(pkt, msg, size);422 423	mutex_lock(&hdev->send_cpu_message_lock);424 425	/* CPU-CP messages can be sent during soft-reset */426	if (hdev->disabled && !hdev->reset_info.in_compute_reset)427		goto out;428 429	if (hdev->device_cpu_disabled) {430		rc = -EAGAIN;431		goto out;432	}433 434	/* set fence to a non valid value */435	pkt->fence = cpu_to_le32(UINT_MAX);436	pi = queue->pi;437 438	/*439	 * The CPU queue is a synchronous queue with an effective depth of440	 * a single entry (although it is allocated with room for multiple441	 * entries). We lock on it using 'send_cpu_message_lock' which442	 * serializes accesses to the CPU queue.443	 * Which means that we don't need to lock the access to the entire H/W444	 * queues module when submitting a JOB to the CPU queue.445	 */446	hl_hw_queue_submit_bd(hdev, queue, hl_queue_inc_ptr(queue->pi), size, pkt_dma_addr);447 448	if (prop->fw_app_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN)449		expected_ack_val = queue->pi;450	else451		expected_ack_val = CPUCP_PACKET_FENCE_VAL;452 453	rc = hl_poll_timeout_memory(hdev, &pkt->fence, tmp,454				(tmp == expected_ack_val), 1000,455				timeout_us, true);456 457	hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);458 459	if (rc == -ETIMEDOUT) {460		/* If FW performed reset just before sending it a packet, we will get a timeout.461		 * This is expected behavior, hence no need for error message.462		 */463		if (!hl_device_operational(hdev, NULL) && !hdev->reset_info.in_compute_reset) {464			dev_dbg(hdev->dev, "Device CPU packet timeout (0x%x) due to FW reset\n",465					tmp);466		} else {467			struct hl_bd *bd = queue->kernel_address;468 469			bd += hl_pi_2_offset(pi);470 471			dev_err(hdev->dev, "Device CPU packet timeout (status = 0x%x)\n"472				"Pkt info[%u]: dma_addr: 0x%llx, kernel_addr: %p, len:0x%x, ctl: 0x%x, ptr:0x%llx, dram_bd:%u\n",473				tmp, pi, pkt_dma_addr, (void *)pkt, bd->len, bd->ctl, bd->ptr,474				queue->dram_bd);475		}476		hdev->device_cpu_disabled = true;477		goto out;478	}479 480	tmp = le32_to_cpu(pkt->ctl);481 482	fw_rc = (tmp & CPUCP_PKT_CTL_RC_MASK) >> CPUCP_PKT_CTL_RC_SHIFT;483	if (fw_rc) {484		opcode = (tmp & CPUCP_PKT_CTL_OPCODE_MASK) >> CPUCP_PKT_CTL_OPCODE_SHIFT;485 486		if (!prop->supports_advanced_cpucp_rc) {487			dev_dbg(hdev->dev, "F/W ERROR %d for CPU packet %d\n", rc, opcode);488			rc = -EIO;489			goto scrub_descriptor;490		}491 492		switch (fw_rc) {493		case cpucp_packet_invalid:494			dev_err(hdev->dev,495				"CPU packet %d is not supported by F/W\n", opcode);496			break;497		case cpucp_packet_fault:498			dev_err(hdev->dev,499				"F/W failed processing CPU packet %d\n", opcode);500			break;501		case cpucp_packet_invalid_pkt:502			dev_dbg(hdev->dev,503				"CPU packet %d is not supported by F/W\n", opcode);504			break;505		case cpucp_packet_invalid_params:506			dev_err(hdev->dev,507				"F/W reports invalid parameters for CPU packet %d\n", opcode);508			break;509 510		default:511			dev_err(hdev->dev,512				"Unknown F/W ERROR %d for CPU packet %d\n", rc, opcode);513		}514 515		/* propagate the return code from the f/w to the callers who want to check it */516		if (result)517			*result = fw_rc;518 519		rc = -EIO;520 521	} else if (result) {522		*result = le64_to_cpu(pkt->result);523	}524 525scrub_descriptor:526	/* Scrub previous buffer descriptor 'ctl' field which contains the527	 * previous PI value written during packet submission.528	 * We must do this or else F/W can read an old value upon queue wraparound.529	 */530	sent_bd = queue->kernel_address;531	sent_bd += hl_pi_2_offset(pi);532	sent_bd->ctl = cpu_to_le32(UINT_MAX);533 534out:535	mutex_unlock(&hdev->send_cpu_message_lock);536 537	hl_cpu_accessible_dma_pool_free(hdev, size, pkt);538 539	return rc;540}541 542int hl_fw_unmask_irq(struct hl_device *hdev, u16 event_type)543{544	struct cpucp_packet pkt;545	u64 result;546	int rc;547 548	memset(&pkt, 0, sizeof(pkt));549 550	pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ <<551				CPUCP_PKT_CTL_OPCODE_SHIFT);552	pkt.value = cpu_to_le64(event_type);553 554	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),555						0, &result);556 557	if (rc)558		dev_err(hdev->dev, "failed to unmask event %d", event_type);559 560	return rc;561}562 563int hl_fw_unmask_irq_arr(struct hl_device *hdev, const u32 *irq_arr,564		size_t irq_arr_size)565{566	struct cpucp_unmask_irq_arr_packet *pkt;567	size_t total_pkt_size;568	u64 result;569	int rc;570 571	total_pkt_size = sizeof(struct cpucp_unmask_irq_arr_packet) +572			irq_arr_size;573 574	/* data should be aligned to 8 bytes in order to CPU-CP to copy it */575	total_pkt_size = (total_pkt_size + 0x7) & ~0x7;576 577	/* total_pkt_size is casted to u16 later on */578	if (total_pkt_size > USHRT_MAX) {579		dev_err(hdev->dev, "too many elements in IRQ array\n");580		return -EINVAL;581	}582 583	pkt = kzalloc(total_pkt_size, GFP_KERNEL);584	if (!pkt)585		return -ENOMEM;586 587	pkt->length = cpu_to_le32(irq_arr_size / sizeof(irq_arr[0]));588	memcpy(&pkt->irqs, irq_arr, irq_arr_size);589 590	pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_UNMASK_RAZWI_IRQ_ARRAY <<591						CPUCP_PKT_CTL_OPCODE_SHIFT);592 593	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) pkt,594						total_pkt_size, 0, &result);595 596	if (rc)597		dev_err(hdev->dev, "failed to unmask event array\n");598 599	kfree(pkt);600 601	return rc;602}603 604int hl_fw_test_cpu_queue(struct hl_device *hdev)605{606	struct cpucp_packet test_pkt = {};607	u64 result = 0;608	int rc;609 610	test_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<611					CPUCP_PKT_CTL_OPCODE_SHIFT);612	test_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);613 614	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &test_pkt,615						sizeof(test_pkt), 0, &result);616 617	if (!rc) {618		if (result != CPUCP_PACKET_FENCE_VAL)619			dev_err(hdev->dev,620				"CPU queue test failed (%#08llx)\n", result);621	} else {622		dev_err(hdev->dev, "CPU queue test failed, error %d\n", rc);623	}624 625	return rc;626}627 628void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size,629						dma_addr_t *dma_handle)630{631	u64 kernel_addr;632 633	kernel_addr = gen_pool_alloc(hdev->cpu_accessible_dma_pool, size);634 635	*dma_handle = hdev->cpu_accessible_dma_address +636		(kernel_addr - (u64) (uintptr_t) hdev->cpu_accessible_dma_mem);637 638	return (void *) (uintptr_t) kernel_addr;639}640 641void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,642					void *vaddr)643{644	gen_pool_free(hdev->cpu_accessible_dma_pool, (u64) (uintptr_t) vaddr,645			size);646}647 648int hl_fw_send_soft_reset(struct hl_device *hdev)649{650	struct cpucp_packet pkt;651	int rc;652 653	memset(&pkt, 0, sizeof(pkt));654	pkt.ctl = cpu_to_le32(CPUCP_PACKET_SOFT_RESET << CPUCP_PKT_CTL_OPCODE_SHIFT);655	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);656	if (rc)657		dev_err(hdev->dev, "failed to send soft-reset msg (err = %d)\n", rc);658 659	return rc;660}661 662int hl_fw_send_device_activity(struct hl_device *hdev, bool open)663{664	struct cpucp_packet pkt;665	int rc;666 667	memset(&pkt, 0, sizeof(pkt));668	pkt.ctl = cpu_to_le32(CPUCP_PACKET_ACTIVE_STATUS_SET <<	CPUCP_PKT_CTL_OPCODE_SHIFT);669	pkt.value = cpu_to_le64(open);670	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);671	if (rc)672		dev_err(hdev->dev, "failed to send device activity msg(%u)\n", open);673 674	return rc;675}676 677int hl_fw_send_heartbeat(struct hl_device *hdev)678{679	struct cpucp_packet hb_pkt;680	u64 result = 0;681	int rc;682 683	memset(&hb_pkt, 0, sizeof(hb_pkt));684	hb_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST << CPUCP_PKT_CTL_OPCODE_SHIFT);685	hb_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);686 687	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &hb_pkt, sizeof(hb_pkt), 0, &result);688 689	if ((rc) || (result != CPUCP_PACKET_FENCE_VAL))690		return -EIO;691 692	if (le32_to_cpu(hb_pkt.status_mask) &693					CPUCP_PKT_HB_STATUS_EQ_FAULT_MASK) {694		dev_warn(hdev->dev, "FW reported EQ fault during heartbeat\n");695		rc = -EIO;696	}697 698	hdev->heartbeat_debug_info.last_pq_heartbeat_ts = ktime_get_real_seconds();699 700	return rc;701}702 703static bool fw_report_boot_dev0(struct hl_device *hdev, u32 err_val, u32 sts_val)704{705	bool err_exists = false;706 707	if (!(err_val & CPU_BOOT_ERR0_ENABLED))708		return false;709 710	if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL)711		dev_err(hdev->dev, "Device boot error - DRAM initialization failed\n");712 713	if (err_val & CPU_BOOT_ERR0_FIT_CORRUPTED)714		dev_err(hdev->dev, "Device boot error - FIT image corrupted\n");715 716	if (err_val & CPU_BOOT_ERR0_TS_INIT_FAIL)717		dev_err(hdev->dev, "Device boot error - Thermal Sensor initialization failed\n");718 719	if (err_val & CPU_BOOT_ERR0_BMC_WAIT_SKIPPED) {720		if (hdev->bmc_enable) {721			dev_err(hdev->dev, "Device boot error - Skipped waiting for BMC\n");722		} else {723			dev_info(hdev->dev, "Device boot message - Skipped waiting for BMC\n");724			/* This is an info so we don't want it to disable the725			 * device726			 */727			err_val &= ~CPU_BOOT_ERR0_BMC_WAIT_SKIPPED;728		}729	}730 731	if (err_val & CPU_BOOT_ERR0_NIC_DATA_NOT_RDY)732		dev_err(hdev->dev, "Device boot error - Serdes data from BMC not available\n");733 734	if (err_val & CPU_BOOT_ERR0_NIC_FW_FAIL)735		dev_err(hdev->dev, "Device boot error - NIC F/W initialization failed\n");736 737	if (err_val & CPU_BOOT_ERR0_SECURITY_NOT_RDY)738		dev_err(hdev->dev, "Device boot warning - security not ready\n");739 740	if (err_val & CPU_BOOT_ERR0_SECURITY_FAIL)741		dev_err(hdev->dev, "Device boot error - security failure\n");742 743	if (err_val & CPU_BOOT_ERR0_EFUSE_FAIL)744		dev_err(hdev->dev, "Device boot error - eFuse failure\n");745 746	if (err_val & CPU_BOOT_ERR0_SEC_IMG_VER_FAIL)747		dev_err(hdev->dev, "Device boot error - Failed to load preboot secondary image\n");748 749	if (err_val & CPU_BOOT_ERR0_PLL_FAIL)750		dev_err(hdev->dev, "Device boot error - PLL failure\n");751 752	if (err_val & CPU_BOOT_ERR0_TMP_THRESH_INIT_FAIL)753		dev_err(hdev->dev, "Device boot error - Failed to set threshold for temperature sensor\n");754 755	if (err_val & CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL) {756		/* Ignore this bit, don't prevent driver loading */757		dev_dbg(hdev->dev, "device unusable status is set\n");758		err_val &= ~CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL;759	}760 761	if (err_val & CPU_BOOT_ERR0_BINNING_FAIL)762		dev_err(hdev->dev, "Device boot error - binning failure\n");763 764	if (sts_val & CPU_BOOT_DEV_STS0_ENABLED)765		dev_dbg(hdev->dev, "Device status0 %#x\n", sts_val);766 767	if (err_val & CPU_BOOT_ERR0_DRAM_SKIPPED)768		dev_err(hdev->dev, "Device boot warning - Skipped DRAM initialization\n");769 770	if (err_val & CPU_BOOT_ERR_ENG_ARC_MEM_SCRUB_FAIL)771		dev_err(hdev->dev, "Device boot error - ARC memory scrub failed\n");772 773	/* All warnings should go here in order not to reach the unknown error validation */774	if (err_val & CPU_BOOT_ERR0_EEPROM_FAIL) {775		dev_err(hdev->dev, "Device boot error - EEPROM failure detected\n");776		err_exists = true;777	}778 779	if (err_val & CPU_BOOT_ERR0_PRI_IMG_VER_FAIL)780		dev_warn(hdev->dev, "Device boot warning - Failed to load preboot primary image\n");781 782	if (err_val & CPU_BOOT_ERR0_TPM_FAIL)783		dev_warn(hdev->dev, "Device boot warning - TPM failure\n");784 785	if (err_val & CPU_BOOT_ERR_FATAL_MASK)786		err_exists = true;787 788	/* return error only if it's in the predefined mask */789	if (err_exists && ((err_val & ~CPU_BOOT_ERR0_ENABLED) &790				lower_32_bits(hdev->boot_error_status_mask)))791		return true;792 793	return false;794}795 796/* placeholder for ERR1 as no errors defined there yet */797static bool fw_report_boot_dev1(struct hl_device *hdev, u32 err_val,798								u32 sts_val)799{800	/*801	 * keep this variable to preserve the logic of the function.802	 * this way it would require less modifications when error will be803	 * added to DEV_ERR1804	 */805	bool err_exists = false;806 807	if (!(err_val & CPU_BOOT_ERR1_ENABLED))808		return false;809 810	if (sts_val & CPU_BOOT_DEV_STS1_ENABLED)811		dev_dbg(hdev->dev, "Device status1 %#x\n", sts_val);812 813	if (!err_exists && (err_val & ~CPU_BOOT_ERR1_ENABLED)) {814		dev_err(hdev->dev,815			"Device boot error - unknown ERR1 error 0x%08x\n",816								err_val);817		err_exists = true;818	}819 820	/* return error only if it's in the predefined mask */821	if (err_exists && ((err_val & ~CPU_BOOT_ERR1_ENABLED) &822				upper_32_bits(hdev->boot_error_status_mask)))823		return true;824 825	return false;826}827 828static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg,829				u32 boot_err1_reg, u32 cpu_boot_dev_status0_reg,830				u32 cpu_boot_dev_status1_reg)831{832	u32 err_val, status_val;833	bool err_exists = false;834 835	/* Some of the firmware status codes are deprecated in newer f/w836	 * versions. In those versions, the errors are reported837	 * in different registers. Therefore, we need to check those838	 * registers and print the exact errors. Moreover, there839	 * may be multiple errors, so we need to report on each error840	 * separately. Some of the error codes might indicate a state841	 * that is not an error per-se, but it is an error in production842	 * environment843	 */844	err_val = RREG32(boot_err0_reg);845	status_val = RREG32(cpu_boot_dev_status0_reg);846	err_exists = fw_report_boot_dev0(hdev, err_val, status_val);847 848	err_val = RREG32(boot_err1_reg);849	status_val = RREG32(cpu_boot_dev_status1_reg);850	err_exists |= fw_report_boot_dev1(hdev, err_val, status_val);851 852	if (err_exists)853		return -EIO;854 855	return 0;856}857 858int hl_fw_cpucp_info_get(struct hl_device *hdev,859				u32 sts_boot_dev_sts0_reg,860				u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,861				u32 boot_err1_reg)862{863	struct asic_fixed_properties *prop = &hdev->asic_prop;864	struct cpucp_packet pkt = {};865	dma_addr_t cpucp_info_dma_addr;866	void *cpucp_info_cpu_addr;867	char *kernel_ver;868	u64 result;869	int rc;870 871	cpucp_info_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, sizeof(struct cpucp_info),872								&cpucp_info_dma_addr);873	if (!cpucp_info_cpu_addr) {874		dev_err(hdev->dev,875			"Failed to allocate DMA memory for CPU-CP info packet\n");876		return -ENOMEM;877	}878 879	memset(cpucp_info_cpu_addr, 0, sizeof(struct cpucp_info));880 881	pkt.ctl = cpu_to_le32(CPUCP_PACKET_INFO_GET <<882				CPUCP_PKT_CTL_OPCODE_SHIFT);883	pkt.addr = cpu_to_le64(cpucp_info_dma_addr);884	pkt.data_max_size = cpu_to_le32(sizeof(struct cpucp_info));885 886	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),887					HL_CPUCP_INFO_TIMEOUT_USEC, &result);888	if (rc) {889		dev_err(hdev->dev,890			"Failed to handle CPU-CP info pkt, error %d\n", rc);891		goto out;892	}893 894	rc = fw_read_errors(hdev, boot_err0_reg, boot_err1_reg,895				sts_boot_dev_sts0_reg, sts_boot_dev_sts1_reg);896	if (rc) {897		dev_err(hdev->dev, "Errors in device boot\n");898		goto out;899	}900 901	memcpy(&prop->cpucp_info, cpucp_info_cpu_addr,902			sizeof(prop->cpucp_info));903 904	rc = hl_build_hwmon_channel_info(hdev, prop->cpucp_info.sensors);905	if (rc) {906		dev_err(hdev->dev,907			"Failed to build hwmon channel info, error %d\n", rc);908		rc = -EFAULT;909		goto out;910	}911 912	kernel_ver = extract_fw_ver_from_str(prop->cpucp_info.kernel_version);913	if (kernel_ver) {914		dev_info(hdev->dev, "Linux version %s", kernel_ver);915		kfree(kernel_ver);916	}917 918	/* assume EQ code doesn't need to check eqe index */919	hdev->event_queue.check_eqe_index = false;920 921	/* Read FW application security bits again */922	if (prop->fw_cpu_boot_dev_sts0_valid) {923		prop->fw_app_cpu_boot_dev_sts0 = RREG32(sts_boot_dev_sts0_reg);924		if (prop->fw_app_cpu_boot_dev_sts0 &925				CPU_BOOT_DEV_STS0_EQ_INDEX_EN)926			hdev->event_queue.check_eqe_index = true;927	}928 929	if (prop->fw_cpu_boot_dev_sts1_valid)930		prop->fw_app_cpu_boot_dev_sts1 = RREG32(sts_boot_dev_sts1_reg);931 932out:933	hl_cpu_accessible_dma_pool_free(hdev, sizeof(struct cpucp_info), cpucp_info_cpu_addr);934 935	return rc;936}937 938static int hl_fw_send_msi_info_msg(struct hl_device *hdev)939{940	struct cpucp_array_data_packet *pkt;941	size_t total_pkt_size, data_size;942	u64 result = 0;943	int rc;944 945	/* skip sending this info for unsupported ASICs */946	if (!hdev->asic_funcs->get_msi_info)947		return 0;948 949	data_size = CPUCP_NUM_OF_MSI_TYPES * sizeof(u32);950	total_pkt_size = sizeof(struct cpucp_array_data_packet) + data_size;951 952	/* data should be aligned to 8 bytes in order to CPU-CP to copy it */953	total_pkt_size = (total_pkt_size + 0x7) & ~0x7;954 955	/* total_pkt_size is casted to u16 later on */956	if (total_pkt_size > USHRT_MAX) {957		dev_err(hdev->dev, "CPUCP array data is too big\n");958		return -EINVAL;959	}960 961	pkt = kzalloc(total_pkt_size, GFP_KERNEL);962	if (!pkt)963		return -ENOMEM;964 965	pkt->length = cpu_to_le32(CPUCP_NUM_OF_MSI_TYPES);966 967	memset((void *) &pkt->data, 0xFF, data_size);968	hdev->asic_funcs->get_msi_info(pkt->data);969 970	pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_MSI_INFO_SET <<971						CPUCP_PKT_CTL_OPCODE_SHIFT);972 973	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *)pkt,974						total_pkt_size, 0, &result);975 976	/*977	 * in case packet result is invalid it means that FW does not support978	 * this feature and will use default/hard coded MSI values. no reason979	 * to stop the boot980	 */981	if (rc && result == cpucp_packet_invalid)982		rc = 0;983 984	if (rc)985		dev_err(hdev->dev, "failed to send CPUCP array data\n");986 987	kfree(pkt);988 989	return rc;990}991 992int hl_fw_cpucp_handshake(struct hl_device *hdev,993				u32 sts_boot_dev_sts0_reg,994				u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,995				u32 boot_err1_reg)996{997	int rc;998 999	rc = hl_fw_cpucp_info_get(hdev, sts_boot_dev_sts0_reg,1000					sts_boot_dev_sts1_reg, boot_err0_reg,1001					boot_err1_reg);1002	if (rc)1003		return rc;1004 1005	return hl_fw_send_msi_info_msg(hdev);1006}1007 1008int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size)1009{1010	struct cpucp_packet pkt = {};1011	void *eeprom_info_cpu_addr;1012	dma_addr_t eeprom_info_dma_addr;1013	u64 result;1014	int rc;1015 1016	eeprom_info_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, max_size,1017									&eeprom_info_dma_addr);1018	if (!eeprom_info_cpu_addr) {1019		dev_err(hdev->dev,1020			"Failed to allocate DMA memory for CPU-CP EEPROM packet\n");1021		return -ENOMEM;1022	}1023 1024	memset(eeprom_info_cpu_addr, 0, max_size);1025 1026	pkt.ctl = cpu_to_le32(CPUCP_PACKET_EEPROM_DATA_GET <<1027				CPUCP_PKT_CTL_OPCODE_SHIFT);1028	pkt.addr = cpu_to_le64(eeprom_info_dma_addr);1029	pkt.data_max_size = cpu_to_le32(max_size);1030 1031	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),1032			HL_CPUCP_EEPROM_TIMEOUT_USEC, &result);1033	if (rc) {1034		if (rc != -EAGAIN)1035			dev_err(hdev->dev,1036				"Failed to handle CPU-CP EEPROM packet, error %d\n", rc);1037		goto out;1038	}1039 1040	/* result contains the actual size */1041	memcpy(data, eeprom_info_cpu_addr, min((size_t)result, max_size));1042 1043out:1044	hl_cpu_accessible_dma_pool_free(hdev, max_size, eeprom_info_cpu_addr);1045 1046	return rc;1047}1048 1049int hl_fw_get_monitor_dump(struct hl_device *hdev, void *data)1050{1051	struct cpucp_monitor_dump *mon_dump_cpu_addr;1052	dma_addr_t mon_dump_dma_addr;1053	struct cpucp_packet pkt = {};1054	size_t data_size;1055	__le32 *src_ptr;1056	u32 *dst_ptr;1057	u64 result;1058	int i, rc;1059 1060	data_size = sizeof(struct cpucp_monitor_dump);1061	mon_dump_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, data_size, &mon_dump_dma_addr);1062	if (!mon_dump_cpu_addr) {1063		dev_err(hdev->dev,1064			"Failed to allocate DMA memory for CPU-CP monitor-dump packet\n");1065		return -ENOMEM;1066	}1067 1068	memset(mon_dump_cpu_addr, 0, data_size);1069 1070	pkt.ctl = cpu_to_le32(CPUCP_PACKET_MONITOR_DUMP_GET << CPUCP_PKT_CTL_OPCODE_SHIFT);1071	pkt.addr = cpu_to_le64(mon_dump_dma_addr);1072	pkt.data_max_size = cpu_to_le32(data_size);1073 1074	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),1075							HL_CPUCP_MON_DUMP_TIMEOUT_USEC, &result);1076	if (rc) {1077		if (rc != -EAGAIN)1078			dev_err(hdev->dev,1079				"Failed to handle CPU-CP monitor-dump packet, error %d\n", rc);1080		goto out;1081	}1082 1083	/* result contains the actual size */1084	src_ptr = (__le32 *) mon_dump_cpu_addr;1085	dst_ptr = data;1086	for (i = 0; i < (data_size / sizeof(u32)); i++) {1087		*dst_ptr = le32_to_cpu(*src_ptr);1088		src_ptr++;1089		dst_ptr++;1090	}1091 1092out:1093	hl_cpu_accessible_dma_pool_free(hdev, data_size, mon_dump_cpu_addr);1094 1095	return rc;1096}1097 1098int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev,1099		struct hl_info_pci_counters *counters)1100{1101	struct cpucp_packet pkt = {};1102	u64 result;1103	int rc;1104 1105	pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<1106			CPUCP_PKT_CTL_OPCODE_SHIFT);1107 1108	/* Fetch PCI rx counter */1109	pkt.index = cpu_to_le32(cpucp_pcie_throughput_rx);1110	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),1111					HL_CPUCP_INFO_TIMEOUT_USEC, &result);1112	if (rc) {1113		if (rc != -EAGAIN)1114			dev_err(hdev->dev,1115				"Failed to handle CPU-CP PCI info pkt, error %d\n", rc);1116		return rc;1117	}1118	counters->rx_throughput = result;1119 1120	memset(&pkt, 0, sizeof(pkt));1121	pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_THROUGHPUT_GET <<1122			CPUCP_PKT_CTL_OPCODE_SHIFT);1123 1124	/* Fetch PCI tx counter */1125	pkt.index = cpu_to_le32(cpucp_pcie_throughput_tx);1126	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),1127					HL_CPUCP_INFO_TIMEOUT_USEC, &result);1128	if (rc) {1129		if (rc != -EAGAIN)1130			dev_err(hdev->dev,1131				"Failed to handle CPU-CP PCI info pkt, error %d\n", rc);1132		return rc;1133	}1134	counters->tx_throughput = result;1135 1136	/* Fetch PCI replay counter */1137	memset(&pkt, 0, sizeof(pkt));1138	pkt.ctl = cpu_to_le32(CPUCP_PACKET_PCIE_REPLAY_CNT_GET <<1139			CPUCP_PKT_CTL_OPCODE_SHIFT);1140 1141	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),1142			HL_CPUCP_INFO_TIMEOUT_USEC, &result);1143	if (rc) {1144		if (rc != -EAGAIN)1145			dev_err(hdev->dev,1146				"Failed to handle CPU-CP PCI info pkt, error %d\n", rc);1147		return rc;1148	}1149	counters->replay_cnt = (u32) result;1150 1151	return rc;1152}1153 1154int hl_fw_cpucp_total_energy_get(struct hl_device *hdev, u64 *total_energy)1155{1156	struct cpucp_packet pkt = {};1157	u64 result;1158	int rc;1159 1160	pkt.ctl = cpu_to_le32(CPUCP_PACKET_TOTAL_ENERGY_GET <<1161				CPUCP_PKT_CTL_OPCODE_SHIFT);1162 1163	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),1164					HL_CPUCP_INFO_TIMEOUT_USEC, &result);1165	if (rc) {1166		if (rc != -EAGAIN)1167			dev_err(hdev->dev,1168				"Failed to handle CpuCP total energy pkt, error %d\n", rc);1169		return rc;1170	}1171 1172	*total_energy = result;1173 1174	return rc;1175}1176 1177int get_used_pll_index(struct hl_device *hdev, u32 input_pll_index,1178						enum pll_index *pll_index)1179{1180	struct asic_fixed_properties *prop = &hdev->asic_prop;1181	u8 pll_byte, pll_bit_off;1182	bool dynamic_pll;1183	int fw_pll_idx;1184 1185	dynamic_pll = !!(prop->fw_app_cpu_boot_dev_sts0 &1186						CPU_BOOT_DEV_STS0_DYN_PLL_EN);1187 1188	if (!dynamic_pll) {1189		/*1190		 * in case we are working with legacy FW (each asic has unique1191		 * PLL numbering) use the driver based index as they are1192		 * aligned with fw legacy numbering1193		 */1194		*pll_index = input_pll_index;1195		return 0;1196	}1197 1198	/* retrieve a FW compatible PLL index based on1199	 * ASIC specific user request1200	 */1201	fw_pll_idx = hdev->asic_funcs->map_pll_idx_to_fw_idx(input_pll_index);1202	if (fw_pll_idx < 0) {1203		dev_err(hdev->dev, "Invalid PLL index (%u) error %d\n",1204			input_pll_index, fw_pll_idx);1205		return -EINVAL;1206	}1207 1208	/* PLL map is a u8 array */1209	pll_byte = prop->cpucp_info.pll_map[fw_pll_idx >> 3];1210	pll_bit_off = fw_pll_idx & 0x7;1211 1212	if (!(pll_byte & BIT(pll_bit_off))) {1213		dev_err(hdev->dev, "PLL index %d is not supported\n",1214			fw_pll_idx);1215		return -EINVAL;1216	}1217 1218	*pll_index = fw_pll_idx;1219 1220	return 0;1221}1222 1223int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u32 pll_index,1224		u16 *pll_freq_arr)1225{1226	struct cpucp_packet pkt;1227	enum pll_index used_pll_idx;1228	u64 result;1229	int rc;1230 1231	rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);1232	if (rc)1233		return rc;1234 1235	memset(&pkt, 0, sizeof(pkt));1236 1237	pkt.ctl = cpu_to_le32(CPUCP_PACKET_PLL_INFO_GET <<1238				CPUCP_PKT_CTL_OPCODE_SHIFT);1239	pkt.pll_type = __cpu_to_le16((u16)used_pll_idx);1240 1241	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),1242			HL_CPUCP_INFO_TIMEOUT_USEC, &result);1243	if (rc) {1244		if (rc != -EAGAIN)1245			dev_err(hdev->dev, "Failed to read PLL info, error %d\n", rc);1246		return rc;1247	}1248 1249	pll_freq_arr[0] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT0_MASK, result);1250	pll_freq_arr[1] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT1_MASK, result);1251	pll_freq_arr[2] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT2_MASK, result);1252	pll_freq_arr[3] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT3_MASK, result);1253 1254	return 0;1255}1256 1257int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power)1258{1259	struct cpucp_packet pkt;1260	u64 result;1261	int rc;1262 1263	memset(&pkt, 0, sizeof(pkt));1264 1265	pkt.ctl = cpu_to_le32(CPUCP_PACKET_POWER_GET <<1266				CPUCP_PKT_CTL_OPCODE_SHIFT);1267	pkt.type = cpu_to_le16(CPUCP_POWER_INPUT);1268 1269	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),1270			HL_CPUCP_INFO_TIMEOUT_USEC, &result);1271	if (rc) {1272		if (rc != -EAGAIN)1273			dev_err(hdev->dev, "Failed to read power, error %d\n", rc);1274		return rc;1275	}1276 1277	*power = result;1278 1279	return rc;1280}1281 1282int hl_fw_dram_replaced_row_get(struct hl_device *hdev,1283				struct cpucp_hbm_row_info *info)1284{1285	struct cpucp_hbm_row_info *cpucp_repl_rows_info_cpu_addr;1286	dma_addr_t cpucp_repl_rows_info_dma_addr;1287	struct cpucp_packet pkt = {};1288	u64 result;1289	int rc;1290 1291	cpucp_repl_rows_info_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev,1292							sizeof(struct cpucp_hbm_row_info),1293							&cpucp_repl_rows_info_dma_addr);1294	if (!cpucp_repl_rows_info_cpu_addr) {1295		dev_err(hdev->dev,1296			"Failed to allocate DMA memory for CPU-CP replaced rows info packet\n");1297		return -ENOMEM;1298	}1299 1300	memset(cpucp_repl_rows_info_cpu_addr, 0, sizeof(struct cpucp_hbm_row_info));1301 1302	pkt.ctl = cpu_to_le32(CPUCP_PACKET_HBM_REPLACED_ROWS_INFO_GET <<1303					CPUCP_PKT_CTL_OPCODE_SHIFT);1304	pkt.addr = cpu_to_le64(cpucp_repl_rows_info_dma_addr);1305	pkt.data_max_size = cpu_to_le32(sizeof(struct cpucp_hbm_row_info));1306 1307	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),1308					HL_CPUCP_INFO_TIMEOUT_USEC, &result);1309	if (rc) {1310		if (rc != -EAGAIN)1311			dev_err(hdev->dev,1312				"Failed to handle CPU-CP replaced rows info pkt, error %d\n", rc);1313		goto out;1314	}1315 1316	memcpy(info, cpucp_repl_rows_info_cpu_addr, sizeof(*info));1317 1318out:1319	hl_cpu_accessible_dma_pool_free(hdev, sizeof(struct cpucp_hbm_row_info),1320						cpucp_repl_rows_info_cpu_addr);1321 1322	return rc;1323}1324 1325int hl_fw_dram_pending_row_get(struct hl_device *hdev, u32 *pend_rows_num)1326{1327	struct cpucp_packet pkt;1328	u64 result;1329	int rc;1330 1331	memset(&pkt, 0, sizeof(pkt));1332 1333	pkt.ctl = cpu_to_le32(CPUCP_PACKET_HBM_PENDING_ROWS_STATUS << CPUCP_PKT_CTL_OPCODE_SHIFT);1334 1335	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);1336	if (rc) {1337		if (rc != -EAGAIN)1338			dev_err(hdev->dev,1339				"Failed to handle CPU-CP pending rows info pkt, error %d\n", rc);1340		goto out;1341	}1342 1343	*pend_rows_num = (u32) result;1344out:1345	return rc;1346}1347 1348int hl_fw_cpucp_engine_core_asid_set(struct hl_device *hdev, u32 asid)1349{1350	struct cpucp_packet pkt;1351	int rc;1352 1353	memset(&pkt, 0, sizeof(pkt));1354 1355	pkt.ctl = cpu_to_le32(CPUCP_PACKET_ENGINE_CORE_ASID_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);1356	pkt.value = cpu_to_le64(asid);1357 1358	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt),1359						HL_CPUCP_INFO_TIMEOUT_USEC, NULL);1360	if (rc)1361		dev_err(hdev->dev,1362			"Failed on ASID configuration request for engine core, error %d\n",1363			rc);1364 1365	return rc;1366}1367 1368void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev)1369{1370	struct static_fw_load_mgr *static_loader =1371			&hdev->fw_loader.static_loader;1372	int rc;1373 1374	if (hdev->asic_prop.dynamic_fw_load) {1375		rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,1376				COMMS_RST_DEV, 0, false,1377				hdev->fw_loader.cpu_timeout);1378		if (rc)1379			dev_err(hdev->dev, "Failed sending COMMS_RST_DEV\n");1380	} else {1381		WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_RST_DEV);1382	}1383}1384 1385void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev)1386{1387	struct fw_load_mgr *fw_loader = &hdev->fw_loader;1388	u32 status, cpu_boot_status_reg, cpu_timeout;1389	struct static_fw_load_mgr *static_loader;1390	struct pre_fw_load_props *pre_fw_load;1391	int rc;1392 1393	if (hdev->device_cpu_is_halted)1394		return;1395 1396	/* Stop device CPU to make sure nothing bad happens */1397	if (hdev->asic_prop.dynamic_fw_load) {1398		pre_fw_load = &fw_loader->pre_fw_load;1399		cpu_timeout = fw_loader->cpu_timeout;1400		cpu_boot_status_reg = pre_fw_load->cpu_boot_status_reg;1401 1402		rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,1403				COMMS_GOTO_WFE, 0, false, cpu_timeout);1404		if (rc) {1405			dev_err(hdev->dev, "Failed sending COMMS_GOTO_WFE\n");1406		} else {1407			rc = hl_poll_timeout(1408				hdev,1409				cpu_boot_status_reg,1410				status,1411				status == CPU_BOOT_STATUS_IN_WFE,1412				hdev->fw_poll_interval_usec,1413				cpu_timeout);1414			if (rc)1415				dev_err(hdev->dev, "Current status=%u. Timed-out updating to WFE\n",1416						status);1417		}1418	} else {1419		static_loader = &hdev->fw_loader.static_loader;1420		WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_GOTO_WFE);1421		msleep(static_loader->cpu_reset_wait_msec);1422 1423		/* Must clear this register in order to prevent preboot1424		 * from reading WFE after reboot1425		 */1426		WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_NA);1427	}1428 1429	hdev->device_cpu_is_halted = true;1430}1431 1432static void detect_cpu_boot_status(struct hl_device *hdev, u32 status)1433{1434	/* Some of the status codes below are deprecated in newer f/w1435	 * versions but we keep them here for backward compatibility1436	 */1437	switch (status) {1438	case CPU_BOOT_STATUS_NA:1439		dev_err(hdev->dev,1440			"Device boot progress - BTL/ROM did NOT run\n");1441		break;1442	case CPU_BOOT_STATUS_IN_WFE:1443		dev_err(hdev->dev,1444			"Device boot progress - Stuck inside WFE loop\n");1445		break;1446	case CPU_BOOT_STATUS_IN_BTL:1447		dev_err(hdev->dev,1448			"Device boot progress - Stuck in BTL\n");1449		break;1450	case CPU_BOOT_STATUS_IN_PREBOOT:1451		dev_err(hdev->dev,1452			"Device boot progress - Stuck in Preboot\n");1453		break;1454	case CPU_BOOT_STATUS_IN_SPL:1455		dev_err(hdev->dev,1456			"Device boot progress - Stuck in SPL\n");1457		break;1458	case CPU_BOOT_STATUS_IN_UBOOT:1459		dev_err(hdev->dev,1460			"Device boot progress - Stuck in u-boot\n");1461		break;1462	case CPU_BOOT_STATUS_DRAM_INIT_FAIL:1463		dev_err(hdev->dev,1464			"Device boot progress - DRAM initialization failed\n");1465		break;1466	case CPU_BOOT_STATUS_UBOOT_NOT_READY:1467		dev_err(hdev->dev,1468			"Device boot progress - Cannot boot\n");1469		break;1470	case CPU_BOOT_STATUS_TS_INIT_FAIL:1471		dev_err(hdev->dev,1472			"Device boot progress - Thermal Sensor initialization failed\n");1473		break;1474	case CPU_BOOT_STATUS_SECURITY_READY:1475		dev_err(hdev->dev,1476			"Device boot progress - Stuck in preboot after security initialization\n");1477		break;1478	case CPU_BOOT_STATUS_FW_SHUTDOWN_PREP:1479		dev_err(hdev->dev,1480			"Device boot progress - Stuck in preparation for shutdown\n");1481		break;1482	default:1483		dev_err(hdev->dev,1484			"Device boot progress - Invalid or unexpected status code %d\n", status);1485		break;1486	}1487}1488 1489int hl_fw_wait_preboot_ready(struct hl_device *hdev)1490{1491	struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load;1492	u32 status = 0, timeout;1493	int rc, tries = 1, fw_err = 0;1494	bool preboot_still_runs;1495 1496	/* Need to check two possible scenarios:1497	 *1498	 * CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT - for newer firmwares where1499	 * the preboot is waiting for the boot fit1500	 *1501	 * All other status values - for older firmwares where the uboot was1502	 * loaded from the FLASH1503	 */1504	timeout = pre_fw_load->wait_for_preboot_timeout;1505retry:1506	rc = hl_poll_timeout(1507		hdev,1508		pre_fw_load->cpu_boot_status_reg,1509		status,1510		(status == CPU_BOOT_STATUS_NIC_FW_RDY) ||1511		(status == CPU_BOOT_STATUS_READY_TO_BOOT) ||1512		(status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT),1513		hdev->fw_poll_interval_usec,1514		timeout);1515	/*1516	 * if F/W reports "security-ready" it means preboot might take longer.1517	 * If the field 'wait_for_preboot_extended_timeout' is non 0 we wait again1518	 * with that timeout1519	 */1520	preboot_still_runs = (status == CPU_BOOT_STATUS_SECURITY_READY ||1521				status == CPU_BOOT_STATUS_IN_PREBOOT ||1522				status == CPU_BOOT_STATUS_FW_SHUTDOWN_PREP ||1523				status == CPU_BOOT_STATUS_DRAM_RDY);1524 1525	if (rc && tries && preboot_still_runs) {1526		tries--;1527		if (pre_fw_load->wait_for_preboot_extended_timeout) {1528			timeout = pre_fw_load->wait_for_preboot_extended_timeout;1529			goto retry;1530		}1531	}1532 1533	/* If we read all FF, then something is totally wrong, no point1534	 * of reading specific errors1535	 */1536	if (status != -1)1537		fw_err = fw_read_errors(hdev, pre_fw_load->boot_err0_reg,1538					pre_fw_load->boot_err1_reg,1539					pre_fw_load->sts_boot_dev_sts0_reg,1540					pre_fw_load->sts_boot_dev_sts1_reg);1541	if (rc || fw_err) {1542		detect_cpu_boot_status(hdev, status);1543		dev_err(hdev->dev, "CPU boot %s (status = %d)\n",1544				fw_err ? "failed due to an error" : "ready timeout", status);1545		return -EIO;1546	}1547 1548	hdev->fw_loader.fw_comp_loaded |= FW_TYPE_PREBOOT_CPU;1549 1550	return 0;1551}1552 1553static int hl_fw_read_preboot_caps(struct hl_device *hdev)1554{1555	struct pre_fw_load_props *pre_fw_load;1556	struct asic_fixed_properties *prop;1557	u32 reg_val;1558	int rc;1559 1560	prop = &hdev->asic_prop;1561	pre_fw_load = &hdev->fw_loader.pre_fw_load;1562 1563	rc = hl_fw_wait_preboot_ready(hdev);1564	if (rc)1565		return rc;1566 1567	/*1568	 * the registers DEV_STS* contain FW capabilities/features.1569	 * We can rely on this registers only if bit CPU_BOOT_DEV_STS*_ENABLED1570	 * is set.1571	 * In the first read of this register we store the value of this1572	 * register ONLY if the register is enabled (which will be propagated1573	 * to next stages) and also mark the register as valid.1574	 * In case it is not enabled the stored value will be left 0- all1575	 * caps/features are off1576	 */1577	reg_val = RREG32(pre_fw_load->sts_boot_dev_sts0_reg);1578	if (reg_val & CPU_BOOT_DEV_STS0_ENABLED) {1579		prop->fw_cpu_boot_dev_sts0_valid = true;1580		prop->fw_preboot_cpu_boot_dev_sts0 = reg_val;1581	}1582 1583	reg_val = RREG32(pre_fw_load->sts_boot_dev_sts1_reg);1584	if (reg_val & CPU_BOOT_DEV_STS1_ENABLED) {1585		prop->fw_cpu_boot_dev_sts1_valid = true;1586		prop->fw_preboot_cpu_boot_dev_sts1 = reg_val;1587	}1588 1589	prop->dynamic_fw_load = !!(prop->fw_preboot_cpu_boot_dev_sts0 &1590						CPU_BOOT_DEV_STS0_FW_LD_COM_EN);1591 1592	/* initialize FW loader once we know what load protocol is used */1593	hdev->asic_funcs->init_firmware_loader(hdev);1594 1595	dev_dbg(hdev->dev, "Attempting %s FW load\n",1596			prop->dynamic_fw_load ? "dynamic" : "legacy");1597	return 0;1598}1599 1600static int hl_fw_static_read_device_fw_version(struct hl_device *hdev,1601					enum hl_fw_component fwc)1602{1603	struct asic_fixed_properties *prop = &hdev->asic_prop;1604	struct fw_load_mgr *fw_loader = &hdev->fw_loader;1605	struct static_fw_load_mgr *static_loader;1606	char *dest, *boot_ver, *preboot_ver;1607	u32 ver_off, limit;1608	const char *name;1609	char btl_ver[32];1610 1611	static_loader = &hdev->fw_loader.static_loader;1612 1613	switch (fwc) {1614	case FW_COMP_BOOT_FIT:1615		ver_off = RREG32(static_loader->boot_fit_version_offset_reg);1616		dest = prop->uboot_ver;1617		name = "Boot-fit";1618		limit = static_loader->boot_fit_version_max_off;1619		break;1620	case FW_COMP_PREBOOT:1621		ver_off = RREG32(static_loader->preboot_version_offset_reg);1622		dest = prop->preboot_ver;1623		name = "Preboot";1624		limit = static_loader->preboot_version_max_off;1625		break;1626	default:1627		dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);1628		return -EIO;1629	}1630 1631	ver_off &= static_loader->sram_offset_mask;1632 1633	if (ver_off < limit) {1634		memcpy_fromio(dest,1635			hdev->pcie_bar[fw_loader->sram_bar_id] + ver_off,1636			VERSION_MAX_LEN);1637	} else {1638		dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n",1639								name, ver_off);1640		strscpy(dest, "unavailable", VERSION_MAX_LEN);1641		return -EIO;1642	}1643 1644	if (fwc == FW_COMP_BOOT_FIT) {1645		boot_ver = extract_fw_ver_from_str(prop->uboot_ver);1646		if (boot_ver) {1647			dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);1648			kfree(boot_ver);1649		}1650	} else if (fwc == FW_COMP_PREBOOT) {1651		preboot_ver = strnstr(prop->preboot_ver, "Preboot",1652						VERSION_MAX_LEN);1653		if (preboot_ver && preboot_ver != prop->preboot_ver) {1654			strscpy(btl_ver, prop->preboot_ver,1655				min((int) (preboot_ver - prop->preboot_ver),1656									31));1657			dev_info(hdev->dev, "%s\n", btl_ver);1658		}1659 1660		preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);1661		if (preboot_ver) {1662			dev_info(hdev->dev, "preboot version %s\n",1663								preboot_ver);1664			kfree(preboot_ver);1665		}1666	}1667 1668	return 0;1669}1670 1671/**1672 * hl_fw_preboot_update_state - update internal data structures during1673 *                              handshake with preboot1674 *1675 *1676 * @hdev: pointer to the habanalabs device structure1677 *1678 * @return 0 on success, otherwise non-zero error code1679 */1680static void hl_fw_preboot_update_state(struct hl_device *hdev)1681{1682	struct asic_fixed_properties *prop = &hdev->asic_prop;1683	u32 cpu_boot_dev_sts0, cpu_boot_dev_sts1;1684 1685	cpu_boot_dev_sts0 = prop->fw_preboot_cpu_boot_dev_sts0;1686	cpu_boot_dev_sts1 = prop->fw_preboot_cpu_boot_dev_sts1;1687 1688	/* We read boot_dev_sts registers multiple times during boot:1689	 * 1. preboot - a. Check whether the security status bits are valid1690	 *              b. Check whether fw security is enabled1691	 *              c. Check whether hard reset is done by preboot1692	 * 2. boot cpu - a. Fetch boot cpu security status1693	 *               b. Check whether hard reset is done by boot cpu1694	 * 3. FW application - a. Fetch fw application security status1695	 *                     b. Check whether hard reset is done by fw app1696	 */1697	prop->hard_reset_done_by_fw = !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);1698 1699	prop->fw_security_enabled = !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_SECURITY_EN);1700 1701	dev_dbg(hdev->dev, "Firmware preboot boot device status0 %#x\n",1702							cpu_boot_dev_sts0);1703 1704	dev_dbg(hdev->dev, "Firmware preboot boot device status1 %#x\n",1705							cpu_boot_dev_sts1);1706 1707	dev_dbg(hdev->dev, "Firmware preboot hard-reset is %s\n",1708			prop->hard_reset_done_by_fw ? "enabled" : "disabled");1709 1710	dev_dbg(hdev->dev, "firmware-level security is %s\n",1711			prop->fw_security_enabled ? "enabled" : "disabled");1712 1713	dev_dbg(hdev->dev, "GIC controller is %s\n",1714			prop->gic_interrupts_enable ? "enabled" : "disabled");1715}1716 1717static int hl_fw_static_read_preboot_status(struct hl_device *hdev)1718{1719	int rc;1720 1721	rc = hl_fw_static_read_device_fw_version(hdev, FW_COMP_PREBOOT);1722	if (rc)1723		return rc;1724 1725	return 0;1726}1727 1728int hl_fw_read_preboot_status(struct hl_device *hdev)1729{1730	int rc;1731 1732	if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU))1733		return 0;1734 1735	/* get FW pre-load parameters  */1736	hdev->asic_funcs->init_firmware_preload_params(hdev);1737 1738	/*1739	 * In order to determine boot method (static VS dynamic) we need to1740	 * read the boot caps register1741	 */1742	rc = hl_fw_read_preboot_caps(hdev);1743	if (rc)1744		return rc;1745 1746	hl_fw_preboot_update_state(hdev);1747 1748	/* no need to read preboot status in dynamic load */1749	if (hdev->asic_prop.dynamic_fw_load)1750		return 0;1751 1752	return hl_fw_static_read_preboot_status(hdev);1753}1754 1755/* associate string with COMM status */1756static char *hl_dynamic_fw_status_str[COMMS_STS_INVLD_LAST] = {1757	[COMMS_STS_NOOP] = "NOOP",1758	[COMMS_STS_ACK] = "ACK",1759	[COMMS_STS_OK] = "OK",1760	[COMMS_STS_ERR] = "ERR",1761	[COMMS_STS_VALID_ERR] = "VALID_ERR",1762	[COMMS_STS_TIMEOUT_ERR] = "TIMEOUT_ERR",1763};1764 1765/**1766 * hl_fw_dynamic_report_error_status - report error status1767 *1768 * @hdev: pointer to the habanalabs device structure1769 * @status: value of FW status register1770 * @expected_status: the expected status1771 */1772static void hl_fw_dynamic_report_error_status(struct hl_device *hdev,1773						u32 status,1774						enum comms_sts expected_status)1775{1776	enum comms_sts comm_status =1777				FIELD_GET(COMMS_STATUS_STATUS_MASK, status);1778 1779	if (comm_status < COMMS_STS_INVLD_LAST)1780		dev_err(hdev->dev, "Device status %s, expected status: %s\n",1781				hl_dynamic_fw_status_str[comm_status],1782				hl_dynamic_fw_status_str[expected_status]);1783	else1784		dev_err(hdev->dev, "Device status unknown %d, expected status: %s\n",1785				comm_status,1786				hl_dynamic_fw_status_str[expected_status]);1787}1788 1789/**1790 * hl_fw_dynamic_send_cmd - send LKD to FW cmd1791 *1792 * @hdev: pointer to the habanalabs device structure1793 * @fw_loader: managing structure for loading device's FW1794 * @cmd: LKD to FW cmd code1795 * @size: size of next FW component to be loaded (0 if not necessary)1796 *1797 * LDK to FW exact command layout is defined at struct comms_command.1798 * note: the size argument is used only when the next FW component should be1799 *       loaded, otherwise it shall be 0. the size is used by the FW in later1800 *       protocol stages and when sending only indicating the amount of memory1801 *       to be allocated by the FW to receive the next boot component.1802 */1803static void hl_fw_dynamic_send_cmd(struct hl_device *hdev,1804				struct fw_load_mgr *fw_loader,1805				enum comms_cmd cmd, unsigned int size)1806{1807	struct cpu_dyn_regs *dyn_regs;1808	u32 val;1809 1810	dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;1811 1812	val = FIELD_PREP(COMMS_COMMAND_CMD_MASK, cmd);1813	val |= FIELD_PREP(COMMS_COMMAND_SIZE_MASK, size);1814 1815	trace_habanalabs_comms_send_cmd(&hdev->pdev->dev, comms_cmd_str_arr[cmd]);1816	WREG32(le32_to_cpu(dyn_regs->kmd_msg_to_cpu), val);1817}1818 1819/**1820 * hl_fw_dynamic_extract_fw_response - update the FW response1821 *1822 * @hdev: pointer to the habanalabs device structure1823 * @fw_loader: managing structure for loading device's FW1824 * @response: FW response1825 * @status: the status read from CPU status register1826 *1827 * @return 0 on success, otherwise non-zero error code1828 */1829static int hl_fw_dynamic_extract_fw_response(struct hl_device *hdev,1830						struct fw_load_mgr *fw_loader,1831						struct fw_response *response,1832						u32 status)1833{1834	response->status = FIELD_GET(COMMS_STATUS_STATUS_MASK, status);1835	response->ram_offset = FIELD_GET(COMMS_STATUS_OFFSET_MASK, status) <<1836						COMMS_STATUS_OFFSET_ALIGN_SHIFT;1837	response->ram_type = FIELD_GET(COMMS_STATUS_RAM_TYPE_MASK, status);1838 1839	if ((response->ram_type != COMMS_SRAM) &&1840					(response->ram_type != COMMS_DRAM)) {1841		dev_err(hdev->dev, "FW status: invalid RAM type %u\n",1842							response->ram_type);1843		return -EIO;1844	}1845 1846	return 0;1847}1848 1849/**1850 * hl_fw_dynamic_wait_for_status - wait for status in dynamic FW load1851 *1852 * @hdev: pointer to the habanalabs device structure1853 * @fw_loader: managing structure for loading device's FW1854 * @expected_status: expected status to wait for1855 * @timeout: timeout for status wait1856 *1857 * @return 0 on success, otherwise non-zero error code1858 *1859 * waiting for status from FW include polling the FW status register until1860 * expected status is received or timeout occurs (whatever occurs first).1861 */1862static int hl_fw_dynamic_wait_for_status(struct hl_device *hdev,1863						struct fw_load_mgr *fw_loader,1864						enum comms_sts expected_status,1865						u32 timeout)1866{1867	struct cpu_dyn_regs *dyn_regs;1868	u32 status;1869	int rc;1870 1871	dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;1872 1873	trace_habanalabs_comms_wait_status(&hdev->pdev->dev, comms_sts_str_arr[expected_status]);1874 1875	/* Wait for expected status */1876	rc = hl_poll_timeout(1877		hdev,1878		le32_to_cpu(dyn_regs->cpu_cmd_status_to_host),1879		status,1880		FIELD_GET(COMMS_STATUS_STATUS_MASK, status) == expected_status,1881		hdev->fw_comms_poll_interval_usec,1882		timeout);1883 1884	if (rc) {1885		hl_fw_dynamic_report_error_status(hdev, status,1886							expected_status);1887		return -EIO;1888	}1889 1890	trace_habanalabs_comms_wait_status_done(&hdev->pdev->dev,1891						comms_sts_str_arr[expected_status]);1892 1893	/*1894	 * skip storing FW response for NOOP to preserve the actual desired1895	 * FW status1896	 */1897	if (expected_status == COMMS_STS_NOOP)1898		return 0;1899 1900	rc = hl_fw_dynamic_extract_fw_response(hdev, fw_loader,1901					&fw_loader->dynamic_loader.response,1902					status);1903	return rc;1904}1905 1906/**1907 * hl_fw_dynamic_send_clear_cmd - send clear command to FW1908 *1909 * @hdev: pointer to the habanalabs device structure1910 * @fw_loader: managing structure for loading device's FW1911 *1912 * @return 0 on success, otherwise non-zero error code1913 *1914 * after command cycle between LKD to FW CPU (i.e. LKD got an expected status1915 * from FW) we need to clear the CPU status register in order to avoid garbage1916 * between command cycles.1917 * This is done by sending clear command and polling the CPU to LKD status1918 * register to hold the status NOOP1919 */1920static int hl_fw_dynamic_send_clear_cmd(struct hl_device *hdev,1921						struct fw_load_mgr *fw_loader)1922{1923	hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_CLR_STS, 0);1924 1925	return hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_NOOP,1926							fw_loader->cpu_timeout);1927}1928 1929/**1930 * hl_fw_dynamic_send_protocol_cmd - send LKD to FW cmd and wait for ACK1931 *1932 * @hdev: pointer to the habanalabs device structure1933 * @fw_loader: managing structure for loading device's FW1934 * @cmd: LKD to FW cmd code1935 * @size: size of next FW component to be loaded (0 if not necessary)1936 * @wait_ok: if true also wait for OK response from FW1937 * @timeout: timeout for status wait1938 *1939 * @return 0 on success, otherwise non-zero error code1940 *1941 * brief:1942 * when sending protocol command we have the following steps:1943 * - send clear (clear command and verify clear status register)1944 * - send the actual protocol command1945 * - wait for ACK on the protocol command1946 * - send clear1947 * - send NOOP1948 * if, in addition, the specific protocol command should wait for OK then:1949 * - wait for OK1950 * - send clear1951 * - send NOOP1952 *1953 * NOTES:1954 * send clear: this is necessary in order to clear the status register to avoid1955 *             leftovers between command1956 * NOOP command: necessary to avoid loop on the clear command by the FW1957 */1958int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev,1959				struct fw_load_mgr *fw_loader,1960				enum comms_cmd cmd, unsigned int size,1961				bool wait_ok, u32 timeout)1962{1963	int rc;1964 1965	trace_habanalabs_comms_protocol_cmd(&hdev->pdev->dev, comms_cmd_str_arr[cmd]);1966 1967	/* first send clear command to clean former commands */1968	rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);1969	if (rc)1970		return rc;1971 1972	/* send the actual command */1973	hl_fw_dynamic_send_cmd(hdev, fw_loader, cmd, size);1974 1975	/* wait for ACK for the command */1976	rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_ACK,1977								timeout);1978	if (rc)1979		return rc;1980 1981	/* clear command to prepare for NOOP command */1982	rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);1983	if (rc)1984		return rc;1985 1986	/* send the actual NOOP command */1987	hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);1988 1989	if (!wait_ok)1990		return 0;1991 1992	rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_OK,1993								timeout);1994	if (rc)1995		return rc;1996 1997	/* clear command to prepare for NOOP command */1998	rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);1999	if (rc)2000		return rc;2001 2002	/* send the actual NOOP command */2003	hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);2004 2005	return 0;2006}2007 2008/**2009 * hl_fw_compat_crc32 - CRC compatible with FW2010 *2011 * @data: pointer to the data2012 * @size: size of the data2013 *2014 * @return the CRC32 result2015 *2016 * NOTE: kernel's CRC32 differs from standard CRC32 calculation.2017 *       in order to be aligned we need to flip the bits of both the input2018 *       initial CRC and kernel's CRC32 result.2019 *       in addition both sides use initial CRC of 0,2020 */2021static u32 hl_fw_compat_crc32(u8 *data, size_t size)2022{2023	return ~crc32_le(~((u32)0), data, size);2024}2025 2026/**2027 * hl_fw_dynamic_validate_memory_bound - validate memory bounds for memory2028 *                                        transfer (image or descriptor) between2029 *                                        host and FW2030 *2031 * @hdev: pointer to the habanalabs device structure2032 * @addr: device address of memory transfer2033 * @size: memory transfer size2034 * @region: PCI memory region2035 *2036 * @return 0 on success, otherwise non-zero error code2037 */2038static int hl_fw_dynamic_validate_memory_bound(struct hl_device *hdev,2039						u64 addr, size_t size,2040						struct pci_mem_region *region)2041{2042	u64 end_addr;2043 2044	/* now make sure that the memory transfer is within region's bounds */2045	end_addr = addr + size;2046	if (end_addr >= region->region_base + region->region_size) {2047		dev_err(hdev->dev,2048			"dynamic FW load: memory transfer end address out of memory region bounds. addr: %llx\n",2049							end_addr);2050		return -EIO;2051	}2052 2053	/*2054	 * now make sure memory transfer is within predefined BAR bounds.2055	 * this is to make sure we do not need to set the bar (e.g. for DRAM2056	 * memory transfers)2057	 */2058	if (end_addr >= region->region_base - region->offset_in_bar +2059							region->bar_size) {2060		dev_err(hdev->dev,2061			"FW image beyond PCI BAR bounds\n");2062		return -EIO;2063	}2064 2065	return 0;2066}2067 2068/**2069 * hl_fw_dynamic_validate_descriptor - validate FW descriptor2070 *2071 * @hdev: pointer to the habanalabs device structure2072 * @fw_loader: managing structure for loading device's FW2073 * @fw_desc: the descriptor from FW2074 *2075 * @return 0 on success, otherwise non-zero error code2076 */2077static int hl_fw_dynamic_validate_descriptor(struct hl_device *hdev,2078					struct fw_load_mgr *fw_loader,2079					struct lkd_fw_comms_desc *fw_desc)2080{2081	struct pci_mem_region *region;2082	enum pci_region region_id;2083	size_t data_size;2084	u32 data_crc32;2085	u8 *data_ptr;2086	u64 addr;2087	int rc;2088 2089	if (le32_to_cpu(fw_desc->header.magic) != HL_COMMS_DESC_MAGIC)2090		dev_dbg(hdev->dev, "Invalid magic for dynamic FW descriptor (%x)\n",2091				fw_desc->header.magic);2092 2093	if (fw_desc->header.version != HL_COMMS_DESC_VER)2094		dev_dbg(hdev->dev, "Invalid version for dynamic FW descriptor (%x)\n",2095				fw_desc->header.version);2096 2097	/*2098	 * Calc CRC32 of data without header. use the size of the descriptor2099	 * reported by firmware, without calculating it ourself, to allow adding2100	 * more fields to the lkd_fw_comms_desc structure.2101	 * note that no alignment/stride address issues here as all structures2102	 * are 64 bit padded.2103	 */2104	data_ptr = (u8 *)fw_desc + sizeof(struct comms_msg_header);2105	data_size = le16_to_cpu(fw_desc->header.size);2106 2107	data_crc32 = hl_fw_compat_crc32(data_ptr, data_size);2108	if (data_crc32 != le32_to_cpu(fw_desc->header.crc32)) {2109		dev_err(hdev->dev, "CRC32 mismatch for dynamic FW descriptor (%x:%x)\n",2110			data_crc32, fw_desc->header.crc32);2111		return -EIO;2112	}2113 2114	/* find memory region to which to copy the image */2115	addr = le64_to_cpu(fw_desc->img_addr);2116	region_id = hl_get_pci_memory_region(hdev, addr);2117	if ((region_id != PCI_REGION_SRAM) && ((region_id != PCI_REGION_DRAM))) {2118		dev_err(hdev->dev, "Invalid region to copy FW image address=%llx\n", addr);2119		return -EIO;2120	}2121 2122	region = &hdev->pci_mem_region[region_id];2123 2124	/* store the region for the copy stage */2125	fw_loader->dynamic_loader.image_region = region;2126 2127	/*2128	 * here we know that the start address is valid, now make sure that the2129	 * image is within region's bounds2130	 */2131	rc = hl_fw_dynamic_validate_memory_bound(hdev, addr,2132					fw_loader->dynamic_loader.fw_image_size,2133					region);2134	if (rc) {2135		dev_err(hdev->dev, "invalid mem transfer request for FW image\n");2136		return rc;2137	}2138 2139	/* here we can mark the descriptor as valid as the content has been validated */2140	fw_loader->dynamic_loader.fw_desc_valid = true;2141 2142	return 0;2143}2144 2145static int hl_fw_dynamic_validate_response(struct hl_device *hdev,2146						struct fw_response *response,2147						struct pci_mem_region *region)2148{2149	u64 device_addr;2150	int rc;2151 2152	device_addr = region->region_base + response->ram_offset;2153 2154	/*2155	 * validate that the descriptor is within region's bounds2156	 * Note that as the start address was supplied according to the RAM2157	 * type- testing only the end address is enough2158	 */2159	rc = hl_fw_dynamic_validate_memory_bound(hdev, device_addr,2160					sizeof(struct lkd_fw_comms_desc),2161					region);2162	return rc;2163}2164 2165/*2166 * hl_fw_dynamic_read_descriptor_msg - read and show the ascii msg that sent by fw2167 *2168 * @hdev: pointer to the habanalabs device structure2169 * @fw_desc: the descriptor from FW2170 */2171static void hl_fw_dynamic_read_descriptor_msg(struct hl_device *hdev,2172					struct lkd_fw_comms_desc *fw_desc)2173{2174	int i;2175	char *msg;2176 2177	for (i = 0 ; i < LKD_FW_ASCII_MSG_MAX ; i++) {2178		if (!fw_desc->ascii_msg[i].valid)2179			return;2180 2181		/* force NULL termination */2182		msg = fw_desc->ascii_msg[i].msg;2183		msg[LKD_FW_ASCII_MSG_MAX_LEN - 1] = '\0';2184 2185		switch (fw_desc->ascii_msg[i].msg_lvl) {2186		case LKD_FW_ASCII_MSG_ERR:2187			dev_err(hdev->dev, "fw: %s", fw_desc->ascii_msg[i].msg);2188			break;2189		case LKD_FW_ASCII_MSG_WRN:2190			dev_warn(hdev->dev, "fw: %s", fw_desc->ascii_msg[i].msg);2191			break;2192		case LKD_FW_ASCII_MSG_INF:2193			dev_info(hdev->dev, "fw: %s", fw_desc->ascii_msg[i].msg);2194			break;2195		default:2196			dev_dbg(hdev->dev, "fw: %s", fw_desc->ascii_msg[i].msg);2197			break;2198		}2199	}2200}2201 2202/**2203 * hl_fw_dynamic_read_and_validate_descriptor - read and validate FW descriptor2204 *2205 * @hdev: pointer to the habanalabs device structure2206 * @fw_loader: managing structure for loading device's FW2207 *2208 * @return 0 on success, otherwise non-zero error code2209 */2210static int hl_fw_dynamic_read_and_validate_descriptor(struct hl_device *hdev,2211						struct fw_load_mgr *fw_loader)2212{2213	struct lkd_fw_comms_desc *fw_desc;2214	struct pci_mem_region *region;2215	struct fw_response *response;2216	void *temp_fw_desc;2217	void __iomem *src;2218	u16 fw_data_size;2219	enum pci_region region_id;2220	int rc;2221 2222	fw_desc = &fw_loader->dynamic_loader.comm_desc;2223	response = &fw_loader->dynamic_loader.response;2224 2225	region_id = (response->ram_type == COMMS_SRAM) ?2226					PCI_REGION_SRAM : PCI_REGION_DRAM;2227 2228	region = &hdev->pci_mem_region[region_id];2229 2230	rc = hl_fw_dynamic_validate_response(hdev, response, region);2231	if (rc) {2232		dev_err(hdev->dev,2233			"invalid mem transfer request for FW descriptor\n");2234		return rc;2235	}2236 2237	/*2238	 * extract address to copy the descriptor from2239	 * in addition, as the descriptor value is going to be over-ridden by new data- we mark it2240	 * as invalid.2241	 * it will be marked again as valid once validated2242	 */2243	fw_loader->dynamic_loader.fw_desc_valid = false;2244	src = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +2245							response->ram_offset;2246 2247	/*2248	 * We do the copy of the fw descriptor in 2 phases:2249	 * 1. copy the header + data info according to our lkd_fw_comms_desc definition.2250	 *    then we're able to read the actual data size provided by fw.2251	 *    this is needed for cases where data in descriptor was changed(add/remove)2252	 *    in embedded specs header file before updating lkd copy of the header file2253	 * 2. copy descriptor to temporary buffer with aligned size and send it to validation2254	 */2255	memcpy_fromio(fw_desc, src, sizeof(struct lkd_fw_comms_desc));2256	fw_data_size = le16_to_cpu(fw_desc->header.size);2257 2258	temp_fw_desc = vzalloc(sizeof(struct comms_msg_header) + fw_data_size);2259	if (!temp_fw_desc)2260		return -ENOMEM;2261 2262	memcpy_fromio(temp_fw_desc, src, sizeof(struct comms_msg_header) + fw_data_size);2263 2264	rc = hl_fw_dynamic_validate_descriptor(hdev, fw_loader,2265					(struct lkd_fw_comms_desc *) temp_fw_desc);2266 2267	if (!rc)2268		hl_fw_dynamic_read_descriptor_msg(hdev, temp_fw_desc);2269 2270	vfree(temp_fw_desc);2271 2272	return rc;2273}2274 2275/**2276 * hl_fw_dynamic_request_descriptor - handshake with CPU to get FW descriptor2277 *2278 * @hdev: pointer to the habanalabs device structure2279 * @fw_loader: managing structure for loading device's FW2280 * @next_image_size: size to allocate for next FW component2281 *2282 * @return 0 on success, otherwise non-zero error code2283 */2284static int hl_fw_dynamic_request_descriptor(struct hl_device *hdev,2285						struct fw_load_mgr *fw_loader,2286						size_t next_image_size)2287{2288	int rc;2289 2290	rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_PREP_DESC,2291						next_image_size, true,2292						fw_loader->cpu_timeout);2293	if (rc)2294		return rc;2295 2296	return hl_fw_dynamic_read_and_validate_descriptor(hdev, fw_loader);2297}2298 2299/**2300 * hl_fw_dynamic_read_device_fw_version - read FW version to exposed properties2301 *2302 * @hdev: pointer to the habanalabs device structure2303 * @fwc: the firmware component2304 * @fw_version: fw component's version string2305 */2306static int hl_fw_dynamic_read_device_fw_version(struct hl_device *hdev,2307					enum hl_fw_component fwc,2308					const char *fw_version)2309{2310	struct asic_fixed_properties *prop = &hdev->asic_prop;2311	char *preboot_ver, *boot_ver;2312	char btl_ver[32];2313	int rc;2314 2315	switch (fwc) {2316	case FW_COMP_BOOT_FIT:2317		strscpy(prop->uboot_ver, fw_version, VERSION_MAX_LEN);2318		boot_ver = extract_fw_ver_from_str(prop->uboot_ver);2319		if (boot_ver) {2320			dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);2321			kfree(boot_ver);2322		}2323 2324		break;2325	case FW_COMP_PREBOOT:2326		strscpy(prop->preboot_ver, fw_version, VERSION_MAX_LEN);2327		preboot_ver = strnstr(prop->preboot_ver, "Preboot", VERSION_MAX_LEN);2328		dev_info(hdev->dev, "preboot full version: '%s'\n", preboot_ver);2329 2330		if (preboot_ver && preboot_ver != prop->preboot_ver) {2331			strscpy(btl_ver, prop->preboot_ver,2332				min((int) (preboot_ver - prop->preboot_ver), 31));2333			dev_info(hdev->dev, "%s\n", btl_ver);2334		}2335 2336		rc = hl_get_sw_major_minor_subminor(hdev, preboot_ver);2337		if (rc)2338			return rc;2339		preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);2340		if (preboot_ver) {2341			rc = hl_get_preboot_major_minor(hdev, preboot_ver);2342			kfree(preboot_ver);2343			if (rc)2344				return rc;2345		}2346 2347		break;2348	default:2349		dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);2350		return -EINVAL;2351	}2352 2353	return 0;2354}2355 2356/**2357 * hl_fw_dynamic_copy_image - copy image to memory allocated by the FW2358 *2359 * @hdev: pointer to the habanalabs device structure2360 * @fw: fw descriptor2361 * @fw_loader: managing structure for loading device's FW2362 */2363static int hl_fw_dynamic_copy_image(struct hl_device *hdev,2364						const struct firmware *fw,2365						struct fw_load_mgr *fw_loader)2366{2367	struct lkd_fw_comms_desc *fw_desc;2368	struct pci_mem_region *region;2369	void __iomem *dest;2370	u64 addr;2371	int rc;2372 2373	fw_desc = &fw_loader->dynamic_loader.comm_desc;2374	addr = le64_to_cpu(fw_desc->img_addr);2375 2376	/* find memory region to which to copy the image */2377	region = fw_loader->dynamic_loader.image_region;2378 2379	dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +2380					(addr - region->region_base);2381 2382	rc = hl_fw_copy_fw_to_device(hdev, fw, dest,2383					fw_loader->boot_fit_img.src_off,2384					fw_loader->boot_fit_img.copy_size);2385 2386	return rc;2387}2388 2389/**2390 * hl_fw_dynamic_copy_msg - copy msg to memory allocated by the FW2391 *2392 * @hdev: pointer to the habanalabs device structure2393 * @msg: message2394 * @fw_loader: managing structure for loading device's FW2395 */2396static int hl_fw_dynamic_copy_msg(struct hl_device *hdev,2397		struct lkd_msg_comms *msg, struct fw_load_mgr *fw_loader)2398{2399	struct lkd_fw_comms_desc *fw_desc;2400	struct pci_mem_region *region;2401	void __iomem *dest;2402	u64 addr;2403	int rc;2404 2405	fw_desc = &fw_loader->dynamic_loader.comm_desc;2406	addr = le64_to_cpu(fw_desc->img_addr);2407 2408	/* find memory region to which to copy the image */2409	region = fw_loader->dynamic_loader.image_region;2410 2411	dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +2412					(addr - region->region_base);2413 2414	rc = hl_fw_copy_msg_to_device(hdev, msg, dest, 0, 0);2415 2416	return rc;2417}2418 2419/**2420 * hl_fw_boot_fit_update_state - update internal data structures after boot-fit2421 *                               is loaded2422 *2423 * @hdev: pointer to the habanalabs device structure2424 * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 02425 * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 12426 *2427 * @return 0 on success, otherwise non-zero error code2428 */2429static void hl_fw_boot_fit_update_state(struct hl_device *hdev,2430						u32 cpu_boot_dev_sts0_reg,2431						u32 cpu_boot_dev_sts1_reg)2432{2433	struct asic_fixed_properties *prop = &hdev->asic_prop;2434 2435	hdev->fw_loader.fw_comp_loaded |= FW_TYPE_BOOT_CPU;2436 2437	/* Read boot_cpu status bits */2438	if (prop->fw_preboot_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_ENABLED) {2439		prop->fw_bootfit_cpu_boot_dev_sts0 =2440				RREG32(cpu_boot_dev_sts0_reg);2441 2442		prop->hard_reset_done_by_fw = !!(prop->fw_bootfit_cpu_boot_dev_sts0 &2443							CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);2444 2445		dev_dbg(hdev->dev, "Firmware boot CPU status0 %#x\n",2446					prop->fw_bootfit_cpu_boot_dev_sts0);2447	}2448 2449	if (prop->fw_cpu_boot_dev_sts1_valid) {2450		prop->fw_bootfit_cpu_boot_dev_sts1 =2451				RREG32(cpu_boot_dev_sts1_reg);2452 2453		dev_dbg(hdev->dev, "Firmware boot CPU status1 %#x\n",2454					prop->fw_bootfit_cpu_boot_dev_sts1);2455	}2456 2457	dev_dbg(hdev->dev, "Firmware boot CPU hard-reset is %s\n",2458			prop->hard_reset_done_by_fw ? "enabled" : "disabled");2459}2460 2461static void hl_fw_dynamic_update_linux_interrupt_if(struct hl_device *hdev)2462{2463	struct cpu_dyn_regs *dyn_regs =2464			&hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;2465 2466	/* Check whether all 3 interrupt interfaces are set, if not use a2467	 * single interface2468	 */2469	if (!hdev->asic_prop.gic_interrupts_enable &&2470			!(hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &2471				CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN)) {2472		dyn_regs->gic_host_halt_irq = dyn_regs->gic_host_pi_upd_irq;2473		dyn_regs->gic_host_ints_irq = dyn_regs->gic_host_pi_upd_irq;2474 2475		dev_warn(hdev->dev,2476			"Using a single interrupt interface towards cpucp");2477	}2478}2479/**2480 * hl_fw_dynamic_load_image - load FW image using dynamic protocol2481 *2482 * @hdev: pointer to the habanalabs device structure2483 * @fw_loader: managing structure for loading device's FW2484 * @load_fwc: the FW component to be loaded2485 * @img_ld_timeout: image load timeout2486 *2487 * @return 0 on success, otherwise non-zero error code2488 */2489static int hl_fw_dynamic_load_image(struct hl_device *hdev,2490						struct fw_load_mgr *fw_loader,2491						enum hl_fw_component load_fwc,2492						u32 img_ld_timeout)2493{2494	enum hl_fw_component cur_fwc;2495	const struct firmware *fw;2496	char *fw_name;2497	int rc = 0;2498 2499	/*2500	 * when loading image we have one of 2 scenarios:2501	 * 1. current FW component is preboot and we want to load boot-fit2502	 * 2. current FW component is boot-fit and we want to load linux2503	 */2504	if (load_fwc == FW_COMP_BOOT_FIT) {2505		cur_fwc = FW_COMP_PREBOOT;2506		fw_name = fw_loader->boot_fit_img.image_name;2507	} else {2508		cur_fwc = FW_COMP_BOOT_FIT;2509		fw_name = fw_loader->linux_img.image_name;2510	}2511 2512	/* request FW in order to communicate to FW the size to be allocated */2513	rc = hl_request_fw(hdev, &fw, fw_name);2514	if (rc)2515		return rc;2516 2517	/* store the image size for future validation */2518	fw_loader->dynamic_loader.fw_image_size = fw->size;2519 2520	rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, fw->size);2521	if (rc)2522		goto release_fw;2523 2524	/* read preboot version */2525	rc = hl_fw_dynamic_read_device_fw_version(hdev, cur_fwc,2526				fw_loader->dynamic_loader.comm_desc.cur_fw_ver);2527	if (rc)2528		goto release_fw;2529 2530	/* copy boot fit to space allocated by FW */2531	rc = hl_fw_dynamic_copy_image(hdev, fw, fw_loader);2532	if (rc)2533		goto release_fw;2534 2535	rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,2536						0, true,2537						fw_loader->cpu_timeout);2538	if (rc)2539		goto release_fw;2540 2541	rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,2542						0, false,2543						img_ld_timeout);2544 2545release_fw:2546	hl_release_firmware(fw);2547	return rc;2548}2549 2550static int hl_fw_dynamic_wait_for_boot_fit_active(struct hl_device *hdev,2551					struct fw_load_mgr *fw_loader)2552{2553	struct dynamic_fw_load_mgr *dyn_loader;2554	u32 status;2555	int rc;2556 2557	dyn_loader = &fw_loader->dynamic_loader;2558 2559	/*2560	 * Make sure CPU boot-loader is running2561	 * Note that the CPU_BOOT_STATUS_SRAM_AVAIL is generally set by Linux2562	 * yet there is a debug scenario in which we loading uboot (without Linux)2563	 * which at later stage is relocated to DRAM. In this case we expect2564	 * uboot to set the CPU_BOOT_STATUS_SRAM_AVAIL and so we add it to the2565	 * poll flags2566	 */2567	rc = hl_poll_timeout(2568		hdev,2569		le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),2570		status,2571		(status == CPU_BOOT_STATUS_READY_TO_BOOT) ||2572		(status == CPU_BOOT_STATUS_SRAM_AVAIL),2573		hdev->fw_poll_interval_usec,2574		dyn_loader->wait_for_bl_timeout);2575	if (rc) {2576		dev_err(hdev->dev, "failed to wait for boot (status = %d)\n", status);2577		return rc;2578	}2579 2580	dev_dbg(hdev->dev, "uboot status = %d\n", status);2581	return 0;2582}2583 2584static int hl_fw_dynamic_wait_for_linux_active(struct hl_device *hdev,2585						struct fw_load_mgr *fw_loader)2586{2587	struct dynamic_fw_load_mgr *dyn_loader;2588	u32 status;2589	int rc;2590 2591	dyn_loader = &fw_loader->dynamic_loader;2592 2593	/* Make sure CPU linux is running */2594 2595	rc = hl_poll_timeout(2596		hdev,2597		le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),2598		status,2599		(status == CPU_BOOT_STATUS_SRAM_AVAIL),2600		hdev->fw_poll_interval_usec,2601		fw_loader->cpu_timeout);2602	if (rc) {2603		dev_err(hdev->dev, "failed to wait for Linux (status = %d)\n", status);2604		return rc;2605	}2606 2607	dev_dbg(hdev->dev, "Boot status = %d\n", status);2608	return 0;2609}2610 2611/**2612 * hl_fw_linux_update_state -	update internal data structures after Linux2613 *				is loaded.2614 *				Note: Linux initialization is comprised mainly2615 *				of two stages - loading kernel (SRAM_AVAIL)2616 *				& loading ARMCP.2617 *				Therefore reading boot device status in any of2618 *				these stages might result in different values.2619 *2620 * @hdev: pointer to the habanalabs device structure2621 * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 02622 * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 12623 *2624 * @return 0 on success, otherwise non-zero error code2625 */2626static void hl_fw_linux_update_state(struct hl_device *hdev,2627						u32 cpu_boot_dev_sts0_reg,2628						u32 cpu_boot_dev_sts1_reg)2629{2630	struct asic_fixed_properties *prop = &hdev->asic_prop;2631 2632	hdev->fw_loader.fw_comp_loaded |= FW_TYPE_LINUX;2633 2634	/* Read FW application security bits */2635	if (prop->fw_cpu_boot_dev_sts0_valid) {2636		prop->fw_app_cpu_boot_dev_sts0 = RREG32(cpu_boot_dev_sts0_reg);2637 2638		prop->hard_reset_done_by_fw = !!(prop->fw_app_cpu_boot_dev_sts0 &2639							CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);2640 2641		if (prop->fw_app_cpu_boot_dev_sts0 &2642				CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN)2643			prop->gic_interrupts_enable = false;2644 2645		dev_dbg(hdev->dev,2646			"Firmware application CPU status0 %#x\n",2647			prop->fw_app_cpu_boot_dev_sts0);2648 2649		dev_dbg(hdev->dev, "GIC controller is %s\n",2650				prop->gic_interrupts_enable ?2651						"enabled" : "disabled");2652	}2653 2654	if (prop->fw_cpu_boot_dev_sts1_valid) {2655		prop->fw_app_cpu_boot_dev_sts1 = RREG32(cpu_boot_dev_sts1_reg);2656 2657		dev_dbg(hdev->dev,2658			"Firmware application CPU status1 %#x\n",2659			prop->fw_app_cpu_boot_dev_sts1);2660	}2661 2662	dev_dbg(hdev->dev, "Firmware application CPU hard-reset is %s\n",2663			prop->hard_reset_done_by_fw ? "enabled" : "disabled");2664 2665	dev_info(hdev->dev, "Successfully loaded firmware to device\n");2666}2667 2668/**2669 * hl_fw_dynamic_send_msg - send a COMMS message with attached data2670 *2671 * @hdev: pointer to the habanalabs device structure2672 * @fw_loader: managing structure for loading device's FW2673 * @msg_type: message type2674 * @data: data to be sent2675 *2676 * @return 0 on success, otherwise non-zero error code2677 */2678static int hl_fw_dynamic_send_msg(struct hl_device *hdev,2679		struct fw_load_mgr *fw_loader, u8 msg_type, void *data)2680{2681	struct lkd_msg_comms *msg;2682	int rc;2683 2684	msg = kzalloc(sizeof(*msg), GFP_KERNEL);2685	if (!msg)2686		return -ENOMEM;2687 2688	/* create message to be sent */2689	msg->header.type = msg_type;2690	msg->header.size = cpu_to_le16(sizeof(struct comms_msg_header));2691	msg->header.magic = cpu_to_le32(HL_COMMS_MSG_MAGIC);2692 2693	switch (msg_type) {2694	case HL_COMMS_RESET_CAUSE_TYPE:2695		msg->reset_cause = *(__u8 *) data;2696		break;2697 2698	default:2699		dev_err(hdev->dev,2700			"Send COMMS message - invalid message type %u\n",2701			msg_type);2702		rc = -EINVAL;2703		goto out;2704	}2705 2706	rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader,2707			sizeof(struct lkd_msg_comms));2708	if (rc)2709		goto out;2710 2711	/* copy message to space allocated by FW */2712	rc = hl_fw_dynamic_copy_msg(hdev, msg, fw_loader);2713	if (rc)2714		goto out;2715 2716	rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,2717						0, true,2718						fw_loader->cpu_timeout);2719	if (rc)2720		goto out;2721 2722	rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,2723						0, true,2724						fw_loader->cpu_timeout);2725 2726out:2727	kfree(msg);2728	return rc;2729}2730 2731/**2732 * hl_fw_dynamic_init_cpu - initialize the device CPU using dynamic protocol2733 *2734 * @hdev: pointer to the habanalabs device structure2735 * @fw_loader: managing structure for loading device's FW2736 *2737 * @return 0 on success, otherwise non-zero error code2738 *2739 * brief: the dynamic protocol is master (LKD) slave (FW CPU) protocol.2740 * the communication is done using registers:2741 * - LKD command register2742 * - FW status register2743 * the protocol is race free. this goal is achieved by splitting the requests2744 * and response to known synchronization points between the LKD and the FW.2745 * each response to LKD request is known and bound to a predefined timeout.2746 * in case of timeout expiration without the desired status from FW- the2747 * protocol (and hence the boot) will fail.2748 */2749static int hl_fw_dynamic_init_cpu(struct hl_device *hdev,2750					struct fw_load_mgr *fw_loader)2751{2752	struct cpu_dyn_regs *dyn_regs;2753	int rc, fw_error_rc;2754 2755	dev_info(hdev->dev,2756		"Loading %sfirmware to device, may take some time...\n",2757		hdev->asic_prop.fw_security_enabled ? "secured " : "");2758 2759	/* initialize FW descriptor as invalid */2760	fw_loader->dynamic_loader.fw_desc_valid = false;2761 2762	/*2763	 * In this stage, "cpu_dyn_regs" contains only LKD's hard coded values!2764	 * It will be updated from FW after hl_fw_dynamic_request_descriptor().2765	 */2766	dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;2767 2768	rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_RST_STATE,2769						0, true,2770						fw_loader->cpu_timeout);2771	if (rc)2772		goto protocol_err;2773 2774	if (hdev->reset_info.curr_reset_cause) {2775		rc = hl_fw_dynamic_send_msg(hdev, fw_loader,2776				HL_COMMS_RESET_CAUSE_TYPE, &hdev->reset_info.curr_reset_cause);2777		if (rc)2778			goto protocol_err;2779 2780		/* Clear current reset cause */2781		hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;2782	}2783 2784	rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, sizeof(struct lkd_msg_comms));2785	if (rc)2786		goto protocol_err;2787 2788	if (hdev->asic_prop.support_dynamic_resereved_fw_size)2789		hdev->asic_prop.reserved_fw_mem_size =2790			le32_to_cpu(fw_loader->dynamic_loader.comm_desc.rsvd_mem_size_mb) * SZ_1M;2791 2792	if (!(hdev->fw_components & FW_TYPE_BOOT_CPU)) {2793		struct lkd_fw_binning_info *binning_info;2794 2795		/* read preboot version */2796		rc = hl_fw_dynamic_read_device_fw_version(hdev, FW_COMP_PREBOOT,2797				fw_loader->dynamic_loader.comm_desc.cur_fw_ver);2798		if (rc)2799			return rc;2800 2801		/* read binning info from preboot */2802		if (hdev->support_preboot_binning) {2803			binning_info = &fw_loader->dynamic_loader.comm_desc.binning_info;2804			hdev->tpc_binning = le64_to_cpu(binning_info->tpc_mask_l);2805			hdev->dram_binning = le32_to_cpu(binning_info->dram_mask);2806			hdev->edma_binning = le32_to_cpu(binning_info->edma_mask);2807			hdev->decoder_binning = le32_to_cpu(binning_info->dec_mask);2808			hdev->rotator_binning = le32_to_cpu(binning_info->rot_mask);2809 2810			rc = hdev->asic_funcs->set_dram_properties(hdev);2811			if (rc)2812				return rc;2813 2814			rc = hdev->asic_funcs->set_binning_masks(hdev);2815			if (rc)2816				return rc;2817 2818			dev_dbg(hdev->dev,2819				"Read binning masks: tpc: 0x%llx, dram: 0x%llx, edma: 0x%x, dec: 0x%x, rot:0x%x\n",2820				hdev->tpc_binning, hdev->dram_binning, hdev->edma_binning,2821				hdev->decoder_binning, hdev->rotator_binning);2822		}2823 2824		return 0;2825	}2826 2827	/* load boot fit to FW */2828	rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_BOOT_FIT,2829						fw_loader->boot_fit_timeout);2830	if (rc) {2831		dev_err(hdev->dev, "failed to load boot fit\n");2832		goto protocol_err;2833	}2834 2835	rc = hl_fw_dynamic_wait_for_boot_fit_active(hdev, fw_loader);2836	if (rc)2837		goto protocol_err;2838 2839	hl_fw_boot_fit_update_state(hdev,2840			le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),2841			le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));2842 2843	/*2844	 * when testing FW load (without Linux) on PLDM we don't want to2845	 * wait until boot fit is active as it may take several hours.2846	 * instead, we load the bootfit and let it do all initialization in2847	 * the background.2848	 */2849	if (hdev->pldm && !(hdev->fw_components & FW_TYPE_LINUX))2850		return 0;2851 2852	/* Enable DRAM scrambling before Linux boot and after successful2853	 *  UBoot2854	 */2855	hdev->asic_funcs->init_cpu_scrambler_dram(hdev);2856 2857	if (!(hdev->fw_components & FW_TYPE_LINUX)) {2858		dev_dbg(hdev->dev, "Skip loading Linux F/W\n");2859		return 0;2860	}2861 2862	if (fw_loader->skip_bmc) {2863		rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader,2864							COMMS_SKIP_BMC, 0,2865							true,2866							fw_loader->cpu_timeout);2867		if (rc) {2868			dev_err(hdev->dev, "failed to load boot fit\n");2869			goto protocol_err;2870		}2871	}2872 2873	/* load Linux image to FW */2874	rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_LINUX,2875							fw_loader->cpu_timeout);2876	if (rc) {2877		dev_err(hdev->dev, "failed to load Linux\n");2878		goto protocol_err;2879	}2880 2881	rc = hl_fw_dynamic_wait_for_linux_active(hdev, fw_loader);2882	if (rc)2883		goto protocol_err;2884 2885	hl_fw_linux_update_state(hdev,2886				le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),2887				le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));2888 2889	hl_fw_dynamic_update_linux_interrupt_if(hdev);2890 2891protocol_err:2892	if (fw_loader->dynamic_loader.fw_desc_valid) {2893		fw_error_rc = fw_read_errors(hdev, le32_to_cpu(dyn_regs->cpu_boot_err0),2894				le32_to_cpu(dyn_regs->cpu_boot_err1),2895				le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),2896				le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));2897 2898		if (fw_error_rc)2899			return fw_error_rc;2900	}2901 2902	return rc;2903}2904 2905/**2906 * hl_fw_static_init_cpu - initialize the device CPU using static protocol2907 *2908 * @hdev: pointer to the habanalabs device structure2909 * @fw_loader: managing structure for loading device's FW2910 *2911 * @return 0 on success, otherwise non-zero error code2912 */2913static int hl_fw_static_init_cpu(struct hl_device *hdev,2914					struct fw_load_mgr *fw_loader)2915{2916	u32 cpu_msg_status_reg, cpu_timeout, msg_to_cpu_reg, status;2917	u32 cpu_boot_dev_status0_reg, cpu_boot_dev_status1_reg;2918	struct static_fw_load_mgr *static_loader;2919	u32 cpu_boot_status_reg;2920	int rc;2921 2922	if (!(hdev->fw_components & FW_TYPE_BOOT_CPU))2923		return 0;2924 2925	/* init common loader parameters */2926	cpu_timeout = fw_loader->cpu_timeout;2927 2928	/* init static loader parameters */2929	static_loader = &fw_loader->static_loader;2930	cpu_msg_status_reg = static_loader->cpu_cmd_status_to_host_reg;2931	msg_to_cpu_reg = static_loader->kmd_msg_to_cpu_reg;2932	cpu_boot_dev_status0_reg = static_loader->cpu_boot_dev_status0_reg;2933	cpu_boot_dev_status1_reg = static_loader->cpu_boot_dev_status1_reg;2934	cpu_boot_status_reg = static_loader->cpu_boot_status_reg;2935 2936	dev_info(hdev->dev, "Going to wait for device boot (up to %lds)\n",2937		cpu_timeout / USEC_PER_SEC);2938 2939	/* Wait for boot FIT request */2940	rc = hl_poll_timeout(2941		hdev,2942		cpu_boot_status_reg,2943		status,2944		status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT,2945		hdev->fw_poll_interval_usec,2946		fw_loader->boot_fit_timeout);2947 2948	if (rc) {2949		dev_dbg(hdev->dev,2950			"No boot fit request received (status = %d), resuming boot\n", status);2951	} else {2952		rc = hdev->asic_funcs->load_boot_fit_to_device(hdev);2953		if (rc)2954			goto out;2955 2956		/* Clear device CPU message status */2957		WREG32(cpu_msg_status_reg, CPU_MSG_CLR);2958 2959		/* Signal device CPU that boot loader is ready */2960		WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);2961 2962		/* Poll for CPU device ack */2963		rc = hl_poll_timeout(2964			hdev,2965			cpu_msg_status_reg,2966			status,2967			status == CPU_MSG_OK,2968			hdev->fw_poll_interval_usec,2969			fw_loader->boot_fit_timeout);2970 2971		if (rc) {2972			dev_err(hdev->dev,2973				"Timeout waiting for boot fit load ack (status = %d)\n", status);2974			goto out;2975		}2976 2977		/* Clear message */2978		WREG32(msg_to_cpu_reg, KMD_MSG_NA);2979	}2980 2981	/*2982	 * Make sure CPU boot-loader is running2983	 * Note that the CPU_BOOT_STATUS_SRAM_AVAIL is generally set by Linux2984	 * yet there is a debug scenario in which we loading uboot (without Linux)2985	 * which at later stage is relocated to DRAM. In this case we expect2986	 * uboot to set the CPU_BOOT_STATUS_SRAM_AVAIL and so we add it to the2987	 * poll flags2988	 */2989	rc = hl_poll_timeout(2990		hdev,2991		cpu_boot_status_reg,2992		status,2993		(status == CPU_BOOT_STATUS_DRAM_RDY) ||2994		(status == CPU_BOOT_STATUS_NIC_FW_RDY) ||2995		(status == CPU_BOOT_STATUS_READY_TO_BOOT) ||2996		(status == CPU_BOOT_STATUS_SRAM_AVAIL),2997		hdev->fw_poll_interval_usec,2998		cpu_timeout);2999 3000	dev_dbg(hdev->dev, "uboot status = %d\n", status);3001 3002	/* Read U-Boot version now in case we will later fail */3003	hl_fw_static_read_device_fw_version(hdev, FW_COMP_BOOT_FIT);3004 3005	/* update state according to boot stage */3006	hl_fw_boot_fit_update_state(hdev, cpu_boot_dev_status0_reg,3007						cpu_boot_dev_status1_reg);3008 3009	if (rc) {3010		detect_cpu_boot_status(hdev, status);3011		rc = -EIO;3012		goto out;3013	}3014 3015	/* Enable DRAM scrambling before Linux boot and after successful3016	 *  UBoot3017	 */3018	hdev->asic_funcs->init_cpu_scrambler_dram(hdev);3019 3020	if (!(hdev->fw_components & FW_TYPE_LINUX)) {3021		dev_info(hdev->dev, "Skip loading Linux F/W\n");3022		rc = 0;3023		goto out;3024	}3025 3026	if (status == CPU_BOOT_STATUS_SRAM_AVAIL) {3027		rc = 0;3028		goto out;3029	}3030 3031	dev_info(hdev->dev,3032		"Loading firmware to device, may take some time...\n");3033 3034	rc = hdev->asic_funcs->load_firmware_to_device(hdev);3035	if (rc)3036		goto out;3037 3038	if (fw_loader->skip_bmc) {3039		WREG32(msg_to_cpu_reg, KMD_MSG_SKIP_BMC);3040 3041		rc = hl_poll_timeout(3042			hdev,3043			cpu_boot_status_reg,3044			status,3045			(status == CPU_BOOT_STATUS_BMC_WAITING_SKIPPED),3046			hdev->fw_poll_interval_usec,3047			cpu_timeout);3048 3049		if (rc) {3050			dev_err(hdev->dev,3051				"Failed to get ACK on skipping BMC (status = %d)\n",3052				status);3053			WREG32(msg_to_cpu_reg, KMD_MSG_NA);3054			rc = -EIO;3055			goto out;3056		}3057	}3058 3059	WREG32(msg_to_cpu_reg, KMD_MSG_FIT_RDY);3060 3061	rc = hl_poll_timeout(3062		hdev,3063		cpu_boot_status_reg,3064		status,3065		(status == CPU_BOOT_STATUS_SRAM_AVAIL),3066		hdev->fw_poll_interval_usec,3067		cpu_timeout);3068 3069	/* Clear message */3070	WREG32(msg_to_cpu_reg, KMD_MSG_NA);3071 3072	if (rc) {3073		if (status == CPU_BOOT_STATUS_FIT_CORRUPTED)3074			dev_err(hdev->dev,3075				"Device reports FIT image is corrupted\n");3076		else3077			dev_err(hdev->dev,3078				"Failed to load firmware to device (status = %d)\n",3079				status);3080 3081		rc = -EIO;3082		goto out;3083	}3084 3085	rc = fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,3086					fw_loader->static_loader.boot_err1_reg,3087					cpu_boot_dev_status0_reg,3088					cpu_boot_dev_status1_reg);3089	if (rc)3090		return rc;3091 3092	hl_fw_linux_update_state(hdev, cpu_boot_dev_status0_reg,3093						cpu_boot_dev_status1_reg);3094 3095	return 0;3096 3097out:3098	fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,3099					fw_loader->static_loader.boot_err1_reg,3100					cpu_boot_dev_status0_reg,3101					cpu_boot_dev_status1_reg);3102 3103	return rc;3104}3105 3106/**3107 * hl_fw_init_cpu - initialize the device CPU3108 *3109 * @hdev: pointer to the habanalabs device structure3110 *3111 * @return 0 on success, otherwise non-zero error code3112 *3113 * perform necessary initializations for device's CPU. takes into account if3114 * init protocol is static or dynamic.3115 */3116int hl_fw_init_cpu(struct hl_device *hdev)3117{3118	struct asic_fixed_properties *prop = &hdev->asic_prop;3119	struct fw_load_mgr *fw_loader = &hdev->fw_loader;3120 3121	return  prop->dynamic_fw_load ?3122			hl_fw_dynamic_init_cpu(hdev, fw_loader) :3123			hl_fw_static_init_cpu(hdev, fw_loader);3124}3125 3126void hl_fw_set_pll_profile(struct hl_device *hdev)3127{3128	hl_fw_set_frequency(hdev, hdev->asic_prop.clk_pll_index,3129				hdev->asic_prop.max_freq_value);3130}3131 3132int hl_fw_get_clk_rate(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk)3133{3134	long value;3135 3136	if (!hl_device_operational(hdev, NULL))3137		return -ENODEV;3138 3139	if (!hdev->pdev) {3140		*cur_clk = 0;3141		*max_clk = 0;3142		return 0;3143	}3144 3145	value = hl_fw_get_frequency(hdev, hdev->asic_prop.clk_pll_index, false);3146 3147	if (value < 0) {3148		dev_err(hdev->dev, "Failed to retrieve device max clock %ld\n", value);3149		return value;3150	}3151 3152	*max_clk = (value / 1000 / 1000);3153 3154	value = hl_fw_get_frequency(hdev, hdev->asic_prop.clk_pll_index, true);3155 3156	if (value < 0) {3157		dev_err(hdev->dev, "Failed to retrieve device current clock %ld\n", value);3158		return value;3159	}3160 3161	*cur_clk = (value / 1000 / 1000);3162 3163	return 0;3164}3165 3166long hl_fw_get_frequency(struct hl_device *hdev, u32 pll_index, bool curr)3167{3168	struct cpucp_packet pkt;3169	u32 used_pll_idx;3170	u64 result;3171	int rc;3172 3173	rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);3174	if (rc)3175		return rc;3176 3177	memset(&pkt, 0, sizeof(pkt));3178 3179	if (curr)3180		pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_CURR_GET <<3181						CPUCP_PKT_CTL_OPCODE_SHIFT);3182	else3183		pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_GET << CPUCP_PKT_CTL_OPCODE_SHIFT);3184 3185	pkt.pll_index = cpu_to_le32((u32)used_pll_idx);3186 3187	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);3188	if (rc) {3189		if (rc != -EAGAIN)3190			dev_err(hdev->dev, "Failed to get frequency of PLL %d, error %d\n",3191				used_pll_idx, rc);3192		return rc;3193	}3194 3195	return (long) result;3196}3197 3198void hl_fw_set_frequency(struct hl_device *hdev, u32 pll_index, u64 freq)3199{3200	struct cpucp_packet pkt;3201	u32 used_pll_idx;3202	int rc;3203 3204	rc = get_used_pll_index(hdev, pll_index, &used_pll_idx);3205	if (rc)3206		return;3207 3208	memset(&pkt, 0, sizeof(pkt));3209 3210	pkt.ctl = cpu_to_le32(CPUCP_PACKET_FREQUENCY_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);3211	pkt.pll_index = cpu_to_le32((u32)used_pll_idx);3212	pkt.value = cpu_to_le64(freq);3213 3214	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);3215	if (rc && rc != -EAGAIN)3216		dev_err(hdev->dev, "Failed to set frequency to PLL %d, error %d\n",3217			used_pll_idx, rc);3218}3219 3220long hl_fw_get_max_power(struct hl_device *hdev)3221{3222	struct cpucp_packet pkt;3223	u64 result;3224	int rc;3225 3226	memset(&pkt, 0, sizeof(pkt));3227 3228	pkt.ctl = cpu_to_le32(CPUCP_PACKET_MAX_POWER_GET << CPUCP_PKT_CTL_OPCODE_SHIFT);3229 3230	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, &result);3231	if (rc) {3232		if (rc != -EAGAIN)3233			dev_err(hdev->dev, "Failed to get max power, error %d\n", rc);3234		return rc;3235	}3236 3237	return result;3238}3239 3240void hl_fw_set_max_power(struct hl_device *hdev)3241{3242	struct cpucp_packet pkt;3243	int rc;3244 3245	/* TODO: remove this after simulator supports this packet */3246	if (!hdev->pdev)3247		return;3248 3249	memset(&pkt, 0, sizeof(pkt));3250 3251	pkt.ctl = cpu_to_le32(CPUCP_PACKET_MAX_POWER_SET << CPUCP_PKT_CTL_OPCODE_SHIFT);3252	pkt.value = cpu_to_le64(hdev->max_power);3253 3254	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), 0, NULL);3255	if (rc && rc != -EAGAIN)3256		dev_err(hdev->dev, "Failed to set max power, error %d\n", rc);3257}3258 3259static int hl_fw_get_sec_attest_data(struct hl_device *hdev, u32 packet_id, void *data, u32 size,3260					u32 nonce, u32 timeout)3261{3262	struct cpucp_packet pkt = {};3263	dma_addr_t req_dma_addr;3264	void *req_cpu_addr;3265	int rc;3266 3267	req_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, size, &req_dma_addr);3268	if (!req_cpu_addr) {3269		dev_err(hdev->dev,3270			"Failed to allocate DMA memory for CPU-CP packet %u\n", packet_id);3271		return -ENOMEM;3272	}3273 3274	memset(data, 0, size);3275 3276	pkt.ctl = cpu_to_le32(packet_id << CPUCP_PKT_CTL_OPCODE_SHIFT);3277	pkt.addr = cpu_to_le64(req_dma_addr);3278	pkt.data_max_size = cpu_to_le32(size);3279	pkt.nonce = cpu_to_le32(nonce);3280 3281	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), timeout, NULL);3282	if (rc) {3283		if (rc != -EAGAIN)3284			dev_err(hdev->dev,3285				"Failed to handle CPU-CP pkt %u, error %d\n", packet_id, rc);3286		goto out;3287	}3288 3289	memcpy(data, req_cpu_addr, size);3290 3291out:3292	hl_cpu_accessible_dma_pool_free(hdev, size, req_cpu_addr);3293 3294	return rc;3295}3296 3297int hl_fw_get_sec_attest_info(struct hl_device *hdev, struct cpucp_sec_attest_info *sec_attest_info,3298				u32 nonce)3299{3300	return hl_fw_get_sec_attest_data(hdev, CPUCP_PACKET_SEC_ATTEST_GET, sec_attest_info,3301					sizeof(struct cpucp_sec_attest_info), nonce,3302					HL_CPUCP_SEC_ATTEST_INFO_TINEOUT_USEC);3303}3304 3305int hl_fw_get_dev_info_signed(struct hl_device *hdev,3306			      struct cpucp_dev_info_signed *dev_info_signed, u32 nonce)3307{3308	return hl_fw_get_sec_attest_data(hdev, CPUCP_PACKET_INFO_SIGNED_GET, dev_info_signed,3309					 sizeof(struct cpucp_dev_info_signed), nonce,3310					 HL_CPUCP_SEC_ATTEST_INFO_TINEOUT_USEC);3311}3312 3313int hl_fw_send_generic_request(struct hl_device *hdev, enum hl_passthrough_type sub_opcode,3314						dma_addr_t buff, u32 *size)3315{3316	struct cpucp_packet pkt = {};3317	u64 result;3318	int rc = 0;3319 3320	pkt.ctl = cpu_to_le32(CPUCP_PACKET_GENERIC_PASSTHROUGH << CPUCP_PKT_CTL_OPCODE_SHIFT);3321	pkt.addr = cpu_to_le64(buff);3322	pkt.data_max_size = cpu_to_le32(*size);3323	pkt.pkt_subidx = cpu_to_le32(sub_opcode);3324 3325	rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *)&pkt, sizeof(pkt),3326						HL_CPUCP_INFO_TIMEOUT_USEC, &result);3327	if (rc) {3328		if (rc != -EAGAIN)3329			dev_err(hdev->dev, "failed to send CPUCP data of generic fw pkt\n");3330	} else {3331		dev_dbg(hdev->dev, "generic pkt was successful, result: 0x%llx\n", result);3332	}3333 3334	*size = (u32)result;3335 3336	return rc;3337}3338