brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.02 3/*4 * Copyright 2016-2022 HabanaLabs, Ltd.5 * All Rights Reserved.6 */7 8#include "gaudiP.h"9#include "../include/hw_ip/mmu/mmu_general.h"10#include "../include/hw_ip/mmu/mmu_v1_1.h"11#include "../include/gaudi/gaudi_masks.h"12#include "../include/gaudi/gaudi_fw_if.h"13#include "../include/gaudi/gaudi_reg_map.h"14#include "../include/gaudi/gaudi_async_ids_map_extended.h"15 16#include <linux/module.h>17#include <linux/pci.h>18#include <linux/firmware.h>19#include <linux/hwmon.h>20#include <linux/iommu.h>21#include <linux/seq_file.h>22 23/*24 * Gaudi security scheme:25 *26 * 1. Host is protected by:27 *        - Range registers28 *        - MMU29 *30 * 2. DDR is protected by:31 *        - Range registers (protect the first 512MB)32 *33 * 3. Configuration is protected by:34 *        - Range registers35 *        - Protection bits36 *37 * MMU is always enabled.38 *39 * QMAN DMA channels 0,1 (PCI DMAN):40 *     - DMA is not secured.41 *     - PQ and CQ are secured.42 *     - CP is secured: The driver needs to parse CB but WREG should be allowed43 *                      because of TDMA (tensor DMA). Hence, WREG is always not44 *                      secured.45 *46 * When the driver needs to use DMA it will check that Gaudi is idle, set DMA47 * channel 0 to be secured, execute the DMA and change it back to not secured.48 * Currently, the driver doesn't use the DMA while there are compute jobs49 * running.50 *51 * The current use cases for the driver to use the DMA are:52 *     - Clear SRAM on context switch (happens on context switch when device is53 *       idle)54 *     - MMU page tables area clear (happens on init)55 *56 * QMAN DMA 2-7, TPC, MME, NIC:57 * PQ is secured and is located on the Host (HBM CON TPC3 bug)58 * CQ, CP and the engine are not secured59 *60 */61 62#define GAUDI_BOOT_FIT_FILE	"habanalabs/gaudi/gaudi-boot-fit.itb"63#define GAUDI_LINUX_FW_FILE	"habanalabs/gaudi/gaudi-fit.itb"64#define GAUDI_TPC_FW_FILE	"habanalabs/gaudi/gaudi_tpc.bin"65 66MODULE_FIRMWARE(GAUDI_BOOT_FIT_FILE);67MODULE_FIRMWARE(GAUDI_LINUX_FW_FILE);68MODULE_FIRMWARE(GAUDI_TPC_FW_FILE);69 70#define GAUDI_DMA_POOL_BLK_SIZE		0x100 /* 256 bytes */71 72#define GAUDI_RESET_TIMEOUT_MSEC	2000		/* 2000ms */73#define GAUDI_RESET_WAIT_MSEC		1		/* 1ms */74#define GAUDI_CPU_RESET_WAIT_MSEC	200		/* 200ms */75#define GAUDI_TEST_QUEUE_WAIT_USEC	100000		/* 100ms */76 77#define GAUDI_PLDM_RESET_WAIT_MSEC	1000		/* 1s */78#define GAUDI_PLDM_HRESET_TIMEOUT_MSEC	20000		/* 20s */79#define GAUDI_PLDM_TEST_QUEUE_WAIT_USEC	1000000		/* 1s */80#define GAUDI_PLDM_MMU_TIMEOUT_USEC	(MMU_CONFIG_TIMEOUT_USEC * 100)81#define GAUDI_PLDM_QMAN0_TIMEOUT_USEC	(HL_DEVICE_TIMEOUT_USEC * 30)82#define GAUDI_PLDM_TPC_KERNEL_WAIT_USEC	(HL_DEVICE_TIMEOUT_USEC * 30)83#define GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC	4000000		/* 4s */84#define GAUDI_MSG_TO_CPU_TIMEOUT_USEC	4000000		/* 4s */85#define GAUDI_WAIT_FOR_BL_TIMEOUT_USEC	15000000	/* 15s */86 87#define GAUDI_QMAN0_FENCE_VAL		0x72E91AB988 89#define GAUDI_MAX_STRING_LEN		2090 91#define GAUDI_CB_POOL_CB_CNT		51292#define GAUDI_CB_POOL_CB_SIZE		0x20000 /* 128KB */93 94#define GAUDI_ALLOC_CPU_MEM_RETRY_CNT	395 96#define GAUDI_NUM_OF_TPC_INTR_CAUSE	2097 98#define GAUDI_NUM_OF_QM_ERR_CAUSE	1699 100#define GAUDI_NUM_OF_QM_ARB_ERR_CAUSE	3101 102#define GAUDI_ARB_WDT_TIMEOUT		0xEE6b27FF /* 8 seconds */103 104#define HBM_SCRUBBING_TIMEOUT_US	1000000 /* 1s */105 106#define BIN_REG_STRING_SIZE	sizeof("0b10101010101010101010101010101010")107 108#define MONITOR_SOB_STRING_SIZE		256109 110static u32 gaudi_stream_master[GAUDI_STREAM_MASTER_ARR_SIZE] = {111	GAUDI_QUEUE_ID_DMA_0_0,112	GAUDI_QUEUE_ID_DMA_0_1,113	GAUDI_QUEUE_ID_DMA_0_2,114	GAUDI_QUEUE_ID_DMA_0_3,115	GAUDI_QUEUE_ID_DMA_1_0,116	GAUDI_QUEUE_ID_DMA_1_1,117	GAUDI_QUEUE_ID_DMA_1_2,118	GAUDI_QUEUE_ID_DMA_1_3119};120 121static const u8 gaudi_dma_assignment[GAUDI_DMA_MAX] = {122	[GAUDI_PCI_DMA_1] = GAUDI_ENGINE_ID_DMA_0,123	[GAUDI_PCI_DMA_2] = GAUDI_ENGINE_ID_DMA_1,124	[GAUDI_HBM_DMA_1] = GAUDI_ENGINE_ID_DMA_2,125	[GAUDI_HBM_DMA_2] = GAUDI_ENGINE_ID_DMA_3,126	[GAUDI_HBM_DMA_3] = GAUDI_ENGINE_ID_DMA_4,127	[GAUDI_HBM_DMA_4] = GAUDI_ENGINE_ID_DMA_5,128	[GAUDI_HBM_DMA_5] = GAUDI_ENGINE_ID_DMA_6,129	[GAUDI_HBM_DMA_6] = GAUDI_ENGINE_ID_DMA_7130};131 132static const u8 gaudi_cq_assignment[NUMBER_OF_CMPLT_QUEUES] = {133	[0] = GAUDI_QUEUE_ID_DMA_0_0,134	[1] = GAUDI_QUEUE_ID_DMA_0_1,135	[2] = GAUDI_QUEUE_ID_DMA_0_2,136	[3] = GAUDI_QUEUE_ID_DMA_0_3,137	[4] = GAUDI_QUEUE_ID_DMA_1_0,138	[5] = GAUDI_QUEUE_ID_DMA_1_1,139	[6] = GAUDI_QUEUE_ID_DMA_1_2,140	[7] = GAUDI_QUEUE_ID_DMA_1_3,141};142 143static const u16 gaudi_packet_sizes[MAX_PACKET_ID] = {144	[PACKET_WREG_32]	= sizeof(struct packet_wreg32),145	[PACKET_WREG_BULK]	= sizeof(struct packet_wreg_bulk),146	[PACKET_MSG_LONG]	= sizeof(struct packet_msg_long),147	[PACKET_MSG_SHORT]	= sizeof(struct packet_msg_short),148	[PACKET_CP_DMA]		= sizeof(struct packet_cp_dma),149	[PACKET_REPEAT]		= sizeof(struct packet_repeat),150	[PACKET_MSG_PROT]	= sizeof(struct packet_msg_prot),151	[PACKET_FENCE]		= sizeof(struct packet_fence),152	[PACKET_LIN_DMA]	= sizeof(struct packet_lin_dma),153	[PACKET_NOP]		= sizeof(struct packet_nop),154	[PACKET_STOP]		= sizeof(struct packet_stop),155	[PACKET_ARB_POINT]	= sizeof(struct packet_arb_point),156	[PACKET_WAIT]		= sizeof(struct packet_wait),157	[PACKET_LOAD_AND_EXE]	= sizeof(struct packet_load_and_exe)158};159 160static inline bool validate_packet_id(enum packet_id id)161{162	switch (id) {163	case PACKET_WREG_32:164	case PACKET_WREG_BULK:165	case PACKET_MSG_LONG:166	case PACKET_MSG_SHORT:167	case PACKET_CP_DMA:168	case PACKET_REPEAT:169	case PACKET_MSG_PROT:170	case PACKET_FENCE:171	case PACKET_LIN_DMA:172	case PACKET_NOP:173	case PACKET_STOP:174	case PACKET_ARB_POINT:175	case PACKET_WAIT:176	case PACKET_LOAD_AND_EXE:177		return true;178	default:179		return false;180	}181}182 183static const char * const184gaudi_tpc_interrupts_cause[GAUDI_NUM_OF_TPC_INTR_CAUSE] = {185	"tpc_address_exceed_slm",186	"tpc_div_by_0",187	"tpc_spu_mac_overflow",188	"tpc_spu_addsub_overflow",189	"tpc_spu_abs_overflow",190	"tpc_spu_fp_dst_nan_inf",191	"tpc_spu_fp_dst_denorm",192	"tpc_vpu_mac_overflow",193	"tpc_vpu_addsub_overflow",194	"tpc_vpu_abs_overflow",195	"tpc_vpu_fp_dst_nan_inf",196	"tpc_vpu_fp_dst_denorm",197	"tpc_assertions",198	"tpc_illegal_instruction",199	"tpc_pc_wrap_around",200	"tpc_qm_sw_err",201	"tpc_hbw_rresp_err",202	"tpc_hbw_bresp_err",203	"tpc_lbw_rresp_err",204	"tpc_lbw_bresp_err"205};206 207static const char * const208gaudi_qman_error_cause[GAUDI_NUM_OF_QM_ERR_CAUSE] = {209	"PQ AXI HBW error",210	"CQ AXI HBW error",211	"CP AXI HBW error",212	"CP error due to undefined OPCODE",213	"CP encountered STOP OPCODE",214	"CP AXI LBW error",215	"CP WRREG32 or WRBULK returned error",216	"N/A",217	"FENCE 0 inc over max value and clipped",218	"FENCE 1 inc over max value and clipped",219	"FENCE 2 inc over max value and clipped",220	"FENCE 3 inc over max value and clipped",221	"FENCE 0 dec under min value and clipped",222	"FENCE 1 dec under min value and clipped",223	"FENCE 2 dec under min value and clipped",224	"FENCE 3 dec under min value and clipped"225};226 227static const char * const228gaudi_qman_arb_error_cause[GAUDI_NUM_OF_QM_ARB_ERR_CAUSE] = {229	"Choice push while full error",230	"Choice Q watchdog error",231	"MSG AXI LBW returned with error"232};233 234static enum hl_queue_type gaudi_queue_type[GAUDI_QUEUE_ID_SIZE] = {235	QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_0 */236	QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_1 */237	QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_2 */238	QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_0_3 */239	QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_1_0 */240	QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_1_1 */241	QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_1_2 */242	QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_1_3 */243	QUEUE_TYPE_CPU, /* GAUDI_QUEUE_ID_CPU_PQ */244	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_2_0 */245	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_2_1 */246	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_2_2 */247	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_2_3 */248	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_3_0 */249	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_3_1 */250	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_3_2 */251	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_3_3 */252	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_4_0 */253	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_4_1 */254	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_4_2 */255	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_4_3 */256	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_5_0 */257	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_5_1 */258	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_5_2 */259	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_5_3 */260	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_0 */261	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_1 */262	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_2 */263	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_3 */264	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_7_0 */265	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_7_1 */266	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_7_2 */267	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_7_3 */268	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_0_0 */269	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_0_1 */270	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_0_2 */271	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_0_3 */272	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_1_0 */273	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_1_1 */274	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_1_2 */275	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_MME_1_3 */276	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_0_0 */277	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_0_1 */278	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_0_2 */279	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_0_3 */280	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_1_0 */281	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_1_1 */282	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_1_2 */283	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_1_3 */284	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_2_0 */285	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_2_1 */286	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_2_2 */287	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_2_3 */288	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_3_0 */289	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_3_1 */290	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_3_2 */291	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_3_3 */292	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_4_0 */293	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_4_1 */294	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_4_2 */295	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_4_3 */296	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_5_0 */297	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_5_1 */298	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_5_2 */299	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_5_3 */300	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_6_0 */301	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_6_1 */302	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_6_2 */303	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_6_3 */304	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_7_0 */305	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_7_1 */306	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_7_2 */307	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_TPC_7_3 */308	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_0_0 */309	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_0_1 */310	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_0_2 */311	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_0_3 */312	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_1_0 */313	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_1_1 */314	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_1_2 */315	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_1_3 */316	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_2_0 */317	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_2_1 */318	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_2_2 */319	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_2_3 */320	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_3_0 */321	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_3_1 */322	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_3_2 */323	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_3_3 */324	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_4_0 */325	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_4_1 */326	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_4_2 */327	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_4_3 */328	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_5_0 */329	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_5_1 */330	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_5_2 */331	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_5_3 */332	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_6_0 */333	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_6_1 */334	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_6_2 */335	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_6_3 */336	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_7_0 */337	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_7_1 */338	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_7_2 */339	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_7_3 */340	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_8_0 */341	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_8_1 */342	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_8_2 */343	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_8_3 */344	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_9_0 */345	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_9_1 */346	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_9_2 */347	QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_NIC_9_3 */348};349 350static struct hl_hw_obj_name_entry gaudi_so_id_to_str[] = {351	{ .id = 0,  .name = "SYNC_OBJ_DMA_DOWN_FEEDBACK" },352	{ .id = 1,  .name = "SYNC_OBJ_DMA_UP_FEEDBACK" },353	{ .id = 2,  .name = "SYNC_OBJ_DMA_STATIC_DRAM_SRAM_FEEDBACK" },354	{ .id = 3,  .name = "SYNC_OBJ_DMA_SRAM_DRAM_FEEDBACK" },355	{ .id = 4,  .name = "SYNC_OBJ_FIRST_COMPUTE_FINISH" },356	{ .id = 5,  .name = "SYNC_OBJ_HOST_DRAM_DONE" },357	{ .id = 6,  .name = "SYNC_OBJ_DBG_CTR_DEPRECATED" },358	{ .id = 7,  .name = "SYNC_OBJ_DMA_ACTIVATIONS_DRAM_SRAM_FEEDBACK" },359	{ .id = 8,  .name = "SYNC_OBJ_ENGINE_SEM_MME_0" },360	{ .id = 9,  .name = "SYNC_OBJ_ENGINE_SEM_MME_1" },361	{ .id = 10, .name = "SYNC_OBJ_ENGINE_SEM_TPC_0" },362	{ .id = 11, .name = "SYNC_OBJ_ENGINE_SEM_TPC_1" },363	{ .id = 12, .name = "SYNC_OBJ_ENGINE_SEM_TPC_2" },364	{ .id = 13, .name = "SYNC_OBJ_ENGINE_SEM_TPC_3" },365	{ .id = 14, .name = "SYNC_OBJ_ENGINE_SEM_TPC_4" },366	{ .id = 15, .name = "SYNC_OBJ_ENGINE_SEM_TPC_5" },367	{ .id = 16, .name = "SYNC_OBJ_ENGINE_SEM_TPC_6" },368	{ .id = 17, .name = "SYNC_OBJ_ENGINE_SEM_TPC_7" },369	{ .id = 18, .name = "SYNC_OBJ_ENGINE_SEM_DMA_1" },370	{ .id = 19, .name = "SYNC_OBJ_ENGINE_SEM_DMA_2" },371	{ .id = 20, .name = "SYNC_OBJ_ENGINE_SEM_DMA_3" },372	{ .id = 21, .name = "SYNC_OBJ_ENGINE_SEM_DMA_4" },373	{ .id = 22, .name = "SYNC_OBJ_ENGINE_SEM_DMA_5" },374	{ .id = 23, .name = "SYNC_OBJ_ENGINE_SEM_DMA_6" },375	{ .id = 24, .name = "SYNC_OBJ_ENGINE_SEM_DMA_7" },376	{ .id = 25, .name = "SYNC_OBJ_DBG_CTR_0" },377	{ .id = 26, .name = "SYNC_OBJ_DBG_CTR_1" },378};379 380static struct hl_hw_obj_name_entry gaudi_monitor_id_to_str[] = {381	{ .id = 200, .name = "MON_OBJ_DMA_DOWN_FEEDBACK_RESET" },382	{ .id = 201, .name = "MON_OBJ_DMA_UP_FEEDBACK_RESET" },383	{ .id = 203, .name = "MON_OBJ_DRAM_TO_SRAM_QUEUE_FENCE" },384	{ .id = 204, .name = "MON_OBJ_TPC_0_CLK_GATE" },385	{ .id = 205, .name = "MON_OBJ_TPC_1_CLK_GATE" },386	{ .id = 206, .name = "MON_OBJ_TPC_2_CLK_GATE" },387	{ .id = 207, .name = "MON_OBJ_TPC_3_CLK_GATE" },388	{ .id = 208, .name = "MON_OBJ_TPC_4_CLK_GATE" },389	{ .id = 209, .name = "MON_OBJ_TPC_5_CLK_GATE" },390	{ .id = 210, .name = "MON_OBJ_TPC_6_CLK_GATE" },391	{ .id = 211, .name = "MON_OBJ_TPC_7_CLK_GATE" },392};393 394static s64 gaudi_state_dump_specs_props[] = {395	[SP_SYNC_OBJ_BASE_ADDR] = mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0,396	[SP_NEXT_SYNC_OBJ_ADDR] = NEXT_SYNC_OBJ_ADDR_INTERVAL,397	[SP_SYNC_OBJ_AMOUNT] = NUM_OF_SOB_IN_BLOCK,398	[SP_MON_OBJ_WR_ADDR_LOW] =399		mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0,400	[SP_MON_OBJ_WR_ADDR_HIGH] =401		mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0,402	[SP_MON_OBJ_WR_DATA] = mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_DATA_0,403	[SP_MON_OBJ_ARM_DATA] = mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_ARM_0,404	[SP_MON_OBJ_STATUS] = mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0,405	[SP_MONITORS_AMOUNT] = NUM_OF_MONITORS_IN_BLOCK,406	[SP_TPC0_CMDQ] = mmTPC0_QM_GLBL_CFG0,407	[SP_TPC0_CFG_SO] = mmTPC0_CFG_QM_SYNC_OBJECT_ADDR,408	[SP_NEXT_TPC] = mmTPC1_QM_GLBL_CFG0 - mmTPC0_QM_GLBL_CFG0,409	[SP_MME_CMDQ] = mmMME0_QM_GLBL_CFG0,410	[SP_MME_CFG_SO] = mmMME0_CTRL_ARCH_DESC_SYNC_OBJECT_ADDR_LOW_LOCAL,411	[SP_NEXT_MME] = mmMME2_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0,412	[SP_DMA_CMDQ] = mmDMA0_QM_GLBL_CFG0,413	[SP_DMA_CFG_SO] = mmDMA0_CORE_WR_COMP_ADDR_LO,414	[SP_DMA_QUEUES_OFFSET] = mmDMA1_QM_GLBL_CFG0 - mmDMA0_QM_GLBL_CFG0,415	[SP_NUM_OF_MME_ENGINES] = NUM_OF_MME_ENGINES,416	[SP_SUB_MME_ENG_NUM] = NUM_OF_MME_SUB_ENGINES,417	[SP_NUM_OF_DMA_ENGINES] = NUM_OF_DMA_ENGINES,418	[SP_NUM_OF_TPC_ENGINES] = NUM_OF_TPC_ENGINES,419	[SP_ENGINE_NUM_OF_QUEUES] = NUM_OF_QUEUES,420	[SP_ENGINE_NUM_OF_STREAMS] = NUM_OF_STREAMS,421	[SP_ENGINE_NUM_OF_FENCES] = NUM_OF_FENCES,422	[SP_FENCE0_CNT_OFFSET] =423		mmDMA0_QM_CP_FENCE0_CNT_0 - mmDMA0_QM_GLBL_CFG0,424	[SP_FENCE0_RDATA_OFFSET] =425		mmDMA0_QM_CP_FENCE0_RDATA_0 - mmDMA0_QM_GLBL_CFG0,426	[SP_CP_STS_OFFSET] = mmDMA0_QM_CP_STS_0 - mmDMA0_QM_GLBL_CFG0,427	[SP_NUM_CORES] = 1,428};429 430static const int gaudi_queue_id_to_engine_id[] = {431	[GAUDI_QUEUE_ID_DMA_0_0...GAUDI_QUEUE_ID_DMA_0_3] = GAUDI_ENGINE_ID_DMA_0,432	[GAUDI_QUEUE_ID_DMA_1_0...GAUDI_QUEUE_ID_DMA_1_3] = GAUDI_ENGINE_ID_DMA_1,433	[GAUDI_QUEUE_ID_CPU_PQ] = GAUDI_ENGINE_ID_SIZE,434	[GAUDI_QUEUE_ID_DMA_2_0...GAUDI_QUEUE_ID_DMA_2_3] = GAUDI_ENGINE_ID_DMA_2,435	[GAUDI_QUEUE_ID_DMA_3_0...GAUDI_QUEUE_ID_DMA_3_3] = GAUDI_ENGINE_ID_DMA_3,436	[GAUDI_QUEUE_ID_DMA_4_0...GAUDI_QUEUE_ID_DMA_4_3] = GAUDI_ENGINE_ID_DMA_4,437	[GAUDI_QUEUE_ID_DMA_5_0...GAUDI_QUEUE_ID_DMA_5_3] = GAUDI_ENGINE_ID_DMA_5,438	[GAUDI_QUEUE_ID_DMA_6_0...GAUDI_QUEUE_ID_DMA_6_3] = GAUDI_ENGINE_ID_DMA_6,439	[GAUDI_QUEUE_ID_DMA_7_0...GAUDI_QUEUE_ID_DMA_7_3] = GAUDI_ENGINE_ID_DMA_7,440	[GAUDI_QUEUE_ID_MME_0_0...GAUDI_QUEUE_ID_MME_0_3] = GAUDI_ENGINE_ID_MME_0,441	[GAUDI_QUEUE_ID_MME_1_0...GAUDI_QUEUE_ID_MME_1_3] = GAUDI_ENGINE_ID_MME_2,442	[GAUDI_QUEUE_ID_TPC_0_0...GAUDI_QUEUE_ID_TPC_0_3] = GAUDI_ENGINE_ID_TPC_0,443	[GAUDI_QUEUE_ID_TPC_1_0...GAUDI_QUEUE_ID_TPC_1_3] = GAUDI_ENGINE_ID_TPC_1,444	[GAUDI_QUEUE_ID_TPC_2_0...GAUDI_QUEUE_ID_TPC_2_3] = GAUDI_ENGINE_ID_TPC_2,445	[GAUDI_QUEUE_ID_TPC_3_0...GAUDI_QUEUE_ID_TPC_3_3] = GAUDI_ENGINE_ID_TPC_3,446	[GAUDI_QUEUE_ID_TPC_4_0...GAUDI_QUEUE_ID_TPC_4_3] = GAUDI_ENGINE_ID_TPC_4,447	[GAUDI_QUEUE_ID_TPC_5_0...GAUDI_QUEUE_ID_TPC_5_3] = GAUDI_ENGINE_ID_TPC_5,448	[GAUDI_QUEUE_ID_TPC_6_0...GAUDI_QUEUE_ID_TPC_6_3] = GAUDI_ENGINE_ID_TPC_6,449	[GAUDI_QUEUE_ID_TPC_7_0...GAUDI_QUEUE_ID_TPC_7_3] = GAUDI_ENGINE_ID_TPC_7,450	[GAUDI_QUEUE_ID_NIC_0_0...GAUDI_QUEUE_ID_NIC_0_3] = GAUDI_ENGINE_ID_NIC_0,451	[GAUDI_QUEUE_ID_NIC_1_0...GAUDI_QUEUE_ID_NIC_1_3] = GAUDI_ENGINE_ID_NIC_1,452	[GAUDI_QUEUE_ID_NIC_2_0...GAUDI_QUEUE_ID_NIC_2_3] = GAUDI_ENGINE_ID_NIC_2,453	[GAUDI_QUEUE_ID_NIC_3_0...GAUDI_QUEUE_ID_NIC_3_3] = GAUDI_ENGINE_ID_NIC_3,454	[GAUDI_QUEUE_ID_NIC_4_0...GAUDI_QUEUE_ID_NIC_4_3] = GAUDI_ENGINE_ID_NIC_4,455	[GAUDI_QUEUE_ID_NIC_5_0...GAUDI_QUEUE_ID_NIC_5_3] = GAUDI_ENGINE_ID_NIC_5,456	[GAUDI_QUEUE_ID_NIC_6_0...GAUDI_QUEUE_ID_NIC_6_3] = GAUDI_ENGINE_ID_NIC_6,457	[GAUDI_QUEUE_ID_NIC_7_0...GAUDI_QUEUE_ID_NIC_7_3] = GAUDI_ENGINE_ID_NIC_7,458	[GAUDI_QUEUE_ID_NIC_8_0...GAUDI_QUEUE_ID_NIC_8_3] = GAUDI_ENGINE_ID_NIC_8,459	[GAUDI_QUEUE_ID_NIC_9_0...GAUDI_QUEUE_ID_NIC_9_3] = GAUDI_ENGINE_ID_NIC_9,460};461 462/* The order here is opposite to the order of the indexing in the h/w.463 * i.e. SYNC_MGR_W_S is actually 0, SYNC_MGR_E_S is 1, etc.464 */465static const char * const gaudi_sync_manager_names[] = {466	"SYNC_MGR_E_N",467	"SYNC_MGR_W_N",468	"SYNC_MGR_E_S",469	"SYNC_MGR_W_S",470	NULL471};472 473struct ecc_info_extract_params {474	u64 block_address;475	u32 num_memories;476	bool derr;477};478 479static int gaudi_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid,480								u64 phys_addr);481static int gaudi_send_job_on_qman0(struct hl_device *hdev,482					struct hl_cs_job *job);483static int gaudi_memset_device_memory(struct hl_device *hdev, u64 addr,484					u32 size, u64 val);485static int gaudi_memset_registers(struct hl_device *hdev, u64 reg_base,486					u32 num_regs, u32 val);487static int gaudi_run_tpc_kernel(struct hl_device *hdev, u64 tpc_kernel,488				u32 tpc_id);489static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev);490static int gaudi_cpucp_info_get(struct hl_device *hdev);491static void gaudi_disable_clock_gating(struct hl_device *hdev);492static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid);493static u32 gaudi_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id,494				u32 size, bool eb);495static u32 gaudi_gen_wait_cb(struct hl_device *hdev,496				struct hl_gen_wait_properties *prop);497static inline enum hl_collective_mode498get_collective_mode(struct hl_device *hdev, u32 queue_id)499{500	if (gaudi_queue_type[queue_id] == QUEUE_TYPE_EXT)501		return HL_COLLECTIVE_MASTER;502 503	if (queue_id >= GAUDI_QUEUE_ID_DMA_5_0 &&504			queue_id <= GAUDI_QUEUE_ID_DMA_5_3)505		return HL_COLLECTIVE_SLAVE;506 507	if (queue_id >= GAUDI_QUEUE_ID_TPC_7_0 &&508			queue_id <= GAUDI_QUEUE_ID_TPC_7_3)509		return HL_COLLECTIVE_SLAVE;510 511	if (queue_id >= GAUDI_QUEUE_ID_NIC_0_0 &&512			queue_id <= GAUDI_QUEUE_ID_NIC_9_3)513		return HL_COLLECTIVE_SLAVE;514 515	return HL_COLLECTIVE_NOT_SUPPORTED;516}517 518static inline void set_default_power_values(struct hl_device *hdev)519{520	struct asic_fixed_properties *prop = &hdev->asic_prop;521 522	if (hdev->card_type == cpucp_card_type_pmc) {523		prop->max_power_default = MAX_POWER_DEFAULT_PMC;524 525		if (prop->fw_security_enabled)526			prop->dc_power_default = DC_POWER_DEFAULT_PMC_SEC;527		else528			prop->dc_power_default = DC_POWER_DEFAULT_PMC;529	} else {530		prop->max_power_default = MAX_POWER_DEFAULT_PCI;531		prop->dc_power_default = DC_POWER_DEFAULT_PCI;532	}533}534 535static int gaudi_set_fixed_properties(struct hl_device *hdev)536{537	struct asic_fixed_properties *prop = &hdev->asic_prop;538	u32 num_sync_stream_queues = 0;539	int i;540 541	prop->max_queues = GAUDI_QUEUE_ID_SIZE;542	prop->hw_queues_props = kcalloc(prop->max_queues,543			sizeof(struct hw_queue_properties),544			GFP_KERNEL);545 546	if (!prop->hw_queues_props)547		return -ENOMEM;548 549	for (i = 0 ; i < prop->max_queues ; i++) {550		if (gaudi_queue_type[i] == QUEUE_TYPE_EXT) {551			prop->hw_queues_props[i].type = QUEUE_TYPE_EXT;552			prop->hw_queues_props[i].driver_only = 0;553			prop->hw_queues_props[i].supports_sync_stream = 1;554			prop->hw_queues_props[i].cb_alloc_flags =555				CB_ALLOC_KERNEL;556			num_sync_stream_queues++;557		} else if (gaudi_queue_type[i] == QUEUE_TYPE_CPU) {558			prop->hw_queues_props[i].type = QUEUE_TYPE_CPU;559			prop->hw_queues_props[i].driver_only = 1;560			prop->hw_queues_props[i].supports_sync_stream = 0;561			prop->hw_queues_props[i].cb_alloc_flags =562				CB_ALLOC_KERNEL;563		} else if (gaudi_queue_type[i] == QUEUE_TYPE_INT) {564			prop->hw_queues_props[i].type = QUEUE_TYPE_INT;565			prop->hw_queues_props[i].driver_only = 0;566			prop->hw_queues_props[i].supports_sync_stream = 0;567			prop->hw_queues_props[i].cb_alloc_flags =568				CB_ALLOC_USER;569 570		}571		prop->hw_queues_props[i].collective_mode =572						get_collective_mode(hdev, i);573	}574 575	prop->cache_line_size = DEVICE_CACHE_LINE_SIZE;576	prop->cfg_base_address = CFG_BASE;577	prop->device_dma_offset_for_host_access = HOST_PHYS_BASE;578	prop->host_base_address = HOST_PHYS_BASE;579	prop->host_end_address = prop->host_base_address + HOST_PHYS_SIZE;580	prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES;581	prop->completion_mode = HL_COMPLETION_MODE_JOB;582	prop->collective_first_sob = 0;583	prop->collective_first_mon = 0;584 585	/* 2 SOBs per internal queue stream are reserved for collective */586	prop->sync_stream_first_sob =587			ALIGN(NUMBER_OF_SOBS_IN_GRP, HL_MAX_SOBS_PER_MONITOR)588			* QMAN_STREAMS * HL_RSVD_SOBS;589 590	/* 1 monitor per internal queue stream are reserved for collective591	 * 2 monitors per external queue stream are reserved for collective592	 */593	prop->sync_stream_first_mon =594			(NUMBER_OF_COLLECTIVE_QUEUES * QMAN_STREAMS) +595			(NUMBER_OF_EXT_HW_QUEUES * 2);596 597	prop->dram_base_address = DRAM_PHYS_BASE;598	prop->dram_size = GAUDI_HBM_SIZE_32GB;599	prop->dram_end_address = prop->dram_base_address + prop->dram_size;600	prop->dram_user_base_address = DRAM_BASE_ADDR_USER;601 602	prop->sram_base_address = SRAM_BASE_ADDR;603	prop->sram_size = SRAM_SIZE;604	prop->sram_end_address = prop->sram_base_address + prop->sram_size;605	prop->sram_user_base_address =606			prop->sram_base_address + SRAM_USER_BASE_OFFSET;607 608	prop->mmu_cache_mng_addr = MMU_CACHE_MNG_ADDR;609	prop->mmu_cache_mng_size = MMU_CACHE_MNG_SIZE;610 611	prop->mmu_pgt_addr = MMU_PAGE_TABLES_ADDR;612	if (hdev->pldm)613		prop->mmu_pgt_size = 0x800000; /* 8MB */614	else615		prop->mmu_pgt_size = MMU_PAGE_TABLES_SIZE;616	prop->mmu_pte_size = HL_PTE_SIZE;617	prop->dram_page_size = PAGE_SIZE_2MB;618	prop->device_mem_alloc_default_page_size = prop->dram_page_size;619	prop->dram_supports_virtual_memory = false;620 621	prop->pmmu.hop_shifts[MMU_HOP0] = MMU_V1_1_HOP0_SHIFT;622	prop->pmmu.hop_shifts[MMU_HOP1] = MMU_V1_1_HOP1_SHIFT;623	prop->pmmu.hop_shifts[MMU_HOP2] = MMU_V1_1_HOP2_SHIFT;624	prop->pmmu.hop_shifts[MMU_HOP3] = MMU_V1_1_HOP3_SHIFT;625	prop->pmmu.hop_shifts[MMU_HOP4] = MMU_V1_1_HOP4_SHIFT;626	prop->pmmu.hop_masks[MMU_HOP0] = MMU_V1_1_HOP0_MASK;627	prop->pmmu.hop_masks[MMU_HOP1] = MMU_V1_1_HOP1_MASK;628	prop->pmmu.hop_masks[MMU_HOP2] = MMU_V1_1_HOP2_MASK;629	prop->pmmu.hop_masks[MMU_HOP3] = MMU_V1_1_HOP3_MASK;630	prop->pmmu.hop_masks[MMU_HOP4] = MMU_V1_1_HOP4_MASK;631	prop->pmmu.start_addr = VA_HOST_SPACE_START;632	prop->pmmu.end_addr =633			(VA_HOST_SPACE_START + VA_HOST_SPACE_SIZE / 2) - 1;634	prop->pmmu.page_size = PAGE_SIZE_4KB;635	prop->pmmu.num_hops = MMU_ARCH_5_HOPS;636	prop->pmmu.last_mask = LAST_MASK;637	/* TODO: will be duplicated until implementing per-MMU props */638	prop->pmmu.hop_table_size = HOP_TABLE_SIZE_512_PTE;639	prop->pmmu.hop0_tables_total_size = HOP0_512_PTE_TABLES_TOTAL_SIZE;640 641	/* PMMU and HPMMU are the same except of page size */642	memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu));643	prop->pmmu_huge.page_size = PAGE_SIZE_2MB;644 645	/* shifts and masks are the same in PMMU and DMMU */646	memcpy(&prop->dmmu, &prop->pmmu, sizeof(prop->pmmu));647	prop->dmmu.start_addr = (VA_HOST_SPACE_START + VA_HOST_SPACE_SIZE / 2);648	prop->dmmu.end_addr = VA_HOST_SPACE_END;649	prop->dmmu.page_size = PAGE_SIZE_2MB;650	prop->dmmu.pgt_size = prop->mmu_pgt_size;651 652	prop->cfg_size = CFG_SIZE;653	prop->max_asid = MAX_ASID;654	prop->num_of_events = GAUDI_EVENT_SIZE;655	prop->max_num_of_engines = GAUDI_ENGINE_ID_SIZE;656	prop->tpc_enabled_mask = TPC_ENABLED_MASK;657 658	set_default_power_values(hdev);659 660	prop->cb_pool_cb_cnt = GAUDI_CB_POOL_CB_CNT;661	prop->cb_pool_cb_size = GAUDI_CB_POOL_CB_SIZE;662 663	prop->pcie_dbi_base_address = mmPCIE_DBI_BASE;664	prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI;665 666	strscpy_pad(prop->cpucp_info.card_name, GAUDI_DEFAULT_CARD_NAME,667					CARD_NAME_MAX_LEN);668 669	prop->max_pending_cs = GAUDI_MAX_PENDING_CS;670 671	prop->first_available_user_sob[HL_GAUDI_WS_DCORE] =672			prop->sync_stream_first_sob +673			(num_sync_stream_queues * HL_RSVD_SOBS);674	prop->first_available_user_mon[HL_GAUDI_WS_DCORE] =675			prop->sync_stream_first_mon +676			(num_sync_stream_queues * HL_RSVD_MONS);677 678	prop->first_available_user_interrupt = USHRT_MAX;679	prop->tpc_interrupt_id = USHRT_MAX;680 681	/* single msi */682	prop->eq_interrupt_id = 0;683 684	for (i = 0 ; i < HL_MAX_DCORES ; i++)685		prop->first_available_cq[i] = USHRT_MAX;686 687	prop->fw_cpu_boot_dev_sts0_valid = false;688	prop->fw_cpu_boot_dev_sts1_valid = false;689	prop->hard_reset_done_by_fw = false;690	prop->gic_interrupts_enable = true;691 692	prop->server_type = HL_SERVER_TYPE_UNKNOWN;693 694	prop->clk_pll_index = HL_GAUDI_MME_PLL;695	prop->max_freq_value = GAUDI_MAX_CLK_FREQ;696 697	prop->use_get_power_for_reset_history = true;698 699	prop->configurable_stop_on_err = true;700 701	prop->set_max_power_on_device_init = true;702 703	prop->dma_mask = 48;704 705	prop->hbw_flush_reg = mmPCIE_WRAP_RR_ELBI_RD_SEC_REG_CTRL;706 707	return 0;708}709 710static int gaudi_pci_bars_map(struct hl_device *hdev)711{712	static const char * const name[] = {"SRAM", "CFG", "HBM"};713	bool is_wc[3] = {false, false, true};714	int rc;715 716	rc = hl_pci_bars_map(hdev, name, is_wc);717	if (rc)718		return rc;719 720	hdev->rmmio = hdev->pcie_bar[CFG_BAR_ID] +721			(CFG_BASE - SPI_FLASH_BASE_ADDR);722 723	return 0;724}725 726static u64 gaudi_set_hbm_bar_base(struct hl_device *hdev, u64 addr)727{728	struct gaudi_device *gaudi = hdev->asic_specific;729	struct hl_inbound_pci_region pci_region;730	u64 old_addr = addr;731	int rc;732 733	if ((gaudi) && (gaudi->hbm_bar_cur_addr == addr))734		return old_addr;735 736	if (hdev->asic_prop.iatu_done_by_fw)737		return U64_MAX;738 739	/* Inbound Region 2 - Bar 4 - Point to HBM */740	pci_region.mode = PCI_BAR_MATCH_MODE;741	pci_region.bar = HBM_BAR_ID;742	pci_region.addr = addr;743	rc = hl_pci_set_inbound_region(hdev, 2, &pci_region);744	if (rc)745		return U64_MAX;746 747	if (gaudi) {748		old_addr = gaudi->hbm_bar_cur_addr;749		gaudi->hbm_bar_cur_addr = addr;750	}751 752	return old_addr;753}754 755static int gaudi_init_iatu(struct hl_device *hdev)756{757	struct hl_inbound_pci_region inbound_region;758	struct hl_outbound_pci_region outbound_region;759	int rc;760 761	if (hdev->asic_prop.iatu_done_by_fw)762		return 0;763 764	/* Inbound Region 0 - Bar 0 - Point to SRAM + CFG */765	inbound_region.mode = PCI_BAR_MATCH_MODE;766	inbound_region.bar = SRAM_BAR_ID;767	inbound_region.addr = SRAM_BASE_ADDR;768	rc = hl_pci_set_inbound_region(hdev, 0, &inbound_region);769	if (rc)770		goto done;771 772	/* Inbound Region 1 - Bar 2 - Point to SPI FLASH */773	inbound_region.mode = PCI_BAR_MATCH_MODE;774	inbound_region.bar = CFG_BAR_ID;775	inbound_region.addr = SPI_FLASH_BASE_ADDR;776	rc = hl_pci_set_inbound_region(hdev, 1, &inbound_region);777	if (rc)778		goto done;779 780	/* Inbound Region 2 - Bar 4 - Point to HBM */781	inbound_region.mode = PCI_BAR_MATCH_MODE;782	inbound_region.bar = HBM_BAR_ID;783	inbound_region.addr = DRAM_PHYS_BASE;784	rc = hl_pci_set_inbound_region(hdev, 2, &inbound_region);785	if (rc)786		goto done;787 788	/* Outbound Region 0 - Point to Host */789	outbound_region.addr = HOST_PHYS_BASE;790	outbound_region.size = HOST_PHYS_SIZE;791	rc = hl_pci_set_outbound_region(hdev, &outbound_region);792 793done:794	return rc;795}796 797static enum hl_device_hw_state gaudi_get_hw_state(struct hl_device *hdev)798{799	return RREG32(mmHW_STATE);800}801 802static int gaudi_early_init(struct hl_device *hdev)803{804	struct asic_fixed_properties *prop = &hdev->asic_prop;805	struct pci_dev *pdev = hdev->pdev;806	resource_size_t pci_bar_size;807	u32 fw_boot_status;808	int rc;809 810	rc = gaudi_set_fixed_properties(hdev);811	if (rc) {812		dev_err(hdev->dev, "Failed setting fixed properties\n");813		return rc;814	}815 816	/* Check BAR sizes */817	pci_bar_size = pci_resource_len(pdev, SRAM_BAR_ID);818 819	if (pci_bar_size != SRAM_BAR_SIZE) {820		dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n",821			SRAM_BAR_ID, &pci_bar_size, SRAM_BAR_SIZE);822		rc = -ENODEV;823		goto free_queue_props;824	}825 826	pci_bar_size = pci_resource_len(pdev, CFG_BAR_ID);827 828	if (pci_bar_size != CFG_BAR_SIZE) {829		dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n",830			CFG_BAR_ID, &pci_bar_size, CFG_BAR_SIZE);831		rc = -ENODEV;832		goto free_queue_props;833	}834 835	prop->dram_pci_bar_size = pci_resource_len(pdev, HBM_BAR_ID);836	hdev->dram_pci_bar_start = pci_resource_start(pdev, HBM_BAR_ID);837 838	/* If FW security is enabled at this point it means no access to ELBI */839	if (hdev->asic_prop.fw_security_enabled) {840		hdev->asic_prop.iatu_done_by_fw = true;841 842		/*843		 * GIC-security-bit can ONLY be set by CPUCP, so in this stage844		 * decision can only be taken based on PCI ID security.845		 */846		hdev->asic_prop.gic_interrupts_enable = false;847		goto pci_init;848	}849 850	rc = hl_pci_elbi_read(hdev, CFG_BASE + mmCPU_BOOT_DEV_STS0,851				&fw_boot_status);852	if (rc)853		goto free_queue_props;854 855	/* Check whether FW is configuring iATU */856	if ((fw_boot_status & CPU_BOOT_DEV_STS0_ENABLED) &&857			(fw_boot_status & CPU_BOOT_DEV_STS0_FW_IATU_CONF_EN))858		hdev->asic_prop.iatu_done_by_fw = true;859 860pci_init:861	rc = hl_pci_init(hdev);862	if (rc)863		goto free_queue_props;864 865	/* Before continuing in the initialization, we need to read the preboot866	 * version to determine whether we run with a security-enabled firmware867	 */868	rc = hl_fw_read_preboot_status(hdev);869	if (rc) {870		if (hdev->reset_on_preboot_fail)871			/* we are already on failure flow, so don't check if hw_fini fails. */872			hdev->asic_funcs->hw_fini(hdev, true, false);873		goto pci_fini;874	}875 876	if (gaudi_get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {877		dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n");878		rc = hdev->asic_funcs->hw_fini(hdev, true, false);879		if (rc) {880			dev_err(hdev->dev, "failed to reset HW in dirty state (%d)\n", rc);881			goto pci_fini;882		}883	}884 885	return 0;886 887pci_fini:888	hl_pci_fini(hdev);889free_queue_props:890	kfree(hdev->asic_prop.hw_queues_props);891	return rc;892}893 894static int gaudi_early_fini(struct hl_device *hdev)895{896	kfree(hdev->asic_prop.hw_queues_props);897	hl_pci_fini(hdev);898 899	return 0;900}901 902/**903 * gaudi_fetch_psoc_frequency - Fetch PSOC frequency values904 *905 * @hdev: pointer to hl_device structure906 *907 */908static int gaudi_fetch_psoc_frequency(struct hl_device *hdev)909{910	u32 nr = 0, nf = 0, od = 0, div_fctr = 0, pll_clk, div_sel;911	struct asic_fixed_properties *prop = &hdev->asic_prop;912	u16 pll_freq_arr[HL_PLL_NUM_OUTPUTS], freq;913	int rc;914 915	if ((hdev->fw_components & FW_TYPE_LINUX) &&916			(prop->fw_app_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_PLL_INFO_EN)) {917		struct gaudi_device *gaudi = hdev->asic_specific;918 919		if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))920			return 0;921 922		rc = hl_fw_cpucp_pll_info_get(hdev, HL_GAUDI_CPU_PLL, pll_freq_arr);923 924		if (rc)925			return rc;926 927		freq = pll_freq_arr[2];928	} else {929		/* Backward compatibility */930		div_fctr = RREG32(mmPSOC_CPU_PLL_DIV_FACTOR_2);931		div_sel = RREG32(mmPSOC_CPU_PLL_DIV_SEL_2);932		nr = RREG32(mmPSOC_CPU_PLL_NR);933		nf = RREG32(mmPSOC_CPU_PLL_NF);934		od = RREG32(mmPSOC_CPU_PLL_OD);935 936		if (div_sel == DIV_SEL_REF_CLK ||937				div_sel == DIV_SEL_DIVIDED_REF) {938			if (div_sel == DIV_SEL_REF_CLK)939				freq = PLL_REF_CLK;940			else941				freq = PLL_REF_CLK / (div_fctr + 1);942		} else if (div_sel == DIV_SEL_PLL_CLK ||943			div_sel == DIV_SEL_DIVIDED_PLL) {944			pll_clk = PLL_REF_CLK * (nf + 1) /945					((nr + 1) * (od + 1));946			if (div_sel == DIV_SEL_PLL_CLK)947				freq = pll_clk;948			else949				freq = pll_clk / (div_fctr + 1);950		} else {951			dev_warn(hdev->dev, "Received invalid div select value: %#x", div_sel);952			freq = 0;953		}954	}955 956	prop->psoc_timestamp_frequency = freq;957	prop->psoc_pci_pll_nr = nr;958	prop->psoc_pci_pll_nf = nf;959	prop->psoc_pci_pll_od = od;960	prop->psoc_pci_pll_div_factor = div_fctr;961 962	return 0;963}964 965static int _gaudi_init_tpc_mem(struct hl_device *hdev,966		dma_addr_t tpc_kernel_src_addr, u32 tpc_kernel_size)967{968	struct asic_fixed_properties *prop = &hdev->asic_prop;969	struct packet_lin_dma *init_tpc_mem_pkt;970	struct hl_cs_job *job;971	struct hl_cb *cb;972	u64 dst_addr;973	u32 cb_size, ctl;974	u8 tpc_id;975	int rc;976 977	cb = hl_cb_kernel_create(hdev, PAGE_SIZE, false);978	if (!cb)979		return -EFAULT;980 981	init_tpc_mem_pkt = cb->kernel_address;982	cb_size = sizeof(*init_tpc_mem_pkt);983	memset(init_tpc_mem_pkt, 0, cb_size);984 985	init_tpc_mem_pkt->tsize = cpu_to_le32(tpc_kernel_size);986 987	ctl = FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_LIN_DMA);988	ctl |= FIELD_PREP(GAUDI_PKT_LIN_DMA_CTL_LIN_MASK, 1);989	ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1);990	ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1);991 992	init_tpc_mem_pkt->ctl = cpu_to_le32(ctl);993 994	init_tpc_mem_pkt->src_addr = cpu_to_le64(tpc_kernel_src_addr);995 996	/* TPC_CMD is configured with I$ prefetch enabled, so address should be aligned to 8KB */997	dst_addr = FIELD_PREP(GAUDI_PKT_LIN_DMA_DST_ADDR_MASK,998				round_up(prop->sram_user_base_address, SZ_8K));999	init_tpc_mem_pkt->dst_addr |= cpu_to_le64(dst_addr);1000 1001	job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true);1002	if (!job) {1003		dev_err(hdev->dev, "Failed to allocate a new job\n");1004		rc = -ENOMEM;1005		goto release_cb;1006	}1007 1008	job->id = 0;1009	job->user_cb = cb;1010	atomic_inc(&job->user_cb->cs_cnt);1011	job->user_cb_size = cb_size;1012	job->hw_queue_id = GAUDI_QUEUE_ID_DMA_0_0;1013	job->patched_cb = job->user_cb;1014	job->job_cb_size = job->user_cb_size + sizeof(struct packet_msg_prot);1015 1016	hl_debugfs_add_job(hdev, job);1017 1018	rc = gaudi_send_job_on_qman0(hdev, job);1019 1020	if (rc)1021		goto free_job;1022 1023	for (tpc_id = 0 ; tpc_id < TPC_NUMBER_OF_ENGINES ; tpc_id++) {1024		rc = gaudi_run_tpc_kernel(hdev, dst_addr, tpc_id);1025		if (rc)1026			break;1027	}1028 1029free_job:1030	hl_userptr_delete_list(hdev, &job->userptr_list);1031	hl_debugfs_remove_job(hdev, job);1032	kfree(job);1033	atomic_dec(&cb->cs_cnt);1034 1035release_cb:1036	hl_cb_put(cb);1037	hl_cb_destroy(&hdev->kernel_mem_mgr, cb->buf->handle);1038 1039	return rc;1040}1041 1042/*1043 * gaudi_init_tpc_mem() - Initialize TPC memories.1044 * @hdev: Pointer to hl_device structure.1045 *1046 * Copy TPC kernel fw from firmware file and run it to initialize TPC memories.1047 *1048 * Return: 0 for success, negative value for error.1049 */1050static int gaudi_init_tpc_mem(struct hl_device *hdev)1051{1052	const struct firmware *fw;1053	size_t fw_size;1054	void *cpu_addr;1055	dma_addr_t dma_handle;1056	int rc, count = 5;1057 1058again:1059	rc = request_firmware(&fw, GAUDI_TPC_FW_FILE, hdev->dev);1060	if (rc == -EINTR && count-- > 0) {1061		msleep(50);1062		goto again;1063	}1064 1065	if (rc) {1066		dev_err(hdev->dev, "Failed to load firmware file %s\n",1067				GAUDI_TPC_FW_FILE);1068		goto out;1069	}1070 1071	fw_size = fw->size;1072	cpu_addr = hl_asic_dma_alloc_coherent(hdev, fw_size, &dma_handle, GFP_KERNEL | __GFP_ZERO);1073	if (!cpu_addr) {1074		dev_err(hdev->dev,1075			"Failed to allocate %zu of dma memory for TPC kernel\n",1076			fw_size);1077		rc = -ENOMEM;1078		goto out;1079	}1080 1081	memcpy(cpu_addr, fw->data, fw_size);1082 1083	rc = _gaudi_init_tpc_mem(hdev, dma_handle, fw_size);1084 1085	hl_asic_dma_free_coherent(hdev, fw->size, cpu_addr, dma_handle);1086 1087out:1088	release_firmware(fw);1089	return rc;1090}1091 1092static void gaudi_collective_map_sobs(struct hl_device *hdev, u32 stream)1093{1094	struct gaudi_device *gaudi = hdev->asic_specific;1095	struct gaudi_collective_properties *prop = &gaudi->collective_props;1096	struct hl_hw_queue *q;1097	u32 i, sob_id, sob_group_id, queue_id;1098 1099	/* Iterate through SOB groups and assign a SOB for each slave queue */1100	sob_group_id =1101		stream * HL_RSVD_SOBS + prop->curr_sob_group_idx[stream];1102	sob_id = prop->hw_sob_group[sob_group_id].base_sob_id;1103 1104	queue_id = GAUDI_QUEUE_ID_NIC_0_0 + stream;1105	for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++) {1106		q = &hdev->kernel_queues[queue_id + (4 * i)];1107		q->sync_stream_prop.collective_sob_id = sob_id + i;1108	}1109 1110	/* Both DMA5 and TPC7 use the same resources since only a single1111	 * engine need to participate in the reduction process1112	 */1113	queue_id = GAUDI_QUEUE_ID_DMA_5_0 + stream;1114	q = &hdev->kernel_queues[queue_id];1115	q->sync_stream_prop.collective_sob_id =1116			sob_id + NIC_NUMBER_OF_ENGINES;1117 1118	queue_id = GAUDI_QUEUE_ID_TPC_7_0 + stream;1119	q = &hdev->kernel_queues[queue_id];1120	q->sync_stream_prop.collective_sob_id =1121			sob_id + NIC_NUMBER_OF_ENGINES;1122}1123 1124static void gaudi_sob_group_hw_reset(struct kref *ref)1125{1126	struct gaudi_hw_sob_group *hw_sob_group =1127		container_of(ref, struct gaudi_hw_sob_group, kref);1128	struct hl_device *hdev = hw_sob_group->hdev;1129	int i;1130 1131	for (i = 0 ; i < NUMBER_OF_SOBS_IN_GRP ; i++)1132		WREG32((mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 +1133			(hw_sob_group->base_sob_id * 4) + (i * 4)), 0);1134 1135	kref_init(&hw_sob_group->kref);1136}1137 1138static void gaudi_sob_group_reset_error(struct kref *ref)1139{1140	struct gaudi_hw_sob_group *hw_sob_group =1141		container_of(ref, struct gaudi_hw_sob_group, kref);1142	struct hl_device *hdev = hw_sob_group->hdev;1143 1144	dev_crit(hdev->dev,1145		"SOB release shouldn't be called here, base_sob_id: %d\n",1146		hw_sob_group->base_sob_id);1147}1148 1149static void gaudi_collective_mstr_sob_mask_set(struct gaudi_device *gaudi)1150{1151	struct gaudi_collective_properties *prop;1152	int i;1153 1154	prop = &gaudi->collective_props;1155 1156	memset(prop->mstr_sob_mask, 0, sizeof(prop->mstr_sob_mask));1157 1158	for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++)1159		if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + i))1160			prop->mstr_sob_mask[i / HL_MAX_SOBS_PER_MONITOR] |=1161					BIT(i % HL_MAX_SOBS_PER_MONITOR);1162	/* Set collective engine bit */1163	prop->mstr_sob_mask[i / HL_MAX_SOBS_PER_MONITOR] |=1164				BIT(i % HL_MAX_SOBS_PER_MONITOR);1165}1166 1167static int gaudi_collective_init(struct hl_device *hdev)1168{1169	u32 i, sob_id, reserved_sobs_per_group;1170	struct gaudi_collective_properties *prop;1171	struct gaudi_device *gaudi;1172 1173	gaudi = hdev->asic_specific;1174	prop = &gaudi->collective_props;1175	sob_id = hdev->asic_prop.collective_first_sob;1176 1177	/* First sob in group must be aligned to HL_MAX_SOBS_PER_MONITOR */1178	reserved_sobs_per_group =1179		ALIGN(NUMBER_OF_SOBS_IN_GRP, HL_MAX_SOBS_PER_MONITOR);1180 1181	/* Init SOB groups */1182	for (i = 0 ; i < NUM_SOB_GROUPS; i++) {1183		prop->hw_sob_group[i].hdev = hdev;1184		prop->hw_sob_group[i].base_sob_id = sob_id;1185		sob_id += reserved_sobs_per_group;1186		gaudi_sob_group_hw_reset(&prop->hw_sob_group[i].kref);1187	}1188 1189	for (i = 0 ; i < QMAN_STREAMS; i++) {1190		prop->next_sob_group_val[i] = 1;1191		prop->curr_sob_group_idx[i] = 0;1192		gaudi_collective_map_sobs(hdev, i);1193	}1194 1195	gaudi_collective_mstr_sob_mask_set(gaudi);1196 1197	return 0;1198}1199 1200static void gaudi_reset_sob_group(struct hl_device *hdev, u16 sob_group)1201{1202	struct gaudi_device *gaudi = hdev->asic_specific;1203	struct gaudi_collective_properties *cprop = &gaudi->collective_props;1204 1205	kref_put(&cprop->hw_sob_group[sob_group].kref,1206					gaudi_sob_group_hw_reset);1207}1208 1209static void gaudi_collective_master_init_job(struct hl_device *hdev,1210		struct hl_cs_job *job, u32 stream, u32 sob_group_offset)1211{1212	u32 master_sob_base, master_monitor, queue_id, cb_size = 0;1213	struct gaudi_collective_properties *cprop;1214	struct hl_gen_wait_properties wait_prop;1215	struct hl_sync_stream_properties *prop;1216	struct gaudi_device *gaudi;1217 1218	gaudi = hdev->asic_specific;1219	cprop = &gaudi->collective_props;1220	queue_id = job->hw_queue_id;1221	prop = &hdev->kernel_queues[queue_id].sync_stream_prop;1222 1223	master_sob_base =1224		cprop->hw_sob_group[sob_group_offset].base_sob_id;1225	master_monitor = prop->collective_mstr_mon_id[0];1226 1227	cprop->hw_sob_group[sob_group_offset].queue_id = queue_id;1228 1229	dev_dbg(hdev->dev,1230		"Generate master wait CBs, sob %d (mask %#x), val:0x%x, mon %u, q %d\n",1231		master_sob_base, cprop->mstr_sob_mask[0],1232		cprop->next_sob_group_val[stream],1233		master_monitor, queue_id);1234 1235	wait_prop.data = (void *) job->patched_cb;1236	wait_prop.sob_base = master_sob_base;1237	wait_prop.sob_mask = cprop->mstr_sob_mask[0];1238	wait_prop.sob_val = cprop->next_sob_group_val[stream];1239	wait_prop.mon_id = master_monitor;1240	wait_prop.q_idx = queue_id;1241	wait_prop.size = cb_size;1242	cb_size += gaudi_gen_wait_cb(hdev, &wait_prop);1243 1244	master_sob_base += HL_MAX_SOBS_PER_MONITOR;1245	master_monitor = prop->collective_mstr_mon_id[1];1246 1247	dev_dbg(hdev->dev,1248		"Generate master wait CBs, sob %d (mask %#x), val:0x%x, mon %u, q %d\n",1249		master_sob_base, cprop->mstr_sob_mask[1],1250		cprop->next_sob_group_val[stream],1251		master_monitor, queue_id);1252 1253	wait_prop.sob_base = master_sob_base;1254	wait_prop.sob_mask = cprop->mstr_sob_mask[1];1255	wait_prop.mon_id = master_monitor;1256	wait_prop.size = cb_size;1257	cb_size += gaudi_gen_wait_cb(hdev, &wait_prop);1258}1259 1260static void gaudi_collective_slave_init_job(struct hl_device *hdev,1261		struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl)1262{1263	struct hl_gen_wait_properties wait_prop;1264	struct hl_sync_stream_properties *prop;1265	u32 queue_id, cb_size = 0;1266 1267	queue_id = job->hw_queue_id;1268	prop = &hdev->kernel_queues[queue_id].sync_stream_prop;1269 1270	if (job->cs->encaps_signals) {1271		/* use the encaps signal handle store earlier in the flow1272		 * and set the SOB information from the encaps1273		 * signals handle1274		 */1275		hl_hw_queue_encaps_sig_set_sob_info(hdev, job->cs, job,1276						cs_cmpl);1277 1278		dev_dbg(hdev->dev, "collective wait: Sequence %llu found, sob_id: %u,  wait for sob_val: %u\n",1279				job->cs->sequence,1280				cs_cmpl->hw_sob->sob_id,1281				cs_cmpl->sob_val);1282	}1283 1284	/* Add to wait CBs using slave monitor */1285	wait_prop.data = (void *) job->user_cb;1286	wait_prop.sob_base = cs_cmpl->hw_sob->sob_id;1287	wait_prop.sob_mask = 0x1;1288	wait_prop.sob_val = cs_cmpl->sob_val;1289	wait_prop.mon_id = prop->collective_slave_mon_id;1290	wait_prop.q_idx = queue_id;1291	wait_prop.size = cb_size;1292 1293	dev_dbg(hdev->dev,1294		"Generate slave wait CB, sob %d, val:%x, mon %d, q %d\n",1295		cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val,1296		prop->collective_slave_mon_id, queue_id);1297 1298	cb_size += gaudi_gen_wait_cb(hdev, &wait_prop);1299 1300	dev_dbg(hdev->dev,1301		"generate signal CB, sob_id: %d, sob val: 1, q_idx: %d\n",1302		prop->collective_sob_id, queue_id);1303 1304	cb_size += gaudi_gen_signal_cb(hdev, job->user_cb,1305			prop->collective_sob_id, cb_size, false);1306}1307 1308static int gaudi_collective_wait_init_cs(struct hl_cs *cs)1309{1310	struct hl_cs_compl *signal_cs_cmpl =1311		container_of(cs->signal_fence, struct hl_cs_compl, base_fence);1312	struct hl_cs_compl *cs_cmpl =1313		container_of(cs->fence, struct hl_cs_compl, base_fence);1314	struct hl_cs_encaps_sig_handle *handle = cs->encaps_sig_hdl;1315	struct gaudi_collective_properties *cprop;1316	u32 stream, queue_id, sob_group_offset;1317	struct gaudi_device *gaudi;1318	struct hl_device *hdev;1319	struct hl_cs_job *job;1320	struct hl_ctx *ctx;1321 1322	ctx = cs->ctx;1323	hdev = ctx->hdev;1324	gaudi = hdev->asic_specific;1325	cprop = &gaudi->collective_props;1326 1327	if (cs->encaps_signals) {1328		cs_cmpl->hw_sob = handle->hw_sob;1329		/* at this checkpoint we only need the hw_sob pointer1330		 * for the completion check before start going over the jobs1331		 * of the master/slaves, the sob_value will be taken later on1332		 * in gaudi_collective_slave_init_job depends on each1333		 * job wait offset value.1334		 */1335		cs_cmpl->sob_val = 0;1336	} else {1337		/* copy the SOB id and value of the signal CS */1338		cs_cmpl->hw_sob = signal_cs_cmpl->hw_sob;1339		cs_cmpl->sob_val = signal_cs_cmpl->sob_val;1340	}1341 1342	/* check again if the signal cs already completed.1343	 * if yes then don't send any wait cs since the hw_sob1344	 * could be in reset already. if signal is not completed1345	 * then get refcount to hw_sob to prevent resetting the sob1346	 * while wait cs is not submitted.1347	 * note that this check is protected by two locks,1348	 * hw queue lock and completion object lock,1349	 * and the same completion object lock also protects1350	 * the hw_sob reset handler function.1351	 * The hw_queue lock prevent out of sync of hw_sob1352	 * refcount value, changed by signal/wait flows.1353	 */1354	spin_lock(&signal_cs_cmpl->lock);1355 1356	if (completion_done(&cs->signal_fence->completion)) {1357		spin_unlock(&signal_cs_cmpl->lock);1358		return -EINVAL;1359	}1360	/* Increment kref since all slave queues are now waiting on it */1361	kref_get(&cs_cmpl->hw_sob->kref);1362 1363	spin_unlock(&signal_cs_cmpl->lock);1364 1365	/* Calculate the stream from collective master queue (1st job) */1366	job = list_first_entry(&cs->job_list, struct hl_cs_job, cs_node);1367	stream = job->hw_queue_id % 4;1368	sob_group_offset =1369		stream * HL_RSVD_SOBS + cprop->curr_sob_group_idx[stream];1370 1371	list_for_each_entry(job, &cs->job_list, cs_node) {1372		queue_id = job->hw_queue_id;1373 1374		if (hdev->kernel_queues[queue_id].collective_mode ==1375				HL_COLLECTIVE_MASTER)1376			gaudi_collective_master_init_job(hdev, job, stream,1377						sob_group_offset);1378		else1379			gaudi_collective_slave_init_job(hdev, job, cs_cmpl);1380	}1381 1382	cs_cmpl->sob_group = sob_group_offset;1383 1384	/* Handle sob group kref and wraparound */1385	kref_get(&cprop->hw_sob_group[sob_group_offset].kref);1386	cprop->next_sob_group_val[stream]++;1387 1388	if (cprop->next_sob_group_val[stream] == HL_MAX_SOB_VAL) {1389		/*1390		 * Decrement as we reached the max value.1391		 * The release function won't be called here as we've1392		 * just incremented the refcount.1393		 */1394		kref_put(&cprop->hw_sob_group[sob_group_offset].kref,1395				gaudi_sob_group_reset_error);1396		cprop->next_sob_group_val[stream] = 1;1397		/* only two SOBs are currently in use */1398		cprop->curr_sob_group_idx[stream] =1399			(cprop->curr_sob_group_idx[stream] + 1) &1400							(HL_RSVD_SOBS - 1);1401 1402		gaudi_collective_map_sobs(hdev, stream);1403 1404		dev_dbg(hdev->dev, "switched to SOB group %d, stream: %d\n",1405				cprop->curr_sob_group_idx[stream], stream);1406	}1407 1408	mb();1409	hl_fence_put(cs->signal_fence);1410	cs->signal_fence = NULL;1411 1412	return 0;1413}1414 1415static u32 gaudi_get_patched_cb_extra_size(u32 user_cb_size)1416{1417	u32 cacheline_end, additional_commands;1418 1419	cacheline_end = round_up(user_cb_size, DEVICE_CACHE_LINE_SIZE);1420	additional_commands = sizeof(struct packet_msg_prot) * 2;1421 1422	if (user_cb_size + additional_commands > cacheline_end)1423		return cacheline_end - user_cb_size + additional_commands;1424	else1425		return additional_commands;1426}1427 1428static int gaudi_collective_wait_create_job(struct hl_device *hdev,1429		struct hl_ctx *ctx, struct hl_cs *cs,1430		enum hl_collective_mode mode, u32 queue_id, u32 wait_queue_id,1431		u32 encaps_signal_offset)1432{1433	struct hw_queue_properties *hw_queue_prop;1434	struct hl_cs_counters_atomic *cntr;1435	struct hl_cs_job *job;1436	struct hl_cb *cb;1437	u32 cb_size;1438	bool patched_cb;1439 1440	cntr = &hdev->aggregated_cs_counters;1441 1442	if (mode == HL_COLLECTIVE_MASTER) {1443		/* CB size of collective master queue contains1444		 * 4 msg short packets for monitor 1 configuration1445		 * 1 fence packet1446		 * 4 msg short packets for monitor 2 configuration1447		 * 1 fence packet1448		 * 2 msg prot packets for completion and MSI1449		 */1450		cb_size = sizeof(struct packet_msg_short) * 8 +1451				sizeof(struct packet_fence) * 2 +1452				sizeof(struct packet_msg_prot) * 2;1453		patched_cb = true;1454	} else {1455		/* CB size of collective slave queues contains1456		 * 4 msg short packets for monitor configuration1457		 * 1 fence packet1458		 * 1 additional msg short packet for sob signal1459		 */1460		cb_size = sizeof(struct packet_msg_short) * 5 +1461				sizeof(struct packet_fence);1462		patched_cb = false;1463	}1464 1465	hw_queue_prop = &hdev->asic_prop.hw_queues_props[queue_id];1466	job = hl_cs_allocate_job(hdev, hw_queue_prop->type, true);1467	if (!job) {1468		atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);1469		atomic64_inc(&cntr->out_of_mem_drop_cnt);1470		dev_err(hdev->dev, "Failed to allocate a new job\n");1471		return -ENOMEM;1472	}1473 1474	/* Allocate internal mapped CB for non patched CBs */1475	cb = hl_cb_kernel_create(hdev, cb_size, !patched_cb);1476	if (!cb) {1477		atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt);1478		atomic64_inc(&cntr->out_of_mem_drop_cnt);1479		kfree(job);1480		return -EFAULT;1481	}1482 1483	job->id = 0;1484	job->cs = cs;1485	job->user_cb = cb;1486	atomic_inc(&job->user_cb->cs_cnt);1487	job->user_cb_size = cb_size;1488	job->hw_queue_id = queue_id;1489 1490	/* since its guaranteed to have only one chunk in the collective wait1491	 * cs, we can use this chunk to set the encapsulated signal offset1492	 * in the jobs.1493	 */1494	if (cs->encaps_signals)1495		job->encaps_sig_wait_offset = encaps_signal_offset;1496 1497	/*1498	 * No need in parsing, user CB is the patched CB.1499	 * We call hl_cb_destroy() out of two reasons - we don't need1500	 * the CB in the CB idr anymore and to decrement its refcount as1501	 * it was incremented inside hl_cb_kernel_create().1502	 */1503	if (patched_cb)1504		job->patched_cb = job->user_cb;1505	else1506		job->patched_cb = NULL;1507 1508	job->job_cb_size = job->user_cb_size;1509	hl_cb_destroy(&hdev->kernel_mem_mgr, cb->buf->handle);1510 1511	/* increment refcount as for external queues we get completion */1512	if (hw_queue_prop->type == QUEUE_TYPE_EXT)1513		cs_get(cs);1514 1515	cs->jobs_in_queue_cnt[job->hw_queue_id]++;1516 1517	list_add_tail(&job->cs_node, &cs->job_list);1518 1519	hl_debugfs_add_job(hdev, job);1520 1521	return 0;1522}1523 1524static int gaudi_collective_wait_create_jobs(struct hl_device *hdev,1525		struct hl_ctx *ctx, struct hl_cs *cs,1526		u32 wait_queue_id, u32 collective_engine_id,1527		u32 encaps_signal_offset)1528{1529	struct gaudi_device *gaudi = hdev->asic_specific;1530	struct hw_queue_properties *hw_queue_prop;1531	u32 queue_id, collective_queue, num_jobs;1532	u32 stream, nic_queue, nic_idx = 0;1533	bool skip;1534	int i, rc = 0;1535 1536	/* Verify wait queue id is configured as master */1537	hw_queue_prop = &hdev->asic_prop.hw_queues_props[wait_queue_id];1538	if (!(hw_queue_prop->collective_mode == HL_COLLECTIVE_MASTER)) {1539		dev_err(hdev->dev,1540			"Queue %d is not configured as collective master\n",1541			wait_queue_id);1542		return -EINVAL;1543	}1544 1545	/* Verify engine id is supported */1546	if (collective_engine_id != GAUDI_ENGINE_ID_DMA_5 &&1547			collective_engine_id != GAUDI_ENGINE_ID_TPC_7) {1548		dev_err(hdev->dev,1549			"Collective wait does not support engine %u\n",1550			collective_engine_id);1551		return -EINVAL;1552	}1553 1554	stream = wait_queue_id % 4;1555 1556	if (collective_engine_id == GAUDI_ENGINE_ID_DMA_5)1557		collective_queue = GAUDI_QUEUE_ID_DMA_5_0 + stream;1558	else1559		collective_queue = GAUDI_QUEUE_ID_TPC_7_0 + stream;1560 1561	num_jobs = NUMBER_OF_SOBS_IN_GRP + 1;1562	nic_queue = GAUDI_QUEUE_ID_NIC_0_0 + stream;1563 1564	/* First job goes to the collective master queue, it will wait for1565	 * the collective slave queues to finish execution.1566	 * The synchronization is done using two monitors:1567	 * First monitor for NICs 0-7, second monitor for NICs 8-9 and the1568	 * reduction engine (DMA5/TPC7).1569	 *1570	 * Rest of the jobs goes to the collective slave queues which will1571	 * all wait for the user to signal sob 'cs_cmpl->sob_val'.1572	 */1573	for (i = 0 ; i < num_jobs ; i++) {1574		if (i == 0) {1575			queue_id = wait_queue_id;1576			rc = gaudi_collective_wait_create_job(hdev, ctx, cs,1577				HL_COLLECTIVE_MASTER, queue_id,1578				wait_queue_id, encaps_signal_offset);1579		} else {1580			if (nic_idx < NIC_NUMBER_OF_ENGINES) {1581				if (gaudi->hw_cap_initialized &1582					BIT(HW_CAP_NIC_SHIFT + nic_idx))1583					skip = false;1584				else1585					skip = true;1586 1587				queue_id = nic_queue;1588				nic_queue += 4;1589				nic_idx++;1590 1591				if (skip)1592					continue;1593			} else {1594				queue_id = collective_queue;1595			}1596 1597			rc = gaudi_collective_wait_create_job(hdev, ctx, cs,1598				HL_COLLECTIVE_SLAVE, queue_id,1599				wait_queue_id, encaps_signal_offset);1600		}1601 1602		if (rc)1603			return rc;1604	}1605 1606	return rc;1607}1608 1609static int gaudi_late_init(struct hl_device *hdev)1610{1611	struct gaudi_device *gaudi = hdev->asic_specific;1612	int rc;1613 1614	rc = gaudi->cpucp_info_get(hdev);1615	if (rc) {1616		dev_err(hdev->dev, "Failed to get cpucp info\n");1617		return rc;1618	}1619 1620	if ((hdev->card_type == cpucp_card_type_pci) &&1621			(hdev->nic_ports_mask & 0x3)) {1622		dev_info(hdev->dev,1623			"PCI card detected, only 8 ports are enabled\n");1624		hdev->nic_ports_mask &= ~0x3;1625 1626		/* Stop and disable unused NIC QMANs */1627		WREG32(mmNIC0_QM0_GLBL_CFG1, NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK |1628					NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK |1629					NIC0_QM0_GLBL_CFG1_CP_STOP_MASK);1630 1631		WREG32(mmNIC0_QM1_GLBL_CFG1, NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK |1632					NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK |1633					NIC0_QM0_GLBL_CFG1_CP_STOP_MASK);1634 1635		WREG32(mmNIC0_QM0_GLBL_CFG0, 0);1636		WREG32(mmNIC0_QM1_GLBL_CFG0, 0);1637 1638		gaudi->hw_cap_initialized &= ~(HW_CAP_NIC0 | HW_CAP_NIC1);1639	}1640 1641	rc = hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_ENABLE_PCI_ACCESS, 0x0);1642	if (rc)1643		return rc;1644 1645	/* Scrub both SRAM and DRAM */1646	rc = hdev->asic_funcs->scrub_device_mem(hdev);1647	if (rc)1648		goto disable_pci_access;1649 1650	rc = gaudi_fetch_psoc_frequency(hdev);1651	if (rc) {1652		dev_err(hdev->dev, "Failed to fetch psoc frequency\n");1653		goto disable_pci_access;1654	}1655 1656	rc = gaudi_mmu_clear_pgt_range(hdev);1657	if (rc) {1658		dev_err(hdev->dev, "Failed to clear MMU page tables range\n");1659		goto disable_pci_access;1660	}1661 1662	rc = gaudi_init_tpc_mem(hdev);1663	if (rc) {1664		dev_err(hdev->dev, "Failed to initialize TPC memories\n");1665		goto disable_pci_access;1666	}1667 1668	rc = gaudi_collective_init(hdev);1669	if (rc) {1670		dev_err(hdev->dev, "Failed to init collective\n");1671		goto disable_pci_access;1672	}1673 1674	/* We only support a single ASID for the user, so for the sake of optimization, just1675	 * initialize the ASID one time during device initialization with the fixed value of 11676	 */1677	gaudi_mmu_prepare(hdev, 1);1678 1679	hl_fw_set_pll_profile(hdev);1680 1681	return 0;1682 1683disable_pci_access:1684	hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0);1685 1686	return rc;1687}1688 1689static void gaudi_late_fini(struct hl_device *hdev)1690{1691	hl_hwmon_release_resources(hdev);1692}1693 1694static int gaudi_alloc_cpu_accessible_dma_mem(struct hl_device *hdev)1695{1696	dma_addr_t dma_addr_arr[GAUDI_ALLOC_CPU_MEM_RETRY_CNT] = {}, end_addr;1697	void *virt_addr_arr[GAUDI_ALLOC_CPU_MEM_RETRY_CNT] = {};1698	int i, j, rc = 0;1699 1700	/*1701	 * The device CPU works with 40-bits addresses, while bit 39 must be set1702	 * to '1' when accessing the host.1703	 * Bits 49:39 of the full host address are saved for a later1704	 * configuration of the HW to perform extension to 50 bits.1705	 * Because there is a single HW register that holds the extension bits,1706	 * these bits must be identical in all allocated range.1707	 */1708 1709	for (i = 0 ; i < GAUDI_ALLOC_CPU_MEM_RETRY_CNT ; i++) {1710		virt_addr_arr[i] = hl_asic_dma_alloc_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE,1711								&dma_addr_arr[i],1712								GFP_KERNEL | __GFP_ZERO);1713		if (!virt_addr_arr[i]) {1714			rc = -ENOMEM;1715			goto free_dma_mem_arr;1716		}1717 1718		end_addr = dma_addr_arr[i] + HL_CPU_ACCESSIBLE_MEM_SIZE - 1;1719		if (GAUDI_CPU_PCI_MSB_ADDR(dma_addr_arr[i]) ==1720				GAUDI_CPU_PCI_MSB_ADDR(end_addr))1721			break;1722	}1723 1724	if (i == GAUDI_ALLOC_CPU_MEM_RETRY_CNT) {1725		dev_err(hdev->dev,1726			"MSB of CPU accessible DMA memory are not identical in all range\n");1727		rc = -EFAULT;1728		goto free_dma_mem_arr;1729	}1730 1731	hdev->cpu_accessible_dma_mem = virt_addr_arr[i];1732	hdev->cpu_accessible_dma_address = dma_addr_arr[i];1733	hdev->cpu_pci_msb_addr =1734		GAUDI_CPU_PCI_MSB_ADDR(hdev->cpu_accessible_dma_address);1735 1736	if (!hdev->asic_prop.fw_security_enabled)1737		GAUDI_PCI_TO_CPU_ADDR(hdev->cpu_accessible_dma_address);1738 1739free_dma_mem_arr:1740	for (j = 0 ; j < i ; j++)1741		hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, virt_addr_arr[j],1742						dma_addr_arr[j]);1743 1744	return rc;1745}1746 1747static void gaudi_free_internal_qmans_pq_mem(struct hl_device *hdev)1748{1749	struct gaudi_device *gaudi = hdev->asic_specific;1750	struct gaudi_internal_qman_info *q;1751	u32 i;1752 1753	for (i = 0 ; i < GAUDI_QUEUE_ID_SIZE ; i++) {1754		q = &gaudi->internal_qmans[i];1755		if (!q->pq_kernel_addr)1756			continue;1757		hl_asic_dma_free_coherent(hdev, q->pq_size, q->pq_kernel_addr, q->pq_dma_addr);1758	}1759}1760 1761static int gaudi_alloc_internal_qmans_pq_mem(struct hl_device *hdev)1762{1763	struct gaudi_device *gaudi = hdev->asic_specific;1764	struct gaudi_internal_qman_info *q;1765	int rc, i;1766 1767	for (i = 0 ; i < GAUDI_QUEUE_ID_SIZE ; i++) {1768		if (gaudi_queue_type[i] != QUEUE_TYPE_INT)1769			continue;1770 1771		q = &gaudi->internal_qmans[i];1772 1773		switch (i) {1774		case GAUDI_QUEUE_ID_DMA_2_0 ... GAUDI_QUEUE_ID_DMA_7_3:1775			q->pq_size = HBM_DMA_QMAN_SIZE_IN_BYTES;1776			break;1777		case GAUDI_QUEUE_ID_MME_0_0 ... GAUDI_QUEUE_ID_MME_1_3:1778			q->pq_size = MME_QMAN_SIZE_IN_BYTES;1779			break;1780		case GAUDI_QUEUE_ID_TPC_0_0 ... GAUDI_QUEUE_ID_TPC_7_3:1781			q->pq_size = TPC_QMAN_SIZE_IN_BYTES;1782			break;1783		case GAUDI_QUEUE_ID_NIC_0_0 ... GAUDI_QUEUE_ID_NIC_9_3:1784			q->pq_size = NIC_QMAN_SIZE_IN_BYTES;1785			break;1786		default:1787			dev_err(hdev->dev, "Bad internal queue index %d", i);1788			rc = -EINVAL;1789			goto free_internal_qmans_pq_mem;1790		}1791 1792		q->pq_kernel_addr = hl_asic_dma_alloc_coherent(hdev, q->pq_size, &q->pq_dma_addr,1793								GFP_KERNEL | __GFP_ZERO);1794		if (!q->pq_kernel_addr) {1795			rc = -ENOMEM;1796			goto free_internal_qmans_pq_mem;1797		}1798	}1799 1800	return 0;1801 1802free_internal_qmans_pq_mem:1803	gaudi_free_internal_qmans_pq_mem(hdev);1804	return rc;1805}1806 1807static void gaudi_set_pci_memory_regions(struct hl_device *hdev)1808{1809	struct asic_fixed_properties *prop = &hdev->asic_prop;1810	struct pci_mem_region *region;1811 1812	/* CFG */1813	region = &hdev->pci_mem_region[PCI_REGION_CFG];1814	region->region_base = CFG_BASE;1815	region->region_size = CFG_SIZE;1816	region->offset_in_bar = CFG_BASE - SPI_FLASH_BASE_ADDR;1817	region->bar_size = CFG_BAR_SIZE;1818	region->bar_id = CFG_BAR_ID;1819	region->used = 1;1820 1821	/* SRAM */1822	region = &hdev->pci_mem_region[PCI_REGION_SRAM];1823	region->region_base = SRAM_BASE_ADDR;1824	region->region_size = SRAM_SIZE;1825	region->offset_in_bar = 0;1826	region->bar_size = SRAM_BAR_SIZE;1827	region->bar_id = SRAM_BAR_ID;1828	region->used = 1;1829 1830	/* DRAM */1831	region = &hdev->pci_mem_region[PCI_REGION_DRAM];1832	region->region_base = DRAM_PHYS_BASE;1833	region->region_size = hdev->asic_prop.dram_size;1834	region->offset_in_bar = 0;1835	region->bar_size = prop->dram_pci_bar_size;1836	region->bar_id = HBM_BAR_ID;1837	region->used = 1;1838 1839	/* SP SRAM */1840	region = &hdev->pci_mem_region[PCI_REGION_SP_SRAM];1841	region->region_base = PSOC_SCRATCHPAD_ADDR;1842	region->region_size = PSOC_SCRATCHPAD_SIZE;1843	region->offset_in_bar = PSOC_SCRATCHPAD_ADDR - SPI_FLASH_BASE_ADDR;1844	region->bar_size = CFG_BAR_SIZE;1845	region->bar_id = CFG_BAR_ID;1846	region->used = 1;1847}1848 1849static int gaudi_sw_init(struct hl_device *hdev)1850{1851	struct gaudi_device *gaudi;1852	u32 i, event_id = 0;1853	int rc;1854 1855	/* Allocate device structure */1856	gaudi = kzalloc(sizeof(*gaudi), GFP_KERNEL);1857	if (!gaudi)1858		return -ENOMEM;1859 1860	for (i = 0 ; i < ARRAY_SIZE(gaudi_irq_map_table) ; i++) {1861		if (gaudi_irq_map_table[i].valid) {1862			if (event_id == GAUDI_EVENT_SIZE) {1863				dev_err(hdev->dev,1864					"Event array exceeds the limit of %u events\n",1865					GAUDI_EVENT_SIZE);1866				rc = -EINVAL;1867				goto free_gaudi_device;1868			}1869 1870			gaudi->events[event_id++] =1871					gaudi_irq_map_table[i].fc_id;1872		}1873	}1874 1875	gaudi->cpucp_info_get = gaudi_cpucp_info_get;1876 1877	hdev->asic_specific = gaudi;1878 1879	/* Create DMA pool for small allocations */1880	hdev->dma_pool = dma_pool_create(dev_name(hdev->dev),1881			&hdev->pdev->dev, GAUDI_DMA_POOL_BLK_SIZE, 8, 0);1882	if (!hdev->dma_pool) {1883		dev_err(hdev->dev, "failed to create DMA pool\n");1884		rc = -ENOMEM;1885		goto free_gaudi_device;1886	}1887 1888	rc = gaudi_alloc_cpu_accessible_dma_mem(hdev);1889	if (rc)1890		goto free_dma_pool;1891 1892	hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1);1893	if (!hdev->cpu_accessible_dma_pool) {1894		dev_err(hdev->dev,1895			"Failed to create CPU accessible DMA pool\n");1896		rc = -ENOMEM;1897		goto free_cpu_dma_mem;1898	}1899 1900	rc = gen_pool_add(hdev->cpu_accessible_dma_pool,1901				(uintptr_t) hdev->cpu_accessible_dma_mem,1902				HL_CPU_ACCESSIBLE_MEM_SIZE, -1);1903	if (rc) {1904		dev_err(hdev->dev,1905			"Failed to add memory to CPU accessible DMA pool\n");1906		rc = -EFAULT;1907		goto free_cpu_accessible_dma_pool;1908	}1909 1910	rc = gaudi_alloc_internal_qmans_pq_mem(hdev);1911	if (rc)1912		goto free_cpu_accessible_dma_pool;1913 1914	spin_lock_init(&gaudi->hw_queues_lock);1915 1916	hdev->supports_sync_stream = true;1917	hdev->supports_coresight = true;1918	hdev->supports_staged_submission = true;1919	hdev->supports_wait_for_multi_cs = true;1920 1921	hdev->asic_funcs->set_pci_memory_regions(hdev);1922	hdev->stream_master_qid_arr =1923				hdev->asic_funcs->get_stream_master_qid_arr();1924	hdev->stream_master_qid_arr_size = GAUDI_STREAM_MASTER_ARR_SIZE;1925 1926	return 0;1927 1928free_cpu_accessible_dma_pool:1929	gen_pool_destroy(hdev->cpu_accessible_dma_pool);1930free_cpu_dma_mem:1931	if (!hdev->asic_prop.fw_security_enabled)1932		GAUDI_CPU_TO_PCI_ADDR(hdev->cpu_accessible_dma_address,1933					hdev->cpu_pci_msb_addr);1934	hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem,1935					hdev->cpu_accessible_dma_address);1936free_dma_pool:1937	dma_pool_destroy(hdev->dma_pool);1938free_gaudi_device:1939	kfree(gaudi);1940	return rc;1941}1942 1943static int gaudi_sw_fini(struct hl_device *hdev)1944{1945	struct gaudi_device *gaudi = hdev->asic_specific;1946 1947	gaudi_free_internal_qmans_pq_mem(hdev);1948 1949	gen_pool_destroy(hdev->cpu_accessible_dma_pool);1950 1951	if (!hdev->asic_prop.fw_security_enabled)1952		GAUDI_CPU_TO_PCI_ADDR(hdev->cpu_accessible_dma_address,1953					hdev->cpu_pci_msb_addr);1954 1955	hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem,1956					hdev->cpu_accessible_dma_address);1957 1958	dma_pool_destroy(hdev->dma_pool);1959 1960	kfree(gaudi);1961 1962	return 0;1963}1964 1965static irqreturn_t gaudi_irq_handler_single(int irq, void *arg)1966{1967	struct hl_device *hdev = arg;1968	int i;1969 1970	if (hdev->disabled)1971		return IRQ_HANDLED;1972 1973	for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)1974		hl_irq_handler_cq(irq, &hdev->completion_queue[i]);1975 1976	hl_irq_handler_eq(irq, &hdev->event_queue);1977 1978	return IRQ_HANDLED;1979}1980 1981/*1982 * For backward compatibility, new MSI interrupts should be set after the1983 * existing CPU and NIC interrupts.1984 */1985static int gaudi_pci_irq_vector(struct hl_device *hdev, unsigned int nr,1986				bool cpu_eq)1987{1988	int msi_vec;1989 1990	if ((nr != GAUDI_EVENT_QUEUE_MSI_IDX) && (cpu_eq))1991		dev_crit(hdev->dev, "CPU EQ must use IRQ %d\n",1992				GAUDI_EVENT_QUEUE_MSI_IDX);1993 1994	msi_vec = ((nr < GAUDI_EVENT_QUEUE_MSI_IDX) || (cpu_eq)) ? nr :1995			(nr + NIC_NUMBER_OF_ENGINES + 1);1996 1997	return pci_irq_vector(hdev->pdev, msi_vec);1998}1999 2000static int gaudi_enable_msi_single(struct hl_device *hdev)2001{2002	int rc, irq;2003 2004	dev_dbg(hdev->dev, "Working in single MSI IRQ mode\n");2005 2006	irq = gaudi_pci_irq_vector(hdev, 0, false);2007	rc = request_irq(irq, gaudi_irq_handler_single, 0,2008			"gaudi single msi", hdev);2009	if (rc)2010		dev_err(hdev->dev,2011			"Failed to request single MSI IRQ\n");2012 2013	return rc;2014}2015 2016static int gaudi_enable_msi(struct hl_device *hdev)2017{2018	struct gaudi_device *gaudi = hdev->asic_specific;2019	int rc;2020 2021	if (gaudi->hw_cap_initialized & HW_CAP_MSI)2022		return 0;2023 2024	rc = pci_alloc_irq_vectors(hdev->pdev, 1, 1, PCI_IRQ_MSI);2025	if (rc < 0) {2026		dev_err(hdev->dev, "MSI: Failed to enable support %d\n", rc);2027		return rc;2028	}2029 2030	rc = gaudi_enable_msi_single(hdev);2031	if (rc)2032		goto free_pci_irq_vectors;2033 2034	gaudi->hw_cap_initialized |= HW_CAP_MSI;2035 2036	return 0;2037 2038free_pci_irq_vectors:2039	pci_free_irq_vectors(hdev->pdev);2040	return rc;2041}2042 2043static void gaudi_sync_irqs(struct hl_device *hdev)2044{2045	struct gaudi_device *gaudi = hdev->asic_specific;2046 2047	if (!(gaudi->hw_cap_initialized & HW_CAP_MSI))2048		return;2049 2050	/* Wait for all pending IRQs to be finished */2051	synchronize_irq(gaudi_pci_irq_vector(hdev, 0, false));2052}2053 2054static void gaudi_disable_msi(struct hl_device *hdev)2055{2056	struct gaudi_device *gaudi = hdev->asic_specific;2057 2058	if (!(gaudi->hw_cap_initialized & HW_CAP_MSI))2059		return;2060 2061	gaudi_sync_irqs(hdev);2062	free_irq(gaudi_pci_irq_vector(hdev, 0, false), hdev);2063	pci_free_irq_vectors(hdev->pdev);2064 2065	gaudi->hw_cap_initialized &= ~HW_CAP_MSI;2066}2067 2068static void gaudi_init_scrambler_sram(struct hl_device *hdev)2069{2070	struct gaudi_device *gaudi = hdev->asic_specific;2071 2072	if (hdev->asic_prop.fw_security_enabled)2073		return;2074 2075	if (hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &2076						CPU_BOOT_DEV_STS0_SRAM_SCR_EN)2077		return;2078 2079	if (gaudi->hw_cap_initialized & HW_CAP_SRAM_SCRAMBLER)2080		return;2081 2082	WREG32(mmNIF_RTR_CTRL_0_SCRAM_SRAM_EN,2083			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);2084	WREG32(mmNIF_RTR_CTRL_1_SCRAM_SRAM_EN,2085			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);2086	WREG32(mmNIF_RTR_CTRL_2_SCRAM_SRAM_EN,2087			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);2088	WREG32(mmNIF_RTR_CTRL_3_SCRAM_SRAM_EN,2089			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);2090	WREG32(mmNIF_RTR_CTRL_4_SCRAM_SRAM_EN,2091			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);2092	WREG32(mmNIF_RTR_CTRL_5_SCRAM_SRAM_EN,2093			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);2094	WREG32(mmNIF_RTR_CTRL_6_SCRAM_SRAM_EN,2095			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);2096	WREG32(mmNIF_RTR_CTRL_7_SCRAM_SRAM_EN,2097			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);2098 2099	WREG32(mmSIF_RTR_CTRL_0_SCRAM_SRAM_EN,2100			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);2101	WREG32(mmSIF_RTR_CTRL_1_SCRAM_SRAM_EN,2102			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);2103	WREG32(mmSIF_RTR_CTRL_2_SCRAM_SRAM_EN,2104			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);2105	WREG32(mmSIF_RTR_CTRL_3_SCRAM_SRAM_EN,2106			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);2107	WREG32(mmSIF_RTR_CTRL_4_SCRAM_SRAM_EN,2108			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);2109	WREG32(mmSIF_RTR_CTRL_5_SCRAM_SRAM_EN,2110			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);2111	WREG32(mmSIF_RTR_CTRL_6_SCRAM_SRAM_EN,2112			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);2113	WREG32(mmSIF_RTR_CTRL_7_SCRAM_SRAM_EN,2114			1 << IF_RTR_CTRL_SCRAM_SRAM_EN_VAL_SHIFT);2115 2116	WREG32(mmDMA_IF_E_N_DOWN_CH0_SCRAM_SRAM_EN,2117			1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);2118	WREG32(mmDMA_IF_E_N_DOWN_CH1_SCRAM_SRAM_EN,2119			1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);2120	WREG32(mmDMA_IF_E_S_DOWN_CH0_SCRAM_SRAM_EN,2121			1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);2122	WREG32(mmDMA_IF_E_S_DOWN_CH1_SCRAM_SRAM_EN,2123			1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);2124	WREG32(mmDMA_IF_W_N_DOWN_CH0_SCRAM_SRAM_EN,2125			1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);2126	WREG32(mmDMA_IF_W_N_DOWN_CH1_SCRAM_SRAM_EN,2127			1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);2128	WREG32(mmDMA_IF_W_S_DOWN_CH0_SCRAM_SRAM_EN,2129			1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);2130	WREG32(mmDMA_IF_W_S_DOWN_CH1_SCRAM_SRAM_EN,2131			1 << DMA_IF_DOWN_CHX_SCRAM_SRAM_EN_VAL_SHIFT);2132 2133	gaudi->hw_cap_initialized |= HW_CAP_SRAM_SCRAMBLER;2134}2135 2136static void gaudi_init_scrambler_hbm(struct hl_device *hdev)2137{2138	struct gaudi_device *gaudi = hdev->asic_specific;2139 2140	if (hdev->asic_prop.fw_security_enabled)2141		return;2142 2143	if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 &2144					CPU_BOOT_DEV_STS0_DRAM_SCR_EN)2145		return;2146 2147	if (gaudi->hw_cap_initialized & HW_CAP_HBM_SCRAMBLER)2148		return;2149 2150	WREG32(mmNIF_RTR_CTRL_0_SCRAM_HBM_EN,2151			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);2152	WREG32(mmNIF_RTR_CTRL_1_SCRAM_HBM_EN,2153			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);2154	WREG32(mmNIF_RTR_CTRL_2_SCRAM_HBM_EN,2155			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);2156	WREG32(mmNIF_RTR_CTRL_3_SCRAM_HBM_EN,2157			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);2158	WREG32(mmNIF_RTR_CTRL_4_SCRAM_HBM_EN,2159			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);2160	WREG32(mmNIF_RTR_CTRL_5_SCRAM_HBM_EN,2161			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);2162	WREG32(mmNIF_RTR_CTRL_6_SCRAM_HBM_EN,2163			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);2164	WREG32(mmNIF_RTR_CTRL_7_SCRAM_HBM_EN,2165			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);2166 2167	WREG32(mmSIF_RTR_CTRL_0_SCRAM_HBM_EN,2168			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);2169	WREG32(mmSIF_RTR_CTRL_1_SCRAM_HBM_EN,2170			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);2171	WREG32(mmSIF_RTR_CTRL_2_SCRAM_HBM_EN,2172			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);2173	WREG32(mmSIF_RTR_CTRL_3_SCRAM_HBM_EN,2174			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);2175	WREG32(mmSIF_RTR_CTRL_4_SCRAM_HBM_EN,2176			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);2177	WREG32(mmSIF_RTR_CTRL_5_SCRAM_HBM_EN,2178			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);2179	WREG32(mmSIF_RTR_CTRL_6_SCRAM_HBM_EN,2180			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);2181	WREG32(mmSIF_RTR_CTRL_7_SCRAM_HBM_EN,2182			1 << IF_RTR_CTRL_SCRAM_HBM_EN_VAL_SHIFT);2183 2184	WREG32(mmDMA_IF_E_N_DOWN_CH0_SCRAM_HBM_EN,2185			1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);2186	WREG32(mmDMA_IF_E_N_DOWN_CH1_SCRAM_HBM_EN,2187			1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);2188	WREG32(mmDMA_IF_E_S_DOWN_CH0_SCRAM_HBM_EN,2189			1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);2190	WREG32(mmDMA_IF_E_S_DOWN_CH1_SCRAM_HBM_EN,2191			1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);2192	WREG32(mmDMA_IF_W_N_DOWN_CH0_SCRAM_HBM_EN,2193			1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);2194	WREG32(mmDMA_IF_W_N_DOWN_CH1_SCRAM_HBM_EN,2195			1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);2196	WREG32(mmDMA_IF_W_S_DOWN_CH0_SCRAM_HBM_EN,2197			1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);2198	WREG32(mmDMA_IF_W_S_DOWN_CH1_SCRAM_HBM_EN,2199			1 << DMA_IF_DOWN_CHX_SCRAM_HBM_EN_VAL_SHIFT);2200 2201	gaudi->hw_cap_initialized |= HW_CAP_HBM_SCRAMBLER;2202}2203 2204static void gaudi_init_e2e(struct hl_device *hdev)2205{2206	if (hdev->asic_prop.fw_security_enabled)2207		return;2208 2209	if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 &2210					CPU_BOOT_DEV_STS0_E2E_CRED_EN)2211		return;2212 2213	WREG32(mmSIF_RTR_CTRL_0_E2E_HBM_WR_SIZE, 247 >> 3);2214	WREG32(mmSIF_RTR_CTRL_0_E2E_HBM_RD_SIZE, 785 >> 3);2215	WREG32(mmSIF_RTR_CTRL_0_E2E_PCI_WR_SIZE, 49);2216	WREG32(mmSIF_RTR_CTRL_0_E2E_PCI_RD_SIZE, 101);2217 2218	WREG32(mmSIF_RTR_CTRL_1_E2E_HBM_WR_SIZE, 275 >> 3);2219	WREG32(mmSIF_RTR_CTRL_1_E2E_HBM_RD_SIZE, 614 >> 3);2220	WREG32(mmSIF_RTR_CTRL_1_E2E_PCI_WR_SIZE, 1);2221	WREG32(mmSIF_RTR_CTRL_1_E2E_PCI_RD_SIZE, 39);2222 2223	WREG32(mmSIF_RTR_CTRL_2_E2E_HBM_WR_SIZE, 1);2224	WREG32(mmSIF_RTR_CTRL_2_E2E_HBM_RD_SIZE, 1);2225	WREG32(mmSIF_RTR_CTRL_2_E2E_PCI_WR_SIZE, 1);2226	WREG32(mmSIF_RTR_CTRL_2_E2E_PCI_RD_SIZE, 32);2227 2228	WREG32(mmSIF_RTR_CTRL_3_E2E_HBM_WR_SIZE, 176 >> 3);2229	WREG32(mmSIF_RTR_CTRL_3_E2E_HBM_RD_SIZE, 32 >> 3);2230	WREG32(mmSIF_RTR_CTRL_3_E2E_PCI_WR_SIZE, 19);2231	WREG32(mmSIF_RTR_CTRL_3_E2E_PCI_RD_SIZE, 32);2232 2233	WREG32(mmSIF_RTR_CTRL_4_E2E_HBM_WR_SIZE, 176 >> 3);2234	WREG32(mmSIF_RTR_CTRL_4_E2E_HBM_RD_SIZE, 32 >> 3);2235	WREG32(mmSIF_RTR_CTRL_4_E2E_PCI_WR_SIZE, 19);2236	WREG32(mmSIF_RTR_CTRL_4_E2E_PCI_RD_SIZE, 32);2237 2238	WREG32(mmSIF_RTR_CTRL_5_E2E_HBM_WR_SIZE, 1);2239	WREG32(mmSIF_RTR_CTRL_5_E2E_HBM_RD_SIZE, 1);2240	WREG32(mmSIF_RTR_CTRL_5_E2E_PCI_WR_SIZE, 1);2241	WREG32(mmSIF_RTR_CTRL_5_E2E_PCI_RD_SIZE, 32);2242 2243	WREG32(mmSIF_RTR_CTRL_6_E2E_HBM_WR_SIZE, 275 >> 3);2244	WREG32(mmSIF_RTR_CTRL_6_E2E_HBM_RD_SIZE, 614 >> 3);2245	WREG32(mmSIF_RTR_CTRL_6_E2E_PCI_WR_SIZE, 1);2246	WREG32(mmSIF_RTR_CTRL_6_E2E_PCI_RD_SIZE, 39);2247 2248	WREG32(mmSIF_RTR_CTRL_7_E2E_HBM_WR_SIZE, 297 >> 3);2249	WREG32(mmSIF_RTR_CTRL_7_E2E_HBM_RD_SIZE, 908 >> 3);2250	WREG32(mmSIF_RTR_CTRL_7_E2E_PCI_WR_SIZE, 19);2251	WREG32(mmSIF_RTR_CTRL_7_E2E_PCI_RD_SIZE, 19);2252 2253	WREG32(mmNIF_RTR_CTRL_0_E2E_HBM_WR_SIZE, 318 >> 3);2254	WREG32(mmNIF_RTR_CTRL_0_E2E_HBM_RD_SIZE, 956 >> 3);2255	WREG32(mmNIF_RTR_CTRL_0_E2E_PCI_WR_SIZE, 79);2256	WREG32(mmNIF_RTR_CTRL_0_E2E_PCI_RD_SIZE, 163);2257 2258	WREG32(mmNIF_RTR_CTRL_1_E2E_HBM_WR_SIZE, 275 >> 3);2259	WREG32(mmNIF_RTR_CTRL_1_E2E_HBM_RD_SIZE, 614 >> 3);2260	WREG32(mmNIF_RTR_CTRL_1_E2E_PCI_WR_SIZE, 1);2261	WREG32(mmNIF_RTR_CTRL_1_E2E_PCI_RD_SIZE, 39);2262 2263	WREG32(mmNIF_RTR_CTRL_2_E2E_HBM_WR_SIZE, 1);2264	WREG32(mmNIF_RTR_CTRL_2_E2E_HBM_RD_SIZE, 1);2265	WREG32(mmNIF_RTR_CTRL_2_E2E_PCI_WR_SIZE, 1);2266	WREG32(mmNIF_RTR_CTRL_2_E2E_PCI_RD_SIZE, 32);2267 2268	WREG32(mmNIF_RTR_CTRL_3_E2E_HBM_WR_SIZE, 176 >> 3);2269	WREG32(mmNIF_RTR_CTRL_3_E2E_HBM_RD_SIZE, 32 >> 3);2270	WREG32(mmNIF_RTR_CTRL_3_E2E_PCI_WR_SIZE, 19);2271	WREG32(mmNIF_RTR_CTRL_3_E2E_PCI_RD_SIZE, 32);2272 2273	WREG32(mmNIF_RTR_CTRL_4_E2E_HBM_WR_SIZE, 176 >> 3);2274	WREG32(mmNIF_RTR_CTRL_4_E2E_HBM_RD_SIZE, 32 >> 3);2275	WREG32(mmNIF_RTR_CTRL_4_E2E_PCI_WR_SIZE, 19);2276	WREG32(mmNIF_RTR_CTRL_4_E2E_PCI_RD_SIZE, 32);2277 2278	WREG32(mmNIF_RTR_CTRL_5_E2E_HBM_WR_SIZE, 1);2279	WREG32(mmNIF_RTR_CTRL_5_E2E_HBM_RD_SIZE, 1);2280	WREG32(mmNIF_RTR_CTRL_5_E2E_PCI_WR_SIZE, 1);2281	WREG32(mmNIF_RTR_CTRL_5_E2E_PCI_RD_SIZE, 32);2282 2283	WREG32(mmNIF_RTR_CTRL_6_E2E_HBM_WR_SIZE, 275 >> 3);2284	WREG32(mmNIF_RTR_CTRL_6_E2E_HBM_RD_SIZE, 614 >> 3);2285	WREG32(mmNIF_RTR_CTRL_6_E2E_PCI_WR_SIZE, 1);2286	WREG32(mmNIF_RTR_CTRL_6_E2E_PCI_RD_SIZE, 39);2287 2288	WREG32(mmNIF_RTR_CTRL_7_E2E_HBM_WR_SIZE, 318 >> 3);2289	WREG32(mmNIF_RTR_CTRL_7_E2E_HBM_RD_SIZE, 956 >> 3);2290	WREG32(mmNIF_RTR_CTRL_7_E2E_PCI_WR_SIZE, 79);2291	WREG32(mmNIF_RTR_CTRL_7_E2E_PCI_RD_SIZE, 79);2292 2293	WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_HBM_WR_SIZE, 344 >> 3);2294	WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_HBM_RD_SIZE, 1000 >> 3);2295	WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_PCI_WR_SIZE, 162);2296	WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_PCI_RD_SIZE, 338);2297 2298	WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_HBM_WR_SIZE, 344 >> 3);2299	WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_HBM_RD_SIZE, 1000 >> 3);2300	WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_PCI_WR_SIZE, 162);2301	WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_PCI_RD_SIZE, 338);2302 2303	WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_HBM_WR_SIZE, 344 >> 3);2304	WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_HBM_RD_SIZE, 1000 >> 3);2305	WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_PCI_WR_SIZE, 162);2306	WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_PCI_RD_SIZE, 338);2307 2308	WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_HBM_WR_SIZE, 344 >> 3);2309	WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_HBM_RD_SIZE, 1000 >> 3);2310	WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_PCI_WR_SIZE, 162);2311	WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_PCI_RD_SIZE, 338);2312 2313	WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_HBM_WR_SIZE, 344 >> 3);2314	WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_HBM_RD_SIZE, 1000 >> 3);2315	WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_PCI_WR_SIZE, 162);2316	WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_PCI_RD_SIZE, 338);2317 2318	WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_HBM_WR_SIZE, 344 >> 3);2319	WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_HBM_RD_SIZE, 1000 >> 3);2320	WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_PCI_WR_SIZE, 162);2321	WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_PCI_RD_SIZE, 338);2322 2323	WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_HBM_WR_SIZE, 344 >> 3);2324	WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_HBM_RD_SIZE, 1000 >> 3);2325	WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_PCI_WR_SIZE, 162);2326	WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_PCI_RD_SIZE, 338);2327 2328	WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_HBM_WR_SIZE, 344 >> 3);2329	WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_HBM_RD_SIZE, 1000 >> 3);2330	WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_PCI_WR_SIZE, 162);2331	WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_PCI_RD_SIZE, 338);2332 2333	WREG32(mmSIF_RTR_CTRL_0_E2E_HBM_EN,2334			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);2335	WREG32(mmSIF_RTR_CTRL_0_E2E_PCI_EN,2336			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);2337 2338	WREG32(mmSIF_RTR_CTRL_1_E2E_HBM_EN,2339			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);2340	WREG32(mmSIF_RTR_CTRL_1_E2E_PCI_EN,2341			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);2342 2343	WREG32(mmSIF_RTR_CTRL_2_E2E_HBM_EN,2344			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);2345	WREG32(mmSIF_RTR_CTRL_2_E2E_PCI_EN,2346			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);2347 2348	WREG32(mmSIF_RTR_CTRL_3_E2E_HBM_EN,2349			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);2350	WREG32(mmSIF_RTR_CTRL_3_E2E_PCI_EN,2351			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);2352 2353	WREG32(mmSIF_RTR_CTRL_4_E2E_HBM_EN,2354			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);2355	WREG32(mmSIF_RTR_CTRL_4_E2E_PCI_EN,2356			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);2357 2358	WREG32(mmSIF_RTR_CTRL_5_E2E_HBM_EN,2359			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);2360	WREG32(mmSIF_RTR_CTRL_5_E2E_PCI_EN,2361			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);2362 2363	WREG32(mmSIF_RTR_CTRL_6_E2E_HBM_EN,2364			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);2365	WREG32(mmSIF_RTR_CTRL_6_E2E_PCI_EN,2366			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);2367 2368	WREG32(mmSIF_RTR_CTRL_7_E2E_HBM_EN,2369			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);2370	WREG32(mmSIF_RTR_CTRL_7_E2E_PCI_EN,2371			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);2372 2373	WREG32(mmNIF_RTR_CTRL_0_E2E_HBM_EN,2374			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);2375	WREG32(mmNIF_RTR_CTRL_0_E2E_PCI_EN,2376			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);2377 2378	WREG32(mmNIF_RTR_CTRL_1_E2E_HBM_EN,2379			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);2380	WREG32(mmNIF_RTR_CTRL_1_E2E_PCI_EN,2381			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);2382 2383	WREG32(mmNIF_RTR_CTRL_2_E2E_HBM_EN,2384			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);2385	WREG32(mmNIF_RTR_CTRL_2_E2E_PCI_EN,2386			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);2387 2388	WREG32(mmNIF_RTR_CTRL_3_E2E_HBM_EN,2389			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);2390	WREG32(mmNIF_RTR_CTRL_3_E2E_PCI_EN,2391			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);2392 2393	WREG32(mmNIF_RTR_CTRL_4_E2E_HBM_EN,2394			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);2395	WREG32(mmNIF_RTR_CTRL_4_E2E_PCI_EN,2396			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);2397 2398	WREG32(mmNIF_RTR_CTRL_5_E2E_HBM_EN,2399			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);2400	WREG32(mmNIF_RTR_CTRL_5_E2E_PCI_EN,2401			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);2402 2403	WREG32(mmNIF_RTR_CTRL_6_E2E_HBM_EN,2404			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);2405	WREG32(mmNIF_RTR_CTRL_6_E2E_PCI_EN,2406			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);2407 2408	WREG32(mmNIF_RTR_CTRL_7_E2E_HBM_EN,2409			1 << IF_RTR_CTRL_E2E_HBM_EN_VAL_SHIFT);2410	WREG32(mmNIF_RTR_CTRL_7_E2E_PCI_EN,2411			1 << IF_RTR_CTRL_E2E_PCI_EN_VAL_SHIFT);2412 2413	WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_HBM_EN,2414			1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);2415	WREG32(mmDMA_IF_E_N_DOWN_CH0_E2E_PCI_EN,2416			1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);2417 2418	WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_HBM_EN,2419			1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);2420	WREG32(mmDMA_IF_E_N_DOWN_CH1_E2E_PCI_EN,2421			1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);2422 2423	WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_HBM_EN,2424			1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);2425	WREG32(mmDMA_IF_E_S_DOWN_CH0_E2E_PCI_EN,2426			1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);2427 2428	WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_HBM_EN,2429			1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);2430	WREG32(mmDMA_IF_E_S_DOWN_CH1_E2E_PCI_EN,2431			1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);2432 2433	WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_HBM_EN,2434			1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);2435	WREG32(mmDMA_IF_W_N_DOWN_CH0_E2E_PCI_EN,2436			1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);2437 2438	WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_HBM_EN,2439			1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);2440	WREG32(mmDMA_IF_W_N_DOWN_CH1_E2E_PCI_EN,2441			1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);2442 2443	WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_HBM_EN,2444			1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);2445	WREG32(mmDMA_IF_W_S_DOWN_CH0_E2E_PCI_EN,2446			1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);2447 2448	WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_HBM_EN,2449			1 << DMA_IF_DOWN_CHX_E2E_HBM_EN_VAL_SHIFT);2450	WREG32(mmDMA_IF_W_S_DOWN_CH1_E2E_PCI_EN,2451			1 << DMA_IF_DOWN_CHX_E2E_PCI_EN_VAL_SHIFT);2452}2453 2454static void gaudi_init_hbm_cred(struct hl_device *hdev)2455{2456	u32 hbm0_wr, hbm1_wr, hbm0_rd, hbm1_rd;2457 2458	if (hdev->asic_prop.fw_security_enabled)2459		return;2460 2461	if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 &2462						CPU_BOOT_DEV_STS0_HBM_CRED_EN)2463		return;2464 2465	hbm0_wr = 0x33333333;2466	hbm0_rd = 0x77777777;2467	hbm1_wr = 0x55555555;2468	hbm1_rd = 0xDDDDDDDD;2469 2470	WREG32(mmDMA_IF_E_N_HBM0_WR_CRED_CNT, hbm0_wr);2471	WREG32(mmDMA_IF_E_N_HBM1_WR_CRED_CNT, hbm1_wr);2472	WREG32(mmDMA_IF_E_N_HBM0_RD_CRED_CNT, hbm0_rd);2473	WREG32(mmDMA_IF_E_N_HBM1_RD_CRED_CNT, hbm1_rd);2474 2475	WREG32(mmDMA_IF_E_S_HBM0_WR_CRED_CNT, hbm0_wr);2476	WREG32(mmDMA_IF_E_S_HBM1_WR_CRED_CNT, hbm1_wr);2477	WREG32(mmDMA_IF_E_S_HBM0_RD_CRED_CNT, hbm0_rd);2478	WREG32(mmDMA_IF_E_S_HBM1_RD_CRED_CNT, hbm1_rd);2479 2480	WREG32(mmDMA_IF_W_N_HBM0_WR_CRED_CNT, hbm0_wr);2481	WREG32(mmDMA_IF_W_N_HBM1_WR_CRED_CNT, hbm1_wr);2482	WREG32(mmDMA_IF_W_N_HBM0_RD_CRED_CNT, hbm0_rd);2483	WREG32(mmDMA_IF_W_N_HBM1_RD_CRED_CNT, hbm1_rd);2484 2485	WREG32(mmDMA_IF_W_S_HBM0_WR_CRED_CNT, hbm0_wr);2486	WREG32(mmDMA_IF_W_S_HBM1_WR_CRED_CNT, hbm1_wr);2487	WREG32(mmDMA_IF_W_S_HBM0_RD_CRED_CNT, hbm0_rd);2488	WREG32(mmDMA_IF_W_S_HBM1_RD_CRED_CNT, hbm1_rd);2489 2490	WREG32(mmDMA_IF_E_N_HBM_CRED_EN_0,2491			(1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |2492			(1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));2493	WREG32(mmDMA_IF_E_S_HBM_CRED_EN_0,2494			(1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |2495			(1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));2496	WREG32(mmDMA_IF_W_N_HBM_CRED_EN_0,2497			(1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |2498			(1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));2499	WREG32(mmDMA_IF_W_S_HBM_CRED_EN_0,2500			(1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |2501			(1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));2502 2503	WREG32(mmDMA_IF_E_N_HBM_CRED_EN_1,2504			(1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |2505			(1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));2506	WREG32(mmDMA_IF_E_S_HBM_CRED_EN_1,2507			(1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |2508			(1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));2509	WREG32(mmDMA_IF_W_N_HBM_CRED_EN_1,2510			(1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |2511			(1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));2512	WREG32(mmDMA_IF_W_S_HBM_CRED_EN_1,2513			(1 << DMA_IF_HBM_CRED_EN_READ_CREDIT_EN_SHIFT) |2514			(1 << DMA_IF_HBM_CRED_EN_WRITE_CREDIT_EN_SHIFT));2515}2516 2517static void gaudi_init_golden_registers(struct hl_device *hdev)2518{2519	u32 tpc_offset;2520	int tpc_id, i;2521 2522	gaudi_init_e2e(hdev);2523	gaudi_init_hbm_cred(hdev);2524 2525	for (tpc_id = 0, tpc_offset = 0;2526				tpc_id < TPC_NUMBER_OF_ENGINES;2527				tpc_id++, tpc_offset += TPC_CFG_OFFSET) {2528		/* Mask all arithmetic interrupts from TPC */2529		WREG32(mmTPC0_CFG_TPC_INTR_MASK + tpc_offset, 0x8FFE);2530		/* Set 16 cache lines */2531		WREG32_FIELD(TPC0_CFG_MSS_CONFIG, tpc_offset,2532				ICACHE_FETCH_LINE_NUM, 2);2533	}2534 2535	/* Make sure 1st 128 bytes in SRAM are 0 for Tensor DMA */2536	for (i = 0 ; i < 128 ; i += 8)2537		writeq(0, hdev->pcie_bar[SRAM_BAR_ID] + i);2538 2539	WREG32(mmMME0_CTRL_EUS_ROLLUP_CNT_ADD, 3);2540	WREG32(mmMME1_CTRL_EUS_ROLLUP_CNT_ADD, 3);2541	WREG32(mmMME2_CTRL_EUS_ROLLUP_CNT_ADD, 3);2542	WREG32(mmMME3_CTRL_EUS_ROLLUP_CNT_ADD, 3);2543}2544 2545static void gaudi_init_pci_dma_qman(struct hl_device *hdev, int dma_id,2546					int qman_id, dma_addr_t qman_pq_addr)2547{2548	struct cpu_dyn_regs *dyn_regs =2549			&hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;2550	u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi;2551	u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi;2552	u32 q_off, dma_qm_offset;2553	u32 dma_qm_err_cfg, irq_handler_offset;2554 2555	dma_qm_offset = dma_id * DMA_QMAN_OFFSET;2556 2557	mtr_base_en_lo = lower_32_bits(CFG_BASE +2558				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);2559	mtr_base_en_hi = upper_32_bits(CFG_BASE +2560				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);2561	so_base_en_lo = lower_32_bits(CFG_BASE +2562				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);2563	so_base_en_hi = upper_32_bits(CFG_BASE +2564				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);2565	mtr_base_ws_lo = lower_32_bits(CFG_BASE +2566				mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);2567	mtr_base_ws_hi = upper_32_bits(CFG_BASE +2568				mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);2569	so_base_ws_lo = lower_32_bits(CFG_BASE +2570				mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0);2571	so_base_ws_hi = upper_32_bits(CFG_BASE +2572				mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0);2573 2574	q_off = dma_qm_offset + qman_id * 4;2575 2576	WREG32(mmDMA0_QM_PQ_BASE_LO_0 + q_off, lower_32_bits(qman_pq_addr));2577	WREG32(mmDMA0_QM_PQ_BASE_HI_0 + q_off, upper_32_bits(qman_pq_addr));2578 2579	WREG32(mmDMA0_QM_PQ_SIZE_0 + q_off, ilog2(HL_QUEUE_LENGTH));2580	WREG32(mmDMA0_QM_PQ_PI_0 + q_off, 0);2581	WREG32(mmDMA0_QM_PQ_CI_0 + q_off, 0);2582 2583	WREG32(mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off, QMAN_LDMA_SIZE_OFFSET);2584	WREG32(mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off,2585							QMAN_LDMA_SRC_OFFSET);2586	WREG32(mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off,2587							QMAN_LDMA_DST_OFFSET);2588 2589	WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_en_lo);2590	WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_en_hi);2591	WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_en_lo);2592	WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_en_hi);2593	WREG32(mmDMA0_QM_CP_MSG_BASE2_ADDR_LO_0 + q_off, mtr_base_ws_lo);2594	WREG32(mmDMA0_QM_CP_MSG_BASE2_ADDR_HI_0 + q_off, mtr_base_ws_hi);2595	WREG32(mmDMA0_QM_CP_MSG_BASE3_ADDR_LO_0 + q_off, so_base_ws_lo);2596	WREG32(mmDMA0_QM_CP_MSG_BASE3_ADDR_HI_0 + q_off, so_base_ws_hi);2597 2598	WREG32(mmDMA0_QM_CP_BARRIER_CFG_0 + q_off, 0x100);2599 2600	/* The following configuration is needed only once per QMAN */2601	if (qman_id == 0) {2602		irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?2603				mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :2604				le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl);2605 2606		/* Configure RAZWI IRQ */2607		dma_qm_err_cfg = PCI_DMA_QMAN_GLBL_ERR_CFG_MSG_EN_MASK;2608		if (hdev->stop_on_err)2609			dma_qm_err_cfg |=2610				PCI_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK;2611 2612		WREG32(mmDMA0_QM_GLBL_ERR_CFG + dma_qm_offset, dma_qm_err_cfg);2613 2614		WREG32(mmDMA0_QM_GLBL_ERR_ADDR_LO + dma_qm_offset,2615			lower_32_bits(CFG_BASE + irq_handler_offset));2616		WREG32(mmDMA0_QM_GLBL_ERR_ADDR_HI + dma_qm_offset,2617			upper_32_bits(CFG_BASE + irq_handler_offset));2618 2619		WREG32(mmDMA0_QM_GLBL_ERR_WDATA + dma_qm_offset,2620			gaudi_irq_map_table[GAUDI_EVENT_DMA0_QM].cpu_id +2621									dma_id);2622 2623		WREG32(mmDMA0_QM_ARB_ERR_MSG_EN + dma_qm_offset,2624				QM_ARB_ERR_MSG_EN_MASK);2625 2626		/* Set timeout to maximum */2627		WREG32(mmDMA0_QM_ARB_SLV_CHOISE_WDT + dma_qm_offset, GAUDI_ARB_WDT_TIMEOUT);2628 2629		WREG32(mmDMA0_QM_GLBL_PROT + dma_qm_offset,2630				QMAN_EXTERNAL_MAKE_TRUSTED);2631 2632		WREG32(mmDMA0_QM_GLBL_CFG1 + dma_qm_offset, 0);2633	}2634}2635 2636static void gaudi_init_dma_core(struct hl_device *hdev, int dma_id)2637{2638	struct cpu_dyn_regs *dyn_regs =2639			&hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;2640	u32 dma_err_cfg = 1 << DMA0_CORE_ERR_CFG_ERR_MSG_EN_SHIFT;2641	u32 dma_offset = dma_id * DMA_CORE_OFFSET;2642	u32 irq_handler_offset;2643 2644	/* Set to maximum possible according to physical size */2645	WREG32(mmDMA0_CORE_RD_MAX_OUTSTAND + dma_offset, 0);2646	WREG32(mmDMA0_CORE_RD_MAX_SIZE + dma_offset, 0);2647 2648	/* WA for H/W bug H3-2116 */2649	WREG32(mmDMA0_CORE_LBW_MAX_OUTSTAND + dma_offset, 15);2650 2651	/* STOP_ON bit implies no completion to operation in case of RAZWI */2652	if (hdev->stop_on_err)2653		dma_err_cfg |= 1 << DMA0_CORE_ERR_CFG_STOP_ON_ERR_SHIFT;2654 2655	WREG32(mmDMA0_CORE_ERR_CFG + dma_offset, dma_err_cfg);2656 2657	irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?2658			mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :2659			le32_to_cpu(dyn_regs->gic_dma_core_irq_ctrl);2660 2661	WREG32(mmDMA0_CORE_ERRMSG_ADDR_LO + dma_offset,2662		lower_32_bits(CFG_BASE + irq_handler_offset));2663	WREG32(mmDMA0_CORE_ERRMSG_ADDR_HI + dma_offset,2664		upper_32_bits(CFG_BASE + irq_handler_offset));2665 2666	WREG32(mmDMA0_CORE_ERRMSG_WDATA + dma_offset,2667		gaudi_irq_map_table[GAUDI_EVENT_DMA0_CORE].cpu_id + dma_id);2668	WREG32(mmDMA0_CORE_PROT + dma_offset,2669			1 << DMA0_CORE_PROT_ERR_VAL_SHIFT);2670	/* If the channel is secured, it should be in MMU bypass mode */2671	WREG32(mmDMA0_CORE_SECURE_PROPS + dma_offset,2672			1 << DMA0_CORE_SECURE_PROPS_MMBP_SHIFT);2673	WREG32(mmDMA0_CORE_CFG_0 + dma_offset, 1 << DMA0_CORE_CFG_0_EN_SHIFT);2674}2675 2676static void gaudi_enable_qman(struct hl_device *hdev, int dma_id,2677				u32 enable_mask)2678{2679	u32 dma_qm_offset = dma_id * DMA_QMAN_OFFSET;2680 2681	WREG32(mmDMA0_QM_GLBL_CFG0 + dma_qm_offset, enable_mask);2682}2683 2684static void gaudi_init_pci_dma_qmans(struct hl_device *hdev)2685{2686	struct gaudi_device *gaudi = hdev->asic_specific;2687	struct hl_hw_queue *q;2688	int i, j, dma_id, cpu_skip, nic_skip, cq_id = 0, q_idx, msi_vec = 0;2689 2690	if (gaudi->hw_cap_initialized & HW_CAP_PCI_DMA)2691		return;2692 2693	for (i = 0 ; i < PCI_DMA_NUMBER_OF_CHNLS ; i++) {2694		dma_id = gaudi_dma_assignment[i];2695		/*2696		 * For queues after the CPU Q need to add 1 to get the correct2697		 * queue. In addition, need to add the CPU EQ and NIC IRQs in2698		 * order to get the correct MSI register.2699		 */2700		if (dma_id > 1) {2701			cpu_skip = 1;2702			nic_skip = NIC_NUMBER_OF_ENGINES;2703		} else {2704			cpu_skip = 0;2705			nic_skip = 0;2706		}2707 2708		for (j = 0 ; j < QMAN_STREAMS ; j++) {2709			q_idx = 4 * dma_id + j + cpu_skip;2710			q = &hdev->kernel_queues[q_idx];2711			q->cq_id = cq_id++;2712			q->msi_vec = nic_skip + cpu_skip + msi_vec++;2713			gaudi_init_pci_dma_qman(hdev, dma_id, j,2714						q->bus_address);2715		}2716 2717		gaudi_init_dma_core(hdev, dma_id);2718 2719		gaudi_enable_qman(hdev, dma_id, PCI_DMA_QMAN_ENABLE);2720	}2721 2722	gaudi->hw_cap_initialized |= HW_CAP_PCI_DMA;2723}2724 2725static void gaudi_init_hbm_dma_qman(struct hl_device *hdev, int dma_id,2726					int qman_id, u64 qman_base_addr)2727{2728	struct cpu_dyn_regs *dyn_regs =2729			&hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;2730	u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi;2731	u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi;2732	u32 dma_qm_err_cfg, irq_handler_offset;2733	u32 q_off, dma_qm_offset;2734 2735	dma_qm_offset = dma_id * DMA_QMAN_OFFSET;2736 2737	mtr_base_en_lo = lower_32_bits(CFG_BASE +2738			mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);2739	mtr_base_en_hi = upper_32_bits(CFG_BASE +2740				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);2741	so_base_en_lo = lower_32_bits(CFG_BASE +2742				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);2743	so_base_en_hi = upper_32_bits(CFG_BASE +2744				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);2745	mtr_base_ws_lo = lower_32_bits(CFG_BASE +2746				mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);2747	mtr_base_ws_hi = upper_32_bits(CFG_BASE +2748				mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);2749	so_base_ws_lo = lower_32_bits(CFG_BASE +2750				mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0);2751	so_base_ws_hi = upper_32_bits(CFG_BASE +2752				mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0);2753 2754	q_off = dma_qm_offset + qman_id * 4;2755 2756	if (qman_id < 4) {2757		WREG32(mmDMA0_QM_PQ_BASE_LO_0 + q_off,2758					lower_32_bits(qman_base_addr));2759		WREG32(mmDMA0_QM_PQ_BASE_HI_0 + q_off,2760					upper_32_bits(qman_base_addr));2761 2762		WREG32(mmDMA0_QM_PQ_SIZE_0 + q_off, ilog2(HBM_DMA_QMAN_LENGTH));2763		WREG32(mmDMA0_QM_PQ_PI_0 + q_off, 0);2764		WREG32(mmDMA0_QM_PQ_CI_0 + q_off, 0);2765 2766		WREG32(mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off,2767							QMAN_CPDMA_SIZE_OFFSET);2768		WREG32(mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off,2769							QMAN_CPDMA_SRC_OFFSET);2770		WREG32(mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off,2771							QMAN_CPDMA_DST_OFFSET);2772	} else {2773		irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?2774				mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :2775				le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl);2776 2777		WREG32(mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off,2778							QMAN_LDMA_SIZE_OFFSET);2779		WREG32(mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off,2780							QMAN_LDMA_SRC_OFFSET);2781		WREG32(mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off,2782							QMAN_LDMA_DST_OFFSET);2783 2784		/* Configure RAZWI IRQ */2785		dma_qm_err_cfg = HBM_DMA_QMAN_GLBL_ERR_CFG_MSG_EN_MASK;2786		if (hdev->stop_on_err)2787			dma_qm_err_cfg |=2788				HBM_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK;2789 2790		WREG32(mmDMA0_QM_GLBL_ERR_CFG + dma_qm_offset, dma_qm_err_cfg);2791 2792		WREG32(mmDMA0_QM_GLBL_ERR_ADDR_LO + dma_qm_offset,2793			lower_32_bits(CFG_BASE + irq_handler_offset));2794		WREG32(mmDMA0_QM_GLBL_ERR_ADDR_HI + dma_qm_offset,2795			upper_32_bits(CFG_BASE + irq_handler_offset));2796 2797		WREG32(mmDMA0_QM_GLBL_ERR_WDATA + dma_qm_offset,2798			gaudi_irq_map_table[GAUDI_EVENT_DMA0_QM].cpu_id +2799									dma_id);2800 2801		WREG32(mmDMA0_QM_ARB_ERR_MSG_EN + dma_qm_offset,2802				QM_ARB_ERR_MSG_EN_MASK);2803 2804		/* Set timeout to maximum */2805		WREG32(mmDMA0_QM_ARB_SLV_CHOISE_WDT + dma_qm_offset, GAUDI_ARB_WDT_TIMEOUT);2806 2807		WREG32(mmDMA0_QM_GLBL_CFG1 + dma_qm_offset, 0);2808		WREG32(mmDMA0_QM_GLBL_PROT + dma_qm_offset,2809				QMAN_INTERNAL_MAKE_TRUSTED);2810	}2811 2812	WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_en_lo);2813	WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_en_hi);2814	WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_en_lo);2815	WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_en_hi);2816 2817	/* Configure DMA5 CP_MSG_BASE 2/3 for sync stream collective */2818	if (gaudi_dma_assignment[dma_id] == GAUDI_ENGINE_ID_DMA_5) {2819		WREG32(mmDMA0_QM_CP_MSG_BASE2_ADDR_LO_0 + q_off,2820				mtr_base_ws_lo);2821		WREG32(mmDMA0_QM_CP_MSG_BASE2_ADDR_HI_0 + q_off,2822				mtr_base_ws_hi);2823		WREG32(mmDMA0_QM_CP_MSG_BASE3_ADDR_LO_0 + q_off,2824				so_base_ws_lo);2825		WREG32(mmDMA0_QM_CP_MSG_BASE3_ADDR_HI_0 + q_off,2826				so_base_ws_hi);2827	}2828}2829 2830static void gaudi_init_hbm_dma_qmans(struct hl_device *hdev)2831{2832	struct gaudi_device *gaudi = hdev->asic_specific;2833	struct gaudi_internal_qman_info *q;2834	u64 qman_base_addr;2835	int i, j, dma_id, internal_q_index;2836 2837	if (gaudi->hw_cap_initialized & HW_CAP_HBM_DMA)2838		return;2839 2840	for (i = 0 ; i < HBM_DMA_NUMBER_OF_CHNLS ; i++) {2841		dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_1 + i];2842 2843		for (j = 0 ; j < QMAN_STREAMS ; j++) {2844			 /*2845			  * Add the CPU queue in order to get the correct queue2846			  * number as all internal queue are placed after it2847			  */2848			internal_q_index = dma_id * QMAN_STREAMS + j + 1;2849 2850			q = &gaudi->internal_qmans[internal_q_index];2851			qman_base_addr = (u64) q->pq_dma_addr;2852			gaudi_init_hbm_dma_qman(hdev, dma_id, j,2853						qman_base_addr);2854		}2855 2856		/* Initializing lower CP for HBM DMA QMAN */2857		gaudi_init_hbm_dma_qman(hdev, dma_id, 4, 0);2858 2859		gaudi_init_dma_core(hdev, dma_id);2860 2861		gaudi_enable_qman(hdev, dma_id, HBM_DMA_QMAN_ENABLE);2862	}2863 2864	gaudi->hw_cap_initialized |= HW_CAP_HBM_DMA;2865}2866 2867static void gaudi_init_mme_qman(struct hl_device *hdev, u32 mme_offset,2868					int qman_id, u64 qman_base_addr)2869{2870	struct cpu_dyn_regs *dyn_regs =2871			&hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;2872	u32 mtr_base_lo, mtr_base_hi;2873	u32 so_base_lo, so_base_hi;2874	u32 irq_handler_offset;2875	u32 q_off, mme_id;2876	u32 mme_qm_err_cfg;2877 2878	mtr_base_lo = lower_32_bits(CFG_BASE +2879				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);2880	mtr_base_hi = upper_32_bits(CFG_BASE +2881				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);2882	so_base_lo = lower_32_bits(CFG_BASE +2883				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);2884	so_base_hi = upper_32_bits(CFG_BASE +2885				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);2886 2887	q_off = mme_offset + qman_id * 4;2888 2889	if (qman_id < 4) {2890		WREG32(mmMME0_QM_PQ_BASE_LO_0 + q_off,2891					lower_32_bits(qman_base_addr));2892		WREG32(mmMME0_QM_PQ_BASE_HI_0 + q_off,2893					upper_32_bits(qman_base_addr));2894 2895		WREG32(mmMME0_QM_PQ_SIZE_0 + q_off, ilog2(MME_QMAN_LENGTH));2896		WREG32(mmMME0_QM_PQ_PI_0 + q_off, 0);2897		WREG32(mmMME0_QM_PQ_CI_0 + q_off, 0);2898 2899		WREG32(mmMME0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off,2900							QMAN_CPDMA_SIZE_OFFSET);2901		WREG32(mmMME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off,2902							QMAN_CPDMA_SRC_OFFSET);2903		WREG32(mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off,2904							QMAN_CPDMA_DST_OFFSET);2905	} else {2906		irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?2907				mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :2908				le32_to_cpu(dyn_regs->gic_mme_qm_irq_ctrl);2909 2910		WREG32(mmMME0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off,2911							QMAN_LDMA_SIZE_OFFSET);2912		WREG32(mmMME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off,2913							QMAN_LDMA_SRC_OFFSET);2914		WREG32(mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off,2915							QMAN_LDMA_DST_OFFSET);2916 2917		/* Configure RAZWI IRQ */2918		mme_id = mme_offset /2919				(mmMME1_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0) / 2;2920 2921		mme_qm_err_cfg = MME_QMAN_GLBL_ERR_CFG_MSG_EN_MASK;2922		if (hdev->stop_on_err)2923			mme_qm_err_cfg |=2924				MME_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK;2925 2926		WREG32(mmMME0_QM_GLBL_ERR_CFG + mme_offset, mme_qm_err_cfg);2927 2928		WREG32(mmMME0_QM_GLBL_ERR_ADDR_LO + mme_offset,2929			lower_32_bits(CFG_BASE + irq_handler_offset));2930		WREG32(mmMME0_QM_GLBL_ERR_ADDR_HI + mme_offset,2931			upper_32_bits(CFG_BASE + irq_handler_offset));2932 2933		WREG32(mmMME0_QM_GLBL_ERR_WDATA + mme_offset,2934			gaudi_irq_map_table[GAUDI_EVENT_MME0_QM].cpu_id +2935									mme_id);2936 2937		WREG32(mmMME0_QM_ARB_ERR_MSG_EN + mme_offset,2938				QM_ARB_ERR_MSG_EN_MASK);2939 2940		/* Set timeout to maximum */2941		WREG32(mmMME0_QM_ARB_SLV_CHOISE_WDT + mme_offset, GAUDI_ARB_WDT_TIMEOUT);2942 2943		WREG32(mmMME0_QM_GLBL_CFG1 + mme_offset, 0);2944		WREG32(mmMME0_QM_GLBL_PROT + mme_offset,2945				QMAN_INTERNAL_MAKE_TRUSTED);2946	}2947 2948	WREG32(mmMME0_QM_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_lo);2949	WREG32(mmMME0_QM_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_hi);2950	WREG32(mmMME0_QM_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_lo);2951	WREG32(mmMME0_QM_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_hi);2952}2953 2954static void gaudi_init_mme_qmans(struct hl_device *hdev)2955{2956	struct gaudi_device *gaudi = hdev->asic_specific;2957	struct gaudi_internal_qman_info *q;2958	u64 qman_base_addr;2959	u32 mme_offset;2960	int i, internal_q_index;2961 2962	if (gaudi->hw_cap_initialized & HW_CAP_MME)2963		return;2964 2965	/*2966	 * map GAUDI_QUEUE_ID_MME_0_X to the N_W_MME (mmMME2_QM_BASE)2967	 * and GAUDI_QUEUE_ID_MME_1_X to the S_W_MME (mmMME0_QM_BASE)2968	 */2969 2970	mme_offset = mmMME2_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0;2971 2972	for (i = 0 ; i < MME_NUMBER_OF_QMANS ; i++) {2973		internal_q_index = GAUDI_QUEUE_ID_MME_0_0 + i;2974		q = &gaudi->internal_qmans[internal_q_index];2975		qman_base_addr = (u64) q->pq_dma_addr;2976		gaudi_init_mme_qman(hdev, mme_offset, (i & 0x3),2977					qman_base_addr);2978		if (i == 3)2979			mme_offset = 0;2980	}2981 2982	/* Initializing lower CP for MME QMANs */2983	mme_offset = mmMME2_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0;2984	gaudi_init_mme_qman(hdev, mme_offset, 4, 0);2985	gaudi_init_mme_qman(hdev, 0, 4, 0);2986 2987	WREG32(mmMME2_QM_GLBL_CFG0, QMAN_MME_ENABLE);2988	WREG32(mmMME0_QM_GLBL_CFG0, QMAN_MME_ENABLE);2989 2990	gaudi->hw_cap_initialized |= HW_CAP_MME;2991}2992 2993static void gaudi_init_tpc_qman(struct hl_device *hdev, u32 tpc_offset,2994				int qman_id, u64 qman_base_addr)2995{2996	struct cpu_dyn_regs *dyn_regs =2997			&hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;2998	u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi;2999	u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi;3000	u32 tpc_qm_err_cfg, irq_handler_offset;3001	u32 q_off, tpc_id;3002 3003	mtr_base_en_lo = lower_32_bits(CFG_BASE +3004			mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);3005	mtr_base_en_hi = upper_32_bits(CFG_BASE +3006				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);3007	so_base_en_lo = lower_32_bits(CFG_BASE +3008				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);3009	so_base_en_hi = upper_32_bits(CFG_BASE +3010				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);3011	mtr_base_ws_lo = lower_32_bits(CFG_BASE +3012				mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);3013	mtr_base_ws_hi = upper_32_bits(CFG_BASE +3014				mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);3015	so_base_ws_lo = lower_32_bits(CFG_BASE +3016				mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0);3017	so_base_ws_hi = upper_32_bits(CFG_BASE +3018				mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0);3019 3020	q_off = tpc_offset + qman_id * 4;3021 3022	tpc_id = tpc_offset /3023			(mmTPC1_QM_GLBL_CFG0 - mmTPC0_QM_GLBL_CFG0);3024 3025	if (qman_id < 4) {3026		WREG32(mmTPC0_QM_PQ_BASE_LO_0 + q_off,3027					lower_32_bits(qman_base_addr));3028		WREG32(mmTPC0_QM_PQ_BASE_HI_0 + q_off,3029					upper_32_bits(qman_base_addr));3030 3031		WREG32(mmTPC0_QM_PQ_SIZE_0 + q_off, ilog2(TPC_QMAN_LENGTH));3032		WREG32(mmTPC0_QM_PQ_PI_0 + q_off, 0);3033		WREG32(mmTPC0_QM_PQ_CI_0 + q_off, 0);3034 3035		WREG32(mmTPC0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off,3036							QMAN_CPDMA_SIZE_OFFSET);3037		WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off,3038							QMAN_CPDMA_SRC_OFFSET);3039		WREG32(mmTPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off,3040							QMAN_CPDMA_DST_OFFSET);3041	} else {3042		irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?3043				mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :3044				le32_to_cpu(dyn_regs->gic_tpc_qm_irq_ctrl);3045 3046		WREG32(mmTPC0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off,3047							QMAN_LDMA_SIZE_OFFSET);3048		WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off,3049							QMAN_LDMA_SRC_OFFSET);3050		WREG32(mmTPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off,3051							QMAN_LDMA_DST_OFFSET);3052 3053		/* Configure RAZWI IRQ */3054		tpc_qm_err_cfg = TPC_QMAN_GLBL_ERR_CFG_MSG_EN_MASK;3055		if (hdev->stop_on_err)3056			tpc_qm_err_cfg |=3057				TPC_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK;3058 3059		WREG32(mmTPC0_QM_GLBL_ERR_CFG + tpc_offset, tpc_qm_err_cfg);3060 3061		WREG32(mmTPC0_QM_GLBL_ERR_ADDR_LO + tpc_offset,3062			lower_32_bits(CFG_BASE + irq_handler_offset));3063		WREG32(mmTPC0_QM_GLBL_ERR_ADDR_HI + tpc_offset,3064			upper_32_bits(CFG_BASE + irq_handler_offset));3065 3066		WREG32(mmTPC0_QM_GLBL_ERR_WDATA + tpc_offset,3067			gaudi_irq_map_table[GAUDI_EVENT_TPC0_QM].cpu_id +3068									tpc_id);3069 3070		WREG32(mmTPC0_QM_ARB_ERR_MSG_EN + tpc_offset,3071				QM_ARB_ERR_MSG_EN_MASK);3072 3073		/* Set timeout to maximum */3074		WREG32(mmTPC0_QM_ARB_SLV_CHOISE_WDT + tpc_offset, GAUDI_ARB_WDT_TIMEOUT);3075 3076		WREG32(mmTPC0_QM_GLBL_CFG1 + tpc_offset, 0);3077		WREG32(mmTPC0_QM_GLBL_PROT + tpc_offset,3078				QMAN_INTERNAL_MAKE_TRUSTED);3079	}3080 3081	WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_en_lo);3082	WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_en_hi);3083	WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_en_lo);3084	WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_en_hi);3085 3086	/* Configure TPC7 CP_MSG_BASE 2/3 for sync stream collective */3087	if (tpc_id == 6) {3088		WREG32(mmTPC0_QM_CP_MSG_BASE2_ADDR_LO_0 + q_off,3089				mtr_base_ws_lo);3090		WREG32(mmTPC0_QM_CP_MSG_BASE2_ADDR_HI_0 + q_off,3091				mtr_base_ws_hi);3092		WREG32(mmTPC0_QM_CP_MSG_BASE3_ADDR_LO_0 + q_off,3093				so_base_ws_lo);3094		WREG32(mmTPC0_QM_CP_MSG_BASE3_ADDR_HI_0 + q_off,3095				so_base_ws_hi);3096	}3097}3098 3099static void gaudi_init_tpc_qmans(struct hl_device *hdev)3100{3101	struct gaudi_device *gaudi = hdev->asic_specific;3102	struct gaudi_internal_qman_info *q;3103	u64 qman_base_addr;3104	u32 so_base_hi, tpc_offset = 0;3105	u32 tpc_delta = mmTPC1_CFG_SM_BASE_ADDRESS_HIGH -3106			mmTPC0_CFG_SM_BASE_ADDRESS_HIGH;3107	int i, tpc_id, internal_q_index;3108 3109	if (gaudi->hw_cap_initialized & HW_CAP_TPC_MASK)3110		return;3111 3112	so_base_hi = upper_32_bits(CFG_BASE +3113				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);3114 3115	for (tpc_id = 0 ; tpc_id < TPC_NUMBER_OF_ENGINES ; tpc_id++) {3116		for (i = 0 ; i < QMAN_STREAMS ; i++) {3117			internal_q_index = GAUDI_QUEUE_ID_TPC_0_0 +3118						tpc_id * QMAN_STREAMS + i;3119			q = &gaudi->internal_qmans[internal_q_index];3120			qman_base_addr = (u64) q->pq_dma_addr;3121			gaudi_init_tpc_qman(hdev, tpc_offset, i,3122						qman_base_addr);3123 3124			if (i == 3) {3125				/* Initializing lower CP for TPC QMAN */3126				gaudi_init_tpc_qman(hdev, tpc_offset, 4, 0);3127 3128				/* Enable the QMAN and TPC channel */3129				WREG32(mmTPC0_QM_GLBL_CFG0 + tpc_offset,3130						QMAN_TPC_ENABLE);3131			}3132		}3133 3134		WREG32(mmTPC0_CFG_SM_BASE_ADDRESS_HIGH + tpc_id * tpc_delta,3135				so_base_hi);3136 3137		tpc_offset += mmTPC1_QM_GLBL_CFG0 - mmTPC0_QM_GLBL_CFG0;3138 3139		gaudi->hw_cap_initialized |=3140				FIELD_PREP(HW_CAP_TPC_MASK, 1 << tpc_id);3141	}3142}3143 3144static void gaudi_init_nic_qman(struct hl_device *hdev, u32 nic_offset,3145				int qman_id, u64 qman_base_addr, int nic_id)3146{3147	struct cpu_dyn_regs *dyn_regs =3148			&hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;3149	u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi;3150	u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi;3151	u32 nic_qm_err_cfg, irq_handler_offset;3152	u32 q_off;3153 3154	mtr_base_en_lo = lower_32_bits((CFG_BASE & U32_MAX) +3155			mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);3156	mtr_base_en_hi = upper_32_bits(CFG_BASE +3157				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);3158	so_base_en_lo = lower_32_bits((CFG_BASE & U32_MAX) +3159				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);3160	so_base_en_hi = upper_32_bits(CFG_BASE +3161				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0);3162	mtr_base_ws_lo = lower_32_bits((CFG_BASE & U32_MAX) +3163				mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);3164	mtr_base_ws_hi = upper_32_bits(CFG_BASE +3165				mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);3166	so_base_ws_lo = lower_32_bits((CFG_BASE & U32_MAX) +3167				mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0);3168	so_base_ws_hi = upper_32_bits(CFG_BASE +3169				mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0);3170 3171	q_off = nic_offset + qman_id * 4;3172 3173	WREG32(mmNIC0_QM0_PQ_BASE_LO_0 + q_off, lower_32_bits(qman_base_addr));3174	WREG32(mmNIC0_QM0_PQ_BASE_HI_0 + q_off, upper_32_bits(qman_base_addr));3175 3176	WREG32(mmNIC0_QM0_PQ_SIZE_0 + q_off, ilog2(NIC_QMAN_LENGTH));3177	WREG32(mmNIC0_QM0_PQ_PI_0 + q_off, 0);3178	WREG32(mmNIC0_QM0_PQ_CI_0 + q_off, 0);3179 3180	WREG32(mmNIC0_QM0_CP_LDMA_TSIZE_OFFSET_0 + q_off,3181							QMAN_LDMA_SIZE_OFFSET);3182	WREG32(mmNIC0_QM0_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off,3183							QMAN_LDMA_SRC_OFFSET);3184	WREG32(mmNIC0_QM0_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off,3185							QMAN_LDMA_DST_OFFSET);3186 3187	WREG32(mmNIC0_QM0_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_en_lo);3188	WREG32(mmNIC0_QM0_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_en_hi);3189	WREG32(mmNIC0_QM0_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_en_lo);3190	WREG32(mmNIC0_QM0_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_en_hi);3191 3192	/* Configure NIC CP_MSG_BASE 2/3 for sync stream collective */3193	WREG32(mmNIC0_QM0_CP_MSG_BASE2_ADDR_LO_0 + q_off, mtr_base_ws_lo);3194	WREG32(mmNIC0_QM0_CP_MSG_BASE2_ADDR_HI_0 + q_off, mtr_base_ws_hi);3195	WREG32(mmNIC0_QM0_CP_MSG_BASE3_ADDR_LO_0 + q_off, so_base_ws_lo);3196	WREG32(mmNIC0_QM0_CP_MSG_BASE3_ADDR_HI_0 + q_off, so_base_ws_hi);3197 3198	if (qman_id == 0) {3199		irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?3200				mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :3201				le32_to_cpu(dyn_regs->gic_nic_qm_irq_ctrl);3202 3203		/* Configure RAZWI IRQ */3204		nic_qm_err_cfg = NIC_QMAN_GLBL_ERR_CFG_MSG_EN_MASK;3205		if (hdev->stop_on_err)3206			nic_qm_err_cfg |=3207				NIC_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK;3208 3209		WREG32(mmNIC0_QM0_GLBL_ERR_CFG + nic_offset, nic_qm_err_cfg);3210 3211		WREG32(mmNIC0_QM0_GLBL_ERR_ADDR_LO + nic_offset,3212			lower_32_bits(CFG_BASE + irq_handler_offset));3213		WREG32(mmNIC0_QM0_GLBL_ERR_ADDR_HI + nic_offset,3214			upper_32_bits(CFG_BASE + irq_handler_offset));3215 3216		WREG32(mmNIC0_QM0_GLBL_ERR_WDATA + nic_offset,3217			gaudi_irq_map_table[GAUDI_EVENT_NIC0_QM0].cpu_id +3218									nic_id);3219 3220		WREG32(mmNIC0_QM0_ARB_ERR_MSG_EN + nic_offset,3221				QM_ARB_ERR_MSG_EN_MASK);3222 3223		/* Set timeout to maximum */3224		WREG32(mmNIC0_QM0_ARB_SLV_CHOISE_WDT + nic_offset, GAUDI_ARB_WDT_TIMEOUT);3225 3226		WREG32(mmNIC0_QM0_GLBL_CFG1 + nic_offset, 0);3227		WREG32(mmNIC0_QM0_GLBL_PROT + nic_offset,3228				QMAN_INTERNAL_MAKE_TRUSTED);3229	}3230}3231 3232static void gaudi_init_nic_qmans(struct hl_device *hdev)3233{3234	struct gaudi_device *gaudi = hdev->asic_specific;3235	struct gaudi_internal_qman_info *q;3236	u64 qman_base_addr;3237	u32 nic_offset = 0;3238	u32 nic_delta_between_qmans =3239			mmNIC0_QM1_GLBL_CFG0 - mmNIC0_QM0_GLBL_CFG0;3240	u32 nic_delta_between_nics =3241			mmNIC1_QM0_GLBL_CFG0 - mmNIC0_QM0_GLBL_CFG0;3242	int i, nic_id, internal_q_index;3243 3244	if (!hdev->nic_ports_mask)3245		return;3246 3247	if (gaudi->hw_cap_initialized & HW_CAP_NIC_MASK)3248		return;3249 3250	dev_dbg(hdev->dev, "Initializing NIC QMANs\n");3251 3252	for (nic_id = 0 ; nic_id < NIC_NUMBER_OF_ENGINES ; nic_id++) {3253		if (!(hdev->nic_ports_mask & (1 << nic_id))) {3254			nic_offset += nic_delta_between_qmans;3255			if (nic_id & 1) {3256				nic_offset -= (nic_delta_between_qmans * 2);3257				nic_offset += nic_delta_between_nics;3258			}3259			continue;3260		}3261 3262		for (i = 0 ; i < QMAN_STREAMS ; i++) {3263			internal_q_index = GAUDI_QUEUE_ID_NIC_0_0 +3264						nic_id * QMAN_STREAMS + i;3265			q = &gaudi->internal_qmans[internal_q_index];3266			qman_base_addr = (u64) q->pq_dma_addr;3267			gaudi_init_nic_qman(hdev, nic_offset, (i & 0x3),3268						qman_base_addr, nic_id);3269		}3270 3271		/* Enable the QMAN */3272		WREG32(mmNIC0_QM0_GLBL_CFG0 + nic_offset, NIC_QMAN_ENABLE);3273 3274		nic_offset += nic_delta_between_qmans;3275		if (nic_id & 1) {3276			nic_offset -= (nic_delta_between_qmans * 2);3277			nic_offset += nic_delta_between_nics;3278		}3279 3280		gaudi->hw_cap_initialized |= 1 << (HW_CAP_NIC_SHIFT + nic_id);3281	}3282}3283 3284static void gaudi_disable_pci_dma_qmans(struct hl_device *hdev)3285{3286	struct gaudi_device *gaudi = hdev->asic_specific;3287 3288	if (!(gaudi->hw_cap_initialized & HW_CAP_PCI_DMA))3289		return;3290 3291	WREG32(mmDMA0_QM_GLBL_CFG0, 0);3292	WREG32(mmDMA1_QM_GLBL_CFG0, 0);3293	WREG32(mmDMA5_QM_GLBL_CFG0, 0);3294}3295 3296static void gaudi_disable_hbm_dma_qmans(struct hl_device *hdev)3297{3298	struct gaudi_device *gaudi = hdev->asic_specific;3299 3300	if (!(gaudi->hw_cap_initialized & HW_CAP_HBM_DMA))3301		return;3302 3303	WREG32(mmDMA2_QM_GLBL_CFG0, 0);3304	WREG32(mmDMA3_QM_GLBL_CFG0, 0);3305	WREG32(mmDMA4_QM_GLBL_CFG0, 0);3306	WREG32(mmDMA6_QM_GLBL_CFG0, 0);3307	WREG32(mmDMA7_QM_GLBL_CFG0, 0);3308}3309 3310static void gaudi_disable_mme_qmans(struct hl_device *hdev)3311{3312	struct gaudi_device *gaudi = hdev->asic_specific;3313 3314	if (!(gaudi->hw_cap_initialized & HW_CAP_MME))3315		return;3316 3317	WREG32(mmMME2_QM_GLBL_CFG0, 0);3318	WREG32(mmMME0_QM_GLBL_CFG0, 0);3319}3320 3321static void gaudi_disable_tpc_qmans(struct hl_device *hdev)3322{3323	struct gaudi_device *gaudi = hdev->asic_specific;3324	u32 tpc_offset = 0;3325	int tpc_id;3326 3327	if (!(gaudi->hw_cap_initialized & HW_CAP_TPC_MASK))3328		return;3329 3330	for (tpc_id = 0 ; tpc_id < TPC_NUMBER_OF_ENGINES ; tpc_id++) {3331		WREG32(mmTPC0_QM_GLBL_CFG0 + tpc_offset, 0);3332		tpc_offset += mmTPC1_QM_GLBL_CFG0 - mmTPC0_QM_GLBL_CFG0;3333	}3334}3335 3336static void gaudi_disable_nic_qmans(struct hl_device *hdev)3337{3338	struct gaudi_device *gaudi = hdev->asic_specific;3339	u32 nic_mask, nic_offset = 0;3340	u32 nic_delta_between_qmans =3341			mmNIC0_QM1_GLBL_CFG0 - mmNIC0_QM0_GLBL_CFG0;3342	u32 nic_delta_between_nics =3343			mmNIC1_QM0_GLBL_CFG0 - mmNIC0_QM0_GLBL_CFG0;3344	int nic_id;3345 3346	for (nic_id = 0 ; nic_id < NIC_NUMBER_OF_ENGINES ; nic_id++) {3347		nic_mask = 1 << (HW_CAP_NIC_SHIFT + nic_id);3348 3349		if (gaudi->hw_cap_initialized & nic_mask)3350			WREG32(mmNIC0_QM0_GLBL_CFG0 + nic_offset, 0);3351 3352		nic_offset += nic_delta_between_qmans;3353		if (nic_id & 1) {3354			nic_offset -= (nic_delta_between_qmans * 2);3355			nic_offset += nic_delta_between_nics;3356		}3357	}3358}3359 3360static void gaudi_stop_pci_dma_qmans(struct hl_device *hdev)3361{3362	struct gaudi_device *gaudi = hdev->asic_specific;3363 3364	if (!(gaudi->hw_cap_initialized & HW_CAP_PCI_DMA))3365		return;3366 3367	/* Stop upper CPs of QMANs 0.0 to 1.3 and 5.0 to 5.3 */3368	WREG32(mmDMA0_QM_GLBL_CFG1, 0xF << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);3369	WREG32(mmDMA1_QM_GLBL_CFG1, 0xF << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);3370	WREG32(mmDMA5_QM_GLBL_CFG1, 0xF << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);3371}3372 3373static void gaudi_stop_hbm_dma_qmans(struct hl_device *hdev)3374{3375	struct gaudi_device *gaudi = hdev->asic_specific;3376 3377	if (!(gaudi->hw_cap_initialized & HW_CAP_HBM_DMA))3378		return;3379 3380	/* Stop CPs of HBM DMA QMANs */3381 3382	WREG32(mmDMA2_QM_GLBL_CFG1, 0x1F << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);3383	WREG32(mmDMA3_QM_GLBL_CFG1, 0x1F << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);3384	WREG32(mmDMA4_QM_GLBL_CFG1, 0x1F << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);3385	WREG32(mmDMA6_QM_GLBL_CFG1, 0x1F << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);3386	WREG32(mmDMA7_QM_GLBL_CFG1, 0x1F << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);3387}3388 3389static void gaudi_stop_mme_qmans(struct hl_device *hdev)3390{3391	struct gaudi_device *gaudi = hdev->asic_specific;3392 3393	if (!(gaudi->hw_cap_initialized & HW_CAP_MME))3394		return;3395 3396	/* Stop CPs of MME QMANs */3397	WREG32(mmMME2_QM_GLBL_CFG1, 0x1F << MME0_QM_GLBL_CFG1_CP_STOP_SHIFT);3398	WREG32(mmMME0_QM_GLBL_CFG1, 0x1F << MME0_QM_GLBL_CFG1_CP_STOP_SHIFT);3399}3400 3401static void gaudi_stop_tpc_qmans(struct hl_device *hdev)3402{3403	struct gaudi_device *gaudi = hdev->asic_specific;3404 3405	if (!(gaudi->hw_cap_initialized & HW_CAP_TPC_MASK))3406		return;3407 3408	WREG32(mmTPC0_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);3409	WREG32(mmTPC1_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);3410	WREG32(mmTPC2_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);3411	WREG32(mmTPC3_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);3412	WREG32(mmTPC4_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);3413	WREG32(mmTPC5_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);3414	WREG32(mmTPC6_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);3415	WREG32(mmTPC7_QM_GLBL_CFG1, 0x1F << TPC0_QM_GLBL_CFG1_CP_STOP_SHIFT);3416}3417 3418static void gaudi_stop_nic_qmans(struct hl_device *hdev)3419{3420	struct gaudi_device *gaudi = hdev->asic_specific;3421 3422	/* Stop upper CPs of QMANs */3423 3424	if (gaudi->hw_cap_initialized & HW_CAP_NIC0)3425		WREG32(mmNIC0_QM0_GLBL_CFG1,3426				NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK |3427				NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK |3428				NIC0_QM0_GLBL_CFG1_CP_STOP_MASK);3429 3430	if (gaudi->hw_cap_initialized & HW_CAP_NIC1)3431		WREG32(mmNIC0_QM1_GLBL_CFG1,3432				NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK |3433				NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK |3434				NIC0_QM0_GLBL_CFG1_CP_STOP_MASK);3435 3436	if (gaudi->hw_cap_initialized & HW_CAP_NIC2)3437		WREG32(mmNIC1_QM0_GLBL_CFG1,3438				NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK |3439				NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK |3440				NIC0_QM0_GLBL_CFG1_CP_STOP_MASK);3441 3442	if (gaudi->hw_cap_initialized & HW_CAP_NIC3)3443		WREG32(mmNIC1_QM1_GLBL_CFG1,3444				NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK |3445				NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK |3446				NIC0_QM0_GLBL_CFG1_CP_STOP_MASK);3447 3448	if (gaudi->hw_cap_initialized & HW_CAP_NIC4)3449		WREG32(mmNIC2_QM0_GLBL_CFG1,3450				NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK |3451				NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK |3452				NIC0_QM0_GLBL_CFG1_CP_STOP_MASK);3453 3454	if (gaudi->hw_cap_initialized & HW_CAP_NIC5)3455		WREG32(mmNIC2_QM1_GLBL_CFG1,3456				NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK |3457				NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK |3458				NIC0_QM0_GLBL_CFG1_CP_STOP_MASK);3459 3460	if (gaudi->hw_cap_initialized & HW_CAP_NIC6)3461		WREG32(mmNIC3_QM0_GLBL_CFG1,3462				NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK |3463				NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK |3464				NIC0_QM0_GLBL_CFG1_CP_STOP_MASK);3465 3466	if (gaudi->hw_cap_initialized & HW_CAP_NIC7)3467		WREG32(mmNIC3_QM1_GLBL_CFG1,3468				NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK |3469				NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK |3470				NIC0_QM0_GLBL_CFG1_CP_STOP_MASK);3471 3472	if (gaudi->hw_cap_initialized & HW_CAP_NIC8)3473		WREG32(mmNIC4_QM0_GLBL_CFG1,3474				NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK |3475				NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK |3476				NIC0_QM0_GLBL_CFG1_CP_STOP_MASK);3477 3478	if (gaudi->hw_cap_initialized & HW_CAP_NIC9)3479		WREG32(mmNIC4_QM1_GLBL_CFG1,3480				NIC0_QM0_GLBL_CFG1_PQF_STOP_MASK |3481				NIC0_QM0_GLBL_CFG1_CQF_STOP_MASK |3482				NIC0_QM0_GLBL_CFG1_CP_STOP_MASK);3483}3484 3485static void gaudi_pci_dma_stall(struct hl_device *hdev)3486{3487	struct gaudi_device *gaudi = hdev->asic_specific;3488 3489	if (!(gaudi->hw_cap_initialized & HW_CAP_PCI_DMA))3490		return;3491 3492	WREG32(mmDMA0_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);3493	WREG32(mmDMA1_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);3494	WREG32(mmDMA5_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);3495}3496 3497static void gaudi_hbm_dma_stall(struct hl_device *hdev)3498{3499	struct gaudi_device *gaudi = hdev->asic_specific;3500 3501	if (!(gaudi->hw_cap_initialized & HW_CAP_HBM_DMA))3502		return;3503 3504	WREG32(mmDMA2_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);3505	WREG32(mmDMA3_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);3506	WREG32(mmDMA4_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);3507	WREG32(mmDMA6_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);3508	WREG32(mmDMA7_CORE_CFG_1, 1 << DMA0_CORE_CFG_1_HALT_SHIFT);3509}3510 3511static void gaudi_mme_stall(struct hl_device *hdev)3512{3513	struct gaudi_device *gaudi = hdev->asic_specific;3514 3515	if (!(gaudi->hw_cap_initialized & HW_CAP_MME))3516		return;3517 3518	/* WA for H3-1800 bug: do ACC and SBAB writes twice */3519	WREG32(mmMME0_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);3520	WREG32(mmMME0_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);3521	WREG32(mmMME0_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);3522	WREG32(mmMME0_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);3523	WREG32(mmMME1_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);3524	WREG32(mmMME1_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);3525	WREG32(mmMME1_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);3526	WREG32(mmMME1_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);3527	WREG32(mmMME2_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);3528	WREG32(mmMME2_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);3529	WREG32(mmMME2_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);3530	WREG32(mmMME2_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);3531	WREG32(mmMME3_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);3532	WREG32(mmMME3_ACC_ACC_STALL, 1 << MME_ACC_ACC_STALL_R_SHIFT);3533	WREG32(mmMME3_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);3534	WREG32(mmMME3_SBAB_SB_STALL, 1 << MME_SBAB_SB_STALL_R_SHIFT);3535}3536 3537static void gaudi_tpc_stall(struct hl_device *hdev)3538{3539	struct gaudi_device *gaudi = hdev->asic_specific;3540 3541	if (!(gaudi->hw_cap_initialized & HW_CAP_TPC_MASK))3542		return;3543 3544	WREG32(mmTPC0_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);3545	WREG32(mmTPC1_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);3546	WREG32(mmTPC2_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);3547	WREG32(mmTPC3_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);3548	WREG32(mmTPC4_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);3549	WREG32(mmTPC5_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);3550	WREG32(mmTPC6_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);3551	WREG32(mmTPC7_CFG_TPC_STALL, 1 << TPC0_CFG_TPC_STALL_V_SHIFT);3552}3553 3554static void gaudi_disable_clock_gating(struct hl_device *hdev)3555{3556	u32 qman_offset;3557	int i;3558 3559	if (hdev->asic_prop.fw_security_enabled)3560		return;3561 3562	for (i = 0, qman_offset = 0 ; i < DMA_NUMBER_OF_CHANNELS ; i++) {3563		WREG32(mmDMA0_QM_CGM_CFG + qman_offset, 0);3564		WREG32(mmDMA0_QM_CGM_CFG1 + qman_offset, 0);3565 3566		qman_offset += (mmDMA1_QM_CGM_CFG - mmDMA0_QM_CGM_CFG);3567	}3568 3569	WREG32(mmMME0_QM_CGM_CFG, 0);3570	WREG32(mmMME0_QM_CGM_CFG1, 0);3571	WREG32(mmMME2_QM_CGM_CFG, 0);3572	WREG32(mmMME2_QM_CGM_CFG1, 0);3573 3574	for (i = 0, qman_offset = 0 ; i < TPC_NUMBER_OF_ENGINES ; i++) {3575		WREG32(mmTPC0_QM_CGM_CFG + qman_offset, 0);3576		WREG32(mmTPC0_QM_CGM_CFG1 + qman_offset, 0);3577 3578		qman_offset += (mmTPC1_QM_CGM_CFG - mmTPC0_QM_CGM_CFG);3579	}3580}3581 3582static void gaudi_enable_timestamp(struct hl_device *hdev)3583{3584	/* Disable the timestamp counter */3585	WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0);3586 3587	/* Zero the lower/upper parts of the 64-bit counter */3588	WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0xC, 0);3589	WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE + 0x8, 0);3590 3591	/* Enable the counter */3592	WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 1);3593}3594 3595static void gaudi_disable_timestamp(struct hl_device *hdev)3596{3597	/* Disable the timestamp counter */3598	WREG32(mmPSOC_TIMESTAMP_BASE - CFG_BASE, 0);3599}3600 3601static void gaudi_halt_engines(struct hl_device *hdev, bool hard_reset, bool fw_reset)3602{3603	u32 wait_timeout_ms;3604 3605	if (hdev->pldm)3606		wait_timeout_ms = GAUDI_PLDM_RESET_WAIT_MSEC;3607	else3608		wait_timeout_ms = GAUDI_RESET_WAIT_MSEC;3609 3610	if (fw_reset)3611		goto skip_engines;3612 3613	gaudi_stop_nic_qmans(hdev);3614	gaudi_stop_mme_qmans(hdev);3615	gaudi_stop_tpc_qmans(hdev);3616	gaudi_stop_hbm_dma_qmans(hdev);3617	gaudi_stop_pci_dma_qmans(hdev);3618 3619	msleep(wait_timeout_ms);3620 3621	gaudi_pci_dma_stall(hdev);3622	gaudi_hbm_dma_stall(hdev);3623	gaudi_tpc_stall(hdev);3624	gaudi_mme_stall(hdev);3625 3626	msleep(wait_timeout_ms);3627 3628	gaudi_disable_nic_qmans(hdev);3629	gaudi_disable_mme_qmans(hdev);3630	gaudi_disable_tpc_qmans(hdev);3631	gaudi_disable_hbm_dma_qmans(hdev);3632	gaudi_disable_pci_dma_qmans(hdev);3633 3634	gaudi_disable_timestamp(hdev);3635 3636skip_engines:3637	gaudi_disable_msi(hdev);3638}3639 3640static int gaudi_mmu_init(struct hl_device *hdev)3641{3642	struct asic_fixed_properties *prop = &hdev->asic_prop;3643	struct gaudi_device *gaudi = hdev->asic_specific;3644	u64 hop0_addr;3645	int rc, i;3646 3647	if (gaudi->hw_cap_initialized & HW_CAP_MMU)3648		return 0;3649 3650	for (i = 0 ; i < prop->max_asid ; i++) {3651		hop0_addr = prop->mmu_pgt_addr +3652				(i * prop->dmmu.hop_table_size);3653 3654		rc = gaudi_mmu_update_asid_hop0_addr(hdev, i, hop0_addr);3655		if (rc) {3656			dev_err(hdev->dev,3657				"failed to set hop0 addr for asid %d\n", i);3658			return rc;3659		}3660	}3661 3662	/* init MMU cache manage page */3663	WREG32(mmSTLB_CACHE_INV_BASE_39_8, prop->mmu_cache_mng_addr >> 8);3664	WREG32(mmSTLB_CACHE_INV_BASE_49_40, prop->mmu_cache_mng_addr >> 40);3665 3666	/* mem cache invalidation */3667	WREG32(mmSTLB_MEM_CACHE_INVALIDATION, 1);3668 3669	rc = hl_mmu_invalidate_cache(hdev, true, 0);3670	if (rc)3671		return rc;3672 3673	WREG32(mmMMU_UP_MMU_ENABLE, 1);3674	WREG32(mmMMU_UP_SPI_MASK, 0xF);3675 3676	WREG32(mmSTLB_HOP_CONFIGURATION, 0x30440);3677 3678	/*3679	 * The H/W expects the first PI after init to be 1. After wraparound3680	 * we'll write 0.3681	 */3682	gaudi->mmu_cache_inv_pi = 1;3683 3684	gaudi->hw_cap_initialized |= HW_CAP_MMU;3685 3686	return 0;3687}3688 3689static int gaudi_load_firmware_to_device(struct hl_device *hdev)3690{3691	void __iomem *dst;3692 3693	dst = hdev->pcie_bar[HBM_BAR_ID] + LINUX_FW_OFFSET;3694 3695	return hl_fw_load_fw_to_device(hdev, GAUDI_LINUX_FW_FILE, dst, 0, 0);3696}3697 3698static int gaudi_load_boot_fit_to_device(struct hl_device *hdev)3699{3700	void __iomem *dst;3701 3702	dst = hdev->pcie_bar[SRAM_BAR_ID] + BOOT_FIT_SRAM_OFFSET;3703 3704	return hl_fw_load_fw_to_device(hdev, GAUDI_BOOT_FIT_FILE, dst, 0, 0);3705}3706 3707static void gaudi_init_dynamic_firmware_loader(struct hl_device *hdev)3708{3709	struct dynamic_fw_load_mgr *dynamic_loader;3710	struct cpu_dyn_regs *dyn_regs;3711 3712	dynamic_loader = &hdev->fw_loader.dynamic_loader;3713 3714	/*3715	 * here we update initial values for few specific dynamic regs (as3716	 * before reading the first descriptor from FW those value has to be3717	 * hard-coded) in later stages of the protocol those values will be3718	 * updated automatically by reading the FW descriptor so data there3719	 * will always be up-to-date3720	 */3721	dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs;3722	dyn_regs->kmd_msg_to_cpu =3723				cpu_to_le32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU);3724	dyn_regs->cpu_cmd_status_to_host =3725				cpu_to_le32(mmCPU_CMD_STATUS_TO_HOST);3726 3727	dynamic_loader->wait_for_bl_timeout = GAUDI_WAIT_FOR_BL_TIMEOUT_USEC;3728}3729 3730static void gaudi_init_static_firmware_loader(struct hl_device *hdev)3731{3732	struct static_fw_load_mgr *static_loader;3733 3734	static_loader = &hdev->fw_loader.static_loader;3735 3736	static_loader->preboot_version_max_off = SRAM_SIZE - VERSION_MAX_LEN;3737	static_loader->boot_fit_version_max_off = SRAM_SIZE - VERSION_MAX_LEN;3738	static_loader->kmd_msg_to_cpu_reg = mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU;3739	static_loader->cpu_cmd_status_to_host_reg = mmCPU_CMD_STATUS_TO_HOST;3740	static_loader->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS;3741	static_loader->cpu_boot_dev_status0_reg = mmCPU_BOOT_DEV_STS0;3742	static_loader->cpu_boot_dev_status1_reg = mmCPU_BOOT_DEV_STS1;3743	static_loader->boot_err0_reg = mmCPU_BOOT_ERR0;3744	static_loader->boot_err1_reg = mmCPU_BOOT_ERR1;3745	static_loader->preboot_version_offset_reg = mmPREBOOT_VER_OFFSET;3746	static_loader->boot_fit_version_offset_reg = mmUBOOT_VER_OFFSET;3747	static_loader->sram_offset_mask = ~(lower_32_bits(SRAM_BASE_ADDR));3748	static_loader->cpu_reset_wait_msec = hdev->pldm ?3749			GAUDI_PLDM_RESET_WAIT_MSEC :3750			GAUDI_CPU_RESET_WAIT_MSEC;3751}3752 3753static void gaudi_init_firmware_preload_params(struct hl_device *hdev)3754{3755	struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load;3756 3757	pre_fw_load->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS;3758	pre_fw_load->sts_boot_dev_sts0_reg = mmCPU_BOOT_DEV_STS0;3759	pre_fw_load->sts_boot_dev_sts1_reg = mmCPU_BOOT_DEV_STS1;3760	pre_fw_load->boot_err0_reg = mmCPU_BOOT_ERR0;3761	pre_fw_load->boot_err1_reg = mmCPU_BOOT_ERR1;3762	pre_fw_load->wait_for_preboot_timeout = GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC;3763}3764 3765static void gaudi_init_firmware_loader(struct hl_device *hdev)3766{3767	struct asic_fixed_properties *prop = &hdev->asic_prop;3768	struct fw_load_mgr *fw_loader = &hdev->fw_loader;3769 3770	/* fill common fields */3771	fw_loader->fw_comp_loaded = FW_TYPE_NONE;3772	fw_loader->boot_fit_img.image_name = GAUDI_BOOT_FIT_FILE;3773	fw_loader->linux_img.image_name = GAUDI_LINUX_FW_FILE;3774	fw_loader->cpu_timeout = GAUDI_CPU_TIMEOUT_USEC;3775	fw_loader->boot_fit_timeout = GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC;3776	fw_loader->skip_bmc = !hdev->bmc_enable;3777	fw_loader->sram_bar_id = SRAM_BAR_ID;3778	fw_loader->dram_bar_id = HBM_BAR_ID;3779 3780	if (prop->dynamic_fw_load)3781		gaudi_init_dynamic_firmware_loader(hdev);3782	else3783		gaudi_init_static_firmware_loader(hdev);3784}3785 3786static int gaudi_init_cpu(struct hl_device *hdev)3787{3788	struct gaudi_device *gaudi = hdev->asic_specific;3789	int rc;3790 3791	if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU))3792		return 0;3793 3794	if (gaudi->hw_cap_initialized & HW_CAP_CPU)3795		return 0;3796 3797	/*3798	 * The device CPU works with 40 bits addresses.3799	 * This register sets the extension to 50 bits.3800	 */3801	if (!hdev->asic_prop.fw_security_enabled)3802		WREG32(mmCPU_IF_CPU_MSB_ADDR, hdev->cpu_pci_msb_addr);3803 3804	rc = hl_fw_init_cpu(hdev);3805 3806	if (rc)3807		return rc;3808 3809	gaudi->hw_cap_initialized |= HW_CAP_CPU;3810 3811	return 0;3812}3813 3814static int gaudi_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout)3815{3816	struct cpu_dyn_regs *dyn_regs =3817			&hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;3818	struct asic_fixed_properties *prop = &hdev->asic_prop;3819	struct gaudi_device *gaudi = hdev->asic_specific;3820	u32 status, irq_handler_offset;3821	struct hl_eq *eq;3822	struct hl_hw_queue *cpu_pq =3823			&hdev->kernel_queues[GAUDI_QUEUE_ID_CPU_PQ];3824	int err;3825 3826	if (!hdev->cpu_queues_enable)3827		return 0;3828 3829	if (gaudi->hw_cap_initialized & HW_CAP_CPU_Q)3830		return 0;3831 3832	eq = &hdev->event_queue;3833 3834	WREG32(mmCPU_IF_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address));3835	WREG32(mmCPU_IF_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address));3836 3837	WREG32(mmCPU_IF_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address));3838	WREG32(mmCPU_IF_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address));3839 3840	WREG32(mmCPU_IF_CQ_BASE_ADDR_LOW,3841			lower_32_bits(hdev->cpu_accessible_dma_address));3842	WREG32(mmCPU_IF_CQ_BASE_ADDR_HIGH,3843			upper_32_bits(hdev->cpu_accessible_dma_address));3844 3845	WREG32(mmCPU_IF_PQ_LENGTH, HL_QUEUE_SIZE_IN_BYTES);3846	WREG32(mmCPU_IF_EQ_LENGTH, HL_EQ_SIZE_IN_BYTES);3847	WREG32(mmCPU_IF_CQ_LENGTH, HL_CPU_ACCESSIBLE_MEM_SIZE);3848 3849	/* Used for EQ CI */3850	WREG32(mmCPU_IF_EQ_RD_OFFS, 0);3851 3852	WREG32(mmCPU_IF_PF_PQ_PI, 0);3853 3854	WREG32(mmCPU_IF_QUEUE_INIT, PQ_INIT_STATUS_READY_FOR_CP_SINGLE_MSI);3855 3856	irq_handler_offset = prop->gic_interrupts_enable ?3857			mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :3858			le32_to_cpu(dyn_regs->gic_host_pi_upd_irq);3859 3860	WREG32(irq_handler_offset,3861		gaudi_irq_map_table[GAUDI_EVENT_PI_UPDATE].cpu_id);3862 3863	err = hl_poll_timeout(3864		hdev,3865		mmCPU_IF_QUEUE_INIT,3866		status,3867		(status == PQ_INIT_STATUS_READY_FOR_HOST),3868		1000,3869		cpu_timeout);3870 3871	if (err) {3872		dev_err(hdev->dev,3873			"Failed to communicate with Device CPU (CPU-CP timeout)\n");3874		return -EIO;3875	}3876 3877	/* update FW application security bits */3878	if (prop->fw_cpu_boot_dev_sts0_valid)3879		prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0);3880	if (prop->fw_cpu_boot_dev_sts1_valid)3881		prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1);3882 3883	gaudi->hw_cap_initialized |= HW_CAP_CPU_Q;3884	return 0;3885}3886 3887static void gaudi_pre_hw_init(struct hl_device *hdev)3888{3889	/* Perform read from the device to make sure device is up */3890	RREG32(mmHW_STATE);3891 3892	if (!hdev->asic_prop.fw_security_enabled) {3893		/* Set the access through PCI bars (Linux driver only) as3894		 * secured3895		 */3896		WREG32(mmPCIE_WRAP_LBW_PROT_OVR,3897				(PCIE_WRAP_LBW_PROT_OVR_RD_EN_MASK |3898				PCIE_WRAP_LBW_PROT_OVR_WR_EN_MASK));3899 3900		/* Perform read to flush the waiting writes to ensure3901		 * configuration was set in the device3902		 */3903		RREG32(mmPCIE_WRAP_LBW_PROT_OVR);3904	}3905 3906	/*3907	 * Let's mark in the H/W that we have reached this point. We check3908	 * this value in the reset_before_init function to understand whether3909	 * we need to reset the chip before doing H/W init. This register is3910	 * cleared by the H/W upon H/W reset3911	 */3912	WREG32(mmHW_STATE, HL_DEVICE_HW_STATE_DIRTY);3913}3914 3915static int gaudi_hw_init(struct hl_device *hdev)3916{3917	struct gaudi_device *gaudi = hdev->asic_specific;3918	int rc;3919 3920	gaudi_pre_hw_init(hdev);3921 3922	/* If iATU is done by FW, the HBM bar ALWAYS points to DRAM_PHYS_BASE.3923	 * So we set it here and if anyone tries to move it later to3924	 * a different address, there will be an error3925	 */3926	if (hdev->asic_prop.iatu_done_by_fw)3927		gaudi->hbm_bar_cur_addr = DRAM_PHYS_BASE;3928 3929	/*3930	 * Before pushing u-boot/linux to device, need to set the hbm bar to3931	 * base address of dram3932	 */3933	if (gaudi_set_hbm_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) {3934		dev_err(hdev->dev,3935			"failed to map HBM bar to DRAM base address\n");3936		return -EIO;3937	}3938 3939	rc = gaudi_init_cpu(hdev);3940	if (rc) {3941		dev_err(hdev->dev, "failed to initialize CPU\n");3942		return rc;3943	}3944 3945	/* In case the clock gating was enabled in preboot we need to disable3946	 * it here before touching the MME/TPC registers.3947	 */3948	gaudi_disable_clock_gating(hdev);3949 3950	/* SRAM scrambler must be initialized after CPU is running from HBM */3951	gaudi_init_scrambler_sram(hdev);3952 3953	/* This is here just in case we are working without CPU */3954	gaudi_init_scrambler_hbm(hdev);3955 3956	gaudi_init_golden_registers(hdev);3957 3958	rc = gaudi_mmu_init(hdev);3959	if (rc)3960		return rc;3961 3962	gaudi_init_security(hdev);3963 3964	gaudi_init_pci_dma_qmans(hdev);3965 3966	gaudi_init_hbm_dma_qmans(hdev);3967 3968	gaudi_init_mme_qmans(hdev);3969 3970	gaudi_init_tpc_qmans(hdev);3971 3972	gaudi_init_nic_qmans(hdev);3973 3974	gaudi_enable_timestamp(hdev);3975 3976	/* MSI must be enabled before CPU queues and NIC are initialized */3977	rc = gaudi_enable_msi(hdev);3978	if (rc)3979		goto disable_queues;3980 3981	/* must be called after MSI was enabled */3982	rc = gaudi_init_cpu_queues(hdev, GAUDI_CPU_TIMEOUT_USEC);3983	if (rc) {3984		dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n",3985			rc);3986		goto disable_msi;3987	}3988 3989	/* Perform read from the device to flush all configuration */3990	RREG32(mmHW_STATE);3991 3992	return 0;3993 3994disable_msi:3995	gaudi_disable_msi(hdev);3996disable_queues:3997	gaudi_disable_mme_qmans(hdev);3998	gaudi_disable_pci_dma_qmans(hdev);3999 4000	return rc;4001}4002 4003static int gaudi_hw_fini(struct hl_device *hdev, bool hard_reset, bool fw_reset)4004{4005	struct cpu_dyn_regs *dyn_regs =4006			&hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;4007	u32 status, reset_timeout_ms, cpu_timeout_ms, irq_handler_offset;4008	struct gaudi_device *gaudi = hdev->asic_specific;4009	bool driver_performs_reset;4010 4011	if (!hard_reset) {4012		dev_err(hdev->dev, "GAUDI doesn't support soft-reset\n");4013		return 0;4014	}4015 4016	if (hdev->pldm) {4017		reset_timeout_ms = GAUDI_PLDM_HRESET_TIMEOUT_MSEC;4018		cpu_timeout_ms = GAUDI_PLDM_RESET_WAIT_MSEC;4019	} else {4020		reset_timeout_ms = GAUDI_RESET_TIMEOUT_MSEC;4021		cpu_timeout_ms = GAUDI_CPU_RESET_WAIT_MSEC;4022	}4023 4024	if (fw_reset) {4025		dev_dbg(hdev->dev,4026			"Firmware performs HARD reset, going to wait %dms\n",4027			reset_timeout_ms);4028 4029		goto skip_reset;4030	}4031 4032	driver_performs_reset = !!(!hdev->asic_prop.fw_security_enabled &&4033					!hdev->asic_prop.hard_reset_done_by_fw);4034 4035	/* Set device to handle FLR by H/W as we will put the device CPU to4036	 * halt mode4037	 */4038	if (driver_performs_reset)4039		WREG32(mmPCIE_AUX_FLR_CTRL, (PCIE_AUX_FLR_CTRL_HW_CTRL_MASK |4040					PCIE_AUX_FLR_CTRL_INT_MASK_MASK));4041 4042	/* If linux is loaded in the device CPU we need to communicate with it4043	 * via the GIC. Otherwise, we need to use COMMS or the MSG_TO_CPU4044	 * registers in case of old F/Ws4045	 */4046	if (hdev->fw_loader.fw_comp_loaded & FW_TYPE_LINUX) {4047		irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?4048				mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :4049				le32_to_cpu(dyn_regs->gic_host_halt_irq);4050 4051		WREG32(irq_handler_offset,4052			gaudi_irq_map_table[GAUDI_EVENT_HALT_MACHINE].cpu_id);4053 4054		/* This is a hail-mary attempt to revive the card in the small chance that the4055		 * f/w has experienced a watchdog event, which caused it to return back to preboot.4056		 * In that case, triggering reset through GIC won't help. We need to trigger the4057		 * reset as if Linux wasn't loaded.4058		 *4059		 * We do it only if the reset cause was HB, because that would be the indication4060		 * of such an event.4061		 *4062		 * In case watchdog hasn't expired but we still got HB, then this won't do any4063		 * damage.4064		 */4065		if (hdev->reset_info.curr_reset_cause == HL_RESET_CAUSE_HEARTBEAT) {4066			if (hdev->asic_prop.hard_reset_done_by_fw)4067				hl_fw_ask_hard_reset_without_linux(hdev);4068			else4069				hl_fw_ask_halt_machine_without_linux(hdev);4070		}4071	} else {4072		if (hdev->asic_prop.hard_reset_done_by_fw)4073			hl_fw_ask_hard_reset_without_linux(hdev);4074		else4075			hl_fw_ask_halt_machine_without_linux(hdev);4076	}4077 4078	if (driver_performs_reset) {4079 4080		/* Configure the reset registers. Must be done as early as4081		 * possible in case we fail during H/W initialization4082		 */4083		WREG32(mmPSOC_GLOBAL_CONF_SOFT_RST_CFG_H,4084						(CFG_RST_H_DMA_MASK |4085						CFG_RST_H_MME_MASK |4086						CFG_RST_H_SM_MASK |4087						CFG_RST_H_TPC_7_MASK));4088 4089		WREG32(mmPSOC_GLOBAL_CONF_SOFT_RST_CFG_L, CFG_RST_L_TPC_MASK);4090 4091		WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST_CFG_H,4092						(CFG_RST_H_HBM_MASK |4093						CFG_RST_H_TPC_7_MASK |4094						CFG_RST_H_NIC_MASK |4095						CFG_RST_H_SM_MASK |4096						CFG_RST_H_DMA_MASK |4097						CFG_RST_H_MME_MASK |4098						CFG_RST_H_CPU_MASK |4099						CFG_RST_H_MMU_MASK));4100 4101		WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST_CFG_L,4102						(CFG_RST_L_IF_MASK |4103						CFG_RST_L_PSOC_MASK |4104						CFG_RST_L_TPC_MASK));4105 4106		msleep(cpu_timeout_ms);4107 4108		/* Tell ASIC not to re-initialize PCIe */4109		WREG32(mmPREBOOT_PCIE_EN, LKD_HARD_RESET_MAGIC);4110 4111		/* Restart BTL/BLR upon hard-reset */4112		WREG32(mmPSOC_GLOBAL_CONF_BOOT_SEQ_RE_START, 1);4113 4114		WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST,4115			1 << PSOC_GLOBAL_CONF_SW_ALL_RST_IND_SHIFT);4116 4117		dev_dbg(hdev->dev,4118			"Issued HARD reset command, going to wait %dms\n",4119			reset_timeout_ms);4120	} else {4121		dev_dbg(hdev->dev,4122			"Firmware performs HARD reset, going to wait %dms\n",4123			reset_timeout_ms);4124	}4125 4126skip_reset:4127	/*4128	 * After hard reset, we can't poll the BTM_FSM register because the PSOC4129	 * itself is in reset. Need to wait until the reset is deasserted4130	 */4131	msleep(reset_timeout_ms);4132 4133	status = RREG32(mmPSOC_GLOBAL_CONF_BTM_FSM);4134	if (status & PSOC_GLOBAL_CONF_BTM_FSM_STATE_MASK) {4135		dev_err(hdev->dev, "Timeout while waiting for device to reset 0x%x\n", status);4136		return -ETIMEDOUT;4137	}4138 4139	if (gaudi) {4140		gaudi->hw_cap_initialized &= ~(HW_CAP_CPU | HW_CAP_CPU_Q | HW_CAP_HBM |4141						HW_CAP_PCI_DMA | HW_CAP_MME | HW_CAP_TPC_MASK |4142						HW_CAP_HBM_DMA | HW_CAP_PLL | HW_CAP_NIC_MASK |4143						HW_CAP_MMU | HW_CAP_SRAM_SCRAMBLER |4144						HW_CAP_HBM_SCRAMBLER);4145 4146		memset(gaudi->events_stat, 0, sizeof(gaudi->events_stat));4147 4148		hdev->device_cpu_is_halted = false;4149	}4150	return 0;4151}4152 4153static int gaudi_suspend(struct hl_device *hdev)4154{4155	return hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0);4156}4157 4158static int gaudi_resume(struct hl_device *hdev)4159{4160	return gaudi_init_iatu(hdev);4161}4162 4163static int gaudi_mmap(struct hl_device *hdev, struct vm_area_struct *vma,4164			void *cpu_addr, dma_addr_t dma_addr, size_t size)4165{4166	int rc;4167 4168	vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP |4169			VM_DONTCOPY | VM_NORESERVE);4170 4171	rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr,4172				(dma_addr - HOST_PHYS_BASE), size);4173	if (rc)4174		dev_err(hdev->dev, "dma_mmap_coherent error %d", rc);4175 4176	return rc;4177}4178 4179static void gaudi_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi)4180{4181	struct cpu_dyn_regs *dyn_regs =4182			&hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;4183	u32 db_reg_offset, db_value, dma_qm_offset, q_off, irq_handler_offset;4184	struct gaudi_device *gaudi = hdev->asic_specific;4185	bool invalid_queue = false;4186	int dma_id;4187 4188	switch (hw_queue_id) {4189	case GAUDI_QUEUE_ID_DMA_0_0...GAUDI_QUEUE_ID_DMA_0_3:4190		dma_id = gaudi_dma_assignment[GAUDI_PCI_DMA_1];4191		dma_qm_offset = dma_id * DMA_QMAN_OFFSET;4192		q_off = dma_qm_offset + (hw_queue_id & 0x3) * 4;4193		db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;4194		break;4195 4196	case GAUDI_QUEUE_ID_DMA_1_0...GAUDI_QUEUE_ID_DMA_1_3:4197		dma_id = gaudi_dma_assignment[GAUDI_PCI_DMA_2];4198		dma_qm_offset = dma_id * DMA_QMAN_OFFSET;4199		q_off = dma_qm_offset + (hw_queue_id & 0x3) * 4;4200		db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;4201		break;4202 4203	case GAUDI_QUEUE_ID_DMA_2_0...GAUDI_QUEUE_ID_DMA_2_3:4204		dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_1];4205		dma_qm_offset = dma_id * DMA_QMAN_OFFSET;4206		q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4;4207		db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;4208		break;4209 4210	case GAUDI_QUEUE_ID_DMA_3_0...GAUDI_QUEUE_ID_DMA_3_3:4211		dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_2];4212		dma_qm_offset = dma_id * DMA_QMAN_OFFSET;4213		q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4;4214		db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;4215		break;4216 4217	case GAUDI_QUEUE_ID_DMA_4_0...GAUDI_QUEUE_ID_DMA_4_3:4218		dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_3];4219		dma_qm_offset = dma_id * DMA_QMAN_OFFSET;4220		q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4;4221		db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;4222		break;4223 4224	case GAUDI_QUEUE_ID_DMA_5_0...GAUDI_QUEUE_ID_DMA_5_3:4225		dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_4];4226		dma_qm_offset = dma_id * DMA_QMAN_OFFSET;4227		q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4;4228		db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;4229		break;4230 4231	case GAUDI_QUEUE_ID_DMA_6_0...GAUDI_QUEUE_ID_DMA_6_3:4232		dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_5];4233		dma_qm_offset = dma_id * DMA_QMAN_OFFSET;4234		q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4;4235		db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;4236		break;4237 4238	case GAUDI_QUEUE_ID_DMA_7_0...GAUDI_QUEUE_ID_DMA_7_3:4239		dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_6];4240		dma_qm_offset = dma_id * DMA_QMAN_OFFSET;4241		q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4;4242		db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off;4243		break;4244 4245	case GAUDI_QUEUE_ID_CPU_PQ:4246		if (gaudi->hw_cap_initialized & HW_CAP_CPU_Q)4247			db_reg_offset = mmCPU_IF_PF_PQ_PI;4248		else4249			invalid_queue = true;4250		break;4251 4252	case GAUDI_QUEUE_ID_MME_0_0:4253		db_reg_offset = mmMME2_QM_PQ_PI_0;4254		break;4255 4256	case GAUDI_QUEUE_ID_MME_0_1:4257		db_reg_offset = mmMME2_QM_PQ_PI_1;4258		break;4259 4260	case GAUDI_QUEUE_ID_MME_0_2:4261		db_reg_offset = mmMME2_QM_PQ_PI_2;4262		break;4263 4264	case GAUDI_QUEUE_ID_MME_0_3:4265		db_reg_offset = mmMME2_QM_PQ_PI_3;4266		break;4267 4268	case GAUDI_QUEUE_ID_MME_1_0:4269		db_reg_offset = mmMME0_QM_PQ_PI_0;4270		break;4271 4272	case GAUDI_QUEUE_ID_MME_1_1:4273		db_reg_offset = mmMME0_QM_PQ_PI_1;4274		break;4275 4276	case GAUDI_QUEUE_ID_MME_1_2:4277		db_reg_offset = mmMME0_QM_PQ_PI_2;4278		break;4279 4280	case GAUDI_QUEUE_ID_MME_1_3:4281		db_reg_offset = mmMME0_QM_PQ_PI_3;4282		break;4283 4284	case GAUDI_QUEUE_ID_TPC_0_0:4285		db_reg_offset = mmTPC0_QM_PQ_PI_0;4286		break;4287 4288	case GAUDI_QUEUE_ID_TPC_0_1:4289		db_reg_offset = mmTPC0_QM_PQ_PI_1;4290		break;4291 4292	case GAUDI_QUEUE_ID_TPC_0_2:4293		db_reg_offset = mmTPC0_QM_PQ_PI_2;4294		break;4295 4296	case GAUDI_QUEUE_ID_TPC_0_3:4297		db_reg_offset = mmTPC0_QM_PQ_PI_3;4298		break;4299 4300	case GAUDI_QUEUE_ID_TPC_1_0:4301		db_reg_offset = mmTPC1_QM_PQ_PI_0;4302		break;4303 4304	case GAUDI_QUEUE_ID_TPC_1_1:4305		db_reg_offset = mmTPC1_QM_PQ_PI_1;4306		break;4307 4308	case GAUDI_QUEUE_ID_TPC_1_2:4309		db_reg_offset = mmTPC1_QM_PQ_PI_2;4310		break;4311 4312	case GAUDI_QUEUE_ID_TPC_1_3:4313		db_reg_offset = mmTPC1_QM_PQ_PI_3;4314		break;4315 4316	case GAUDI_QUEUE_ID_TPC_2_0:4317		db_reg_offset = mmTPC2_QM_PQ_PI_0;4318		break;4319 4320	case GAUDI_QUEUE_ID_TPC_2_1:4321		db_reg_offset = mmTPC2_QM_PQ_PI_1;4322		break;4323 4324	case GAUDI_QUEUE_ID_TPC_2_2:4325		db_reg_offset = mmTPC2_QM_PQ_PI_2;4326		break;4327 4328	case GAUDI_QUEUE_ID_TPC_2_3:4329		db_reg_offset = mmTPC2_QM_PQ_PI_3;4330		break;4331 4332	case GAUDI_QUEUE_ID_TPC_3_0:4333		db_reg_offset = mmTPC3_QM_PQ_PI_0;4334		break;4335 4336	case GAUDI_QUEUE_ID_TPC_3_1:4337		db_reg_offset = mmTPC3_QM_PQ_PI_1;4338		break;4339 4340	case GAUDI_QUEUE_ID_TPC_3_2:4341		db_reg_offset = mmTPC3_QM_PQ_PI_2;4342		break;4343 4344	case GAUDI_QUEUE_ID_TPC_3_3:4345		db_reg_offset = mmTPC3_QM_PQ_PI_3;4346		break;4347 4348	case GAUDI_QUEUE_ID_TPC_4_0:4349		db_reg_offset = mmTPC4_QM_PQ_PI_0;4350		break;4351 4352	case GAUDI_QUEUE_ID_TPC_4_1:4353		db_reg_offset = mmTPC4_QM_PQ_PI_1;4354		break;4355 4356	case GAUDI_QUEUE_ID_TPC_4_2:4357		db_reg_offset = mmTPC4_QM_PQ_PI_2;4358		break;4359 4360	case GAUDI_QUEUE_ID_TPC_4_3:4361		db_reg_offset = mmTPC4_QM_PQ_PI_3;4362		break;4363 4364	case GAUDI_QUEUE_ID_TPC_5_0:4365		db_reg_offset = mmTPC5_QM_PQ_PI_0;4366		break;4367 4368	case GAUDI_QUEUE_ID_TPC_5_1:4369		db_reg_offset = mmTPC5_QM_PQ_PI_1;4370		break;4371 4372	case GAUDI_QUEUE_ID_TPC_5_2:4373		db_reg_offset = mmTPC5_QM_PQ_PI_2;4374		break;4375 4376	case GAUDI_QUEUE_ID_TPC_5_3:4377		db_reg_offset = mmTPC5_QM_PQ_PI_3;4378		break;4379 4380	case GAUDI_QUEUE_ID_TPC_6_0:4381		db_reg_offset = mmTPC6_QM_PQ_PI_0;4382		break;4383 4384	case GAUDI_QUEUE_ID_TPC_6_1:4385		db_reg_offset = mmTPC6_QM_PQ_PI_1;4386		break;4387 4388	case GAUDI_QUEUE_ID_TPC_6_2:4389		db_reg_offset = mmTPC6_QM_PQ_PI_2;4390		break;4391 4392	case GAUDI_QUEUE_ID_TPC_6_3:4393		db_reg_offset = mmTPC6_QM_PQ_PI_3;4394		break;4395 4396	case GAUDI_QUEUE_ID_TPC_7_0:4397		db_reg_offset = mmTPC7_QM_PQ_PI_0;4398		break;4399 4400	case GAUDI_QUEUE_ID_TPC_7_1:4401		db_reg_offset = mmTPC7_QM_PQ_PI_1;4402		break;4403 4404	case GAUDI_QUEUE_ID_TPC_7_2:4405		db_reg_offset = mmTPC7_QM_PQ_PI_2;4406		break;4407 4408	case GAUDI_QUEUE_ID_TPC_7_3:4409		db_reg_offset = mmTPC7_QM_PQ_PI_3;4410		break;4411 4412	case GAUDI_QUEUE_ID_NIC_0_0...GAUDI_QUEUE_ID_NIC_0_3:4413		if (!(gaudi->hw_cap_initialized & HW_CAP_NIC0))4414			invalid_queue = true;4415 4416		q_off = ((hw_queue_id - 1) & 0x3) * 4;4417		db_reg_offset = mmNIC0_QM0_PQ_PI_0 + q_off;4418		break;4419 4420	case GAUDI_QUEUE_ID_NIC_1_0...GAUDI_QUEUE_ID_NIC_1_3:4421		if (!(gaudi->hw_cap_initialized & HW_CAP_NIC1))4422			invalid_queue = true;4423 4424		q_off = ((hw_queue_id - 1) & 0x3) * 4;4425		db_reg_offset = mmNIC0_QM1_PQ_PI_0 + q_off;4426		break;4427 4428	case GAUDI_QUEUE_ID_NIC_2_0...GAUDI_QUEUE_ID_NIC_2_3:4429		if (!(gaudi->hw_cap_initialized & HW_CAP_NIC2))4430			invalid_queue = true;4431 4432		q_off = ((hw_queue_id - 1) & 0x3) * 4;4433		db_reg_offset = mmNIC1_QM0_PQ_PI_0 + q_off;4434		break;4435 4436	case GAUDI_QUEUE_ID_NIC_3_0...GAUDI_QUEUE_ID_NIC_3_3:4437		if (!(gaudi->hw_cap_initialized & HW_CAP_NIC3))4438			invalid_queue = true;4439 4440		q_off = ((hw_queue_id - 1) & 0x3) * 4;4441		db_reg_offset = mmNIC1_QM1_PQ_PI_0 + q_off;4442		break;4443 4444	case GAUDI_QUEUE_ID_NIC_4_0...GAUDI_QUEUE_ID_NIC_4_3:4445		if (!(gaudi->hw_cap_initialized & HW_CAP_NIC4))4446			invalid_queue = true;4447 4448		q_off = ((hw_queue_id - 1) & 0x3) * 4;4449		db_reg_offset = mmNIC2_QM0_PQ_PI_0 + q_off;4450		break;4451 4452	case GAUDI_QUEUE_ID_NIC_5_0...GAUDI_QUEUE_ID_NIC_5_3:4453		if (!(gaudi->hw_cap_initialized & HW_CAP_NIC5))4454			invalid_queue = true;4455 4456		q_off = ((hw_queue_id - 1) & 0x3) * 4;4457		db_reg_offset = mmNIC2_QM1_PQ_PI_0 + q_off;4458		break;4459 4460	case GAUDI_QUEUE_ID_NIC_6_0...GAUDI_QUEUE_ID_NIC_6_3:4461		if (!(gaudi->hw_cap_initialized & HW_CAP_NIC6))4462			invalid_queue = true;4463 4464		q_off = ((hw_queue_id - 1) & 0x3) * 4;4465		db_reg_offset = mmNIC3_QM0_PQ_PI_0 + q_off;4466		break;4467 4468	case GAUDI_QUEUE_ID_NIC_7_0...GAUDI_QUEUE_ID_NIC_7_3:4469		if (!(gaudi->hw_cap_initialized & HW_CAP_NIC7))4470			invalid_queue = true;4471 4472		q_off = ((hw_queue_id - 1) & 0x3) * 4;4473		db_reg_offset = mmNIC3_QM1_PQ_PI_0 + q_off;4474		break;4475 4476	case GAUDI_QUEUE_ID_NIC_8_0...GAUDI_QUEUE_ID_NIC_8_3:4477		if (!(gaudi->hw_cap_initialized & HW_CAP_NIC8))4478			invalid_queue = true;4479 4480		q_off = ((hw_queue_id - 1) & 0x3) * 4;4481		db_reg_offset = mmNIC4_QM0_PQ_PI_0 + q_off;4482		break;4483 4484	case GAUDI_QUEUE_ID_NIC_9_0...GAUDI_QUEUE_ID_NIC_9_3:4485		if (!(gaudi->hw_cap_initialized & HW_CAP_NIC9))4486			invalid_queue = true;4487 4488		q_off = ((hw_queue_id - 1) & 0x3) * 4;4489		db_reg_offset = mmNIC4_QM1_PQ_PI_0 + q_off;4490		break;4491 4492	default:4493		invalid_queue = true;4494	}4495 4496	if (invalid_queue) {4497		/* Should never get here */4498		dev_err(hdev->dev, "h/w queue %d is invalid. Can't set pi\n",4499			hw_queue_id);4500		return;4501	}4502 4503	db_value = pi;4504 4505	/* ring the doorbell */4506	WREG32(db_reg_offset, db_value);4507 4508	if (hw_queue_id == GAUDI_QUEUE_ID_CPU_PQ) {4509		/* make sure device CPU will read latest data from host */4510		mb();4511 4512		irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?4513				mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :4514				le32_to_cpu(dyn_regs->gic_host_pi_upd_irq);4515 4516		WREG32(irq_handler_offset,4517			gaudi_irq_map_table[GAUDI_EVENT_PI_UPDATE].cpu_id);4518	}4519}4520 4521static void gaudi_pqe_write(struct hl_device *hdev, __le64 *pqe,4522				struct hl_bd *bd)4523{4524	__le64 *pbd = (__le64 *) bd;4525 4526	/* The QMANs are on the host memory so a simple copy suffice */4527	pqe[0] = pbd[0];4528	pqe[1] = pbd[1];4529}4530 4531static void *gaudi_dma_alloc_coherent(struct hl_device *hdev, size_t size,4532					dma_addr_t *dma_handle, gfp_t flags)4533{4534	void *kernel_addr = dma_alloc_coherent(&hdev->pdev->dev, size,4535						dma_handle, flags);4536 4537	/* Shift to the device's base physical address of host memory */4538	if (kernel_addr)4539		*dma_handle += HOST_PHYS_BASE;4540 4541	return kernel_addr;4542}4543 4544static void gaudi_dma_free_coherent(struct hl_device *hdev, size_t size,4545		void *cpu_addr, dma_addr_t dma_handle)4546{4547	/* Cancel the device's base physical address of host memory */4548	dma_addr_t fixed_dma_handle = dma_handle - HOST_PHYS_BASE;4549 4550	dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, fixed_dma_handle);4551}4552 4553static int gaudi_scrub_device_dram(struct hl_device *hdev, u64 val)4554{4555	struct asic_fixed_properties *prop = &hdev->asic_prop;4556	u64 cur_addr = prop->dram_user_base_address;4557	u32 chunk_size, busy;4558	int rc, dma_id;4559 4560	while (cur_addr < prop->dram_end_address) {4561		for (dma_id = 0 ; dma_id < DMA_NUMBER_OF_CHANNELS ; dma_id++) {4562			u32 dma_offset = dma_id * DMA_CORE_OFFSET;4563 4564			chunk_size =4565			min((u64)SZ_2G, prop->dram_end_address - cur_addr);4566 4567			dev_dbg(hdev->dev,4568				"Doing HBM scrubbing for 0x%09llx - 0x%09llx\n",4569				cur_addr, cur_addr + chunk_size);4570 4571			WREG32(mmDMA0_CORE_SRC_BASE_LO + dma_offset,4572					lower_32_bits(val));4573			WREG32(mmDMA0_CORE_SRC_BASE_HI + dma_offset,4574					upper_32_bits(val));4575			WREG32(mmDMA0_CORE_DST_BASE_LO + dma_offset,4576						lower_32_bits(cur_addr));4577			WREG32(mmDMA0_CORE_DST_BASE_HI + dma_offset,4578						upper_32_bits(cur_addr));4579			WREG32(mmDMA0_CORE_DST_TSIZE_0 + dma_offset,4580					chunk_size);4581			WREG32(mmDMA0_CORE_COMMIT + dma_offset,4582					((1 << DMA0_CORE_COMMIT_LIN_SHIFT) |4583					(1 << DMA0_CORE_COMMIT_MEM_SET_SHIFT)));4584 4585			cur_addr += chunk_size;4586 4587			if (cur_addr == prop->dram_end_address)4588				break;4589		}4590 4591		for (dma_id = 0 ; dma_id < DMA_NUMBER_OF_CHANNELS ; dma_id++) {4592			u32 dma_offset = dma_id * DMA_CORE_OFFSET;4593 4594			rc = hl_poll_timeout(4595				hdev,4596				mmDMA0_CORE_STS0 + dma_offset,4597				busy,4598				((busy & DMA0_CORE_STS0_BUSY_MASK) == 0),4599				1000,4600				HBM_SCRUBBING_TIMEOUT_US);4601 4602			if (rc) {4603				dev_err(hdev->dev,4604					"DMA Timeout during HBM scrubbing of DMA #%d\n",4605					dma_id);4606				return -EIO;4607			}4608		}4609	}4610 4611	return 0;4612}4613 4614static int gaudi_scrub_device_mem(struct hl_device *hdev)4615{4616	struct asic_fixed_properties *prop = &hdev->asic_prop;4617	u64 wait_to_idle_time = HBM_SCRUBBING_TIMEOUT_US;4618	u64 addr, size, val = hdev->memory_scrub_val;4619	ktime_t timeout;4620	int rc = 0;4621 4622	if (!hdev->memory_scrub)4623		return 0;4624 4625	timeout = ktime_add_us(ktime_get(), wait_to_idle_time);4626	while (!hdev->asic_funcs->is_device_idle(hdev, NULL, 0, NULL)) {4627		if (ktime_compare(ktime_get(), timeout) > 0) {4628			dev_err(hdev->dev, "waiting for idle timeout\n");4629			return -ETIMEDOUT;4630		}4631		usleep_range((1000 >> 2) + 1, 1000);4632	}4633 4634	/* Scrub SRAM */4635	addr = prop->sram_user_base_address;4636	size = hdev->pldm ? 0x10000 : prop->sram_size - SRAM_USER_BASE_OFFSET;4637 4638	dev_dbg(hdev->dev, "Scrubbing SRAM: 0x%09llx - 0x%09llx val: 0x%llx\n",4639			addr, addr + size, val);4640	rc = gaudi_memset_device_memory(hdev, addr, size, val);4641	if (rc) {4642		dev_err(hdev->dev, "Failed to clear SRAM (%d)\n", rc);4643		return rc;4644	}4645 4646	/* Scrub HBM using all DMA channels in parallel */4647	rc = gaudi_scrub_device_dram(hdev, val);4648	if (rc) {4649		dev_err(hdev->dev, "Failed to clear HBM (%d)\n", rc);4650		return rc;4651	}4652 4653	return 0;4654}4655 4656static void *gaudi_get_int_queue_base(struct hl_device *hdev,4657				u32 queue_id, dma_addr_t *dma_handle,4658				u16 *queue_len)4659{4660	struct gaudi_device *gaudi = hdev->asic_specific;4661	struct gaudi_internal_qman_info *q;4662 4663	if (queue_id >= GAUDI_QUEUE_ID_SIZE ||4664			gaudi_queue_type[queue_id] != QUEUE_TYPE_INT) {4665		dev_err(hdev->dev, "Got invalid queue id %d\n", queue_id);4666		return NULL;4667	}4668 4669	q = &gaudi->internal_qmans[queue_id];4670	*dma_handle = q->pq_dma_addr;4671	*queue_len = q->pq_size / QMAN_PQ_ENTRY_SIZE;4672 4673	return q->pq_kernel_addr;4674}4675 4676static int gaudi_send_cpu_message(struct hl_device *hdev, u32 *msg,4677				u16 len, u32 timeout, u64 *result)4678{4679	struct gaudi_device *gaudi = hdev->asic_specific;4680 4681	if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) {4682		if (result)4683			*result = 0;4684		return 0;4685	}4686 4687	if (!timeout)4688		timeout = GAUDI_MSG_TO_CPU_TIMEOUT_USEC;4689 4690	return hl_fw_send_cpu_message(hdev, GAUDI_QUEUE_ID_CPU_PQ, msg, len,4691						timeout, result);4692}4693 4694static int gaudi_test_queue(struct hl_device *hdev, u32 hw_queue_id)4695{4696	struct packet_msg_prot *fence_pkt;4697	dma_addr_t pkt_dma_addr;4698	u32 fence_val, tmp, timeout_usec;4699	dma_addr_t fence_dma_addr;4700	u32 *fence_ptr;4701	int rc;4702 4703	if (hdev->pldm)4704		timeout_usec = GAUDI_PLDM_TEST_QUEUE_WAIT_USEC;4705	else4706		timeout_usec = GAUDI_TEST_QUEUE_WAIT_USEC;4707 4708	fence_val = GAUDI_QMAN0_FENCE_VAL;4709 4710	fence_ptr = hl_asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL, &fence_dma_addr);4711	if (!fence_ptr) {4712		dev_err(hdev->dev,4713			"Failed to allocate memory for H/W queue %d testing\n",4714			hw_queue_id);4715		return -ENOMEM;4716	}4717 4718	*fence_ptr = 0;4719 4720	fence_pkt = hl_asic_dma_pool_zalloc(hdev, sizeof(struct packet_msg_prot), GFP_KERNEL,4721						&pkt_dma_addr);4722	if (!fence_pkt) {4723		dev_err(hdev->dev,4724			"Failed to allocate packet for H/W queue %d testing\n",4725			hw_queue_id);4726		rc = -ENOMEM;4727		goto free_fence_ptr;4728	}4729 4730	tmp = FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_PROT);4731	tmp |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 1);4732	tmp |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1);4733 4734	fence_pkt->ctl = cpu_to_le32(tmp);4735	fence_pkt->value = cpu_to_le32(fence_val);4736	fence_pkt->addr = cpu_to_le64(fence_dma_addr);4737 4738	rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id,4739					sizeof(struct packet_msg_prot),4740					pkt_dma_addr);4741	if (rc) {4742		dev_err(hdev->dev,4743			"Failed to send fence packet to H/W queue %d\n",4744			hw_queue_id);4745		goto free_pkt;4746	}4747 4748	rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp, (tmp == fence_val),4749					1000, timeout_usec, true);4750 4751	hl_hw_queue_inc_ci_kernel(hdev, hw_queue_id);4752 4753	if (rc == -ETIMEDOUT) {4754		dev_err(hdev->dev,4755			"H/W queue %d test failed (scratch(0x%08llX) == 0x%08X)\n",4756			hw_queue_id, (unsigned long long) fence_dma_addr, tmp);4757		rc = -EIO;4758	}4759 4760free_pkt:4761	hl_asic_dma_pool_free(hdev, (void *) fence_pkt, pkt_dma_addr);4762free_fence_ptr:4763	hl_asic_dma_pool_free(hdev, (void *) fence_ptr, fence_dma_addr);4764	return rc;4765}4766 4767static int gaudi_test_cpu_queue(struct hl_device *hdev)4768{4769	struct gaudi_device *gaudi = hdev->asic_specific;4770 4771	/*4772	 * check capability here as send_cpu_message() won't update the result4773	 * value if no capability4774	 */4775	if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))4776		return 0;4777 4778	return hl_fw_test_cpu_queue(hdev);4779}4780 4781static int gaudi_test_queues(struct hl_device *hdev)4782{4783	int i, rc, ret_val = 0;4784 4785	for (i = 0 ; i < hdev->asic_prop.max_queues ; i++) {4786		if (hdev->asic_prop.hw_queues_props[i].type == QUEUE_TYPE_EXT) {4787			rc = gaudi_test_queue(hdev, i);4788			if (rc)4789				ret_val = -EINVAL;4790		}4791	}4792 4793	rc = gaudi_test_cpu_queue(hdev);4794	if (rc)4795		ret_val = -EINVAL;4796 4797	return ret_val;4798}4799 4800static void *gaudi_dma_pool_zalloc(struct hl_device *hdev, size_t size,4801		gfp_t mem_flags, dma_addr_t *dma_handle)4802{4803	void *kernel_addr;4804 4805	if (size > GAUDI_DMA_POOL_BLK_SIZE)4806		return NULL;4807 4808	kernel_addr = dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle);4809 4810	/* Shift to the device's base physical address of host memory */4811	if (kernel_addr)4812		*dma_handle += HOST_PHYS_BASE;4813 4814	return kernel_addr;4815}4816 4817static void gaudi_dma_pool_free(struct hl_device *hdev, void *vaddr,4818			dma_addr_t dma_addr)4819{4820	/* Cancel the device's base physical address of host memory */4821	dma_addr_t fixed_dma_addr = dma_addr - HOST_PHYS_BASE;4822 4823	dma_pool_free(hdev->dma_pool, vaddr, fixed_dma_addr);4824}4825 4826static void *gaudi_cpu_accessible_dma_pool_alloc(struct hl_device *hdev,4827					size_t size, dma_addr_t *dma_handle)4828{4829	return hl_fw_cpu_accessible_dma_pool_alloc(hdev, size, dma_handle);4830}4831 4832static void gaudi_cpu_accessible_dma_pool_free(struct hl_device *hdev,4833						size_t size, void *vaddr)4834{4835	hl_fw_cpu_accessible_dma_pool_free(hdev, size, vaddr);4836}4837 4838static u32 gaudi_get_dma_desc_list_size(struct hl_device *hdev, struct sg_table *sgt)4839{4840	struct scatterlist *sg, *sg_next_iter;4841	u32 count, dma_desc_cnt;4842	u64 len, len_next;4843	dma_addr_t addr, addr_next;4844 4845	dma_desc_cnt = 0;4846 4847	for_each_sgtable_dma_sg(sgt, sg, count) {4848		len = sg_dma_len(sg);4849		addr = sg_dma_address(sg);4850 4851		if (len == 0)4852			break;4853 4854		while ((count + 1) < sgt->nents) {4855			sg_next_iter = sg_next(sg);4856			len_next = sg_dma_len(sg_next_iter);4857			addr_next = sg_dma_address(sg_next_iter);4858 4859			if (len_next == 0)4860				break;4861 4862			if ((addr + len == addr_next) &&4863				(len + len_next <= DMA_MAX_TRANSFER_SIZE)) {4864				len += len_next;4865				count++;4866				sg = sg_next_iter;4867			} else {4868				break;4869			}4870		}4871 4872		dma_desc_cnt++;4873	}4874 4875	return dma_desc_cnt * sizeof(struct packet_lin_dma);4876}4877 4878static int gaudi_pin_memory_before_cs(struct hl_device *hdev,4879				struct hl_cs_parser *parser,4880				struct packet_lin_dma *user_dma_pkt,4881				u64 addr, enum dma_data_direction dir)4882{4883	struct hl_userptr *userptr;4884	int rc;4885 4886	if (hl_userptr_is_pinned(hdev, addr, le32_to_cpu(user_dma_pkt->tsize),4887			parser->job_userptr_list, &userptr))4888		goto already_pinned;4889 4890	userptr = kzalloc(sizeof(*userptr), GFP_KERNEL);4891	if (!userptr)4892		return -ENOMEM;4893 4894	rc = hl_pin_host_memory(hdev, addr, le32_to_cpu(user_dma_pkt->tsize),4895				userptr);4896	if (rc)4897		goto free_userptr;4898 4899	list_add_tail(&userptr->job_node, parser->job_userptr_list);4900 4901	rc = hl_dma_map_sgtable(hdev, userptr->sgt, dir);4902	if (rc) {4903		dev_err(hdev->dev, "failed to map sgt with DMA region\n");4904		goto unpin_memory;4905	}4906 4907	userptr->dma_mapped = true;4908	userptr->dir = dir;4909 4910already_pinned:4911	parser->patched_cb_size +=4912			gaudi_get_dma_desc_list_size(hdev, userptr->sgt);4913 4914	return 0;4915 4916unpin_memory:4917	list_del(&userptr->job_node);4918	hl_unpin_host_memory(hdev, userptr);4919free_userptr:4920	kfree(userptr);4921	return rc;4922}4923 4924static int gaudi_validate_dma_pkt_host(struct hl_device *hdev,4925				struct hl_cs_parser *parser,4926				struct packet_lin_dma *user_dma_pkt,4927				bool src_in_host)4928{4929	enum dma_data_direction dir;4930	bool skip_host_mem_pin = false, user_memset;4931	u64 addr;4932	int rc = 0;4933 4934	user_memset = (le32_to_cpu(user_dma_pkt->ctl) &4935			GAUDI_PKT_LIN_DMA_CTL_MEMSET_MASK) >>4936			GAUDI_PKT_LIN_DMA_CTL_MEMSET_SHIFT;4937 4938	if (src_in_host) {4939		if (user_memset)4940			skip_host_mem_pin = true;4941 4942		dev_dbg(hdev->dev, "DMA direction is HOST --> DEVICE\n");4943		dir = DMA_TO_DEVICE;4944		addr = le64_to_cpu(user_dma_pkt->src_addr);4945	} else {4946		dev_dbg(hdev->dev, "DMA direction is DEVICE --> HOST\n");4947		dir = DMA_FROM_DEVICE;4948		addr = (le64_to_cpu(user_dma_pkt->dst_addr) &4949				GAUDI_PKT_LIN_DMA_DST_ADDR_MASK) >>4950				GAUDI_PKT_LIN_DMA_DST_ADDR_SHIFT;4951	}4952 4953	if (skip_host_mem_pin)4954		parser->patched_cb_size += sizeof(*user_dma_pkt);4955	else4956		rc = gaudi_pin_memory_before_cs(hdev, parser, user_dma_pkt,4957						addr, dir);4958 4959	return rc;4960}4961 4962static int gaudi_validate_dma_pkt_no_mmu(struct hl_device *hdev,4963				struct hl_cs_parser *parser,4964				struct packet_lin_dma *user_dma_pkt)4965{4966	bool src_in_host = false;4967	u64 dst_addr = (le64_to_cpu(user_dma_pkt->dst_addr) &4968			GAUDI_PKT_LIN_DMA_DST_ADDR_MASK) >>4969			GAUDI_PKT_LIN_DMA_DST_ADDR_SHIFT;4970 4971	dev_dbg(hdev->dev, "DMA packet details:\n");4972	dev_dbg(hdev->dev, "source == 0x%llx\n",4973				le64_to_cpu(user_dma_pkt->src_addr));4974	dev_dbg(hdev->dev, "destination == 0x%llx\n", dst_addr);4975	dev_dbg(hdev->dev, "size == %u\n", le32_to_cpu(user_dma_pkt->tsize));4976 4977	/*4978	 * Special handling for DMA with size 0. Bypass all validations4979	 * because no transactions will be done except for WR_COMP, which4980	 * is not a security issue4981	 */4982	if (!le32_to_cpu(user_dma_pkt->tsize)) {4983		parser->patched_cb_size += sizeof(*user_dma_pkt);4984		return 0;4985	}4986 4987	if (parser->hw_queue_id <= GAUDI_QUEUE_ID_DMA_0_3)4988		src_in_host = true;4989 4990	return gaudi_validate_dma_pkt_host(hdev, parser, user_dma_pkt,4991						src_in_host);4992}4993 4994static int gaudi_validate_load_and_exe_pkt(struct hl_device *hdev,4995					struct hl_cs_parser *parser,4996					struct packet_load_and_exe *user_pkt)4997{4998	u32 cfg;4999 5000	cfg = le32_to_cpu(user_pkt->cfg);5001 5002	if (cfg & GAUDI_PKT_LOAD_AND_EXE_CFG_DST_MASK) {5003		dev_err(hdev->dev,5004			"User not allowed to use Load and Execute\n");5005		return -EPERM;5006	}5007 5008	parser->patched_cb_size += sizeof(struct packet_load_and_exe);5009 5010	return 0;5011}5012 5013static int gaudi_validate_cb(struct hl_device *hdev,5014			struct hl_cs_parser *parser, bool is_mmu)5015{5016	u32 cb_parsed_length = 0;5017	int rc = 0;5018 5019	parser->patched_cb_size = 0;5020 5021	/* cb_user_size is more than 0 so loop will always be executed */5022	while (cb_parsed_length < parser->user_cb_size) {5023		enum packet_id pkt_id;5024		u16 pkt_size;5025		struct gaudi_packet *user_pkt;5026 5027		user_pkt = parser->user_cb->kernel_address + cb_parsed_length;5028 5029		pkt_id = (enum packet_id) (5030				(le64_to_cpu(user_pkt->header) &5031				PACKET_HEADER_PACKET_ID_MASK) >>5032					PACKET_HEADER_PACKET_ID_SHIFT);5033 5034		if (!validate_packet_id(pkt_id)) {5035			dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id);5036			rc = -EINVAL;5037			break;5038		}5039 5040		pkt_size = gaudi_packet_sizes[pkt_id];5041		cb_parsed_length += pkt_size;5042		if (cb_parsed_length > parser->user_cb_size) {5043			dev_err(hdev->dev,5044				"packet 0x%x is out of CB boundary\n", pkt_id);5045			rc = -EINVAL;5046			break;5047		}5048 5049		switch (pkt_id) {5050		case PACKET_MSG_PROT:5051			dev_err(hdev->dev,5052				"User not allowed to use MSG_PROT\n");5053			rc = -EPERM;5054			break;5055 5056		case PACKET_CP_DMA:5057			dev_err(hdev->dev, "User not allowed to use CP_DMA\n");5058			rc = -EPERM;5059			break;5060 5061		case PACKET_STOP:5062			dev_err(hdev->dev, "User not allowed to use STOP\n");5063			rc = -EPERM;5064			break;5065 5066		case PACKET_WREG_BULK:5067			dev_err(hdev->dev,5068				"User not allowed to use WREG_BULK\n");5069			rc = -EPERM;5070			break;5071 5072		case PACKET_LOAD_AND_EXE:5073			rc = gaudi_validate_load_and_exe_pkt(hdev, parser,5074				(struct packet_load_and_exe *) user_pkt);5075			break;5076 5077		case PACKET_LIN_DMA:5078			parser->contains_dma_pkt = true;5079			if (is_mmu)5080				parser->patched_cb_size += pkt_size;5081			else5082				rc = gaudi_validate_dma_pkt_no_mmu(hdev, parser,5083					(struct packet_lin_dma *) user_pkt);5084			break;5085 5086		case PACKET_WREG_32:5087		case PACKET_MSG_LONG:5088		case PACKET_MSG_SHORT:5089		case PACKET_REPEAT:5090		case PACKET_FENCE:5091		case PACKET_NOP:5092		case PACKET_ARB_POINT:5093			parser->patched_cb_size += pkt_size;5094			break;5095 5096		default:5097			dev_err(hdev->dev, "Invalid packet header 0x%x\n",5098				pkt_id);5099			rc = -EINVAL;5100			break;5101		}5102 5103		if (rc)5104			break;5105	}5106 5107	/*5108	 * The new CB should have space at the end for two MSG_PROT packets:5109	 * 1. Optional NOP padding for cacheline alignment5110	 * 2. A packet that will act as a completion packet5111	 * 3. A packet that will generate MSI interrupt5112	 */5113	if (parser->completion)5114		parser->patched_cb_size += gaudi_get_patched_cb_extra_size(5115			parser->patched_cb_size);5116 5117	return rc;5118}5119 5120static int gaudi_patch_dma_packet(struct hl_device *hdev,5121				struct hl_cs_parser *parser,5122				struct packet_lin_dma *user_dma_pkt,5123				struct packet_lin_dma *new_dma_pkt,5124				u32 *new_dma_pkt_size)5125{5126	struct hl_userptr *userptr;5127	struct scatterlist *sg, *sg_next_iter;5128	u32 count, dma_desc_cnt, user_wrcomp_en_mask, ctl;5129	u64 len, len_next;5130	dma_addr_t dma_addr, dma_addr_next;5131	u64 device_memory_addr, addr;5132	enum dma_data_direction dir;5133	struct sg_table *sgt;5134	bool src_in_host = false;5135	bool skip_host_mem_pin = false;5136	bool user_memset;5137 5138	ctl = le32_to_cpu(user_dma_pkt->ctl);5139 5140	if (parser->hw_queue_id <= GAUDI_QUEUE_ID_DMA_0_3)5141		src_in_host = true;5142 5143	user_memset = (ctl & GAUDI_PKT_LIN_DMA_CTL_MEMSET_MASK) >>5144			GAUDI_PKT_LIN_DMA_CTL_MEMSET_SHIFT;5145 5146	if (src_in_host) {5147		addr = le64_to_cpu(user_dma_pkt->src_addr);5148		device_memory_addr = le64_to_cpu(user_dma_pkt->dst_addr);5149		dir = DMA_TO_DEVICE;5150		if (user_memset)5151			skip_host_mem_pin = true;5152	} else {5153		addr = le64_to_cpu(user_dma_pkt->dst_addr);5154		device_memory_addr = le64_to_cpu(user_dma_pkt->src_addr);5155		dir = DMA_FROM_DEVICE;5156	}5157 5158	if ((!skip_host_mem_pin) &&5159		(!hl_userptr_is_pinned(hdev, addr,5160					le32_to_cpu(user_dma_pkt->tsize),5161					parser->job_userptr_list, &userptr))) {5162		dev_err(hdev->dev, "Userptr 0x%llx + 0x%x NOT mapped\n",5163				addr, user_dma_pkt->tsize);5164		return -EFAULT;5165	}5166 5167	if ((user_memset) && (dir == DMA_TO_DEVICE)) {5168		memcpy(new_dma_pkt, user_dma_pkt, sizeof(*user_dma_pkt));5169		*new_dma_pkt_size = sizeof(*user_dma_pkt);5170		return 0;5171	}5172 5173	user_wrcomp_en_mask = ctl & GAUDI_PKT_LIN_DMA_CTL_WRCOMP_EN_MASK;5174 5175	sgt = userptr->sgt;5176	dma_desc_cnt = 0;5177 5178	for_each_sgtable_dma_sg(sgt, sg, count) {5179		len = sg_dma_len(sg);5180		dma_addr = sg_dma_address(sg);5181 5182		if (len == 0)5183			break;5184 5185		while ((count + 1) < sgt->nents) {5186			sg_next_iter = sg_next(sg);5187			len_next = sg_dma_len(sg_next_iter);5188			dma_addr_next = sg_dma_address(sg_next_iter);5189 5190			if (len_next == 0)5191				break;5192 5193			if ((dma_addr + len == dma_addr_next) &&5194				(len + len_next <= DMA_MAX_TRANSFER_SIZE)) {5195				len += len_next;5196				count++;5197				sg = sg_next_iter;5198			} else {5199				break;5200			}5201		}5202 5203		ctl = le32_to_cpu(user_dma_pkt->ctl);5204		if (likely(dma_desc_cnt))5205			ctl &= ~GAUDI_PKT_CTL_EB_MASK;5206		ctl &= ~GAUDI_PKT_LIN_DMA_CTL_WRCOMP_EN_MASK;5207		new_dma_pkt->ctl = cpu_to_le32(ctl);5208		new_dma_pkt->tsize = cpu_to_le32(len);5209 5210		if (dir == DMA_TO_DEVICE) {5211			new_dma_pkt->src_addr = cpu_to_le64(dma_addr);5212			new_dma_pkt->dst_addr = cpu_to_le64(device_memory_addr);5213		} else {5214			new_dma_pkt->src_addr = cpu_to_le64(device_memory_addr);5215			new_dma_pkt->dst_addr = cpu_to_le64(dma_addr);5216		}5217 5218		if (!user_memset)5219			device_memory_addr += len;5220		dma_desc_cnt++;5221		new_dma_pkt++;5222	}5223 5224	if (!dma_desc_cnt) {5225		dev_err(hdev->dev,5226			"Error of 0 SG entries when patching DMA packet\n");5227		return -EFAULT;5228	}5229 5230	/* Fix the last dma packet - wrcomp must be as user set it */5231	new_dma_pkt--;5232	new_dma_pkt->ctl |= cpu_to_le32(user_wrcomp_en_mask);5233 5234	*new_dma_pkt_size = dma_desc_cnt * sizeof(struct packet_lin_dma);5235 5236	return 0;5237}5238 5239static int gaudi_patch_cb(struct hl_device *hdev,5240				struct hl_cs_parser *parser)5241{5242	u32 cb_parsed_length = 0;5243	u32 cb_patched_cur_length = 0;5244	int rc = 0;5245 5246	/* cb_user_size is more than 0 so loop will always be executed */5247	while (cb_parsed_length < parser->user_cb_size) {5248		enum packet_id pkt_id;5249		u16 pkt_size;5250		u32 new_pkt_size = 0;5251		struct gaudi_packet *user_pkt, *kernel_pkt;5252 5253		user_pkt = parser->user_cb->kernel_address + cb_parsed_length;5254		kernel_pkt = parser->patched_cb->kernel_address +5255					cb_patched_cur_length;5256 5257		pkt_id = (enum packet_id) (5258				(le64_to_cpu(user_pkt->header) &5259				PACKET_HEADER_PACKET_ID_MASK) >>5260					PACKET_HEADER_PACKET_ID_SHIFT);5261 5262		if (!validate_packet_id(pkt_id)) {5263			dev_err(hdev->dev, "Invalid packet id %u\n", pkt_id);5264			rc = -EINVAL;5265			break;5266		}5267 5268		pkt_size = gaudi_packet_sizes[pkt_id];5269		cb_parsed_length += pkt_size;5270		if (cb_parsed_length > parser->user_cb_size) {5271			dev_err(hdev->dev,5272				"packet 0x%x is out of CB boundary\n", pkt_id);5273			rc = -EINVAL;5274			break;5275		}5276 5277		switch (pkt_id) {5278		case PACKET_LIN_DMA:5279			rc = gaudi_patch_dma_packet(hdev, parser,5280					(struct packet_lin_dma *) user_pkt,5281					(struct packet_lin_dma *) kernel_pkt,5282					&new_pkt_size);5283			cb_patched_cur_length += new_pkt_size;5284			break;5285 5286		case PACKET_MSG_PROT:5287			dev_err(hdev->dev,5288				"User not allowed to use MSG_PROT\n");5289			rc = -EPERM;5290			break;5291 5292		case PACKET_CP_DMA:5293			dev_err(hdev->dev, "User not allowed to use CP_DMA\n");5294			rc = -EPERM;5295			break;5296 5297		case PACKET_STOP:5298			dev_err(hdev->dev, "User not allowed to use STOP\n");5299			rc = -EPERM;5300			break;5301 5302		case PACKET_WREG_32:5303		case PACKET_WREG_BULK:5304		case PACKET_MSG_LONG:5305		case PACKET_MSG_SHORT:5306		case PACKET_REPEAT:5307		case PACKET_FENCE:5308		case PACKET_NOP:5309		case PACKET_ARB_POINT:5310		case PACKET_LOAD_AND_EXE:5311			memcpy(kernel_pkt, user_pkt, pkt_size);5312			cb_patched_cur_length += pkt_size;5313			break;5314 5315		default:5316			dev_err(hdev->dev, "Invalid packet header 0x%x\n",5317				pkt_id);5318			rc = -EINVAL;5319			break;5320		}5321 5322		if (rc)5323			break;5324	}5325 5326	return rc;5327}5328 5329static int gaudi_parse_cb_mmu(struct hl_device *hdev,5330		struct hl_cs_parser *parser)5331{5332	u64 handle;5333	u32 patched_cb_size;5334	struct hl_cb *user_cb;5335	int rc;5336 5337	/*5338	 * The new CB should have space at the end for two MSG_PROT packets:5339	 * 1. Optional NOP padding for cacheline alignment5340	 * 2. A packet that will act as a completion packet5341	 * 3. A packet that will generate MSI interrupt5342	 */5343	if (parser->completion)5344		parser->patched_cb_size = parser->user_cb_size +5345				gaudi_get_patched_cb_extra_size(parser->user_cb_size);5346	else5347		parser->patched_cb_size = parser->user_cb_size;5348 5349	rc = hl_cb_create(hdev, &hdev->kernel_mem_mgr, hdev->kernel_ctx,5350				parser->patched_cb_size, false, false,5351				&handle);5352 5353	if (rc) {5354		dev_err(hdev->dev,5355			"Failed to allocate patched CB for DMA CS %d\n",5356			rc);5357		return rc;5358	}5359 5360	parser->patched_cb = hl_cb_get(&hdev->kernel_mem_mgr, handle);5361	/* hl_cb_get should never fail */5362	if (!parser->patched_cb) {5363		dev_crit(hdev->dev, "DMA CB handle invalid 0x%llx\n", handle);5364		rc = -EFAULT;5365		goto out;5366	}5367 5368	/*5369	 * We are protected from overflow because the check5370	 * "parser->user_cb_size <= parser->user_cb->size" was done in get_cb_from_cs_chunk()5371	 * in the common code. That check is done only if is_kernel_allocated_cb is true.5372	 *5373	 * There is no option to reach here without going through that check because:5374	 * 1. validate_queue_index() assigns true to is_kernel_allocated_cb for any submission to5375	 *    an external queue.5376	 * 2. For Gaudi, we only parse CBs that were submitted to the external queues.5377	 */5378	memcpy(parser->patched_cb->kernel_address,5379		parser->user_cb->kernel_address,5380		parser->user_cb_size);5381 5382	patched_cb_size = parser->patched_cb_size;5383 5384	/* Validate patched CB instead of user CB */5385	user_cb = parser->user_cb;5386	parser->user_cb = parser->patched_cb;5387	rc = gaudi_validate_cb(hdev, parser, true);5388	parser->user_cb = user_cb;5389 5390	if (rc) {5391		hl_cb_put(parser->patched_cb);5392		goto out;5393	}5394 5395	if (patched_cb_size != parser->patched_cb_size) {5396		dev_err(hdev->dev, "user CB size mismatch\n");5397		hl_cb_put(parser->patched_cb);5398		rc = -EINVAL;5399		goto out;5400	}5401 5402out:5403	/*5404	 * Always call cb destroy here because we still have 1 reference5405	 * to it by calling cb_get earlier. After the job will be completed,5406	 * cb_put will release it, but here we want to remove it from the5407	 * idr5408	 */5409	hl_cb_destroy(&hdev->kernel_mem_mgr, handle);5410 5411	return rc;5412}5413 5414static int gaudi_parse_cb_no_mmu(struct hl_device *hdev,5415		struct hl_cs_parser *parser)5416{5417	u64 handle;5418	int rc;5419 5420	rc = gaudi_validate_cb(hdev, parser, false);5421 5422	if (rc)5423		goto free_userptr;5424 5425	rc = hl_cb_create(hdev, &hdev->kernel_mem_mgr, hdev->kernel_ctx,5426				parser->patched_cb_size, false, false,5427				&handle);5428	if (rc) {5429		dev_err(hdev->dev,5430			"Failed to allocate patched CB for DMA CS %d\n", rc);5431		goto free_userptr;5432	}5433 5434	parser->patched_cb = hl_cb_get(&hdev->kernel_mem_mgr, handle);5435	/* hl_cb_get should never fail here */5436	if (!parser->patched_cb) {5437		dev_crit(hdev->dev, "DMA CB handle invalid 0x%llx\n", handle);5438		rc = -EFAULT;5439		goto out;5440	}5441 5442	rc = gaudi_patch_cb(hdev, parser);5443 5444	if (rc)5445		hl_cb_put(parser->patched_cb);5446 5447out:5448	/*5449	 * Always call cb destroy here because we still have 1 reference5450	 * to it by calling cb_get earlier. After the job will be completed,5451	 * cb_put will release it, but here we want to remove it from the5452	 * idr5453	 */5454	hl_cb_destroy(&hdev->kernel_mem_mgr, handle);5455 5456free_userptr:5457	if (rc)5458		hl_userptr_delete_list(hdev, parser->job_userptr_list);5459	return rc;5460}5461 5462static int gaudi_parse_cb_no_ext_queue(struct hl_device *hdev,5463					struct hl_cs_parser *parser)5464{5465	struct asic_fixed_properties *asic_prop = &hdev->asic_prop;5466	struct gaudi_device *gaudi = hdev->asic_specific;5467	u32 nic_queue_offset, nic_mask_q_id;5468 5469	if ((parser->hw_queue_id >= GAUDI_QUEUE_ID_NIC_0_0) &&5470			(parser->hw_queue_id <= GAUDI_QUEUE_ID_NIC_9_3)) {5471		nic_queue_offset = parser->hw_queue_id - GAUDI_QUEUE_ID_NIC_0_0;5472		nic_mask_q_id = 1 << (HW_CAP_NIC_SHIFT + (nic_queue_offset >> 2));5473 5474		if (!(gaudi->hw_cap_initialized & nic_mask_q_id)) {5475			dev_err(hdev->dev, "h/w queue %d is disabled\n", parser->hw_queue_id);5476			return -EINVAL;5477		}5478	}5479 5480	/* For internal queue jobs just check if CB address is valid */5481	if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,5482					parser->user_cb_size,5483					asic_prop->sram_user_base_address,5484					asic_prop->sram_end_address))5485		return 0;5486 5487	if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,5488					parser->user_cb_size,5489					asic_prop->dram_user_base_address,5490					asic_prop->dram_end_address))5491		return 0;5492 5493	/* PMMU and HPMMU addresses are equal, check only one of them */5494	if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb,5495					parser->user_cb_size,5496					asic_prop->pmmu.start_addr,5497					asic_prop->pmmu.end_addr))5498		return 0;5499 5500	dev_err(hdev->dev,5501		"CB address 0x%px + 0x%x for internal QMAN is not valid\n",5502		parser->user_cb, parser->user_cb_size);5503 5504	return -EFAULT;5505}5506 5507static int gaudi_cs_parser(struct hl_device *hdev, struct hl_cs_parser *parser)5508{5509	struct gaudi_device *gaudi = hdev->asic_specific;5510 5511	if (parser->queue_type == QUEUE_TYPE_INT)5512		return gaudi_parse_cb_no_ext_queue(hdev, parser);5513 5514	if (gaudi->hw_cap_initialized & HW_CAP_MMU)5515		return gaudi_parse_cb_mmu(hdev, parser);5516	else5517		return gaudi_parse_cb_no_mmu(hdev, parser);5518}5519 5520static void gaudi_add_end_of_cb_packets(struct hl_device *hdev, void *kernel_address,5521				u32 len, u32 original_len, u64 cq_addr, u32 cq_val,5522				u32 msi_vec, bool eb)5523{5524	struct packet_msg_prot *cq_pkt;5525	struct packet_nop *cq_padding;5526	u64 msi_addr;5527	u32 tmp;5528 5529	cq_padding = kernel_address + original_len;5530	cq_pkt = kernel_address + len - (sizeof(struct packet_msg_prot) * 2);5531 5532	while ((void *)cq_padding < (void *)cq_pkt) {5533		cq_padding->ctl = cpu_to_le32(FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_NOP));5534		cq_padding++;5535	}5536 5537	tmp = FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_PROT);5538	tmp |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1);5539 5540	if (eb)5541		tmp |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 1);5542 5543	cq_pkt->ctl = cpu_to_le32(tmp);5544	cq_pkt->value = cpu_to_le32(cq_val);5545	cq_pkt->addr = cpu_to_le64(cq_addr);5546 5547	cq_pkt++;5548 5549	tmp = FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_PROT);5550	tmp |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1);5551	cq_pkt->ctl = cpu_to_le32(tmp);5552	cq_pkt->value = cpu_to_le32(1);5553	msi_addr = hdev->pdev ? mmPCIE_CORE_MSI_REQ : mmPCIE_MSI_INTR_0 + msi_vec * 4;5554	cq_pkt->addr = cpu_to_le64(CFG_BASE + msi_addr);5555}5556 5557static void gaudi_update_eq_ci(struct hl_device *hdev, u32 val)5558{5559	WREG32(mmCPU_IF_EQ_RD_OFFS, val);5560}5561 5562static int gaudi_memset_device_memory(struct hl_device *hdev, u64 addr,5563					u32 size, u64 val)5564{5565	struct packet_lin_dma *lin_dma_pkt;5566	struct hl_cs_job *job;5567	u32 cb_size, ctl, err_cause;5568	struct hl_cb *cb;5569	int rc;5570 5571	cb = hl_cb_kernel_create(hdev, PAGE_SIZE, false);5572	if (!cb)5573		return -EFAULT;5574 5575	lin_dma_pkt = cb->kernel_address;5576	memset(lin_dma_pkt, 0, sizeof(*lin_dma_pkt));5577	cb_size = sizeof(*lin_dma_pkt);5578 5579	ctl = FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_LIN_DMA);5580	ctl |= FIELD_PREP(GAUDI_PKT_LIN_DMA_CTL_MEMSET_MASK, 1);5581	ctl |= FIELD_PREP(GAUDI_PKT_LIN_DMA_CTL_LIN_MASK, 1);5582	ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1);5583	ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1);5584 5585	lin_dma_pkt->ctl = cpu_to_le32(ctl);5586	lin_dma_pkt->src_addr = cpu_to_le64(val);5587	lin_dma_pkt->dst_addr |= cpu_to_le64(addr);5588	lin_dma_pkt->tsize = cpu_to_le32(size);5589 5590	job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true);5591	if (!job) {5592		dev_err(hdev->dev, "Failed to allocate a new job\n");5593		rc = -ENOMEM;5594		goto release_cb;5595	}5596 5597	/* Verify DMA is OK */5598	err_cause = RREG32(mmDMA0_CORE_ERR_CAUSE);5599	if (err_cause && !hdev->init_done) {5600		dev_dbg(hdev->dev,5601			"Clearing DMA0 engine from errors (cause 0x%x)\n",5602			err_cause);5603		WREG32(mmDMA0_CORE_ERR_CAUSE, err_cause);5604	}5605 5606	job->id = 0;5607	job->user_cb = cb;5608	atomic_inc(&job->user_cb->cs_cnt);5609	job->user_cb_size = cb_size;5610	job->hw_queue_id = GAUDI_QUEUE_ID_DMA_0_0;5611	job->patched_cb = job->user_cb;5612	job->job_cb_size = job->user_cb_size + sizeof(struct packet_msg_prot);5613 5614	hl_debugfs_add_job(hdev, job);5615 5616	rc = gaudi_send_job_on_qman0(hdev, job);5617	hl_debugfs_remove_job(hdev, job);5618	kfree(job);5619	atomic_dec(&cb->cs_cnt);5620 5621	/* Verify DMA is OK */5622	err_cause = RREG32(mmDMA0_CORE_ERR_CAUSE);5623	if (err_cause) {5624		dev_err(hdev->dev, "DMA Failed, cause 0x%x\n", err_cause);5625		rc = -EIO;5626		if (!hdev->init_done) {5627			dev_dbg(hdev->dev,5628				"Clearing DMA0 engine from errors (cause 0x%x)\n",5629				err_cause);5630			WREG32(mmDMA0_CORE_ERR_CAUSE, err_cause);5631		}5632	}5633 5634release_cb:5635	hl_cb_put(cb);5636	hl_cb_destroy(&hdev->kernel_mem_mgr, cb->buf->handle);5637 5638	return rc;5639}5640 5641static int gaudi_memset_registers(struct hl_device *hdev, u64 reg_base,5642					u32 num_regs, u32 val)5643{5644	struct packet_msg_long *pkt;5645	struct hl_cs_job *job;5646	u32 cb_size, ctl;5647	struct hl_cb *cb;5648	int i, rc;5649 5650	cb_size = (sizeof(*pkt) * num_regs) + sizeof(struct packet_msg_prot);5651 5652	if (cb_size > SZ_2M) {5653		dev_err(hdev->dev, "CB size must be smaller than %uMB", SZ_2M);5654		return -ENOMEM;5655	}5656 5657	cb = hl_cb_kernel_create(hdev, cb_size, false);5658	if (!cb)5659		return -EFAULT;5660 5661	pkt = cb->kernel_address;5662 5663	ctl = FIELD_PREP(GAUDI_PKT_LONG_CTL_OP_MASK, 0); /* write the value */5664	ctl |= FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_LONG);5665	ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 1);5666	ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1);5667	ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1);5668 5669	for (i = 0; i < num_regs ; i++, pkt++) {5670		pkt->ctl = cpu_to_le32(ctl);5671		pkt->value = cpu_to_le32(val);5672		pkt->addr = cpu_to_le64(reg_base + (i * 4));5673	}5674 5675	job = hl_cs_allocate_job(hdev, QUEUE_TYPE_EXT, true);5676	if (!job) {5677		dev_err(hdev->dev, "Failed to allocate a new job\n");5678		rc = -ENOMEM;5679		goto release_cb;5680	}5681 5682	job->id = 0;5683	job->user_cb = cb;5684	atomic_inc(&job->user_cb->cs_cnt);5685	job->user_cb_size = cb_size;5686	job->hw_queue_id = GAUDI_QUEUE_ID_DMA_0_0;5687	job->patched_cb = job->user_cb;5688	job->job_cb_size = cb_size;5689 5690	hl_debugfs_add_job(hdev, job);5691 5692	rc = gaudi_send_job_on_qman0(hdev, job);5693	hl_debugfs_remove_job(hdev, job);5694	kfree(job);5695	atomic_dec(&cb->cs_cnt);5696 5697release_cb:5698	hl_cb_put(cb);5699	hl_cb_destroy(&hdev->kernel_mem_mgr, cb->buf->handle);5700 5701	return rc;5702}5703 5704static int gaudi_restore_sm_registers(struct hl_device *hdev)5705{5706	u64 base_addr;5707	u32 num_regs;5708	int rc;5709 5710	base_addr = CFG_BASE + mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0;5711	num_regs = NUM_OF_SOB_IN_BLOCK;5712	rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0);5713	if (rc) {5714		dev_err(hdev->dev, "failed resetting SM registers");5715		return -ENOMEM;5716	}5717 5718	base_addr = CFG_BASE +  mmSYNC_MNGR_E_S_SYNC_MNGR_OBJS_SOB_OBJ_0;5719	num_regs = NUM_OF_SOB_IN_BLOCK;5720	rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0);5721	if (rc) {5722		dev_err(hdev->dev, "failed resetting SM registers");5723		return -ENOMEM;5724	}5725 5726	base_addr = CFG_BASE +  mmSYNC_MNGR_W_N_SYNC_MNGR_OBJS_SOB_OBJ_0;5727	num_regs = NUM_OF_SOB_IN_BLOCK;5728	rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0);5729	if (rc) {5730		dev_err(hdev->dev, "failed resetting SM registers");5731		return -ENOMEM;5732	}5733 5734	base_addr = CFG_BASE +  mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0;5735	num_regs = NUM_OF_MONITORS_IN_BLOCK;5736	rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0);5737	if (rc) {5738		dev_err(hdev->dev, "failed resetting SM registers");5739		return -ENOMEM;5740	}5741 5742	base_addr = CFG_BASE +  mmSYNC_MNGR_E_S_SYNC_MNGR_OBJS_MON_STATUS_0;5743	num_regs = NUM_OF_MONITORS_IN_BLOCK;5744	rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0);5745	if (rc) {5746		dev_err(hdev->dev, "failed resetting SM registers");5747		return -ENOMEM;5748	}5749 5750	base_addr = CFG_BASE +  mmSYNC_MNGR_W_N_SYNC_MNGR_OBJS_MON_STATUS_0;5751	num_regs = NUM_OF_MONITORS_IN_BLOCK;5752	rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0);5753	if (rc) {5754		dev_err(hdev->dev, "failed resetting SM registers");5755		return -ENOMEM;5756	}5757 5758	base_addr = CFG_BASE +  mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 +5759			(GAUDI_FIRST_AVAILABLE_W_S_SYNC_OBJECT * 4);5760	num_regs = NUM_OF_SOB_IN_BLOCK - GAUDI_FIRST_AVAILABLE_W_S_SYNC_OBJECT;5761	rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0);5762	if (rc) {5763		dev_err(hdev->dev, "failed resetting SM registers");5764		return -ENOMEM;5765	}5766 5767	base_addr = CFG_BASE +  mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0 +5768			(GAUDI_FIRST_AVAILABLE_W_S_MONITOR * 4);5769	num_regs = NUM_OF_MONITORS_IN_BLOCK - GAUDI_FIRST_AVAILABLE_W_S_MONITOR;5770	rc = gaudi_memset_registers(hdev, base_addr, num_regs, 0);5771	if (rc) {5772		dev_err(hdev->dev, "failed resetting SM registers");5773		return -ENOMEM;5774	}5775 5776	return 0;5777}5778 5779static void gaudi_restore_dma_registers(struct hl_device *hdev)5780{5781	u32 sob_delta = mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_1 -5782			mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0;5783	int i;5784 5785	for (i = 0 ; i < DMA_NUMBER_OF_CHANNELS ; i++) {5786		u64 sob_addr = CFG_BASE +5787				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0 +5788				(i * sob_delta);5789		u32 dma_offset = i * DMA_CORE_OFFSET;5790 5791		WREG32(mmDMA0_CORE_WR_COMP_ADDR_LO + dma_offset,5792				lower_32_bits(sob_addr));5793		WREG32(mmDMA0_CORE_WR_COMP_ADDR_HI + dma_offset,5794				upper_32_bits(sob_addr));5795		WREG32(mmDMA0_CORE_WR_COMP_WDATA + dma_offset, 0x80000001);5796 5797		/* For DMAs 2-7, need to restore WR_AWUSER_31_11 as it can be5798		 * modified by the user for SRAM reduction5799		 */5800		if (i > 1)5801			WREG32(mmDMA0_CORE_WR_AWUSER_31_11 + dma_offset,5802								0x00000001);5803	}5804}5805 5806static void gaudi_restore_qm_registers(struct hl_device *hdev)5807{5808	u32 qman_offset;5809	int i;5810 5811	for (i = 0 ; i < DMA_NUMBER_OF_CHANNELS ; i++) {5812		qman_offset = i * DMA_QMAN_OFFSET;5813		WREG32(mmDMA0_QM_ARB_CFG_0 + qman_offset, 0);5814	}5815 5816	for (i = 0 ; i < MME_NUMBER_OF_MASTER_ENGINES ; i++) {5817		qman_offset = i * (mmMME2_QM_BASE - mmMME0_QM_BASE);5818		WREG32(mmMME0_QM_ARB_CFG_0 + qman_offset, 0);5819	}5820 5821	for (i = 0 ; i < TPC_NUMBER_OF_ENGINES ; i++) {5822		qman_offset = i * TPC_QMAN_OFFSET;5823		WREG32(mmTPC0_QM_ARB_CFG_0 + qman_offset, 0);5824	}5825 5826	for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++) {5827		qman_offset = (i >> 1) * NIC_MACRO_QMAN_OFFSET +5828				(i & 0x1) * NIC_ENGINE_QMAN_OFFSET;5829		WREG32(mmNIC0_QM0_ARB_CFG_0 + qman_offset, 0);5830	}5831}5832 5833static int gaudi_restore_user_registers(struct hl_device *hdev)5834{5835	int rc;5836 5837	rc = gaudi_restore_sm_registers(hdev);5838	if (rc)5839		return rc;5840 5841	gaudi_restore_dma_registers(hdev);5842	gaudi_restore_qm_registers(hdev);5843 5844	return 0;5845}5846 5847static int gaudi_context_switch(struct hl_device *hdev, u32 asid)5848{5849	return 0;5850}5851 5852static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev)5853{5854	u32 size = hdev->asic_prop.mmu_pgt_size +5855			hdev->asic_prop.mmu_cache_mng_size;5856	struct gaudi_device *gaudi = hdev->asic_specific;5857	u64 addr = hdev->asic_prop.mmu_pgt_addr;5858 5859	if (!(gaudi->hw_cap_initialized & HW_CAP_MMU))5860		return 0;5861 5862	return gaudi_memset_device_memory(hdev, addr, size, 0);5863}5864 5865static void gaudi_restore_phase_topology(struct hl_device *hdev)5866{5867 5868}5869 5870static int gaudi_dma_core_transfer(struct hl_device *hdev, int dma_id, u64 addr,5871					u32 size_to_dma, dma_addr_t dma_addr)5872{5873	u32 err_cause, val;5874	u64 dma_offset;5875	int rc;5876 5877	dma_offset = dma_id * DMA_CORE_OFFSET;5878 5879	WREG32(mmDMA0_CORE_SRC_BASE_LO + dma_offset, lower_32_bits(addr));5880	WREG32(mmDMA0_CORE_SRC_BASE_HI + dma_offset, upper_32_bits(addr));5881	WREG32(mmDMA0_CORE_DST_BASE_LO + dma_offset, lower_32_bits(dma_addr));5882	WREG32(mmDMA0_CORE_DST_BASE_HI + dma_offset, upper_32_bits(dma_addr));5883	WREG32(mmDMA0_CORE_DST_TSIZE_0 + dma_offset, size_to_dma);5884	WREG32(mmDMA0_CORE_COMMIT + dma_offset,5885			(1 << DMA0_CORE_COMMIT_LIN_SHIFT));5886 5887	rc = hl_poll_timeout(5888		hdev,5889		mmDMA0_CORE_STS0 + dma_offset,5890		val,5891		((val & DMA0_CORE_STS0_BUSY_MASK) == 0),5892		0,5893		1000000);5894 5895	if (rc) {5896		dev_err(hdev->dev,5897			"DMA %d timed-out during reading of 0x%llx\n",5898			dma_id, addr);5899		return -EIO;5900	}5901 5902	/* Verify DMA is OK */5903	err_cause = RREG32(mmDMA0_CORE_ERR_CAUSE + dma_offset);5904	if (err_cause) {5905		dev_err(hdev->dev, "DMA Failed, cause 0x%x\n", err_cause);5906		dev_dbg(hdev->dev,5907			"Clearing DMA0 engine from errors (cause 0x%x)\n",5908			err_cause);5909		WREG32(mmDMA0_CORE_ERR_CAUSE + dma_offset, err_cause);5910 5911		return -EIO;5912	}5913 5914	return 0;5915}5916 5917static int gaudi_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size,5918				void *blob_addr)5919{5920	u32 dma_core_sts0, err_cause, cfg1, size_left, pos, size_to_dma;5921	u32 qm_glbl_sts0, qm_cgm_sts;5922	u64 dma_offset, qm_offset;5923	dma_addr_t dma_addr;5924	void *kernel_addr;5925	bool is_eng_idle;5926	int rc = 0, dma_id;5927 5928	kernel_addr = hl_asic_dma_alloc_coherent(hdev, SZ_2M, &dma_addr, GFP_KERNEL | __GFP_ZERO);5929 5930	if (!kernel_addr)5931		return -ENOMEM;5932 5933	hdev->asic_funcs->hw_queues_lock(hdev);5934 5935	dma_id = gaudi_dma_assignment[GAUDI_PCI_DMA_1];5936	dma_offset = dma_id * DMA_CORE_OFFSET;5937	qm_offset = dma_id * DMA_QMAN_OFFSET;5938	dma_core_sts0 = RREG32(mmDMA0_CORE_STS0 + dma_offset);5939	qm_glbl_sts0 = RREG32(mmDMA0_QM_GLBL_STS0 + qm_offset);5940	qm_cgm_sts = RREG32(mmDMA0_QM_CGM_STS + qm_offset);5941	is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts) &&5942		      IS_DMA_IDLE(dma_core_sts0);5943 5944	if (!is_eng_idle) {5945		dma_id = gaudi_dma_assignment[GAUDI_PCI_DMA_2];5946		dma_offset = dma_id * DMA_CORE_OFFSET;5947		qm_offset = dma_id * DMA_QMAN_OFFSET;5948		dma_core_sts0 = RREG32(mmDMA0_CORE_STS0 + dma_offset);5949		qm_glbl_sts0 = RREG32(mmDMA0_QM_GLBL_STS0 + qm_offset);5950		qm_cgm_sts = RREG32(mmDMA0_QM_CGM_STS + qm_offset);5951		is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts) &&5952			      IS_DMA_IDLE(dma_core_sts0);5953 5954		if (!is_eng_idle) {5955			dev_err_ratelimited(hdev->dev,5956				"Can't read via DMA because it is BUSY\n");5957			rc = -EAGAIN;5958			goto out;5959		}5960	}5961 5962	cfg1 = RREG32(mmDMA0_QM_GLBL_CFG1 + qm_offset);5963	WREG32(mmDMA0_QM_GLBL_CFG1 + qm_offset,5964			0xF << DMA0_QM_GLBL_CFG1_CP_STOP_SHIFT);5965 5966	/* TODO: remove this by mapping the DMA temporary buffer to the MMU5967	 * using the compute ctx ASID, if exists. If not, use the kernel ctx5968	 * ASID5969	 */5970	WREG32_OR(mmDMA0_CORE_PROT + dma_offset, BIT(DMA0_CORE_PROT_VAL_SHIFT));5971 5972	/* Verify DMA is OK */5973	err_cause = RREG32(mmDMA0_CORE_ERR_CAUSE + dma_offset);5974	if (err_cause) {5975		dev_dbg(hdev->dev,5976			"Clearing DMA0 engine from errors (cause 0x%x)\n",5977			err_cause);5978		WREG32(mmDMA0_CORE_ERR_CAUSE + dma_offset, err_cause);5979	}5980 5981	pos = 0;5982	size_left = size;5983	size_to_dma = SZ_2M;5984 5985	while (size_left > 0) {5986 5987		if (size_left < SZ_2M)5988			size_to_dma = size_left;5989 5990		rc = gaudi_dma_core_transfer(hdev, dma_id, addr, size_to_dma,5991						dma_addr);5992		if (rc)5993			break;5994 5995		memcpy(blob_addr + pos, kernel_addr, size_to_dma);5996 5997		if (size_left <= SZ_2M)5998			break;5999 6000		pos += SZ_2M;6001		addr += SZ_2M;6002		size_left -= SZ_2M;6003	}6004 6005	/* TODO: remove this by mapping the DMA temporary buffer to the MMU6006	 * using the compute ctx ASID, if exists. If not, use the kernel ctx6007	 * ASID6008	 */6009	WREG32_AND(mmDMA0_CORE_PROT + dma_offset,6010			~BIT(DMA0_CORE_PROT_VAL_SHIFT));6011 6012	WREG32(mmDMA0_QM_GLBL_CFG1 + qm_offset, cfg1);6013 6014out:6015	hdev->asic_funcs->hw_queues_unlock(hdev);6016 6017	hl_asic_dma_free_coherent(hdev, SZ_2M, kernel_addr, dma_addr);6018 6019	return rc;6020}6021 6022static u64 gaudi_read_pte(struct hl_device *hdev, u64 addr)6023{6024	struct gaudi_device *gaudi = hdev->asic_specific;6025 6026	if (hdev->reset_info.hard_reset_pending)6027		return U64_MAX;6028 6029	return readq(hdev->pcie_bar[HBM_BAR_ID] +6030			(addr - gaudi->hbm_bar_cur_addr));6031}6032 6033static void gaudi_write_pte(struct hl_device *hdev, u64 addr, u64 val)6034{6035	struct gaudi_device *gaudi = hdev->asic_specific;6036 6037	if (hdev->reset_info.hard_reset_pending)6038		return;6039 6040	writeq(val, hdev->pcie_bar[HBM_BAR_ID] +6041			(addr - gaudi->hbm_bar_cur_addr));6042}6043 6044void gaudi_mmu_prepare_reg(struct hl_device *hdev, u64 reg, u32 asid)6045{6046	/* mask to zero the MMBP and ASID bits */6047	WREG32_AND(reg, ~0x7FF);6048	WREG32_OR(reg, asid);6049}6050 6051static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid)6052{6053	struct gaudi_device *gaudi = hdev->asic_specific;6054 6055	if (!(gaudi->hw_cap_initialized & HW_CAP_MMU))6056		return;6057 6058	if (asid & ~DMA0_QM_GLBL_NON_SECURE_PROPS_0_ASID_MASK) {6059		dev_crit(hdev->dev, "asid %u is too big\n", asid);6060		return;6061	}6062 6063	gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_0, asid);6064	gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_1, asid);6065	gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_2, asid);6066	gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_3, asid);6067	gaudi_mmu_prepare_reg(hdev, mmDMA0_QM_GLBL_NON_SECURE_PROPS_4, asid);6068 6069	gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_0, asid);6070	gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_1, asid);6071	gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_2, asid);6072	gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_3, asid);6073	gaudi_mmu_prepare_reg(hdev, mmDMA1_QM_GLBL_NON_SECURE_PROPS_4, asid);6074 6075	gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_0, asid);6076	gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_1, asid);6077	gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_2, asid);6078	gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_3, asid);6079	gaudi_mmu_prepare_reg(hdev, mmDMA2_QM_GLBL_NON_SECURE_PROPS_4, asid);6080 6081	gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_0, asid);6082	gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_1, asid);6083	gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_2, asid);6084	gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_3, asid);6085	gaudi_mmu_prepare_reg(hdev, mmDMA3_QM_GLBL_NON_SECURE_PROPS_4, asid);6086 6087	gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_0, asid);6088	gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_1, asid);6089	gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_2, asid);6090	gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_3, asid);6091	gaudi_mmu_prepare_reg(hdev, mmDMA4_QM_GLBL_NON_SECURE_PROPS_4, asid);6092 6093	gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_0, asid);6094	gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_1, asid);6095	gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_2, asid);6096	gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_3, asid);6097	gaudi_mmu_prepare_reg(hdev, mmDMA5_QM_GLBL_NON_SECURE_PROPS_4, asid);6098 6099	gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_0, asid);6100	gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_1, asid);6101	gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_2, asid);6102	gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_3, asid);6103	gaudi_mmu_prepare_reg(hdev, mmDMA6_QM_GLBL_NON_SECURE_PROPS_4, asid);6104 6105	gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_0, asid);6106	gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_1, asid);6107	gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_2, asid);6108	gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_3, asid);6109	gaudi_mmu_prepare_reg(hdev, mmDMA7_QM_GLBL_NON_SECURE_PROPS_4, asid);6110 6111	gaudi_mmu_prepare_reg(hdev, mmDMA0_CORE_NON_SECURE_PROPS, asid);6112	gaudi_mmu_prepare_reg(hdev, mmDMA1_CORE_NON_SECURE_PROPS, asid);6113	gaudi_mmu_prepare_reg(hdev, mmDMA2_CORE_NON_SECURE_PROPS, asid);6114	gaudi_mmu_prepare_reg(hdev, mmDMA3_CORE_NON_SECURE_PROPS, asid);6115	gaudi_mmu_prepare_reg(hdev, mmDMA4_CORE_NON_SECURE_PROPS, asid);6116	gaudi_mmu_prepare_reg(hdev, mmDMA5_CORE_NON_SECURE_PROPS, asid);6117	gaudi_mmu_prepare_reg(hdev, mmDMA6_CORE_NON_SECURE_PROPS, asid);6118	gaudi_mmu_prepare_reg(hdev, mmDMA7_CORE_NON_SECURE_PROPS, asid);6119 6120	gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_0, asid);6121	gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_1, asid);6122	gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_2, asid);6123	gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_3, asid);6124	gaudi_mmu_prepare_reg(hdev, mmTPC0_QM_GLBL_NON_SECURE_PROPS_4, asid);6125	gaudi_mmu_prepare_reg(hdev, mmTPC0_CFG_ARUSER_LO, asid);6126	gaudi_mmu_prepare_reg(hdev, mmTPC0_CFG_AWUSER_LO, asid);6127 6128	gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_0, asid);6129	gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_1, asid);6130	gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_2, asid);6131	gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_3, asid);6132	gaudi_mmu_prepare_reg(hdev, mmTPC1_QM_GLBL_NON_SECURE_PROPS_4, asid);6133	gaudi_mmu_prepare_reg(hdev, mmTPC1_CFG_ARUSER_LO, asid);6134	gaudi_mmu_prepare_reg(hdev, mmTPC1_CFG_AWUSER_LO, asid);6135 6136	gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_0, asid);6137	gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_1, asid);6138	gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_2, asid);6139	gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_3, asid);6140	gaudi_mmu_prepare_reg(hdev, mmTPC2_QM_GLBL_NON_SECURE_PROPS_4, asid);6141	gaudi_mmu_prepare_reg(hdev, mmTPC2_CFG_ARUSER_LO, asid);6142	gaudi_mmu_prepare_reg(hdev, mmTPC2_CFG_AWUSER_LO, asid);6143 6144	gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_0, asid);6145	gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_1, asid);6146	gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_2, asid);6147	gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_3, asid);6148	gaudi_mmu_prepare_reg(hdev, mmTPC3_QM_GLBL_NON_SECURE_PROPS_4, asid);6149	gaudi_mmu_prepare_reg(hdev, mmTPC3_CFG_ARUSER_LO, asid);6150	gaudi_mmu_prepare_reg(hdev, mmTPC3_CFG_AWUSER_LO, asid);6151 6152	gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_0, asid);6153	gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_1, asid);6154	gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_2, asid);6155	gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_3, asid);6156	gaudi_mmu_prepare_reg(hdev, mmTPC4_QM_GLBL_NON_SECURE_PROPS_4, asid);6157	gaudi_mmu_prepare_reg(hdev, mmTPC4_CFG_ARUSER_LO, asid);6158	gaudi_mmu_prepare_reg(hdev, mmTPC4_CFG_AWUSER_LO, asid);6159 6160	gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_0, asid);6161	gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_1, asid);6162	gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_2, asid);6163	gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_3, asid);6164	gaudi_mmu_prepare_reg(hdev, mmTPC5_QM_GLBL_NON_SECURE_PROPS_4, asid);6165	gaudi_mmu_prepare_reg(hdev, mmTPC5_CFG_ARUSER_LO, asid);6166	gaudi_mmu_prepare_reg(hdev, mmTPC5_CFG_AWUSER_LO, asid);6167 6168	gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_0, asid);6169	gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_1, asid);6170	gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_2, asid);6171	gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_3, asid);6172	gaudi_mmu_prepare_reg(hdev, mmTPC6_QM_GLBL_NON_SECURE_PROPS_4, asid);6173	gaudi_mmu_prepare_reg(hdev, mmTPC6_CFG_ARUSER_LO, asid);6174	gaudi_mmu_prepare_reg(hdev, mmTPC6_CFG_AWUSER_LO, asid);6175 6176	gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_0, asid);6177	gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_1, asid);6178	gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_2, asid);6179	gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_3, asid);6180	gaudi_mmu_prepare_reg(hdev, mmTPC7_QM_GLBL_NON_SECURE_PROPS_4, asid);6181	gaudi_mmu_prepare_reg(hdev, mmTPC7_CFG_ARUSER_LO, asid);6182	gaudi_mmu_prepare_reg(hdev, mmTPC7_CFG_AWUSER_LO, asid);6183 6184	gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_0, asid);6185	gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_1, asid);6186	gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_2, asid);6187	gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_3, asid);6188	gaudi_mmu_prepare_reg(hdev, mmMME0_QM_GLBL_NON_SECURE_PROPS_4, asid);6189	gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_0, asid);6190	gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_1, asid);6191	gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_2, asid);6192	gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_3, asid);6193	gaudi_mmu_prepare_reg(hdev, mmMME2_QM_GLBL_NON_SECURE_PROPS_4, asid);6194 6195	gaudi_mmu_prepare_reg(hdev, mmMME0_SBAB_ARUSER0, asid);6196	gaudi_mmu_prepare_reg(hdev, mmMME0_SBAB_ARUSER1, asid);6197	gaudi_mmu_prepare_reg(hdev, mmMME1_SBAB_ARUSER0, asid);6198	gaudi_mmu_prepare_reg(hdev, mmMME1_SBAB_ARUSER1, asid);6199	gaudi_mmu_prepare_reg(hdev, mmMME2_SBAB_ARUSER0, asid);6200	gaudi_mmu_prepare_reg(hdev, mmMME2_SBAB_ARUSER1, asid);6201	gaudi_mmu_prepare_reg(hdev, mmMME3_SBAB_ARUSER0, asid);6202	gaudi_mmu_prepare_reg(hdev, mmMME3_SBAB_ARUSER1, asid);6203	gaudi_mmu_prepare_reg(hdev, mmMME0_ACC_WBC, asid);6204	gaudi_mmu_prepare_reg(hdev, mmMME1_ACC_WBC, asid);6205	gaudi_mmu_prepare_reg(hdev, mmMME2_ACC_WBC, asid);6206	gaudi_mmu_prepare_reg(hdev, mmMME3_ACC_WBC, asid);6207 6208	if (gaudi->hw_cap_initialized & HW_CAP_NIC0) {6209		gaudi_mmu_prepare_reg(hdev, mmNIC0_QM0_GLBL_NON_SECURE_PROPS_0,6210				asid);6211		gaudi_mmu_prepare_reg(hdev, mmNIC0_QM0_GLBL_NON_SECURE_PROPS_1,6212				asid);6213		gaudi_mmu_prepare_reg(hdev, mmNIC0_QM0_GLBL_NON_SECURE_PROPS_2,6214				asid);6215		gaudi_mmu_prepare_reg(hdev, mmNIC0_QM0_GLBL_NON_SECURE_PROPS_3,6216				asid);6217		gaudi_mmu_prepare_reg(hdev, mmNIC0_QM0_GLBL_NON_SECURE_PROPS_4,6218				asid);6219	}6220 6221	if (gaudi->hw_cap_initialized & HW_CAP_NIC1) {6222		gaudi_mmu_prepare_reg(hdev, mmNIC0_QM1_GLBL_NON_SECURE_PROPS_0,6223				asid);6224		gaudi_mmu_prepare_reg(hdev, mmNIC0_QM1_GLBL_NON_SECURE_PROPS_1,6225				asid);6226		gaudi_mmu_prepare_reg(hdev, mmNIC0_QM1_GLBL_NON_SECURE_PROPS_2,6227				asid);6228		gaudi_mmu_prepare_reg(hdev, mmNIC0_QM1_GLBL_NON_SECURE_PROPS_3,6229				asid);6230		gaudi_mmu_prepare_reg(hdev, mmNIC0_QM1_GLBL_NON_SECURE_PROPS_4,6231				asid);6232	}6233 6234	if (gaudi->hw_cap_initialized & HW_CAP_NIC2) {6235		gaudi_mmu_prepare_reg(hdev, mmNIC1_QM0_GLBL_NON_SECURE_PROPS_0,6236				asid);6237		gaudi_mmu_prepare_reg(hdev, mmNIC1_QM0_GLBL_NON_SECURE_PROPS_1,6238				asid);6239		gaudi_mmu_prepare_reg(hdev, mmNIC1_QM0_GLBL_NON_SECURE_PROPS_2,6240				asid);6241		gaudi_mmu_prepare_reg(hdev, mmNIC1_QM0_GLBL_NON_SECURE_PROPS_3,6242				asid);6243		gaudi_mmu_prepare_reg(hdev, mmNIC1_QM0_GLBL_NON_SECURE_PROPS_4,6244				asid);6245	}6246 6247	if (gaudi->hw_cap_initialized & HW_CAP_NIC3) {6248		gaudi_mmu_prepare_reg(hdev, mmNIC1_QM1_GLBL_NON_SECURE_PROPS_0,6249				asid);6250		gaudi_mmu_prepare_reg(hdev, mmNIC1_QM1_GLBL_NON_SECURE_PROPS_1,6251				asid);6252		gaudi_mmu_prepare_reg(hdev, mmNIC1_QM1_GLBL_NON_SECURE_PROPS_2,6253				asid);6254		gaudi_mmu_prepare_reg(hdev, mmNIC1_QM1_GLBL_NON_SECURE_PROPS_3,6255				asid);6256		gaudi_mmu_prepare_reg(hdev, mmNIC1_QM1_GLBL_NON_SECURE_PROPS_4,6257				asid);6258	}6259 6260	if (gaudi->hw_cap_initialized & HW_CAP_NIC4) {6261		gaudi_mmu_prepare_reg(hdev, mmNIC2_QM0_GLBL_NON_SECURE_PROPS_0,6262				asid);6263		gaudi_mmu_prepare_reg(hdev, mmNIC2_QM0_GLBL_NON_SECURE_PROPS_1,6264				asid);6265		gaudi_mmu_prepare_reg(hdev, mmNIC2_QM0_GLBL_NON_SECURE_PROPS_2,6266				asid);6267		gaudi_mmu_prepare_reg(hdev, mmNIC2_QM0_GLBL_NON_SECURE_PROPS_3,6268				asid);6269		gaudi_mmu_prepare_reg(hdev, mmNIC2_QM0_GLBL_NON_SECURE_PROPS_4,6270				asid);6271	}6272 6273	if (gaudi->hw_cap_initialized & HW_CAP_NIC5) {6274		gaudi_mmu_prepare_reg(hdev, mmNIC2_QM1_GLBL_NON_SECURE_PROPS_0,6275				asid);6276		gaudi_mmu_prepare_reg(hdev, mmNIC2_QM1_GLBL_NON_SECURE_PROPS_1,6277				asid);6278		gaudi_mmu_prepare_reg(hdev, mmNIC2_QM1_GLBL_NON_SECURE_PROPS_2,6279				asid);6280		gaudi_mmu_prepare_reg(hdev, mmNIC2_QM1_GLBL_NON_SECURE_PROPS_3,6281				asid);6282		gaudi_mmu_prepare_reg(hdev, mmNIC2_QM1_GLBL_NON_SECURE_PROPS_4,6283				asid);6284	}6285 6286	if (gaudi->hw_cap_initialized & HW_CAP_NIC6) {6287		gaudi_mmu_prepare_reg(hdev, mmNIC3_QM0_GLBL_NON_SECURE_PROPS_0,6288				asid);6289		gaudi_mmu_prepare_reg(hdev, mmNIC3_QM0_GLBL_NON_SECURE_PROPS_1,6290				asid);6291		gaudi_mmu_prepare_reg(hdev, mmNIC3_QM0_GLBL_NON_SECURE_PROPS_2,6292				asid);6293		gaudi_mmu_prepare_reg(hdev, mmNIC3_QM0_GLBL_NON_SECURE_PROPS_3,6294				asid);6295		gaudi_mmu_prepare_reg(hdev, mmNIC3_QM0_GLBL_NON_SECURE_PROPS_4,6296				asid);6297	}6298 6299	if (gaudi->hw_cap_initialized & HW_CAP_NIC7) {6300		gaudi_mmu_prepare_reg(hdev, mmNIC3_QM1_GLBL_NON_SECURE_PROPS_0,6301				asid);6302		gaudi_mmu_prepare_reg(hdev, mmNIC3_QM1_GLBL_NON_SECURE_PROPS_1,6303				asid);6304		gaudi_mmu_prepare_reg(hdev, mmNIC3_QM1_GLBL_NON_SECURE_PROPS_2,6305				asid);6306		gaudi_mmu_prepare_reg(hdev, mmNIC3_QM1_GLBL_NON_SECURE_PROPS_3,6307				asid);6308		gaudi_mmu_prepare_reg(hdev, mmNIC3_QM1_GLBL_NON_SECURE_PROPS_4,6309				asid);6310	}6311 6312	if (gaudi->hw_cap_initialized & HW_CAP_NIC8) {6313		gaudi_mmu_prepare_reg(hdev, mmNIC4_QM0_GLBL_NON_SECURE_PROPS_0,6314				asid);6315		gaudi_mmu_prepare_reg(hdev, mmNIC4_QM0_GLBL_NON_SECURE_PROPS_1,6316				asid);6317		gaudi_mmu_prepare_reg(hdev, mmNIC4_QM0_GLBL_NON_SECURE_PROPS_2,6318				asid);6319		gaudi_mmu_prepare_reg(hdev, mmNIC4_QM0_GLBL_NON_SECURE_PROPS_3,6320				asid);6321		gaudi_mmu_prepare_reg(hdev, mmNIC4_QM0_GLBL_NON_SECURE_PROPS_4,6322				asid);6323	}6324 6325	if (gaudi->hw_cap_initialized & HW_CAP_NIC9) {6326		gaudi_mmu_prepare_reg(hdev, mmNIC4_QM1_GLBL_NON_SECURE_PROPS_0,6327				asid);6328		gaudi_mmu_prepare_reg(hdev, mmNIC4_QM1_GLBL_NON_SECURE_PROPS_1,6329				asid);6330		gaudi_mmu_prepare_reg(hdev, mmNIC4_QM1_GLBL_NON_SECURE_PROPS_2,6331				asid);6332		gaudi_mmu_prepare_reg(hdev, mmNIC4_QM1_GLBL_NON_SECURE_PROPS_3,6333				asid);6334		gaudi_mmu_prepare_reg(hdev, mmNIC4_QM1_GLBL_NON_SECURE_PROPS_4,6335				asid);6336	}6337 6338	gaudi_mmu_prepare_reg(hdev, mmPSOC_GLOBAL_CONF_TRACE_ARUSER, asid);6339	gaudi_mmu_prepare_reg(hdev, mmPSOC_GLOBAL_CONF_TRACE_AWUSER, asid);6340}6341 6342static int gaudi_send_job_on_qman0(struct hl_device *hdev,6343		struct hl_cs_job *job)6344{6345	struct packet_msg_prot *fence_pkt;6346	u32 *fence_ptr;6347	dma_addr_t fence_dma_addr;6348	struct hl_cb *cb;6349	u32 tmp, timeout, dma_offset;6350	int rc;6351 6352	if (hdev->pldm)6353		timeout = GAUDI_PLDM_QMAN0_TIMEOUT_USEC;6354	else6355		timeout = HL_DEVICE_TIMEOUT_USEC;6356 6357	fence_ptr = hl_asic_dma_pool_zalloc(hdev, 4, GFP_KERNEL, &fence_dma_addr);6358	if (!fence_ptr) {6359		dev_err(hdev->dev,6360			"Failed to allocate fence memory for QMAN0\n");6361		return -ENOMEM;6362	}6363 6364	cb = job->patched_cb;6365 6366	fence_pkt = cb->kernel_address +6367			job->job_cb_size - sizeof(struct packet_msg_prot);6368 6369	tmp = FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_PROT);6370	tmp |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 1);6371	tmp |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1);6372 6373	fence_pkt->ctl = cpu_to_le32(tmp);6374	fence_pkt->value = cpu_to_le32(GAUDI_QMAN0_FENCE_VAL);6375	fence_pkt->addr = cpu_to_le64(fence_dma_addr);6376 6377	dma_offset = gaudi_dma_assignment[GAUDI_PCI_DMA_1] * DMA_CORE_OFFSET;6378 6379	WREG32(mmDMA0_CORE_PROT + dma_offset,6380			BIT(DMA0_CORE_PROT_ERR_VAL_SHIFT) | BIT(DMA0_CORE_PROT_VAL_SHIFT));6381 6382	rc = hl_hw_queue_send_cb_no_cmpl(hdev, GAUDI_QUEUE_ID_DMA_0_0,6383					job->job_cb_size, cb->bus_address);6384	if (rc) {6385		dev_err(hdev->dev, "Failed to send CB on QMAN0, %d\n", rc);6386		goto free_fence_ptr;6387	}6388 6389	rc = hl_poll_timeout_memory(hdev, fence_ptr, tmp,6390				(tmp == GAUDI_QMAN0_FENCE_VAL), 1000,6391				timeout, true);6392 6393	hl_hw_queue_inc_ci_kernel(hdev, GAUDI_QUEUE_ID_DMA_0_0);6394 6395	if (rc == -ETIMEDOUT) {6396		dev_err(hdev->dev, "QMAN0 Job timeout (0x%x)\n", tmp);6397		goto free_fence_ptr;6398	}6399 6400free_fence_ptr:6401	WREG32(mmDMA0_CORE_PROT + dma_offset, BIT(DMA0_CORE_PROT_ERR_VAL_SHIFT));6402 6403	hl_asic_dma_pool_free(hdev, (void *) fence_ptr, fence_dma_addr);6404	return rc;6405}6406 6407static void gaudi_get_event_desc(u16 event_type, char *desc, size_t size)6408{6409	if (event_type >= GAUDI_EVENT_SIZE)6410		goto event_not_supported;6411 6412	if (!gaudi_irq_map_table[event_type].valid)6413		goto event_not_supported;6414 6415	snprintf(desc, size, gaudi_irq_map_table[event_type].name);6416 6417	return;6418 6419event_not_supported:6420	snprintf(desc, size, "N/A");6421}6422 6423static const char *gaudi_get_razwi_initiator_dma_name(struct hl_device *hdev, u32 x_y,6424							bool is_write, u16 *engine_id_1,6425							u16 *engine_id_2)6426{6427	u32 dma_id[2], dma_offset, err_cause[2], mask, i;6428 6429	mask = is_write ? DMA0_CORE_ERR_CAUSE_HBW_WR_ERR_MASK :6430				DMA0_CORE_ERR_CAUSE_HBW_RD_ERR_MASK;6431 6432	switch (x_y) {6433	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_0:6434	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_1:6435		dma_id[0] = 0;6436		dma_id[1] = 2;6437		break;6438	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_0:6439	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_1:6440		dma_id[0] = 1;6441		dma_id[1] = 3;6442		break;6443	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_0:6444	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_1:6445		dma_id[0] = 4;6446		dma_id[1] = 6;6447		break;6448	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_0:6449	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_1:6450		dma_id[0] = 5;6451		dma_id[1] = 7;6452		break;6453	default:6454		goto unknown_initiator;6455	}6456 6457	for (i = 0 ; i < 2 ; i++) {6458		dma_offset = dma_id[i] * DMA_CORE_OFFSET;6459		err_cause[i] = RREG32(mmDMA0_CORE_ERR_CAUSE + dma_offset);6460	}6461 6462	switch (x_y) {6463	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_0:6464	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_1:6465		if ((err_cause[0] & mask) && !(err_cause[1] & mask)) {6466			*engine_id_1 = GAUDI_ENGINE_ID_DMA_0;6467			return "DMA0";6468		} else if (!(err_cause[0] & mask) && (err_cause[1] & mask)) {6469			*engine_id_1 = GAUDI_ENGINE_ID_DMA_2;6470			return "DMA2";6471		} else {6472			*engine_id_1 = GAUDI_ENGINE_ID_DMA_0;6473			*engine_id_2 = GAUDI_ENGINE_ID_DMA_2;6474			return "DMA0 or DMA2";6475		}6476	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_0:6477	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_1:6478		if ((err_cause[0] & mask) && !(err_cause[1] & mask)) {6479			*engine_id_1 = GAUDI_ENGINE_ID_DMA_1;6480			return "DMA1";6481		} else if (!(err_cause[0] & mask) && (err_cause[1] & mask)) {6482			*engine_id_1 = GAUDI_ENGINE_ID_DMA_3;6483			return "DMA3";6484		} else {6485			*engine_id_1 = GAUDI_ENGINE_ID_DMA_1;6486			*engine_id_2 = GAUDI_ENGINE_ID_DMA_3;6487			return "DMA1 or DMA3";6488		}6489	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_0:6490	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_1:6491		if ((err_cause[0] & mask) && !(err_cause[1] & mask)) {6492			*engine_id_1 = GAUDI_ENGINE_ID_DMA_4;6493			return "DMA4";6494		} else if (!(err_cause[0] & mask) && (err_cause[1] & mask)) {6495			*engine_id_1 = GAUDI_ENGINE_ID_DMA_6;6496			return "DMA6";6497		} else {6498			*engine_id_1 = GAUDI_ENGINE_ID_DMA_4;6499			*engine_id_2 = GAUDI_ENGINE_ID_DMA_6;6500			return "DMA4 or DMA6";6501		}6502	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_0:6503	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_1:6504		if ((err_cause[0] & mask) && !(err_cause[1] & mask)) {6505			*engine_id_1 = GAUDI_ENGINE_ID_DMA_5;6506			return "DMA5";6507		} else if (!(err_cause[0] & mask) && (err_cause[1] & mask)) {6508			*engine_id_1 = GAUDI_ENGINE_ID_DMA_7;6509			return "DMA7";6510		} else {6511			*engine_id_1 = GAUDI_ENGINE_ID_DMA_5;6512			*engine_id_2 = GAUDI_ENGINE_ID_DMA_7;6513			return "DMA5 or DMA7";6514		}6515	}6516 6517unknown_initiator:6518	return "unknown initiator";6519}6520 6521static const char *gaudi_get_razwi_initiator_name(struct hl_device *hdev, bool is_write,6522							u16 *engine_id_1, u16 *engine_id_2)6523{6524	u32 val, x_y, axi_id;6525 6526	val = is_write ? RREG32(mmMMU_UP_RAZWI_WRITE_ID) :6527				RREG32(mmMMU_UP_RAZWI_READ_ID);6528	x_y = val & ((RAZWI_INITIATOR_Y_MASK << RAZWI_INITIATOR_Y_SHIFT) |6529			(RAZWI_INITIATOR_X_MASK << RAZWI_INITIATOR_X_SHIFT));6530	axi_id = val & (RAZWI_INITIATOR_AXI_ID_MASK <<6531			RAZWI_INITIATOR_AXI_ID_SHIFT);6532 6533	switch (x_y) {6534	case RAZWI_INITIATOR_ID_X_Y_TPC0_NIC0:6535		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_TPC)) {6536			*engine_id_1 = GAUDI_ENGINE_ID_TPC_0;6537			return "TPC0";6538		}6539		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_NIC)) {6540			*engine_id_1 = GAUDI_ENGINE_ID_NIC_0;6541			return "NIC0";6542		}6543		break;6544	case RAZWI_INITIATOR_ID_X_Y_TPC1:6545		*engine_id_1 = GAUDI_ENGINE_ID_TPC_1;6546		return "TPC1";6547	case RAZWI_INITIATOR_ID_X_Y_MME0_0:6548	case RAZWI_INITIATOR_ID_X_Y_MME0_1:6549		*engine_id_1 = GAUDI_ENGINE_ID_MME_0;6550		return "MME0";6551	case RAZWI_INITIATOR_ID_X_Y_MME1_0:6552	case RAZWI_INITIATOR_ID_X_Y_MME1_1:6553		*engine_id_1 = GAUDI_ENGINE_ID_MME_1;6554		return "MME1";6555	case RAZWI_INITIATOR_ID_X_Y_TPC2:6556		*engine_id_1 = GAUDI_ENGINE_ID_TPC_2;6557		return "TPC2";6558	case RAZWI_INITIATOR_ID_X_Y_TPC3_PCI_CPU_PSOC:6559		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_TPC)) {6560			*engine_id_1 = GAUDI_ENGINE_ID_TPC_3;6561			return "TPC3";6562		}6563		/* PCI, CPU or PSOC does not have engine id*/6564		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_PCI))6565			return "PCI";6566		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_CPU))6567			return "CPU";6568		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_PSOC))6569			return "PSOC";6570		break;6571	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_0:6572	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_S_1:6573	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_0:6574	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_S_1:6575	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_0:6576	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_W_N_1:6577	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_0:6578	case RAZWI_INITIATOR_ID_X_Y_DMA_IF_E_N_1:6579		return gaudi_get_razwi_initiator_dma_name(hdev, x_y, is_write,6580				engine_id_1, engine_id_2);6581	case RAZWI_INITIATOR_ID_X_Y_TPC4_NIC1_NIC2:6582		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_TPC)) {6583			*engine_id_1 = GAUDI_ENGINE_ID_TPC_4;6584			return "TPC4";6585		}6586		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_NIC)) {6587			*engine_id_1 = GAUDI_ENGINE_ID_NIC_1;6588			return "NIC1";6589		}6590		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_NIC_FT)) {6591			*engine_id_1 = GAUDI_ENGINE_ID_NIC_2;6592			return "NIC2";6593		}6594		break;6595	case RAZWI_INITIATOR_ID_X_Y_TPC5:6596		*engine_id_1 = GAUDI_ENGINE_ID_TPC_5;6597		return "TPC5";6598	case RAZWI_INITIATOR_ID_X_Y_MME2_0:6599	case RAZWI_INITIATOR_ID_X_Y_MME2_1:6600		*engine_id_1 = GAUDI_ENGINE_ID_MME_2;6601		return "MME2";6602	case RAZWI_INITIATOR_ID_X_Y_MME3_0:6603	case RAZWI_INITIATOR_ID_X_Y_MME3_1:6604		*engine_id_1 = GAUDI_ENGINE_ID_MME_3;6605		return "MME3";6606	case RAZWI_INITIATOR_ID_X_Y_TPC6:6607		*engine_id_1 = GAUDI_ENGINE_ID_TPC_6;6608		return "TPC6";6609	case RAZWI_INITIATOR_ID_X_Y_TPC7_NIC4_NIC5:6610		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_TPC)) {6611			*engine_id_1 = GAUDI_ENGINE_ID_TPC_7;6612			return "TPC7";6613		}6614		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_NIC)) {6615			*engine_id_1 = GAUDI_ENGINE_ID_NIC_4;6616			return "NIC4";6617		}6618		if (axi_id == RAZWI_INITIATOR_ID_AXI_ID(AXI_ID_NIC_FT)) {6619			*engine_id_1 = GAUDI_ENGINE_ID_NIC_5;6620			return "NIC5";6621		}6622		break;6623	default:6624		break;6625	}6626 6627	dev_err(hdev->dev,6628		"Unknown RAZWI initiator ID 0x%x [Y=%d, X=%d, AXI_ID=%d]\n",6629		val,6630		(val >> RAZWI_INITIATOR_Y_SHIFT) & RAZWI_INITIATOR_Y_MASK,6631		(val >> RAZWI_INITIATOR_X_SHIFT) & RAZWI_INITIATOR_X_MASK,6632		(val >> RAZWI_INITIATOR_AXI_ID_SHIFT) &6633			RAZWI_INITIATOR_AXI_ID_MASK);6634 6635	return "unknown initiator";6636}6637 6638static void gaudi_print_and_get_razwi_info(struct hl_device *hdev, u16 *engine_id_1,6639						u16 *engine_id_2, bool *is_read, bool *is_write)6640{6641 6642	if (RREG32(mmMMU_UP_RAZWI_WRITE_VLD)) {6643		dev_err_ratelimited(hdev->dev,6644			"RAZWI event caused by illegal write of %s\n",6645			gaudi_get_razwi_initiator_name(hdev, true, engine_id_1, engine_id_2));6646		WREG32(mmMMU_UP_RAZWI_WRITE_VLD, 0);6647		*is_write = true;6648	}6649 6650	if (RREG32(mmMMU_UP_RAZWI_READ_VLD)) {6651		dev_err_ratelimited(hdev->dev,6652			"RAZWI event caused by illegal read of %s\n",6653			gaudi_get_razwi_initiator_name(hdev, false, engine_id_1, engine_id_2));6654		WREG32(mmMMU_UP_RAZWI_READ_VLD, 0);6655		*is_read = true;6656	}6657}6658 6659static void gaudi_print_and_get_mmu_error_info(struct hl_device *hdev, u64 *addr, u64 *event_mask)6660{6661	struct gaudi_device *gaudi = hdev->asic_specific;6662	u32 val;6663 6664	if (!(gaudi->hw_cap_initialized & HW_CAP_MMU))6665		return;6666 6667	val = RREG32(mmMMU_UP_PAGE_ERROR_CAPTURE);6668	if (val & MMU_UP_PAGE_ERROR_CAPTURE_ENTRY_VALID_MASK) {6669		*addr = val & MMU_UP_PAGE_ERROR_CAPTURE_VA_49_32_MASK;6670		*addr <<= 32;6671		*addr |= RREG32(mmMMU_UP_PAGE_ERROR_CAPTURE_VA);6672 6673		dev_err_ratelimited(hdev->dev, "MMU page fault on va 0x%llx\n", *addr);6674		hl_handle_page_fault(hdev, *addr, 0, true, event_mask);6675 6676		WREG32(mmMMU_UP_PAGE_ERROR_CAPTURE, 0);6677	}6678 6679	val = RREG32(mmMMU_UP_ACCESS_ERROR_CAPTURE);6680	if (val & MMU_UP_ACCESS_ERROR_CAPTURE_ENTRY_VALID_MASK) {6681		*addr = val & MMU_UP_ACCESS_ERROR_CAPTURE_VA_49_32_MASK;6682		*addr <<= 32;6683		*addr |= RREG32(mmMMU_UP_ACCESS_ERROR_CAPTURE_VA);6684 6685		dev_err_ratelimited(hdev->dev, "MMU access error on va 0x%llx\n", *addr);6686 6687		WREG32(mmMMU_UP_ACCESS_ERROR_CAPTURE, 0);6688	}6689}6690 6691/*6692 *  +-------------------+------------------------------------------------------+6693 *  | Configuration Reg |                     Description                      |6694 *  |      Address      |                                                      |6695 *  +-------------------+------------------------------------------------------+6696 *  |  0xF30 - 0xF3F    |ECC single error indication (1 bit per memory wrapper)|6697 *  |                   |0xF30 memory wrappers 31:0 (MSB to LSB)               |6698 *  |                   |0xF34 memory wrappers 63:32                           |6699 *  |                   |0xF38 memory wrappers 95:64                           |6700 *  |                   |0xF3C memory wrappers 127:96                          |6701 *  +-------------------+------------------------------------------------------+6702 *  |  0xF40 - 0xF4F    |ECC double error indication (1 bit per memory wrapper)|6703 *  |                   |0xF40 memory wrappers 31:0 (MSB to LSB)               |6704 *  |                   |0xF44 memory wrappers 63:32                           |6705 *  |                   |0xF48 memory wrappers 95:64                           |6706 *  |                   |0xF4C memory wrappers 127:96                          |6707 *  +-------------------+------------------------------------------------------+6708 */6709static int gaudi_extract_ecc_info(struct hl_device *hdev,6710		struct ecc_info_extract_params *params, u64 *ecc_address,6711		u64 *ecc_syndrom, u8 *memory_wrapper_idx)6712{6713	u32 i, num_mem_regs, reg, err_bit;6714	u64 err_addr, err_word = 0;6715 6716	num_mem_regs = params->num_memories / 32 +6717			((params->num_memories % 32) ? 1 : 0);6718 6719	if (params->block_address >= CFG_BASE)6720		params->block_address -= CFG_BASE;6721 6722	if (params->derr)6723		err_addr = params->block_address + GAUDI_ECC_DERR0_OFFSET;6724	else6725		err_addr = params->block_address + GAUDI_ECC_SERR0_OFFSET;6726 6727	/* Set invalid wrapper index */6728	*memory_wrapper_idx = 0xFF;6729 6730	/* Iterate through memory wrappers, a single bit must be set */6731	for (i = 0 ; i < num_mem_regs ; i++) {6732		err_addr += i * 4;6733		err_word = RREG32(err_addr);6734		if (err_word) {6735			err_bit = __ffs(err_word);6736			*memory_wrapper_idx = err_bit + (32 * i);6737			break;6738		}6739	}6740 6741	if (*memory_wrapper_idx == 0xFF) {6742		dev_err(hdev->dev, "ECC error information cannot be found\n");6743		return -EINVAL;6744	}6745 6746	WREG32(params->block_address + GAUDI_ECC_MEM_SEL_OFFSET,6747			*memory_wrapper_idx);6748 6749	*ecc_address =6750		RREG32(params->block_address + GAUDI_ECC_ADDRESS_OFFSET);6751	*ecc_syndrom =6752		RREG32(params->block_address + GAUDI_ECC_SYNDROME_OFFSET);6753 6754	/* Clear error indication */6755	reg = RREG32(params->block_address + GAUDI_ECC_MEM_INFO_CLR_OFFSET);6756	if (params->derr)6757		reg |= FIELD_PREP(GAUDI_ECC_MEM_INFO_CLR_DERR_MASK, 1);6758	else6759		reg |= FIELD_PREP(GAUDI_ECC_MEM_INFO_CLR_SERR_MASK, 1);6760 6761	WREG32(params->block_address + GAUDI_ECC_MEM_INFO_CLR_OFFSET, reg);6762 6763	return 0;6764}6765 6766/*6767 * gaudi_queue_idx_dec - decrement queue index (pi/ci) and handle wrap6768 *6769 * @idx: the current pi/ci value6770 * @q_len: the queue length (power of 2)6771 *6772 * @return the cyclically decremented index6773 */6774static inline u32 gaudi_queue_idx_dec(u32 idx, u32 q_len)6775{6776	u32 mask = q_len - 1;6777 6778	/*6779	 * modular decrement is equivalent to adding (queue_size -1)6780	 * later we take LSBs to make sure the value is in the6781	 * range [0, queue_len - 1]6782	 */6783	return (idx + q_len - 1) & mask;6784}6785 6786/**6787 * gaudi_handle_sw_config_stream_data - print SW config stream data6788 *6789 * @hdev: pointer to the habanalabs device structure6790 * @stream: the QMAN's stream6791 * @qman_base: base address of QMAN registers block6792 * @event_mask: mask of the last events occurred6793 */6794static void gaudi_handle_sw_config_stream_data(struct hl_device *hdev, u32 stream,6795						u64 qman_base, u64 event_mask)6796{6797	u64 cq_ptr_lo, cq_ptr_hi, cq_tsize, cq_ptr;6798	u32 cq_ptr_lo_off, size;6799 6800	cq_ptr_lo_off = mmTPC0_QM_CQ_PTR_LO_1 - mmTPC0_QM_CQ_PTR_LO_0;6801 6802	cq_ptr_lo = qman_base + (mmTPC0_QM_CQ_PTR_LO_0 - mmTPC0_QM_BASE) +6803						stream * cq_ptr_lo_off;6804	cq_ptr_hi = cq_ptr_lo +6805				(mmTPC0_QM_CQ_PTR_HI_0 - mmTPC0_QM_CQ_PTR_LO_0);6806	cq_tsize = cq_ptr_lo +6807				(mmTPC0_QM_CQ_TSIZE_0 - mmTPC0_QM_CQ_PTR_LO_0);6808 6809	cq_ptr = (((u64) RREG32(cq_ptr_hi)) << 32) | RREG32(cq_ptr_lo);6810	size = RREG32(cq_tsize);6811	dev_info(hdev->dev, "stop on err: stream: %u, addr: %#llx, size: %u\n",6812							stream, cq_ptr, size);6813 6814	if (event_mask & HL_NOTIFIER_EVENT_UNDEFINED_OPCODE) {6815		hdev->captured_err_info.undef_opcode.cq_addr = cq_ptr;6816		hdev->captured_err_info.undef_opcode.cq_size = size;6817		hdev->captured_err_info.undef_opcode.stream_id = stream;6818	}6819}6820 6821/**6822 * gaudi_handle_last_pqes_on_err - print last PQEs on error6823 *6824 * @hdev: pointer to the habanalabs device structure6825 * @qid_base: first QID of the QMAN (out of 4 streams)6826 * @stream: the QMAN's stream6827 * @qman_base: base address of QMAN registers block6828 * @event_mask: mask of the last events occurred6829 * @pr_sw_conf: if true print the SW config stream data (CQ PTR and SIZE)6830 */6831static void gaudi_handle_last_pqes_on_err(struct hl_device *hdev, u32 qid_base,6832						u32 stream, u64 qman_base,6833						u64 event_mask,6834						bool pr_sw_conf)6835{6836	u32 ci, qm_ci_stream_off, queue_len;6837	struct hl_hw_queue *q;6838	u64 pq_ci, addr[PQ_FETCHER_CACHE_SIZE];6839	int i;6840 6841	q = &hdev->kernel_queues[qid_base + stream];6842 6843	qm_ci_stream_off = mmTPC0_QM_PQ_CI_1 - mmTPC0_QM_PQ_CI_0;6844	pq_ci = qman_base + (mmTPC0_QM_PQ_CI_0 - mmTPC0_QM_BASE) +6845						stream * qm_ci_stream_off;6846 6847	queue_len = (q->queue_type == QUEUE_TYPE_INT) ?6848					q->int_queue_len : HL_QUEUE_LENGTH;6849 6850	hdev->asic_funcs->hw_queues_lock(hdev);6851 6852	if (pr_sw_conf)6853		gaudi_handle_sw_config_stream_data(hdev, stream, qman_base, event_mask);6854 6855	ci = RREG32(pq_ci);6856 6857	/* we should start printing form ci -1 */6858	ci = gaudi_queue_idx_dec(ci, queue_len);6859	memset(addr, 0, sizeof(addr));6860 6861	for (i = 0; i < PQ_FETCHER_CACHE_SIZE; i++) {6862		struct hl_bd *bd;6863		u32 len;6864 6865		bd = q->kernel_address;6866		bd += ci;6867 6868		len = le32_to_cpu(bd->len);6869		/* len 0 means uninitialized entry- break */6870		if (!len)6871			break;6872 6873		addr[i] = le64_to_cpu(bd->ptr);6874 6875		dev_info(hdev->dev, "stop on err PQE(stream %u): ci: %u, addr: %#llx, size: %u\n",6876							stream, ci, addr[i], len);6877 6878		/* get previous ci, wrap if needed */6879		ci = gaudi_queue_idx_dec(ci, queue_len);6880	}6881 6882	if (event_mask & HL_NOTIFIER_EVENT_UNDEFINED_OPCODE) {6883		struct undefined_opcode_info *undef_opcode = &hdev->captured_err_info.undef_opcode;6884		u32 arr_idx = undef_opcode->cb_addr_streams_len;6885 6886		if (arr_idx == 0) {6887			undef_opcode->timestamp = ktime_get();6888			undef_opcode->engine_id = gaudi_queue_id_to_engine_id[qid_base];6889		}6890 6891		memcpy(undef_opcode->cb_addr_streams[arr_idx], addr, sizeof(addr));6892		undef_opcode->cb_addr_streams_len++;6893	}6894 6895	hdev->asic_funcs->hw_queues_unlock(hdev);6896}6897 6898/**6899 * handle_qman_data_on_err - extract QMAN data on error6900 *6901 * @hdev: pointer to the habanalabs device structure6902 * @qid_base: first QID of the QMAN (out of 4 streams)6903 * @stream: the QMAN's stream6904 * @qman_base: base address of QMAN registers block6905 * @event_mask: mask of the last events occurred6906 *6907 * This function attempt to exatract as much data as possible on QMAN error.6908 * On upper CP print the SW config stream data and last 8 PQEs.6909 * On lower CP print SW config data and last PQEs of ALL 4 upper CPs6910 */6911static void handle_qman_data_on_err(struct hl_device *hdev, u32 qid_base,6912				   u32 stream, u64 qman_base, u64 event_mask)6913{6914	u32 i;6915 6916	if (stream != QMAN_STREAMS) {6917		gaudi_handle_last_pqes_on_err(hdev, qid_base, stream,6918			qman_base, event_mask, true);6919		return;6920	}6921 6922	/* handle Lower-CP */6923	gaudi_handle_sw_config_stream_data(hdev, stream, qman_base, event_mask);6924 6925	for (i = 0; i < QMAN_STREAMS; i++)6926		gaudi_handle_last_pqes_on_err(hdev, qid_base, i,6927			qman_base, event_mask, false);6928}6929 6930static void gaudi_handle_qman_err_generic(struct hl_device *hdev,6931					  const char *qm_name,6932					  u64 qman_base,6933					  u32 qid_base,6934					  u64 *event_mask)6935{6936	u32 i, j, glbl_sts_val, arb_err_val, glbl_sts_clr_val;6937	u64 glbl_sts_addr, arb_err_addr;6938	char reg_desc[32];6939 6940	glbl_sts_addr = qman_base + (mmTPC0_QM_GLBL_STS1_0 - mmTPC0_QM_BASE);6941	arb_err_addr = qman_base + (mmTPC0_QM_ARB_ERR_CAUSE - mmTPC0_QM_BASE);6942 6943	/* Iterate through all stream GLBL_STS1 registers + Lower CP */6944	for (i = 0 ; i < QMAN_STREAMS + 1 ; i++) {6945		glbl_sts_clr_val = 0;6946		glbl_sts_val = RREG32(glbl_sts_addr + 4 * i);6947 6948		if (!glbl_sts_val)6949			continue;6950 6951		if (i == QMAN_STREAMS)6952			snprintf(reg_desc, ARRAY_SIZE(reg_desc), "LowerCP");6953		else6954			snprintf(reg_desc, ARRAY_SIZE(reg_desc), "stream%u", i);6955 6956		for (j = 0 ; j < GAUDI_NUM_OF_QM_ERR_CAUSE ; j++) {6957			if (glbl_sts_val & BIT(j)) {6958				dev_err_ratelimited(hdev->dev,6959						"%s %s. err cause: %s\n",6960						qm_name, reg_desc,6961						gaudi_qman_error_cause[j]);6962				glbl_sts_clr_val |= BIT(j);6963			}6964		}6965		/* check for undefined opcode */6966		if (glbl_sts_val & TPC0_QM_GLBL_STS1_CP_UNDEF_CMD_ERR_MASK &&6967				hdev->captured_err_info.undef_opcode.write_enable) {6968			memset(&hdev->captured_err_info.undef_opcode, 0,6969						sizeof(hdev->captured_err_info.undef_opcode));6970 6971			hdev->captured_err_info.undef_opcode.write_enable = false;6972			*event_mask |= HL_NOTIFIER_EVENT_UNDEFINED_OPCODE;6973		}6974 6975		/* Write 1 clear errors */6976		if (!hdev->stop_on_err)6977			WREG32(glbl_sts_addr + 4 * i, glbl_sts_clr_val);6978		else6979			handle_qman_data_on_err(hdev, qid_base, i, qman_base, *event_mask);6980	}6981 6982	arb_err_val = RREG32(arb_err_addr);6983 6984	if (!arb_err_val)6985		return;6986 6987	for (j = 0 ; j < GAUDI_NUM_OF_QM_ARB_ERR_CAUSE ; j++) {6988		if (arb_err_val & BIT(j)) {6989			dev_err_ratelimited(hdev->dev,6990					"%s ARB_ERR. err cause: %s\n",6991					qm_name,6992					gaudi_qman_arb_error_cause[j]);6993		}6994	}6995}6996 6997static void gaudi_print_sm_sei_info(struct hl_device *hdev, u16 event_type,6998		struct hl_eq_sm_sei_data *sei_data)6999{7000	u32 index = event_type - GAUDI_EVENT_DMA_IF_SEI_0;7001 7002	/* Flip the bits as the enum is ordered in the opposite way */7003	index = (index ^ 0x3) & 0x3;7004 7005	switch (sei_data->sei_cause) {7006	case SM_SEI_SO_OVERFLOW:7007		dev_err_ratelimited(hdev->dev,7008			"%s SEI Error: SOB Group %u overflow/underflow",7009			gaudi_sync_manager_names[index],7010			le32_to_cpu(sei_data->sei_log));7011		break;7012	case SM_SEI_LBW_4B_UNALIGNED:7013		dev_err_ratelimited(hdev->dev,7014			"%s SEI Error: Unaligned 4B LBW access, monitor agent address low - %#x",7015			gaudi_sync_manager_names[index],7016			le32_to_cpu(sei_data->sei_log));7017		break;7018	case SM_SEI_AXI_RESPONSE_ERR:7019		dev_err_ratelimited(hdev->dev,7020			"%s SEI Error: AXI ID %u response error",7021			gaudi_sync_manager_names[index],7022			le32_to_cpu(sei_data->sei_log));7023		break;7024	default:7025		dev_err_ratelimited(hdev->dev, "Unknown SM SEI cause %u",7026				le32_to_cpu(sei_data->sei_log));7027		break;7028	}7029}7030 7031static void gaudi_handle_ecc_event(struct hl_device *hdev, u16 event_type,7032		struct hl_eq_ecc_data *ecc_data)7033{7034	struct ecc_info_extract_params params;7035	u64 ecc_address = 0, ecc_syndrom = 0;7036	u8 index, memory_wrapper_idx = 0;7037	bool extract_info_from_fw;7038	int rc;7039 7040	if (hdev->asic_prop.fw_security_enabled) {7041		extract_info_from_fw = true;7042		goto extract_ecc_info;7043	}7044 7045	switch (event_type) {7046	case GAUDI_EVENT_PCIE_CORE_SERR ... GAUDI_EVENT_PCIE_PHY_DERR:7047	case GAUDI_EVENT_DMA0_SERR_ECC ... GAUDI_EVENT_MMU_DERR:7048		extract_info_from_fw = true;7049		break;7050	case GAUDI_EVENT_TPC0_SERR ... GAUDI_EVENT_TPC7_SERR:7051		index = event_type - GAUDI_EVENT_TPC0_SERR;7052		params.block_address = mmTPC0_CFG_BASE + index * TPC_CFG_OFFSET;7053		params.num_memories = 90;7054		params.derr = false;7055		extract_info_from_fw = false;7056		break;7057	case GAUDI_EVENT_TPC0_DERR ... GAUDI_EVENT_TPC7_DERR:7058		index = event_type - GAUDI_EVENT_TPC0_DERR;7059		params.block_address =7060			mmTPC0_CFG_BASE + index * TPC_CFG_OFFSET;7061		params.num_memories = 90;7062		params.derr = true;7063		extract_info_from_fw = false;7064		break;7065	case GAUDI_EVENT_MME0_ACC_SERR:7066	case GAUDI_EVENT_MME1_ACC_SERR:7067	case GAUDI_EVENT_MME2_ACC_SERR:7068	case GAUDI_EVENT_MME3_ACC_SERR:7069		index = (event_type - GAUDI_EVENT_MME0_ACC_SERR) / 4;7070		params.block_address = mmMME0_ACC_BASE + index * MME_ACC_OFFSET;7071		params.num_memories = 128;7072		params.derr = false;7073		extract_info_from_fw = false;7074		break;7075	case GAUDI_EVENT_MME0_ACC_DERR:7076	case GAUDI_EVENT_MME1_ACC_DERR:7077	case GAUDI_EVENT_MME2_ACC_DERR:7078	case GAUDI_EVENT_MME3_ACC_DERR:7079		index = (event_type - GAUDI_EVENT_MME0_ACC_DERR) / 4;7080		params.block_address = mmMME0_ACC_BASE + index * MME_ACC_OFFSET;7081		params.num_memories = 128;7082		params.derr = true;7083		extract_info_from_fw = false;7084		break;7085	case GAUDI_EVENT_MME0_SBAB_SERR:7086	case GAUDI_EVENT_MME1_SBAB_SERR:7087	case GAUDI_EVENT_MME2_SBAB_SERR:7088	case GAUDI_EVENT_MME3_SBAB_SERR:7089		index = (event_type - GAUDI_EVENT_MME0_SBAB_SERR) / 4;7090		params.block_address =7091			mmMME0_SBAB_BASE + index * MME_ACC_OFFSET;7092		params.num_memories = 33;7093		params.derr = false;7094		extract_info_from_fw = false;7095		break;7096	case GAUDI_EVENT_MME0_SBAB_DERR:7097	case GAUDI_EVENT_MME1_SBAB_DERR:7098	case GAUDI_EVENT_MME2_SBAB_DERR:7099	case GAUDI_EVENT_MME3_SBAB_DERR:7100		index = (event_type - GAUDI_EVENT_MME0_SBAB_DERR) / 4;7101		params.block_address =7102			mmMME0_SBAB_BASE + index * MME_ACC_OFFSET;7103		params.num_memories = 33;7104		params.derr = true;7105		extract_info_from_fw = false;7106		break;7107	default:7108		return;7109	}7110 7111extract_ecc_info:7112	if (extract_info_from_fw) {7113		ecc_address = le64_to_cpu(ecc_data->ecc_address);7114		ecc_syndrom = le64_to_cpu(ecc_data->ecc_syndrom);7115		memory_wrapper_idx = ecc_data->memory_wrapper_idx;7116	} else {7117		rc = gaudi_extract_ecc_info(hdev, &params, &ecc_address,7118				&ecc_syndrom, &memory_wrapper_idx);7119		if (rc)7120			return;7121	}7122 7123	dev_err(hdev->dev,7124		"ECC error detected. address: %#llx. Syndrom: %#llx. block id %u\n",7125		ecc_address, ecc_syndrom, memory_wrapper_idx);7126}7127 7128static void gaudi_handle_qman_err(struct hl_device *hdev, u16 event_type, u64 *event_mask)7129{7130	u64 qman_base;7131	char desc[32];7132	u32 qid_base;7133	u8 index;7134 7135	switch (event_type) {7136	case GAUDI_EVENT_TPC0_QM ... GAUDI_EVENT_TPC7_QM:7137		index = event_type - GAUDI_EVENT_TPC0_QM;7138		qid_base = GAUDI_QUEUE_ID_TPC_0_0 + index * QMAN_STREAMS;7139		qman_base = mmTPC0_QM_BASE + index * TPC_QMAN_OFFSET;7140		snprintf(desc, ARRAY_SIZE(desc), "%s%d", "TPC_QM", index);7141		break;7142	case GAUDI_EVENT_MME0_QM ... GAUDI_EVENT_MME2_QM:7143		if (event_type == GAUDI_EVENT_MME0_QM) {7144			index = 0;7145			qid_base = GAUDI_QUEUE_ID_MME_0_0;7146		} else { /* event_type == GAUDI_EVENT_MME2_QM */7147			index = 2;7148			qid_base = GAUDI_QUEUE_ID_MME_1_0;7149		}7150		qman_base = mmMME0_QM_BASE + index * MME_QMAN_OFFSET;7151		snprintf(desc, ARRAY_SIZE(desc), "%s%d", "MME_QM", index);7152		break;7153	case GAUDI_EVENT_DMA0_QM ... GAUDI_EVENT_DMA7_QM:7154		index = event_type - GAUDI_EVENT_DMA0_QM;7155		qid_base = GAUDI_QUEUE_ID_DMA_0_0 + index * QMAN_STREAMS;7156		/* skip GAUDI_QUEUE_ID_CPU_PQ if necessary */7157		if (index > 1)7158			qid_base++;7159		qman_base = mmDMA0_QM_BASE + index * DMA_QMAN_OFFSET;7160		snprintf(desc, ARRAY_SIZE(desc), "%s%d", "DMA_QM", index);7161		break;7162	case GAUDI_EVENT_NIC0_QM0:7163		qid_base = GAUDI_QUEUE_ID_NIC_0_0;7164		qman_base = mmNIC0_QM0_BASE;7165		snprintf(desc, ARRAY_SIZE(desc), "NIC0_QM0");7166		break;7167	case GAUDI_EVENT_NIC0_QM1:7168		qid_base = GAUDI_QUEUE_ID_NIC_1_0;7169		qman_base = mmNIC0_QM1_BASE;7170		snprintf(desc, ARRAY_SIZE(desc), "NIC0_QM1");7171		break;7172	case GAUDI_EVENT_NIC1_QM0:7173		qid_base = GAUDI_QUEUE_ID_NIC_2_0;7174		qman_base = mmNIC1_QM0_BASE;7175		snprintf(desc, ARRAY_SIZE(desc), "NIC1_QM0");7176		break;7177	case GAUDI_EVENT_NIC1_QM1:7178		qid_base = GAUDI_QUEUE_ID_NIC_3_0;7179		qman_base = mmNIC1_QM1_BASE;7180		snprintf(desc, ARRAY_SIZE(desc), "NIC1_QM1");7181		break;7182	case GAUDI_EVENT_NIC2_QM0:7183		qid_base = GAUDI_QUEUE_ID_NIC_4_0;7184		qman_base = mmNIC2_QM0_BASE;7185		snprintf(desc, ARRAY_SIZE(desc), "NIC2_QM0");7186		break;7187	case GAUDI_EVENT_NIC2_QM1:7188		qid_base = GAUDI_QUEUE_ID_NIC_5_0;7189		qman_base = mmNIC2_QM1_BASE;7190		snprintf(desc, ARRAY_SIZE(desc), "NIC2_QM1");7191		break;7192	case GAUDI_EVENT_NIC3_QM0:7193		qid_base = GAUDI_QUEUE_ID_NIC_6_0;7194		qman_base = mmNIC3_QM0_BASE;7195		snprintf(desc, ARRAY_SIZE(desc), "NIC3_QM0");7196		break;7197	case GAUDI_EVENT_NIC3_QM1:7198		qid_base = GAUDI_QUEUE_ID_NIC_7_0;7199		qman_base = mmNIC3_QM1_BASE;7200		snprintf(desc, ARRAY_SIZE(desc), "NIC3_QM1");7201		break;7202	case GAUDI_EVENT_NIC4_QM0:7203		qid_base = GAUDI_QUEUE_ID_NIC_8_0;7204		qman_base = mmNIC4_QM0_BASE;7205		snprintf(desc, ARRAY_SIZE(desc), "NIC4_QM0");7206		break;7207	case GAUDI_EVENT_NIC4_QM1:7208		qid_base = GAUDI_QUEUE_ID_NIC_9_0;7209		qman_base = mmNIC4_QM1_BASE;7210		snprintf(desc, ARRAY_SIZE(desc), "NIC4_QM1");7211		break;7212	default:7213		return;7214	}7215 7216	gaudi_handle_qman_err_generic(hdev, desc, qman_base, qid_base, event_mask);7217}7218 7219static void gaudi_print_irq_info(struct hl_device *hdev, u16 event_type,7220					bool check_razwi, u64 *event_mask)7221{7222	bool is_read = false, is_write = false;7223	u16 engine_id[2], num_of_razwi_eng = 0;7224	char desc[64] = "";7225	u64 razwi_addr = 0;7226	u8 razwi_flags = 0;7227 7228	/*7229	 * Init engine id by default as not valid and only if razwi initiated from engine with7230	 * engine id it will get valid value.7231	 */7232	engine_id[0] = HL_RAZWI_NA_ENG_ID;7233	engine_id[1] = HL_RAZWI_NA_ENG_ID;7234 7235	gaudi_get_event_desc(event_type, desc, sizeof(desc));7236	dev_err_ratelimited(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n",7237		event_type, desc);7238 7239	if (check_razwi) {7240		gaudi_print_and_get_razwi_info(hdev, &engine_id[0], &engine_id[1], &is_read,7241						&is_write);7242		gaudi_print_and_get_mmu_error_info(hdev, &razwi_addr, event_mask);7243 7244		if (is_read)7245			razwi_flags |= HL_RAZWI_READ;7246		if (is_write)7247			razwi_flags |= HL_RAZWI_WRITE;7248 7249		if (engine_id[0] != HL_RAZWI_NA_ENG_ID) {7250			if (engine_id[1] != HL_RAZWI_NA_ENG_ID)7251				num_of_razwi_eng = 2;7252			else7253				num_of_razwi_eng = 1;7254		}7255 7256		if (razwi_flags)7257			hl_handle_razwi(hdev, razwi_addr, engine_id, num_of_razwi_eng,7258					razwi_flags, event_mask);7259	}7260}7261 7262static void gaudi_print_out_of_sync_info(struct hl_device *hdev,7263					struct cpucp_pkt_sync_err *sync_err)7264{7265	struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI_QUEUE_ID_CPU_PQ];7266 7267	dev_err(hdev->dev, "Out of sync with FW, FW: pi=%u, ci=%u, LKD: pi=%u, ci=%d\n",7268		le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), q->pi, atomic_read(&q->ci));7269}7270 7271static void gaudi_print_fw_alive_info(struct hl_device *hdev,7272					struct hl_eq_fw_alive *fw_alive)7273{7274	dev_err(hdev->dev,7275		"FW alive report: severity=%s, process_id=%u, thread_id=%u, uptime=%llu seconds\n",7276		(fw_alive->severity == FW_ALIVE_SEVERITY_MINOR) ? "Minor" : "Critical",7277		le32_to_cpu(fw_alive->process_id),7278		le32_to_cpu(fw_alive->thread_id),7279		le64_to_cpu(fw_alive->uptime_seconds));7280}7281 7282static void gaudi_print_nic_axi_irq_info(struct hl_device *hdev, u16 event_type,7283						void *data)7284{7285	char desc[64] = "", *type;7286	struct eq_nic_sei_event *eq_nic_sei = data;7287	u16 nic_id = event_type - GAUDI_EVENT_NIC_SEI_0;7288 7289	switch (eq_nic_sei->axi_error_cause) {7290	case RXB:7291		type = "RXB";7292		break;7293	case RXE:7294		type = "RXE";7295		break;7296	case TXS:7297		type = "TXS";7298		break;7299	case TXE:7300		type = "TXE";7301		break;7302	case QPC_RESP:7303		type = "QPC_RESP";7304		break;7305	case NON_AXI_ERR:7306		type = "NON_AXI_ERR";7307		break;7308	case TMR:7309		type = "TMR";7310		break;7311	default:7312		dev_err(hdev->dev, "unknown NIC AXI cause %d\n",7313			eq_nic_sei->axi_error_cause);7314		type = "N/A";7315		break;7316	}7317 7318	snprintf(desc, sizeof(desc), "NIC%d_%s%d", nic_id, type,7319			eq_nic_sei->id);7320	dev_err_ratelimited(hdev->dev, "Received H/W interrupt %d [\"%s\"]\n",7321		event_type, desc);7322}7323 7324static int gaudi_compute_reset_late_init(struct hl_device *hdev)7325{7326	/* GAUDI doesn't support any reset except hard-reset */7327	return -EPERM;7328}7329 7330static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device,7331			struct hl_eq_hbm_ecc_data *hbm_ecc_data)7332{7333	u32 base, val, val2, wr_par, rd_par, ca_par, derr, serr, type, ch;7334	int rc = 0;7335 7336	if (hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &7337					CPU_BOOT_DEV_STS0_HBM_ECC_EN) {7338		if (!hbm_ecc_data) {7339			dev_err(hdev->dev, "No FW ECC data");7340			return 0;7341		}7342 7343		wr_par = FIELD_GET(CPUCP_PKT_HBM_ECC_INFO_WR_PAR_MASK,7344				le32_to_cpu(hbm_ecc_data->hbm_ecc_info));7345		rd_par = FIELD_GET(CPUCP_PKT_HBM_ECC_INFO_RD_PAR_MASK,7346				le32_to_cpu(hbm_ecc_data->hbm_ecc_info));7347		ca_par = FIELD_GET(CPUCP_PKT_HBM_ECC_INFO_CA_PAR_MASK,7348				le32_to_cpu(hbm_ecc_data->hbm_ecc_info));7349		derr = FIELD_GET(CPUCP_PKT_HBM_ECC_INFO_DERR_MASK,7350				le32_to_cpu(hbm_ecc_data->hbm_ecc_info));7351		serr = FIELD_GET(CPUCP_PKT_HBM_ECC_INFO_SERR_MASK,7352				le32_to_cpu(hbm_ecc_data->hbm_ecc_info));7353		type = FIELD_GET(CPUCP_PKT_HBM_ECC_INFO_TYPE_MASK,7354				le32_to_cpu(hbm_ecc_data->hbm_ecc_info));7355		ch = FIELD_GET(CPUCP_PKT_HBM_ECC_INFO_HBM_CH_MASK,7356				le32_to_cpu(hbm_ecc_data->hbm_ecc_info));7357 7358		dev_err(hdev->dev,7359			"HBM%d pc%d interrupts info: WR_PAR=%d, RD_PAR=%d, CA_PAR=%d, SERR=%d, DERR=%d\n",7360			device, ch, wr_par, rd_par, ca_par, serr, derr);7361		dev_err(hdev->dev,7362			"HBM%d pc%d ECC info: 1ST_ERR_ADDR=0x%x, 1ST_ERR_TYPE=%d, SEC_CONT_CNT=%u, SEC_CNT=%d, DEC_CNT=%d\n",7363			device, ch, hbm_ecc_data->first_addr, type,7364			hbm_ecc_data->sec_cont_cnt, hbm_ecc_data->sec_cnt,7365			hbm_ecc_data->dec_cnt);7366		return 0;7367	}7368 7369	if (hdev->asic_prop.fw_security_enabled) {7370		dev_info(hdev->dev, "Cannot access MC regs for ECC data while security is enabled\n");7371		return 0;7372	}7373 7374	base = GAUDI_HBM_CFG_BASE + device * GAUDI_HBM_CFG_OFFSET;7375	for (ch = 0 ; ch < GAUDI_HBM_CHANNELS ; ch++) {7376		val = RREG32_MASK(base + ch * 0x1000 + 0x06C, 0x0000FFFF);7377		val = (val & 0xFF) | ((val >> 8) & 0xFF);7378		if (val) {7379			rc = -EIO;7380			dev_err(hdev->dev,7381				"HBM%d pc%d interrupts info: WR_PAR=%d, RD_PAR=%d, CA_PAR=%d, SERR=%d, DERR=%d\n",7382				device, ch * 2, val & 0x1, (val >> 1) & 0x1,7383				(val >> 2) & 0x1, (val >> 3) & 0x1,7384				(val >> 4) & 0x1);7385 7386			val2 = RREG32(base + ch * 0x1000 + 0x060);7387			dev_err(hdev->dev,7388				"HBM%d pc%d ECC info: 1ST_ERR_ADDR=0x%x, 1ST_ERR_TYPE=%d, SEC_CONT_CNT=%d, SEC_CNT=%d, DEC_CNT=%d\n",7389				device, ch * 2,7390				RREG32(base + ch * 0x1000 + 0x064),7391				(val2 & 0x200) >> 9, (val2 & 0xFC00) >> 10,7392				(val2 & 0xFF0000) >> 16,7393				(val2 & 0xFF000000) >> 24);7394		}7395 7396		val = RREG32_MASK(base + ch * 0x1000 + 0x07C, 0x0000FFFF);7397		val = (val & 0xFF) | ((val >> 8) & 0xFF);7398		if (val) {7399			rc = -EIO;7400			dev_err(hdev->dev,7401				"HBM%d pc%d interrupts info: WR_PAR=%d, RD_PAR=%d, CA_PAR=%d, SERR=%d, DERR=%d\n",7402				device, ch * 2 + 1, val & 0x1, (val >> 1) & 0x1,7403				(val >> 2) & 0x1, (val >> 3) & 0x1,7404				(val >> 4) & 0x1);7405 7406			val2 = RREG32(base + ch * 0x1000 + 0x070);7407			dev_err(hdev->dev,7408				"HBM%d pc%d ECC info: 1ST_ERR_ADDR=0x%x, 1ST_ERR_TYPE=%d, SEC_CONT_CNT=%d, SEC_CNT=%d, DEC_CNT=%d\n",7409				device, ch * 2 + 1,7410				RREG32(base + ch * 0x1000 + 0x074),7411				(val2 & 0x200) >> 9, (val2 & 0xFC00) >> 10,7412				(val2 & 0xFF0000) >> 16,7413				(val2 & 0xFF000000) >> 24);7414		}7415 7416		/* Clear interrupts */7417		RMWREG32(base + (ch * 0x1000) + 0x060, 0x1C8, 0x1FF);7418		RMWREG32(base + (ch * 0x1000) + 0x070, 0x1C8, 0x1FF);7419		WREG32(base + (ch * 0x1000) + 0x06C, 0x1F1F);7420		WREG32(base + (ch * 0x1000) + 0x07C, 0x1F1F);7421		RMWREG32(base + (ch * 0x1000) + 0x060, 0x0, 0xF);7422		RMWREG32(base + (ch * 0x1000) + 0x070, 0x0, 0xF);7423	}7424 7425	val  = RREG32(base + 0x8F30);7426	val2 = RREG32(base + 0x8F34);7427	if (val | val2) {7428		rc = -EIO;7429		dev_err(hdev->dev,7430			"HBM %d MC SRAM SERR info: Reg 0x8F30=0x%x, Reg 0x8F34=0x%x\n",7431			device, val, val2);7432	}7433	val  = RREG32(base + 0x8F40);7434	val2 = RREG32(base + 0x8F44);7435	if (val | val2) {7436		rc = -EIO;7437		dev_err(hdev->dev,7438			"HBM %d MC SRAM DERR info: Reg 0x8F40=0x%x, Reg 0x8F44=0x%x\n",7439			device, val, val2);7440	}7441 7442	return rc;7443}7444 7445static int gaudi_hbm_event_to_dev(u16 hbm_event_type)7446{7447	switch (hbm_event_type) {7448	case GAUDI_EVENT_HBM0_SPI_0:7449	case GAUDI_EVENT_HBM0_SPI_1:7450		return 0;7451	case GAUDI_EVENT_HBM1_SPI_0:7452	case GAUDI_EVENT_HBM1_SPI_1:7453		return 1;7454	case GAUDI_EVENT_HBM2_SPI_0:7455	case GAUDI_EVENT_HBM2_SPI_1:7456		return 2;7457	case GAUDI_EVENT_HBM3_SPI_0:7458	case GAUDI_EVENT_HBM3_SPI_1:7459		return 3;7460	default:7461		break;7462	}7463 7464	/* Should never happen */7465	return 0;7466}7467 7468static bool gaudi_tpc_read_interrupts(struct hl_device *hdev, u8 tpc_id,7469					char *interrupt_name)7470{7471	u32 tpc_offset = tpc_id * TPC_CFG_OFFSET, tpc_interrupts_cause, i;7472	bool soft_reset_required = false;7473 7474	tpc_interrupts_cause = RREG32(mmTPC0_CFG_TPC_INTR_CAUSE + tpc_offset) &7475				TPC0_CFG_TPC_INTR_CAUSE_CAUSE_MASK;7476 7477	for (i = 0 ; i < GAUDI_NUM_OF_TPC_INTR_CAUSE ; i++)7478		if (tpc_interrupts_cause & BIT(i)) {7479			dev_err_ratelimited(hdev->dev,7480					"TPC%d_%s interrupt cause: %s\n",7481					tpc_id, interrupt_name,7482					gaudi_tpc_interrupts_cause[i]);7483			/* If this is QM error, we need to soft-reset */7484			if (i == 15)7485				soft_reset_required = true;7486		}7487 7488	/* Clear interrupts */7489	WREG32(mmTPC0_CFG_TPC_INTR_CAUSE + tpc_offset, 0);7490 7491	return soft_reset_required;7492}7493 7494static int tpc_dec_event_to_tpc_id(u16 tpc_dec_event_type)7495{7496	return (tpc_dec_event_type - GAUDI_EVENT_TPC0_DEC) >> 1;7497}7498 7499static int tpc_krn_event_to_tpc_id(u16 tpc_dec_event_type)7500{7501	return (tpc_dec_event_type - GAUDI_EVENT_TPC0_KRN_ERR) / 6;7502}7503 7504static void gaudi_print_clk_change_info(struct hl_device *hdev, u16 event_type, u64 *event_mask)7505{7506	ktime_t zero_time = ktime_set(0, 0);7507 7508	mutex_lock(&hdev->clk_throttling.lock);7509 7510	switch (event_type) {7511	case GAUDI_EVENT_FIX_POWER_ENV_S:7512		hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_POWER;7513		hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_POWER;7514		hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].start = ktime_get();7515		hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = zero_time;7516		dev_info_ratelimited(hdev->dev,7517			"Clock throttling due to power consumption\n");7518		break;7519 7520	case GAUDI_EVENT_FIX_POWER_ENV_E:7521		hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_POWER;7522		hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = ktime_get();7523		dev_info_ratelimited(hdev->dev,7524			"Power envelop is safe, back to optimal clock\n");7525		break;7526 7527	case GAUDI_EVENT_FIX_THERMAL_ENV_S:7528		hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_THERMAL;7529		hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_THERMAL;7530		hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].start = ktime_get();7531		hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = zero_time;7532		*event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;7533		dev_info_ratelimited(hdev->dev,7534			"Clock throttling due to overheating\n");7535		break;7536 7537	case GAUDI_EVENT_FIX_THERMAL_ENV_E:7538		hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_THERMAL;7539		hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = ktime_get();7540		*event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;7541		dev_info_ratelimited(hdev->dev,7542			"Thermal envelop is safe, back to optimal clock\n");7543		break;7544 7545	default:7546		dev_err(hdev->dev, "Received invalid clock change event %d\n",7547			event_type);7548		break;7549	}7550 7551	mutex_unlock(&hdev->clk_throttling.lock);7552}7553 7554static void gaudi_handle_eqe(struct hl_device *hdev, struct hl_eq_entry *eq_entry)7555{7556	struct gaudi_device *gaudi = hdev->asic_specific;7557	struct hl_info_fw_err_info fw_err_info;7558	u64 data = le64_to_cpu(eq_entry->data[0]), event_mask = 0;7559	u32 ctl = le32_to_cpu(eq_entry->hdr.ctl);7560	u32 fw_fatal_err_flag = 0, flags = 0;7561	u16 event_type = ((ctl & EQ_CTL_EVENT_TYPE_MASK)7562			>> EQ_CTL_EVENT_TYPE_SHIFT);7563	bool reset_required, reset_direct = false;7564	u8 cause;7565	int rc;7566 7567	if (event_type >= GAUDI_EVENT_SIZE) {7568		dev_err(hdev->dev, "Event type %u exceeds maximum of %u",7569				event_type, GAUDI_EVENT_SIZE - 1);7570		return;7571	}7572 7573	gaudi->events_stat[event_type]++;7574	gaudi->events_stat_aggregate[event_type]++;7575 7576	switch (event_type) {7577	case GAUDI_EVENT_PCIE_CORE_DERR:7578	case GAUDI_EVENT_PCIE_IF_DERR:7579	case GAUDI_EVENT_PCIE_PHY_DERR:7580	case GAUDI_EVENT_TPC0_DERR ... GAUDI_EVENT_TPC7_DERR:7581	case GAUDI_EVENT_MME0_ACC_DERR:7582	case GAUDI_EVENT_MME0_SBAB_DERR:7583	case GAUDI_EVENT_MME1_ACC_DERR:7584	case GAUDI_EVENT_MME1_SBAB_DERR:7585	case GAUDI_EVENT_MME2_ACC_DERR:7586	case GAUDI_EVENT_MME2_SBAB_DERR:7587	case GAUDI_EVENT_MME3_ACC_DERR:7588	case GAUDI_EVENT_MME3_SBAB_DERR:7589	case GAUDI_EVENT_DMA0_DERR_ECC ... GAUDI_EVENT_DMA7_DERR_ECC:7590		fallthrough;7591	case GAUDI_EVENT_CPU_IF_ECC_DERR:7592	case GAUDI_EVENT_PSOC_MEM_DERR:7593	case GAUDI_EVENT_PSOC_CORESIGHT_DERR:7594	case GAUDI_EVENT_SRAM0_DERR ... GAUDI_EVENT_SRAM28_DERR:7595	case GAUDI_EVENT_NIC0_DERR ... GAUDI_EVENT_NIC4_DERR:7596	case GAUDI_EVENT_DMA_IF0_DERR ... GAUDI_EVENT_DMA_IF3_DERR:7597	case GAUDI_EVENT_HBM_0_DERR ... GAUDI_EVENT_HBM_3_DERR:7598	case GAUDI_EVENT_MMU_DERR:7599	case GAUDI_EVENT_NIC0_CS_DBG_DERR ... GAUDI_EVENT_NIC4_CS_DBG_DERR:7600		gaudi_print_irq_info(hdev, event_type, true, &event_mask);7601		gaudi_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data);7602		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;7603		fw_fatal_err_flag = HL_DRV_RESET_FW_FATAL_ERR;7604		goto reset_device;7605 7606	case GAUDI_EVENT_GIC500:7607	case GAUDI_EVENT_AXI_ECC:7608	case GAUDI_EVENT_L2_RAM_ECC:7609	case GAUDI_EVENT_PLL0 ... GAUDI_EVENT_PLL17:7610		gaudi_print_irq_info(hdev, event_type, false, &event_mask);7611		fw_fatal_err_flag = HL_DRV_RESET_FW_FATAL_ERR;7612		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;7613		goto reset_device;7614 7615	case GAUDI_EVENT_HBM0_SPI_0:7616	case GAUDI_EVENT_HBM1_SPI_0:7617	case GAUDI_EVENT_HBM2_SPI_0:7618	case GAUDI_EVENT_HBM3_SPI_0:7619		gaudi_print_irq_info(hdev, event_type, false, &event_mask);7620		gaudi_hbm_read_interrupts(hdev,7621				gaudi_hbm_event_to_dev(event_type),7622				&eq_entry->hbm_ecc_data);7623		fw_fatal_err_flag = HL_DRV_RESET_FW_FATAL_ERR;7624		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;7625		goto reset_device;7626 7627	case GAUDI_EVENT_HBM0_SPI_1:7628	case GAUDI_EVENT_HBM1_SPI_1:7629	case GAUDI_EVENT_HBM2_SPI_1:7630	case GAUDI_EVENT_HBM3_SPI_1:7631		gaudi_print_irq_info(hdev, event_type, false, &event_mask);7632		gaudi_hbm_read_interrupts(hdev,7633				gaudi_hbm_event_to_dev(event_type),7634				&eq_entry->hbm_ecc_data);7635		hl_fw_unmask_irq(hdev, event_type);7636		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;7637		break;7638 7639	case GAUDI_EVENT_TPC0_DEC:7640	case GAUDI_EVENT_TPC1_DEC:7641	case GAUDI_EVENT_TPC2_DEC:7642	case GAUDI_EVENT_TPC3_DEC:7643	case GAUDI_EVENT_TPC4_DEC:7644	case GAUDI_EVENT_TPC5_DEC:7645	case GAUDI_EVENT_TPC6_DEC:7646	case GAUDI_EVENT_TPC7_DEC:7647		/* In TPC DEC event, notify on TPC assertion. While there isn't7648		 * a specific event for assertion yet, the FW generates TPC DEC event.7649		 * The SW upper layer will inspect an internal mapped area to indicate7650		 * if the event is a TPC Assertion or a "real" TPC DEC.7651		 */7652		event_mask |= HL_NOTIFIER_EVENT_TPC_ASSERT;7653		gaudi_print_irq_info(hdev, event_type, true, &event_mask);7654		reset_required = gaudi_tpc_read_interrupts(hdev,7655					tpc_dec_event_to_tpc_id(event_type),7656					"AXI_SLV_DEC_Error");7657		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;7658		if (reset_required) {7659			dev_err(hdev->dev, "reset required due to %s\n",7660				gaudi_irq_map_table[event_type].name);7661 7662			reset_direct = true;7663			goto reset_device;7664		} else {7665			hl_fw_unmask_irq(hdev, event_type);7666			event_mask |= HL_NOTIFIER_EVENT_DEVICE_RESET;7667		}7668		break;7669 7670	case GAUDI_EVENT_TPC0_KRN_ERR:7671	case GAUDI_EVENT_TPC1_KRN_ERR:7672	case GAUDI_EVENT_TPC2_KRN_ERR:7673	case GAUDI_EVENT_TPC3_KRN_ERR:7674	case GAUDI_EVENT_TPC4_KRN_ERR:7675	case GAUDI_EVENT_TPC5_KRN_ERR:7676	case GAUDI_EVENT_TPC6_KRN_ERR:7677	case GAUDI_EVENT_TPC7_KRN_ERR:7678		gaudi_print_irq_info(hdev, event_type, true, &event_mask);7679		reset_required = gaudi_tpc_read_interrupts(hdev,7680					tpc_krn_event_to_tpc_id(event_type),7681					"KRN_ERR");7682		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;7683		if (reset_required) {7684			dev_err(hdev->dev, "reset required due to %s\n",7685				gaudi_irq_map_table[event_type].name);7686 7687			reset_direct = true;7688			goto reset_device;7689		} else {7690			hl_fw_unmask_irq(hdev, event_type);7691			event_mask |= HL_NOTIFIER_EVENT_DEVICE_RESET;7692		}7693		break;7694 7695	case GAUDI_EVENT_PCIE_CORE_SERR:7696	case GAUDI_EVENT_PCIE_IF_SERR:7697	case GAUDI_EVENT_PCIE_PHY_SERR:7698	case GAUDI_EVENT_TPC0_SERR ... GAUDI_EVENT_TPC7_SERR:7699	case GAUDI_EVENT_MME0_ACC_SERR:7700	case GAUDI_EVENT_MME0_SBAB_SERR:7701	case GAUDI_EVENT_MME1_ACC_SERR:7702	case GAUDI_EVENT_MME1_SBAB_SERR:7703	case GAUDI_EVENT_MME2_ACC_SERR:7704	case GAUDI_EVENT_MME2_SBAB_SERR:7705	case GAUDI_EVENT_MME3_ACC_SERR:7706	case GAUDI_EVENT_MME3_SBAB_SERR:7707	case GAUDI_EVENT_DMA0_SERR_ECC ... GAUDI_EVENT_DMA7_SERR_ECC:7708	case GAUDI_EVENT_CPU_IF_ECC_SERR:7709	case GAUDI_EVENT_PSOC_MEM_SERR:7710	case GAUDI_EVENT_PSOC_CORESIGHT_SERR:7711	case GAUDI_EVENT_SRAM0_SERR ... GAUDI_EVENT_SRAM28_SERR:7712	case GAUDI_EVENT_NIC0_SERR ... GAUDI_EVENT_NIC4_SERR:7713	case GAUDI_EVENT_DMA_IF0_SERR ... GAUDI_EVENT_DMA_IF3_SERR:7714	case GAUDI_EVENT_HBM_0_SERR ... GAUDI_EVENT_HBM_3_SERR:7715		fallthrough;7716	case GAUDI_EVENT_MMU_SERR:7717		gaudi_print_irq_info(hdev, event_type, true, &event_mask);7718		gaudi_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data);7719		hl_fw_unmask_irq(hdev, event_type);7720		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;7721		break;7722 7723	case GAUDI_EVENT_PCIE_DEC:7724	case GAUDI_EVENT_CPU_AXI_SPLITTER:7725	case GAUDI_EVENT_PSOC_AXI_DEC:7726	case GAUDI_EVENT_PSOC_PRSTN_FALL:7727		gaudi_print_irq_info(hdev, event_type, true, &event_mask);7728		hl_fw_unmask_irq(hdev, event_type);7729		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;7730		break;7731 7732	case GAUDI_EVENT_MMU_PAGE_FAULT:7733	case GAUDI_EVENT_MMU_WR_PERM:7734		gaudi_print_irq_info(hdev, event_type, true, &event_mask);7735		hl_fw_unmask_irq(hdev, event_type);7736		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;7737		break;7738 7739	case GAUDI_EVENT_MME0_WBC_RSP:7740	case GAUDI_EVENT_MME0_SBAB0_RSP:7741	case GAUDI_EVENT_MME1_WBC_RSP:7742	case GAUDI_EVENT_MME1_SBAB0_RSP:7743	case GAUDI_EVENT_MME2_WBC_RSP:7744	case GAUDI_EVENT_MME2_SBAB0_RSP:7745	case GAUDI_EVENT_MME3_WBC_RSP:7746	case GAUDI_EVENT_MME3_SBAB0_RSP:7747	case GAUDI_EVENT_RAZWI_OR_ADC:7748	case GAUDI_EVENT_MME0_QM ... GAUDI_EVENT_MME2_QM:7749	case GAUDI_EVENT_DMA0_QM ... GAUDI_EVENT_DMA7_QM:7750		fallthrough;7751	case GAUDI_EVENT_NIC0_QM0:7752	case GAUDI_EVENT_NIC0_QM1:7753	case GAUDI_EVENT_NIC1_QM0:7754	case GAUDI_EVENT_NIC1_QM1:7755	case GAUDI_EVENT_NIC2_QM0:7756	case GAUDI_EVENT_NIC2_QM1:7757	case GAUDI_EVENT_NIC3_QM0:7758	case GAUDI_EVENT_NIC3_QM1:7759	case GAUDI_EVENT_NIC4_QM0:7760	case GAUDI_EVENT_NIC4_QM1:7761	case GAUDI_EVENT_DMA0_CORE ... GAUDI_EVENT_DMA7_CORE:7762	case GAUDI_EVENT_TPC0_QM ... GAUDI_EVENT_TPC7_QM:7763		gaudi_print_irq_info(hdev, event_type, true, &event_mask);7764		gaudi_handle_qman_err(hdev, event_type, &event_mask);7765		hl_fw_unmask_irq(hdev, event_type);7766		event_mask |= (HL_NOTIFIER_EVENT_USER_ENGINE_ERR | HL_NOTIFIER_EVENT_DEVICE_RESET);7767		break;7768 7769	case GAUDI_EVENT_RAZWI_OR_ADC_SW:7770		gaudi_print_irq_info(hdev, event_type, true, &event_mask);7771		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;7772		goto reset_device;7773 7774	case GAUDI_EVENT_TPC0_BMON_SPMU:7775	case GAUDI_EVENT_TPC1_BMON_SPMU:7776	case GAUDI_EVENT_TPC2_BMON_SPMU:7777	case GAUDI_EVENT_TPC3_BMON_SPMU:7778	case GAUDI_EVENT_TPC4_BMON_SPMU:7779	case GAUDI_EVENT_TPC5_BMON_SPMU:7780	case GAUDI_EVENT_TPC6_BMON_SPMU:7781	case GAUDI_EVENT_TPC7_BMON_SPMU:7782	case GAUDI_EVENT_DMA_BM_CH0 ... GAUDI_EVENT_DMA_BM_CH7:7783		gaudi_print_irq_info(hdev, event_type, false, &event_mask);7784		hl_fw_unmask_irq(hdev, event_type);7785		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;7786		break;7787 7788	case GAUDI_EVENT_NIC_SEI_0 ... GAUDI_EVENT_NIC_SEI_4:7789		gaudi_print_nic_axi_irq_info(hdev, event_type, &data);7790		hl_fw_unmask_irq(hdev, event_type);7791		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;7792		break;7793 7794	case GAUDI_EVENT_DMA_IF_SEI_0 ... GAUDI_EVENT_DMA_IF_SEI_3:7795		gaudi_print_irq_info(hdev, event_type, false, &event_mask);7796		gaudi_print_sm_sei_info(hdev, event_type,7797					&eq_entry->sm_sei_data);7798		rc = hl_state_dump(hdev);7799		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;7800		if (rc)7801			dev_err(hdev->dev,7802				"Error during system state dump %d\n", rc);7803		hl_fw_unmask_irq(hdev, event_type);7804		break;7805 7806	case GAUDI_EVENT_STATUS_NIC0_ENG0 ... GAUDI_EVENT_STATUS_NIC4_ENG1:7807		break;7808 7809	case GAUDI_EVENT_FIX_POWER_ENV_S ... GAUDI_EVENT_FIX_THERMAL_ENV_E:7810		gaudi_print_clk_change_info(hdev, event_type, &event_mask);7811		hl_fw_unmask_irq(hdev, event_type);7812		break;7813 7814	case GAUDI_EVENT_PSOC_GPIO_U16_0:7815		cause = le64_to_cpu(eq_entry->data[0]) & 0xFF;7816		dev_err(hdev->dev,7817			"Received high temp H/W interrupt %d (cause %d)\n",7818			event_type, cause);7819		event_mask |= HL_NOTIFIER_EVENT_USER_ENGINE_ERR;7820		break;7821 7822	case GAUDI_EVENT_DEV_RESET_REQ:7823		gaudi_print_irq_info(hdev, event_type, false, &event_mask);7824		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;7825		goto reset_device;7826 7827	case GAUDI_EVENT_PKT_QUEUE_OUT_SYNC:7828		gaudi_print_irq_info(hdev, event_type, false, &event_mask);7829		gaudi_print_out_of_sync_info(hdev, &eq_entry->pkt_sync_err);7830		event_mask |= HL_NOTIFIER_EVENT_GENERAL_HW_ERR;7831		goto reset_device;7832 7833	case GAUDI_EVENT_FW_ALIVE_S:7834		gaudi_print_irq_info(hdev, event_type, false, &event_mask);7835		gaudi_print_fw_alive_info(hdev, &eq_entry->fw_alive);7836		fw_err_info.err_type = HL_INFO_FW_REPORTED_ERR;7837		fw_err_info.event_id = event_type;7838		fw_err_info.event_mask = &event_mask;7839		hl_handle_fw_err(hdev, &fw_err_info);7840		goto reset_device;7841 7842	default:7843		dev_err(hdev->dev, "Received invalid H/W interrupt %d\n",7844				event_type);7845		break;7846	}7847 7848	if (event_mask)7849		hl_notifier_event_send_all(hdev, event_mask);7850 7851	return;7852 7853reset_device:7854	reset_required = true;7855 7856	if (hdev->asic_prop.fw_security_enabled && !reset_direct) {7857		flags = HL_DRV_RESET_HARD | HL_DRV_RESET_BYPASS_REQ_TO_FW | fw_fatal_err_flag;7858 7859		/* notify on device unavailable while the reset triggered by fw */7860		event_mask |= (HL_NOTIFIER_EVENT_DEVICE_RESET |7861					HL_NOTIFIER_EVENT_DEVICE_UNAVAILABLE);7862	} else if (hdev->hard_reset_on_fw_events) {7863		flags = HL_DRV_RESET_HARD | HL_DRV_RESET_DELAY | fw_fatal_err_flag;7864		event_mask |= HL_NOTIFIER_EVENT_DEVICE_RESET;7865	} else {7866		reset_required = false;7867	}7868 7869	if (reset_required) {7870		/* escalate general hw errors to critical/fatal error */7871		if (event_mask & HL_NOTIFIER_EVENT_GENERAL_HW_ERR)7872			hl_handle_critical_hw_err(hdev, event_type, &event_mask);7873 7874		hl_device_cond_reset(hdev, flags, event_mask);7875	} else {7876		hl_fw_unmask_irq(hdev, event_type);7877		/* Notification on occurred event needs to be sent although reset is not executed */7878		if (event_mask)7879			hl_notifier_event_send_all(hdev, event_mask);7880	}7881}7882 7883static void *gaudi_get_events_stat(struct hl_device *hdev, bool aggregate, u32 *size)7884{7885	struct gaudi_device *gaudi = hdev->asic_specific;7886 7887	if (aggregate) {7888		*size = (u32) sizeof(gaudi->events_stat_aggregate);7889		return gaudi->events_stat_aggregate;7890	}7891 7892	*size = (u32) sizeof(gaudi->events_stat);7893	return gaudi->events_stat;7894}7895 7896static int gaudi_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags)7897{7898	struct gaudi_device *gaudi = hdev->asic_specific;7899	u32 status, timeout_usec;7900	int rc;7901 7902	if (!(gaudi->hw_cap_initialized & HW_CAP_MMU) ||7903		hdev->reset_info.hard_reset_pending)7904		return 0;7905 7906	if (hdev->pldm)7907		timeout_usec = GAUDI_PLDM_MMU_TIMEOUT_USEC;7908	else7909		timeout_usec = MMU_CONFIG_TIMEOUT_USEC;7910 7911	/* L0 & L1 invalidation */7912	WREG32(mmSTLB_INV_PS, 3);7913	WREG32(mmSTLB_CACHE_INV, gaudi->mmu_cache_inv_pi++);7914	WREG32(mmSTLB_INV_PS, 2);7915 7916	rc = hl_poll_timeout(7917		hdev,7918		mmSTLB_INV_PS,7919		status,7920		!status,7921		1000,7922		timeout_usec);7923 7924	WREG32(mmSTLB_INV_SET, 0);7925 7926	return rc;7927}7928 7929static int gaudi_mmu_invalidate_cache_range(struct hl_device *hdev,7930						bool is_hard, u32 flags,7931						u32 asid, u64 va, u64 size)7932{7933	/* Treat as invalidate all because there is no range invalidation7934	 * in Gaudi7935	 */7936	return hdev->asic_funcs->mmu_invalidate_cache(hdev, is_hard, flags);7937}7938 7939static int gaudi_mmu_update_asid_hop0_addr(struct hl_device *hdev, u32 asid, u64 phys_addr)7940{7941	u32 status, timeout_usec;7942	int rc;7943 7944	if (hdev->pldm)7945		timeout_usec = GAUDI_PLDM_MMU_TIMEOUT_USEC;7946	else7947		timeout_usec = MMU_CONFIG_TIMEOUT_USEC;7948 7949	WREG32(MMU_ASID, asid);7950	WREG32(MMU_HOP0_PA43_12, phys_addr >> MMU_HOP0_PA43_12_SHIFT);7951	WREG32(MMU_HOP0_PA49_44, phys_addr >> MMU_HOP0_PA49_44_SHIFT);7952	WREG32(MMU_BUSY, 0x80000000);7953 7954	rc = hl_poll_timeout(7955		hdev,7956		MMU_BUSY,7957		status,7958		!(status & 0x80000000),7959		1000,7960		timeout_usec);7961 7962	if (rc) {7963		dev_err(hdev->dev,7964			"Timeout during MMU hop0 config of asid %d\n", asid);7965		return rc;7966	}7967 7968	return 0;7969}7970 7971static int gaudi_send_heartbeat(struct hl_device *hdev)7972{7973	struct gaudi_device *gaudi = hdev->asic_specific;7974 7975	if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))7976		return 0;7977 7978	return hl_fw_send_heartbeat(hdev);7979}7980 7981static int gaudi_cpucp_info_get(struct hl_device *hdev)7982{7983	struct gaudi_device *gaudi = hdev->asic_specific;7984	struct asic_fixed_properties *prop = &hdev->asic_prop;7985	int rc;7986 7987	if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))7988		return 0;7989 7990	rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0,7991					mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0,7992					mmCPU_BOOT_ERR1);7993	if (rc)7994		return rc;7995 7996	if (!strlen(prop->cpucp_info.card_name))7997		strscpy_pad(prop->cpucp_info.card_name, GAUDI_DEFAULT_CARD_NAME,7998				CARD_NAME_MAX_LEN);7999 8000	hdev->card_type = le32_to_cpu(hdev->asic_prop.cpucp_info.card_type);8001 8002	set_default_power_values(hdev);8003 8004	return 0;8005}8006 8007static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask_arr, u8 mask_len,8008		struct engines_data *e)8009{8010	struct gaudi_device *gaudi = hdev->asic_specific;8011	const char *fmt = "%-5d%-9s%#-14x%#-12x%#x\n";8012	const char *mme_slave_fmt = "%-5d%-9s%-14s%-12s%#x\n";8013	const char *nic_fmt = "%-5d%-9s%#-14x%#x\n";8014	unsigned long *mask = (unsigned long *)mask_arr;8015	u32 qm_glbl_sts0, qm_cgm_sts, dma_core_sts0, tpc_cfg_sts, mme_arch_sts;8016	bool is_idle = true, is_eng_idle, is_slave;8017	u64 offset;8018	int i, dma_id, port;8019 8020	if (e)8021		hl_engine_data_sprintf(e,8022			"\nDMA  is_idle  QM_GLBL_STS0  QM_CGM_STS  DMA_CORE_STS0\n"8023			"---  -------  ------------  ----------  -------------\n");8024 8025	for (i = 0 ; i < DMA_NUMBER_OF_CHNLS ; i++) {8026		dma_id = gaudi_dma_assignment[i];8027		offset = dma_id * DMA_QMAN_OFFSET;8028 8029		qm_glbl_sts0 = RREG32(mmDMA0_QM_GLBL_STS0 + offset);8030		qm_cgm_sts = RREG32(mmDMA0_QM_CGM_STS + offset);8031		dma_core_sts0 = RREG32(mmDMA0_CORE_STS0 + offset);8032		is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts) &&8033				IS_DMA_IDLE(dma_core_sts0);8034		is_idle &= is_eng_idle;8035 8036		if (mask && !is_eng_idle)8037			set_bit(GAUDI_ENGINE_ID_DMA_0 + dma_id, mask);8038		if (e)8039			hl_engine_data_sprintf(e, fmt, dma_id,8040				is_eng_idle ? "Y" : "N", qm_glbl_sts0,8041				qm_cgm_sts, dma_core_sts0);8042	}8043 8044	if (e)8045		hl_engine_data_sprintf(e,8046			"\nTPC  is_idle  QM_GLBL_STS0  QM_CGM_STS  CFG_STATUS\n"8047			"---  -------  ------------  ----------  ----------\n");8048 8049	for (i = 0 ; i < TPC_NUMBER_OF_ENGINES ; i++) {8050		offset = i * TPC_QMAN_OFFSET;8051		qm_glbl_sts0 = RREG32(mmTPC0_QM_GLBL_STS0 + offset);8052		qm_cgm_sts = RREG32(mmTPC0_QM_CGM_STS + offset);8053		tpc_cfg_sts = RREG32(mmTPC0_CFG_STATUS + offset);8054		is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts) &&8055				IS_TPC_IDLE(tpc_cfg_sts);8056		is_idle &= is_eng_idle;8057 8058		if (mask && !is_eng_idle)8059			set_bit(GAUDI_ENGINE_ID_TPC_0 + i, mask);8060		if (e)8061			hl_engine_data_sprintf(e, fmt, i,8062				is_eng_idle ? "Y" : "N",8063				qm_glbl_sts0, qm_cgm_sts, tpc_cfg_sts);8064	}8065 8066	if (e)8067		hl_engine_data_sprintf(e,8068			"\nMME  is_idle  QM_GLBL_STS0  QM_CGM_STS  ARCH_STATUS\n"8069			"---  -------  ------------  ----------  -----------\n");8070 8071	for (i = 0 ; i < MME_NUMBER_OF_ENGINES ; i++) {8072		offset = i * MME_QMAN_OFFSET;8073		mme_arch_sts = RREG32(mmMME0_CTRL_ARCH_STATUS + offset);8074		is_eng_idle = IS_MME_IDLE(mme_arch_sts);8075 8076		/* MME 1 & 3 are slaves, no need to check their QMANs */8077		is_slave = i % 2;8078		if (!is_slave) {8079			qm_glbl_sts0 = RREG32(mmMME0_QM_GLBL_STS0 + offset);8080			qm_cgm_sts = RREG32(mmMME0_QM_CGM_STS + offset);8081			is_eng_idle &= IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts);8082		}8083 8084		is_idle &= is_eng_idle;8085 8086		if (mask && !is_eng_idle)8087			set_bit(GAUDI_ENGINE_ID_MME_0 + i, mask);8088		if (e) {8089			if (!is_slave)8090				hl_engine_data_sprintf(e, fmt, i,8091					is_eng_idle ? "Y" : "N",8092					qm_glbl_sts0, qm_cgm_sts, mme_arch_sts);8093			else8094				hl_engine_data_sprintf(e, mme_slave_fmt, i,8095					is_eng_idle ? "Y" : "N", "-",8096					"-", mme_arch_sts);8097		}8098	}8099 8100	if (e)8101		hl_engine_data_sprintf(e,8102				"\nNIC  is_idle  QM_GLBL_STS0  QM_CGM_STS\n"8103				"---  -------  ------------  ----------\n");8104 8105	for (i = 0 ; i < (NIC_NUMBER_OF_ENGINES / 2) ; i++) {8106		offset = i * NIC_MACRO_QMAN_OFFSET;8107		port = 2 * i;8108		if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + port)) {8109			qm_glbl_sts0 = RREG32(mmNIC0_QM0_GLBL_STS0 + offset);8110			qm_cgm_sts = RREG32(mmNIC0_QM0_CGM_STS + offset);8111			is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts);8112			is_idle &= is_eng_idle;8113 8114			if (mask && !is_eng_idle)8115				set_bit(GAUDI_ENGINE_ID_NIC_0 + port, mask);8116			if (e)8117				hl_engine_data_sprintf(e, nic_fmt, port,8118						is_eng_idle ? "Y" : "N",8119						qm_glbl_sts0, qm_cgm_sts);8120		}8121 8122		port = 2 * i + 1;8123		if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + port)) {8124			qm_glbl_sts0 = RREG32(mmNIC0_QM1_GLBL_STS0 + offset);8125			qm_cgm_sts = RREG32(mmNIC0_QM1_CGM_STS + offset);8126			is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts);8127			is_idle &= is_eng_idle;8128 8129			if (mask && !is_eng_idle)8130				set_bit(GAUDI_ENGINE_ID_NIC_0 + port, mask);8131			if (e)8132				hl_engine_data_sprintf(e, nic_fmt, port,8133						is_eng_idle ? "Y" : "N",8134						qm_glbl_sts0, qm_cgm_sts);8135		}8136	}8137 8138	if (e)8139		hl_engine_data_sprintf(e, "\n");8140 8141	return is_idle;8142}8143 8144static void gaudi_hw_queues_lock(struct hl_device *hdev)8145	__acquires(&gaudi->hw_queues_lock)8146{8147	struct gaudi_device *gaudi = hdev->asic_specific;8148 8149	spin_lock(&gaudi->hw_queues_lock);8150}8151 8152static void gaudi_hw_queues_unlock(struct hl_device *hdev)8153	__releases(&gaudi->hw_queues_lock)8154{8155	struct gaudi_device *gaudi = hdev->asic_specific;8156 8157	spin_unlock(&gaudi->hw_queues_lock);8158}8159 8160static u32 gaudi_get_pci_id(struct hl_device *hdev)8161{8162	return hdev->pdev->device;8163}8164 8165static int gaudi_get_eeprom_data(struct hl_device *hdev, void *data,8166				size_t max_size)8167{8168	struct gaudi_device *gaudi = hdev->asic_specific;8169 8170	if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))8171		return 0;8172 8173	return hl_fw_get_eeprom_data(hdev, data, max_size);8174}8175 8176static int gaudi_get_monitor_dump(struct hl_device *hdev, void *data)8177{8178	struct gaudi_device *gaudi = hdev->asic_specific;8179 8180	if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))8181		return 0;8182 8183	return hl_fw_get_monitor_dump(hdev, data);8184}8185 8186/*8187 * this function should be used only during initialization and/or after reset,8188 * when there are no active users.8189 */8190static int gaudi_run_tpc_kernel(struct hl_device *hdev, u64 tpc_kernel,	u32 tpc_id)8191{8192	u64 kernel_timeout;8193	u32 status, offset;8194	int rc;8195 8196	offset = tpc_id * (mmTPC1_CFG_STATUS - mmTPC0_CFG_STATUS);8197 8198	if (hdev->pldm)8199		kernel_timeout = GAUDI_PLDM_TPC_KERNEL_WAIT_USEC;8200	else8201		kernel_timeout = HL_DEVICE_TIMEOUT_USEC;8202 8203	WREG32(mmTPC0_CFG_QM_KERNEL_BASE_ADDRESS_LOW + offset,8204			lower_32_bits(tpc_kernel));8205	WREG32(mmTPC0_CFG_QM_KERNEL_BASE_ADDRESS_HIGH + offset,8206			upper_32_bits(tpc_kernel));8207 8208	WREG32(mmTPC0_CFG_ICACHE_BASE_ADDERESS_LOW + offset,8209			lower_32_bits(tpc_kernel));8210	WREG32(mmTPC0_CFG_ICACHE_BASE_ADDERESS_HIGH + offset,8211			upper_32_bits(tpc_kernel));8212	/* set a valid LUT pointer, content is of no significance */8213	WREG32(mmTPC0_CFG_LUT_FUNC256_BASE_ADDR_LO + offset,8214			lower_32_bits(tpc_kernel));8215	WREG32(mmTPC0_CFG_LUT_FUNC256_BASE_ADDR_HI + offset,8216			upper_32_bits(tpc_kernel));8217 8218	WREG32(mmTPC0_CFG_QM_SYNC_OBJECT_ADDR + offset,8219			lower_32_bits(CFG_BASE +8220				mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0));8221 8222	WREG32(mmTPC0_CFG_TPC_CMD + offset,8223			(1 << TPC0_CFG_TPC_CMD_ICACHE_INVALIDATE_SHIFT |8224			1 << TPC0_CFG_TPC_CMD_ICACHE_PREFETCH_64KB_SHIFT));8225	/* wait a bit for the engine to start executing */8226	usleep_range(1000, 1500);8227 8228	/* wait until engine has finished executing */8229	rc = hl_poll_timeout(8230		hdev,8231		mmTPC0_CFG_STATUS + offset,8232		status,8233		(status & TPC0_CFG_STATUS_VECTOR_PIPE_EMPTY_MASK) ==8234				TPC0_CFG_STATUS_VECTOR_PIPE_EMPTY_MASK,8235		1000,8236		kernel_timeout);8237 8238	if (rc) {8239		dev_err(hdev->dev,8240			"Timeout while waiting for TPC%d icache prefetch\n",8241			tpc_id);8242		return -EIO;8243	}8244 8245	WREG32(mmTPC0_CFG_TPC_EXECUTE + offset,8246			1 << TPC0_CFG_TPC_EXECUTE_V_SHIFT);8247 8248	/* wait a bit for the engine to start executing */8249	usleep_range(1000, 1500);8250 8251	/* wait until engine has finished executing */8252	rc = hl_poll_timeout(8253		hdev,8254		mmTPC0_CFG_STATUS + offset,8255		status,8256		(status & TPC0_CFG_STATUS_VECTOR_PIPE_EMPTY_MASK) ==8257				TPC0_CFG_STATUS_VECTOR_PIPE_EMPTY_MASK,8258		1000,8259		kernel_timeout);8260 8261	if (rc) {8262		dev_err(hdev->dev,8263			"Timeout while waiting for TPC%d vector pipe\n",8264			tpc_id);8265		return -EIO;8266	}8267 8268	rc = hl_poll_timeout(8269		hdev,8270		mmTPC0_CFG_WQ_INFLIGHT_CNTR + offset,8271		status,8272		(status == 0),8273		1000,8274		kernel_timeout);8275 8276	if (rc) {8277		dev_err(hdev->dev,8278			"Timeout while waiting for TPC%d kernel to execute\n",8279			tpc_id);8280		return -EIO;8281	}8282 8283	return 0;8284}8285 8286static int gaudi_internal_cb_pool_init(struct hl_device *hdev,8287		struct hl_ctx *ctx)8288{8289	struct gaudi_device *gaudi = hdev->asic_specific;8290	int min_alloc_order, rc, collective_cb_size;8291 8292	if (!(gaudi->hw_cap_initialized & HW_CAP_MMU))8293		return 0;8294 8295	hdev->internal_cb_pool_virt_addr = hl_asic_dma_alloc_coherent(hdev,8296							HOST_SPACE_INTERNAL_CB_SZ,8297							&hdev->internal_cb_pool_dma_addr,8298							GFP_KERNEL | __GFP_ZERO);8299 8300	if (!hdev->internal_cb_pool_virt_addr)8301		return -ENOMEM;8302 8303	collective_cb_size = sizeof(struct packet_msg_short) * 5 +8304			sizeof(struct packet_fence);8305	min_alloc_order = ilog2(collective_cb_size);8306 8307	hdev->internal_cb_pool = gen_pool_create(min_alloc_order, -1);8308	if (!hdev->internal_cb_pool) {8309		dev_err(hdev->dev,8310			"Failed to create internal CB pool\n");8311		rc = -ENOMEM;8312		goto free_internal_cb_pool;8313	}8314 8315	rc = gen_pool_add(hdev->internal_cb_pool,8316				(uintptr_t) hdev->internal_cb_pool_virt_addr,8317				HOST_SPACE_INTERNAL_CB_SZ, -1);8318	if (rc) {8319		dev_err(hdev->dev,8320			"Failed to add memory to internal CB pool\n");8321		rc = -EFAULT;8322		goto destroy_internal_cb_pool;8323	}8324 8325	hdev->internal_cb_va_base = hl_reserve_va_block(hdev, ctx,8326			HL_VA_RANGE_TYPE_HOST, HOST_SPACE_INTERNAL_CB_SZ,8327			HL_MMU_VA_ALIGNMENT_NOT_NEEDED);8328 8329	if (!hdev->internal_cb_va_base) {8330		rc = -ENOMEM;8331		goto destroy_internal_cb_pool;8332	}8333 8334	mutex_lock(&hdev->mmu_lock);8335 8336	rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base,8337			hdev->internal_cb_pool_dma_addr,8338			HOST_SPACE_INTERNAL_CB_SZ);8339	if (rc)8340		goto unreserve_internal_cb_pool;8341 8342	rc = hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR);8343	if (rc)8344		goto unmap_internal_cb_pool;8345 8346	mutex_unlock(&hdev->mmu_lock);8347 8348	return 0;8349 8350unmap_internal_cb_pool:8351	hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base,8352			HOST_SPACE_INTERNAL_CB_SZ);8353unreserve_internal_cb_pool:8354	mutex_unlock(&hdev->mmu_lock);8355	hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base,8356			HOST_SPACE_INTERNAL_CB_SZ);8357destroy_internal_cb_pool:8358	gen_pool_destroy(hdev->internal_cb_pool);8359free_internal_cb_pool:8360	hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr,8361					hdev->internal_cb_pool_dma_addr);8362 8363	return rc;8364}8365 8366static void gaudi_internal_cb_pool_fini(struct hl_device *hdev,8367		struct hl_ctx *ctx)8368{8369	struct gaudi_device *gaudi = hdev->asic_specific;8370 8371	if (!(gaudi->hw_cap_initialized & HW_CAP_MMU))8372		return;8373 8374	mutex_lock(&hdev->mmu_lock);8375	hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base,8376			HOST_SPACE_INTERNAL_CB_SZ);8377	hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base,8378			HOST_SPACE_INTERNAL_CB_SZ);8379	hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR);8380	mutex_unlock(&hdev->mmu_lock);8381 8382	gen_pool_destroy(hdev->internal_cb_pool);8383 8384	hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr,8385					hdev->internal_cb_pool_dma_addr);8386}8387 8388static int gaudi_ctx_init(struct hl_ctx *ctx)8389{8390	int rc;8391 8392	if (ctx->asid == HL_KERNEL_ASID_ID)8393		return 0;8394 8395	rc = gaudi_internal_cb_pool_init(ctx->hdev, ctx);8396	if (rc)8397		return rc;8398 8399	rc = gaudi_restore_user_registers(ctx->hdev);8400	if (rc)8401		gaudi_internal_cb_pool_fini(ctx->hdev, ctx);8402 8403	return rc;8404}8405 8406static void gaudi_ctx_fini(struct hl_ctx *ctx)8407{8408	if (ctx->asid == HL_KERNEL_ASID_ID)8409		return;8410 8411	gaudi_internal_cb_pool_fini(ctx->hdev, ctx);8412}8413 8414static int gaudi_pre_schedule_cs(struct hl_cs *cs)8415{8416	return 0;8417}8418 8419static u32 gaudi_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx)8420{8421	return gaudi_cq_assignment[cq_idx];8422}8423 8424static u32 gaudi_get_signal_cb_size(struct hl_device *hdev)8425{8426	return sizeof(struct packet_msg_short) +8427			sizeof(struct packet_msg_prot) * 2;8428}8429 8430static u32 gaudi_get_wait_cb_size(struct hl_device *hdev)8431{8432	return sizeof(struct packet_msg_short) * 4 +8433			sizeof(struct packet_fence) +8434			sizeof(struct packet_msg_prot) * 2;8435}8436 8437static u32 gaudi_get_sob_addr(struct hl_device *hdev, u32 sob_id)8438{8439	return mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + (sob_id * 4);8440}8441 8442static u32 gaudi_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id,8443				u32 size, bool eb)8444{8445	struct hl_cb *cb = (struct hl_cb *) data;8446	struct packet_msg_short *pkt;8447	u32 value, ctl, pkt_size = sizeof(*pkt);8448 8449	pkt = cb->kernel_address + size;8450	memset(pkt, 0, pkt_size);8451 8452	/* Inc by 1, Mode ADD */8453	value = FIELD_PREP(GAUDI_PKT_SHORT_VAL_SOB_SYNC_VAL_MASK, 1);8454	value |= FIELD_PREP(GAUDI_PKT_SHORT_VAL_SOB_MOD_MASK, 1);8455 8456	ctl = FIELD_PREP(GAUDI_PKT_SHORT_CTL_ADDR_MASK, sob_id * 4);8457	ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_OP_MASK, 0); /* write the value */8458	ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_BASE_MASK, 3); /* W_S SOB base */8459	ctl |= FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT);8460	ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, eb);8461	ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1);8462	ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1);8463 8464	pkt->value = cpu_to_le32(value);8465	pkt->ctl = cpu_to_le32(ctl);8466 8467	return size + pkt_size;8468}8469 8470static u32 gaudi_add_mon_msg_short(struct packet_msg_short *pkt, u32 value,8471					u16 addr)8472{8473	u32 ctl, pkt_size = sizeof(*pkt);8474 8475	memset(pkt, 0, pkt_size);8476 8477	ctl = FIELD_PREP(GAUDI_PKT_SHORT_CTL_ADDR_MASK, addr);8478	ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_BASE_MASK, 2);  /* W_S MON base */8479	ctl |= FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT);8480	ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 0);8481	ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1);8482	ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 0); /* last pkt MB */8483 8484	pkt->value = cpu_to_le32(value);8485	pkt->ctl = cpu_to_le32(ctl);8486 8487	return pkt_size;8488}8489 8490static u32 gaudi_add_arm_monitor_pkt(struct hl_device *hdev,8491		struct packet_msg_short *pkt, u16 sob_base, u8 sob_mask,8492		u16 sob_val, u16 mon_id)8493{8494	u64 monitor_base;8495	u32 ctl, value, pkt_size = sizeof(*pkt);8496	u16 msg_addr_offset;8497	u8 mask;8498 8499	if (hl_gen_sob_mask(sob_base, sob_mask, &mask)) {8500		dev_err(hdev->dev,8501			"sob_base %u (mask %#x) is not valid\n",8502			sob_base, sob_mask);8503		return 0;8504	}8505 8506	/*8507	 * monitor_base should be the content of the base0 address registers,8508	 * so it will be added to the msg short offsets8509	 */8510	monitor_base = mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0;8511 8512	msg_addr_offset =8513		(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0 + mon_id * 4) -8514				monitor_base;8515 8516	memset(pkt, 0, pkt_size);8517 8518	/* Monitor config packet: bind the monitor to a sync object */8519	value = FIELD_PREP(GAUDI_PKT_SHORT_VAL_MON_SYNC_GID_MASK, sob_base / 8);8520	value |= FIELD_PREP(GAUDI_PKT_SHORT_VAL_MON_SYNC_VAL_MASK, sob_val);8521	value |= FIELD_PREP(GAUDI_PKT_SHORT_VAL_MON_MODE_MASK,8522			0); /* GREATER OR EQUAL*/8523	value |= FIELD_PREP(GAUDI_PKT_SHORT_VAL_MON_MASK_MASK, mask);8524 8525	ctl = FIELD_PREP(GAUDI_PKT_SHORT_CTL_ADDR_MASK, msg_addr_offset);8526	ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_OP_MASK, 0); /* write the value */8527	ctl |= FIELD_PREP(GAUDI_PKT_SHORT_CTL_BASE_MASK, 2); /* W_S MON base */8528	ctl |= FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_MSG_SHORT);8529	ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 0);8530	ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1);8531	ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1);8532 8533	pkt->value = cpu_to_le32(value);8534	pkt->ctl = cpu_to_le32(ctl);8535 8536	return pkt_size;8537}8538 8539static u32 gaudi_add_fence_pkt(struct packet_fence *pkt)8540{8541	u32 ctl, cfg, pkt_size = sizeof(*pkt);8542 8543	memset(pkt, 0, pkt_size);8544 8545	cfg = FIELD_PREP(GAUDI_PKT_FENCE_CFG_DEC_VAL_MASK, 1);8546	cfg |= FIELD_PREP(GAUDI_PKT_FENCE_CFG_TARGET_VAL_MASK, 1);8547	cfg |= FIELD_PREP(GAUDI_PKT_FENCE_CFG_ID_MASK, 2);8548 8549	ctl = FIELD_PREP(GAUDI_PKT_CTL_OPCODE_MASK, PACKET_FENCE);8550	ctl |= FIELD_PREP(GAUDI_PKT_CTL_EB_MASK, 0);8551	ctl |= FIELD_PREP(GAUDI_PKT_CTL_RB_MASK, 1);8552	ctl |= FIELD_PREP(GAUDI_PKT_CTL_MB_MASK, 1);8553 8554	pkt->cfg = cpu_to_le32(cfg);8555	pkt->ctl = cpu_to_le32(ctl);8556 8557	return pkt_size;8558}8559 8560static int gaudi_get_fence_addr(struct hl_device *hdev, u32 queue_id, u64 *addr)8561{8562	u32 offset, nic_index;8563 8564	switch (queue_id) {8565	case GAUDI_QUEUE_ID_DMA_0_0:8566		offset = mmDMA0_QM_CP_FENCE2_RDATA_0;8567		break;8568	case GAUDI_QUEUE_ID_DMA_0_1:8569		offset = mmDMA0_QM_CP_FENCE2_RDATA_1;8570		break;8571	case GAUDI_QUEUE_ID_DMA_0_2:8572		offset = mmDMA0_QM_CP_FENCE2_RDATA_2;8573		break;8574	case GAUDI_QUEUE_ID_DMA_0_3:8575		offset = mmDMA0_QM_CP_FENCE2_RDATA_3;8576		break;8577	case GAUDI_QUEUE_ID_DMA_1_0:8578		offset = mmDMA1_QM_CP_FENCE2_RDATA_0;8579		break;8580	case GAUDI_QUEUE_ID_DMA_1_1:8581		offset = mmDMA1_QM_CP_FENCE2_RDATA_1;8582		break;8583	case GAUDI_QUEUE_ID_DMA_1_2:8584		offset = mmDMA1_QM_CP_FENCE2_RDATA_2;8585		break;8586	case GAUDI_QUEUE_ID_DMA_1_3:8587		offset = mmDMA1_QM_CP_FENCE2_RDATA_3;8588		break;8589	case GAUDI_QUEUE_ID_DMA_5_0:8590		offset = mmDMA5_QM_CP_FENCE2_RDATA_0;8591		break;8592	case GAUDI_QUEUE_ID_DMA_5_1:8593		offset = mmDMA5_QM_CP_FENCE2_RDATA_1;8594		break;8595	case GAUDI_QUEUE_ID_DMA_5_2:8596		offset = mmDMA5_QM_CP_FENCE2_RDATA_2;8597		break;8598	case GAUDI_QUEUE_ID_DMA_5_3:8599		offset = mmDMA5_QM_CP_FENCE2_RDATA_3;8600		break;8601	case GAUDI_QUEUE_ID_TPC_7_0:8602		offset = mmTPC7_QM_CP_FENCE2_RDATA_0;8603		break;8604	case GAUDI_QUEUE_ID_TPC_7_1:8605		offset = mmTPC7_QM_CP_FENCE2_RDATA_1;8606		break;8607	case GAUDI_QUEUE_ID_TPC_7_2:8608		offset = mmTPC7_QM_CP_FENCE2_RDATA_2;8609		break;8610	case GAUDI_QUEUE_ID_TPC_7_3:8611		offset = mmTPC7_QM_CP_FENCE2_RDATA_3;8612		break;8613	case GAUDI_QUEUE_ID_NIC_0_0:8614	case GAUDI_QUEUE_ID_NIC_1_0:8615	case GAUDI_QUEUE_ID_NIC_2_0:8616	case GAUDI_QUEUE_ID_NIC_3_0:8617	case GAUDI_QUEUE_ID_NIC_4_0:8618	case GAUDI_QUEUE_ID_NIC_5_0:8619	case GAUDI_QUEUE_ID_NIC_6_0:8620	case GAUDI_QUEUE_ID_NIC_7_0:8621	case GAUDI_QUEUE_ID_NIC_8_0:8622	case GAUDI_QUEUE_ID_NIC_9_0:8623		nic_index = (queue_id - GAUDI_QUEUE_ID_NIC_0_0) >> 2;8624		offset = mmNIC0_QM0_CP_FENCE2_RDATA_0 +8625				(nic_index >> 1) * NIC_MACRO_QMAN_OFFSET +8626				(nic_index & 0x1) * NIC_ENGINE_QMAN_OFFSET;8627		break;8628	case GAUDI_QUEUE_ID_NIC_0_1:8629	case GAUDI_QUEUE_ID_NIC_1_1:8630	case GAUDI_QUEUE_ID_NIC_2_1:8631	case GAUDI_QUEUE_ID_NIC_3_1:8632	case GAUDI_QUEUE_ID_NIC_4_1:8633	case GAUDI_QUEUE_ID_NIC_5_1:8634	case GAUDI_QUEUE_ID_NIC_6_1:8635	case GAUDI_QUEUE_ID_NIC_7_1:8636	case GAUDI_QUEUE_ID_NIC_8_1:8637	case GAUDI_QUEUE_ID_NIC_9_1:8638		nic_index = (queue_id - GAUDI_QUEUE_ID_NIC_0_1) >> 2;8639		offset = mmNIC0_QM0_CP_FENCE2_RDATA_1 +8640				(nic_index >> 1) * NIC_MACRO_QMAN_OFFSET +8641				(nic_index & 0x1) * NIC_ENGINE_QMAN_OFFSET;8642		break;8643	case GAUDI_QUEUE_ID_NIC_0_2:8644	case GAUDI_QUEUE_ID_NIC_1_2:8645	case GAUDI_QUEUE_ID_NIC_2_2:8646	case GAUDI_QUEUE_ID_NIC_3_2:8647	case GAUDI_QUEUE_ID_NIC_4_2:8648	case GAUDI_QUEUE_ID_NIC_5_2:8649	case GAUDI_QUEUE_ID_NIC_6_2:8650	case GAUDI_QUEUE_ID_NIC_7_2:8651	case GAUDI_QUEUE_ID_NIC_8_2:8652	case GAUDI_QUEUE_ID_NIC_9_2:8653		nic_index = (queue_id - GAUDI_QUEUE_ID_NIC_0_2) >> 2;8654		offset = mmNIC0_QM0_CP_FENCE2_RDATA_2 +8655				(nic_index >> 1) * NIC_MACRO_QMAN_OFFSET +8656				(nic_index & 0x1) * NIC_ENGINE_QMAN_OFFSET;8657		break;8658	case GAUDI_QUEUE_ID_NIC_0_3:8659	case GAUDI_QUEUE_ID_NIC_1_3:8660	case GAUDI_QUEUE_ID_NIC_2_3:8661	case GAUDI_QUEUE_ID_NIC_3_3:8662	case GAUDI_QUEUE_ID_NIC_4_3:8663	case GAUDI_QUEUE_ID_NIC_5_3:8664	case GAUDI_QUEUE_ID_NIC_6_3:8665	case GAUDI_QUEUE_ID_NIC_7_3:8666	case GAUDI_QUEUE_ID_NIC_8_3:8667	case GAUDI_QUEUE_ID_NIC_9_3:8668		nic_index = (queue_id - GAUDI_QUEUE_ID_NIC_0_3) >> 2;8669		offset = mmNIC0_QM0_CP_FENCE2_RDATA_3 +8670				(nic_index >> 1) * NIC_MACRO_QMAN_OFFSET +8671				(nic_index & 0x1) * NIC_ENGINE_QMAN_OFFSET;8672		break;8673	default:8674		return -EINVAL;8675	}8676 8677	*addr = CFG_BASE + offset;8678 8679	return 0;8680}8681 8682static u32 gaudi_add_mon_pkts(void *buf, u16 mon_id, u64 fence_addr)8683{8684	u64 monitor_base;8685	u32 size = 0;8686	u16 msg_addr_offset;8687 8688	/*8689	 * monitor_base should be the content of the base0 address registers,8690	 * so it will be added to the msg short offsets8691	 */8692	monitor_base = mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0;8693 8694	/* First monitor config packet: low address of the sync */8695	msg_addr_offset =8696		(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + mon_id * 4) -8697				monitor_base;8698 8699	size += gaudi_add_mon_msg_short(buf + size, (u32) fence_addr,8700					msg_addr_offset);8701 8702	/* Second monitor config packet: high address of the sync */8703	msg_addr_offset =8704		(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + mon_id * 4) -8705				monitor_base;8706 8707	size += gaudi_add_mon_msg_short(buf + size, (u32) (fence_addr >> 32),8708					msg_addr_offset);8709 8710	/*8711	 * Third monitor config packet: the payload, i.e. what to write when the8712	 * sync triggers8713	 */8714	msg_addr_offset =8715		(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + mon_id * 4) -8716				monitor_base;8717 8718	size += gaudi_add_mon_msg_short(buf + size, 1, msg_addr_offset);8719 8720	return size;8721}8722 8723static u32 gaudi_gen_wait_cb(struct hl_device *hdev,8724				struct hl_gen_wait_properties *prop)8725{8726	struct hl_cb *cb = (struct hl_cb *) prop->data;8727	void *buf = cb->kernel_address;8728	u64 fence_addr = 0;8729	u32 size = prop->size;8730 8731	if (gaudi_get_fence_addr(hdev, prop->q_idx, &fence_addr)) {8732		dev_crit(hdev->dev, "wrong queue id %d for wait packet\n",8733				prop->q_idx);8734		return 0;8735	}8736 8737	size += gaudi_add_mon_pkts(buf + size, prop->mon_id, fence_addr);8738	size += gaudi_add_arm_monitor_pkt(hdev, buf + size, prop->sob_base,8739			prop->sob_mask, prop->sob_val, prop->mon_id);8740	size += gaudi_add_fence_pkt(buf + size);8741 8742	return size;8743}8744 8745static void gaudi_reset_sob(struct hl_device *hdev, void *data)8746{8747	struct hl_hw_sob *hw_sob = (struct hl_hw_sob *) data;8748 8749	dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx,8750		hw_sob->sob_id);8751 8752	WREG32(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 +8753			hw_sob->sob_id * 4, 0);8754 8755	kref_init(&hw_sob->kref);8756}8757 8758static u64 gaudi_get_device_time(struct hl_device *hdev)8759{8760	u64 device_time = ((u64) RREG32(mmPSOC_TIMESTAMP_CNTCVU)) << 32;8761 8762	return device_time | RREG32(mmPSOC_TIMESTAMP_CNTCVL);8763}8764 8765static int gaudi_get_hw_block_id(struct hl_device *hdev, u64 block_addr,8766				u32 *block_size, u32 *block_id)8767{8768	return -EPERM;8769}8770 8771static int gaudi_block_mmap(struct hl_device *hdev,8772				struct vm_area_struct *vma,8773				u32 block_id, u32 block_size)8774{8775	return -EPERM;8776}8777 8778static void gaudi_enable_events_from_fw(struct hl_device *hdev)8779{8780	struct cpu_dyn_regs *dyn_regs =8781			&hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;8782	u32 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?8783			mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :8784			le32_to_cpu(dyn_regs->gic_host_ints_irq);8785 8786	WREG32(irq_handler_offset,8787		gaudi_irq_map_table[GAUDI_EVENT_INTS_REGISTER].cpu_id);8788}8789 8790static int gaudi_ack_mmu_page_fault_or_access_error(struct hl_device *hdev, u64 mmu_cap_mask)8791{8792	return -EINVAL;8793}8794 8795static int gaudi_map_pll_idx_to_fw_idx(u32 pll_idx)8796{8797	switch (pll_idx) {8798	case HL_GAUDI_CPU_PLL: return CPU_PLL;8799	case HL_GAUDI_PCI_PLL: return PCI_PLL;8800	case HL_GAUDI_NIC_PLL: return NIC_PLL;8801	case HL_GAUDI_DMA_PLL: return DMA_PLL;8802	case HL_GAUDI_MESH_PLL: return MESH_PLL;8803	case HL_GAUDI_MME_PLL: return MME_PLL;8804	case HL_GAUDI_TPC_PLL: return TPC_PLL;8805	case HL_GAUDI_IF_PLL: return IF_PLL;8806	case HL_GAUDI_SRAM_PLL: return SRAM_PLL;8807	case HL_GAUDI_HBM_PLL: return HBM_PLL;8808	default: return -EINVAL;8809	}8810}8811 8812static int gaudi_add_sync_to_engine_map_entry(8813	struct hl_sync_to_engine_map *map, u32 reg_value,8814	enum hl_sync_engine_type engine_type, u32 engine_id)8815{8816	struct hl_sync_to_engine_map_entry *entry;8817 8818	/* Reg value represents a partial address of sync object,8819	 * it is used as unique identifier. For this we need to8820	 * clear the cutoff cfg base bits from the value.8821	 */8822	if (reg_value == 0 || reg_value == 0xffffffff)8823		return 0;8824	reg_value -= lower_32_bits(CFG_BASE);8825 8826	/* create a new hash entry */8827	entry = kzalloc(sizeof(*entry), GFP_KERNEL);8828	if (!entry)8829		return -ENOMEM;8830	entry->engine_type = engine_type;8831	entry->engine_id = engine_id;8832	entry->sync_id = reg_value;8833	hash_add(map->tb, &entry->node, reg_value);8834 8835	return 0;8836}8837 8838static int gaudi_gen_sync_to_engine_map(struct hl_device *hdev,8839				struct hl_sync_to_engine_map *map)8840{8841	struct hl_state_dump_specs *sds = &hdev->state_dump_specs;8842	int i, j, rc;8843	u32 reg_value;8844 8845	/* Iterate over TPC engines */8846	for (i = 0; i < sds->props[SP_NUM_OF_TPC_ENGINES]; ++i) {8847 8848		reg_value = RREG32(sds->props[SP_TPC0_CFG_SO] +8849					sds->props[SP_NEXT_TPC] * i);8850 8851		rc = gaudi_add_sync_to_engine_map_entry(map, reg_value,8852							ENGINE_TPC, i);8853		if (rc)8854			goto free_sync_to_engine_map;8855	}8856 8857	/* Iterate over MME engines */8858	for (i = 0; i < sds->props[SP_NUM_OF_MME_ENGINES]; ++i) {8859		for (j = 0; j < sds->props[SP_SUB_MME_ENG_NUM]; ++j) {8860 8861			reg_value = RREG32(sds->props[SP_MME_CFG_SO] +8862						sds->props[SP_NEXT_MME] * i +8863						j * sizeof(u32));8864 8865			rc = gaudi_add_sync_to_engine_map_entry(8866				map, reg_value, ENGINE_MME,8867				i * sds->props[SP_SUB_MME_ENG_NUM] + j);8868			if (rc)8869				goto free_sync_to_engine_map;8870		}8871	}8872 8873	/* Iterate over DMA engines */8874	for (i = 0; i < sds->props[SP_NUM_OF_DMA_ENGINES]; ++i) {8875		reg_value = RREG32(sds->props[SP_DMA_CFG_SO] +8876					sds->props[SP_DMA_QUEUES_OFFSET] * i);8877		rc = gaudi_add_sync_to_engine_map_entry(map, reg_value,8878							ENGINE_DMA, i);8879		if (rc)8880			goto free_sync_to_engine_map;8881	}8882 8883	return 0;8884 8885free_sync_to_engine_map:8886	hl_state_dump_free_sync_to_engine_map(map);8887 8888	return rc;8889}8890 8891static int gaudi_monitor_valid(struct hl_mon_state_dump *mon)8892{8893	return FIELD_GET(8894		SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0_VALID_MASK,8895		mon->status);8896}8897 8898static void gaudi_fill_sobs_from_mon(char *sobs, struct hl_mon_state_dump *mon)8899{8900	const size_t max_write = 10;8901	u32 gid, mask, sob;8902	int i, offset;8903 8904	/* Sync object ID is calculated as follows:8905	 * (8 * group_id + cleared bits in mask)8906	 */8907	gid = FIELD_GET(SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SID_MASK,8908			mon->arm_data);8909	mask = FIELD_GET(SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_MASK_MASK,8910			mon->arm_data);8911 8912	for (i = 0, offset = 0; mask && offset < MONITOR_SOB_STRING_SIZE -8913		max_write; mask >>= 1, i++) {8914		if (!(mask & 1)) {8915			sob = gid * MONITOR_MAX_SOBS + i;8916 8917			if (offset > 0)8918				offset += snprintf(sobs + offset, max_write,8919							", ");8920 8921			offset += snprintf(sobs + offset, max_write, "%u", sob);8922		}8923	}8924}8925 8926static int gaudi_print_single_monitor(char **buf, size_t *size, size_t *offset,8927				struct hl_device *hdev,8928				struct hl_mon_state_dump *mon)8929{8930	const char *name;8931	char scratch_buf1[BIN_REG_STRING_SIZE],8932		scratch_buf2[BIN_REG_STRING_SIZE];8933	char monitored_sobs[MONITOR_SOB_STRING_SIZE] = {0};8934 8935	name = hl_state_dump_get_monitor_name(hdev, mon);8936	if (!name)8937		name = "";8938 8939	gaudi_fill_sobs_from_mon(monitored_sobs, mon);8940 8941	return hl_snprintf_resize(8942		buf, size, offset,8943		"Mon id: %u%s, wait for group id: %u mask %s to reach val: %u and write %u to address 0x%llx. Pending: %s. Means sync objects [%s] are being monitored.",8944		mon->id, name,8945		FIELD_GET(SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SID_MASK,8946				mon->arm_data),8947		hl_format_as_binary(8948			scratch_buf1, sizeof(scratch_buf1),8949			FIELD_GET(8950				SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_MASK_MASK,8951				mon->arm_data)),8952		FIELD_GET(SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_ARM_0_SOD_MASK,8953				mon->arm_data),8954		mon->wr_data,8955		(((u64)mon->wr_addr_high) << 32) | mon->wr_addr_low,8956		hl_format_as_binary(8957			scratch_buf2, sizeof(scratch_buf2),8958			FIELD_GET(8959				SYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_STATUS_0_PENDING_MASK,8960				mon->status)),8961		monitored_sobs);8962}8963 8964 8965static int gaudi_print_fences_single_engine(8966	struct hl_device *hdev, u64 base_offset, u64 status_base_offset,8967	enum hl_sync_engine_type engine_type, u32 engine_id, char **buf,8968	size_t *size, size_t *offset)8969{8970	struct hl_state_dump_specs *sds = &hdev->state_dump_specs;8971	int rc = -ENOMEM, i;8972	u32 *statuses, *fences;8973 8974	statuses = kcalloc(sds->props[SP_ENGINE_NUM_OF_QUEUES],8975			sizeof(*statuses), GFP_KERNEL);8976	if (!statuses)8977		goto out;8978 8979	fences = kcalloc(sds->props[SP_ENGINE_NUM_OF_FENCES] *8980				sds->props[SP_ENGINE_NUM_OF_QUEUES],8981			 sizeof(*fences), GFP_KERNEL);8982	if (!fences)8983		goto free_status;8984 8985	for (i = 0; i < sds->props[SP_ENGINE_NUM_OF_FENCES]; ++i)8986		statuses[i] = RREG32(status_base_offset + i * sizeof(u32));8987 8988	for (i = 0; i < sds->props[SP_ENGINE_NUM_OF_FENCES] *8989				sds->props[SP_ENGINE_NUM_OF_QUEUES]; ++i)8990		fences[i] = RREG32(base_offset + i * sizeof(u32));8991 8992	/* The actual print */8993	for (i = 0; i < sds->props[SP_ENGINE_NUM_OF_QUEUES]; ++i) {8994		u32 fence_id;8995		u64 fence_cnt, fence_rdata;8996		const char *engine_name;8997 8998		if (!FIELD_GET(TPC0_QM_CP_STS_0_FENCE_IN_PROGRESS_MASK,8999			statuses[i]))9000			continue;9001 9002		fence_id =9003			FIELD_GET(TPC0_QM_CP_STS_0_FENCE_ID_MASK, statuses[i]);9004		fence_cnt = base_offset + CFG_BASE +9005			sizeof(u32) *9006			(i + fence_id * sds->props[SP_ENGINE_NUM_OF_QUEUES]);9007		fence_rdata = fence_cnt - sds->props[SP_FENCE0_CNT_OFFSET] +9008				sds->props[SP_FENCE0_RDATA_OFFSET];9009		engine_name = hl_sync_engine_to_string(engine_type);9010 9011		rc = hl_snprintf_resize(9012			buf, size, offset,9013			"%s%u, stream %u: fence id %u cnt = 0x%llx (%s%u_QM.CP_FENCE%u_CNT_%u) rdata = 0x%llx (%s%u_QM.CP_FENCE%u_RDATA_%u) value = %u, cp_status = %u\n",9014			engine_name, engine_id,9015			i, fence_id,9016			fence_cnt, engine_name, engine_id, fence_id, i,9017			fence_rdata, engine_name, engine_id, fence_id, i,9018			fences[fence_id],9019			statuses[i]);9020		if (rc)9021			goto free_fences;9022	}9023 9024	rc = 0;9025 9026free_fences:9027	kfree(fences);9028free_status:9029	kfree(statuses);9030out:9031	return rc;9032}9033 9034 9035static struct hl_state_dump_specs_funcs gaudi_state_dump_funcs = {9036	.monitor_valid = gaudi_monitor_valid,9037	.print_single_monitor = gaudi_print_single_monitor,9038	.gen_sync_to_engine_map = gaudi_gen_sync_to_engine_map,9039	.print_fences_single_engine = gaudi_print_fences_single_engine,9040};9041 9042static void gaudi_state_dump_init(struct hl_device *hdev)9043{9044	struct hl_state_dump_specs *sds = &hdev->state_dump_specs;9045	int i;9046 9047	for (i = 0; i < ARRAY_SIZE(gaudi_so_id_to_str); ++i)9048		hash_add(sds->so_id_to_str_tb,9049			&gaudi_so_id_to_str[i].node,9050			gaudi_so_id_to_str[i].id);9051 9052	for (i = 0; i < ARRAY_SIZE(gaudi_monitor_id_to_str); ++i)9053		hash_add(sds->monitor_id_to_str_tb,9054			&gaudi_monitor_id_to_str[i].node,9055			gaudi_monitor_id_to_str[i].id);9056 9057	sds->props = gaudi_state_dump_specs_props;9058 9059	sds->sync_namager_names = gaudi_sync_manager_names;9060 9061	sds->funcs = gaudi_state_dump_funcs;9062}9063 9064static u32 *gaudi_get_stream_master_qid_arr(void)9065{9066	return gaudi_stream_master;9067}9068 9069static int gaudi_set_dram_properties(struct hl_device *hdev)9070{9071	return 0;9072}9073 9074static int gaudi_set_binning_masks(struct hl_device *hdev)9075{9076	return 0;9077}9078 9079static void gaudi_check_if_razwi_happened(struct hl_device *hdev)9080{9081}9082 9083static ssize_t infineon_ver_show(struct device *dev, struct device_attribute *attr, char *buf)9084{9085	struct hl_device *hdev = dev_get_drvdata(dev);9086	struct cpucp_info *cpucp_info;9087 9088	cpucp_info = &hdev->asic_prop.cpucp_info;9089 9090	return sprintf(buf, "%#04x\n", le32_to_cpu(cpucp_info->infineon_version));9091}9092 9093static DEVICE_ATTR_RO(infineon_ver);9094 9095static struct attribute *gaudi_vrm_dev_attrs[] = {9096	&dev_attr_infineon_ver.attr,9097	NULL,9098};9099 9100static void gaudi_add_device_attr(struct hl_device *hdev, struct attribute_group *dev_clk_attr_grp,9101					struct attribute_group *dev_vrm_attr_grp)9102{9103	hl_sysfs_add_dev_clk_attr(hdev, dev_clk_attr_grp);9104	dev_vrm_attr_grp->attrs = gaudi_vrm_dev_attrs;9105}9106 9107static int gaudi_send_device_activity(struct hl_device *hdev, bool open)9108{9109	return 0;9110}9111 9112static const struct hl_asic_funcs gaudi_funcs = {9113	.early_init = gaudi_early_init,9114	.early_fini = gaudi_early_fini,9115	.late_init = gaudi_late_init,9116	.late_fini = gaudi_late_fini,9117	.sw_init = gaudi_sw_init,9118	.sw_fini = gaudi_sw_fini,9119	.hw_init = gaudi_hw_init,9120	.hw_fini = gaudi_hw_fini,9121	.halt_engines = gaudi_halt_engines,9122	.suspend = gaudi_suspend,9123	.resume = gaudi_resume,9124	.mmap = gaudi_mmap,9125	.ring_doorbell = gaudi_ring_doorbell,9126	.pqe_write = gaudi_pqe_write,9127	.asic_dma_alloc_coherent = gaudi_dma_alloc_coherent,9128	.asic_dma_free_coherent = gaudi_dma_free_coherent,9129	.scrub_device_mem = gaudi_scrub_device_mem,9130	.scrub_device_dram = gaudi_scrub_device_dram,9131	.get_int_queue_base = gaudi_get_int_queue_base,9132	.test_queues = gaudi_test_queues,9133	.asic_dma_pool_zalloc = gaudi_dma_pool_zalloc,9134	.asic_dma_pool_free = gaudi_dma_pool_free,9135	.cpu_accessible_dma_pool_alloc = gaudi_cpu_accessible_dma_pool_alloc,9136	.cpu_accessible_dma_pool_free = gaudi_cpu_accessible_dma_pool_free,9137	.dma_unmap_sgtable = hl_asic_dma_unmap_sgtable,9138	.cs_parser = gaudi_cs_parser,9139	.dma_map_sgtable = hl_asic_dma_map_sgtable,9140	.add_end_of_cb_packets = gaudi_add_end_of_cb_packets,9141	.update_eq_ci = gaudi_update_eq_ci,9142	.context_switch = gaudi_context_switch,9143	.restore_phase_topology = gaudi_restore_phase_topology,9144	.debugfs_read_dma = gaudi_debugfs_read_dma,9145	.add_device_attr = gaudi_add_device_attr,9146	.handle_eqe = gaudi_handle_eqe,9147	.get_events_stat = gaudi_get_events_stat,9148	.read_pte = gaudi_read_pte,9149	.write_pte = gaudi_write_pte,9150	.mmu_invalidate_cache = gaudi_mmu_invalidate_cache,9151	.mmu_invalidate_cache_range = gaudi_mmu_invalidate_cache_range,9152	.mmu_prefetch_cache_range = NULL,9153	.send_heartbeat = gaudi_send_heartbeat,9154	.debug_coresight = gaudi_debug_coresight,9155	.is_device_idle = gaudi_is_device_idle,9156	.compute_reset_late_init = gaudi_compute_reset_late_init,9157	.hw_queues_lock = gaudi_hw_queues_lock,9158	.hw_queues_unlock = gaudi_hw_queues_unlock,9159	.get_pci_id = gaudi_get_pci_id,9160	.get_eeprom_data = gaudi_get_eeprom_data,9161	.get_monitor_dump = gaudi_get_monitor_dump,9162	.send_cpu_message = gaudi_send_cpu_message,9163	.pci_bars_map = gaudi_pci_bars_map,9164	.init_iatu = gaudi_init_iatu,9165	.rreg = hl_rreg,9166	.wreg = hl_wreg,9167	.halt_coresight = gaudi_halt_coresight,9168	.ctx_init = gaudi_ctx_init,9169	.ctx_fini = gaudi_ctx_fini,9170	.pre_schedule_cs = gaudi_pre_schedule_cs,9171	.get_queue_id_for_cq = gaudi_get_queue_id_for_cq,9172	.load_firmware_to_device = gaudi_load_firmware_to_device,9173	.load_boot_fit_to_device = gaudi_load_boot_fit_to_device,9174	.get_signal_cb_size = gaudi_get_signal_cb_size,9175	.get_wait_cb_size = gaudi_get_wait_cb_size,9176	.gen_signal_cb = gaudi_gen_signal_cb,9177	.gen_wait_cb = gaudi_gen_wait_cb,9178	.reset_sob = gaudi_reset_sob,9179	.reset_sob_group = gaudi_reset_sob_group,9180	.get_device_time = gaudi_get_device_time,9181	.pb_print_security_errors = NULL,9182	.collective_wait_init_cs = gaudi_collective_wait_init_cs,9183	.collective_wait_create_jobs = gaudi_collective_wait_create_jobs,9184	.get_dec_base_addr = NULL,9185	.scramble_addr = hl_mmu_scramble_addr,9186	.descramble_addr = hl_mmu_descramble_addr,9187	.ack_protection_bits_errors = gaudi_ack_protection_bits_errors,9188	.get_hw_block_id = gaudi_get_hw_block_id,9189	.hw_block_mmap = gaudi_block_mmap,9190	.enable_events_from_fw = gaudi_enable_events_from_fw,9191	.ack_mmu_errors = gaudi_ack_mmu_page_fault_or_access_error,9192	.map_pll_idx_to_fw_idx = gaudi_map_pll_idx_to_fw_idx,9193	.init_firmware_preload_params = gaudi_init_firmware_preload_params,9194	.init_firmware_loader = gaudi_init_firmware_loader,9195	.init_cpu_scrambler_dram = gaudi_init_scrambler_hbm,9196	.state_dump_init = gaudi_state_dump_init,9197	.get_sob_addr = gaudi_get_sob_addr,9198	.set_pci_memory_regions = gaudi_set_pci_memory_regions,9199	.get_stream_master_qid_arr = gaudi_get_stream_master_qid_arr,9200	.check_if_razwi_happened = gaudi_check_if_razwi_happened,9201	.mmu_get_real_page_size = hl_mmu_get_real_page_size,9202	.access_dev_mem = hl_access_dev_mem,9203	.set_dram_bar_base = gaudi_set_hbm_bar_base,9204	.send_device_activity = gaudi_send_device_activity,9205	.set_dram_properties = gaudi_set_dram_properties,9206	.set_binning_masks = gaudi_set_binning_masks,9207};9208 9209/**9210 * gaudi_set_asic_funcs - set GAUDI function pointers9211 *9212 * @hdev: pointer to hl_device structure9213 *9214 */9215void gaudi_set_asic_funcs(struct hl_device *hdev)9216{9217	hdev->asic_funcs = &gaudi_funcs;9218}9219