344 lines · c
1/* SPDX-License-Identifier: GPL-2.02 *3 * Copyright 2019-2022 HabanaLabs, Ltd.4 * All Rights Reserved.5 *6 */7 8#ifndef GAUDIP_H_9#define GAUDIP_H_10 11#include <uapi/drm/habanalabs_accel.h>12#include "../common/habanalabs.h"13#include <linux/habanalabs/hl_boot_if.h>14#include "../include/gaudi/gaudi_packets.h"15#include "../include/gaudi/gaudi.h"16#include "../include/gaudi/gaudi_async_events.h"17#include "../include/gaudi/gaudi_fw_if.h"18 19#define NUMBER_OF_EXT_HW_QUEUES 820#define NUMBER_OF_CMPLT_QUEUES NUMBER_OF_EXT_HW_QUEUES21#define NUMBER_OF_CPU_HW_QUEUES 122#define NUMBER_OF_INT_HW_QUEUES 10023#define NUMBER_OF_HW_QUEUES (NUMBER_OF_EXT_HW_QUEUES + \24 NUMBER_OF_CPU_HW_QUEUES + \25 NUMBER_OF_INT_HW_QUEUES)26 27/* 10 NIC QMANs, DMA5 QMAN, TPC7 QMAN */28#define NUMBER_OF_COLLECTIVE_QUEUES 1229#define NUMBER_OF_SOBS_IN_GRP 1130 31#define GAUDI_STREAM_MASTER_ARR_SIZE 832 33#define CORESIGHT_TIMEOUT_USEC 100000 /* 100 ms */34 35#define GAUDI_MAX_CLK_FREQ 2200000000ull /* 2200 MHz */36 37#define MAX_POWER_DEFAULT_PCI 200000 /* 200W */38#define MAX_POWER_DEFAULT_PMC 350000 /* 350W */39 40#define DC_POWER_DEFAULT_PCI 60000 /* 60W */41#define DC_POWER_DEFAULT_PMC 60000 /* 60W */42 43#define DC_POWER_DEFAULT_PMC_SEC 97000 /* 97W */44 45#define GAUDI_CPU_TIMEOUT_USEC 30000000 /* 30s */46 47#define TPC_ENABLED_MASK 0xFF48 49#define GAUDI_HBM_SIZE_32GB 0x800000000ull50#define GAUDI_HBM_DEVICES 451#define GAUDI_HBM_CHANNELS 852#define GAUDI_HBM_CFG_BASE (mmHBM0_BASE - CFG_BASE)53#define GAUDI_HBM_CFG_OFFSET (mmHBM1_BASE - mmHBM0_BASE)54 55#define DMA_MAX_TRANSFER_SIZE U32_MAX56 57#define GAUDI_DEFAULT_CARD_NAME "HL205"58 59#define GAUDI_MAX_PENDING_CS SZ_16K60 61#if !IS_MAX_PENDING_CS_VALID(GAUDI_MAX_PENDING_CS)62#error "GAUDI_MAX_PENDING_CS must be power of 2 and greater than 1"63#endif64 65#define PCI_DMA_NUMBER_OF_CHNLS 266#define HBM_DMA_NUMBER_OF_CHNLS 667#define DMA_NUMBER_OF_CHNLS (PCI_DMA_NUMBER_OF_CHNLS + \68 HBM_DMA_NUMBER_OF_CHNLS)69 70#define MME_NUMBER_OF_SLAVE_ENGINES 271#define MME_NUMBER_OF_ENGINES (MME_NUMBER_OF_MASTER_ENGINES + \72 MME_NUMBER_OF_SLAVE_ENGINES)73#define MME_NUMBER_OF_QMANS (MME_NUMBER_OF_MASTER_ENGINES * \74 QMAN_STREAMS)75 76#define QMAN_STREAMS 477#define PQ_FETCHER_CACHE_SIZE 878 79#define DMA_QMAN_OFFSET (mmDMA1_QM_BASE - mmDMA0_QM_BASE)80#define TPC_QMAN_OFFSET (mmTPC1_QM_BASE - mmTPC0_QM_BASE)81#define MME_QMAN_OFFSET (mmMME1_QM_BASE - mmMME0_QM_BASE)82#define NIC_MACRO_QMAN_OFFSET (mmNIC1_QM0_BASE - mmNIC0_QM0_BASE)83#define NIC_ENGINE_QMAN_OFFSET (mmNIC0_QM1_BASE - mmNIC0_QM0_BASE)84 85#define TPC_CFG_OFFSET (mmTPC1_CFG_BASE - mmTPC0_CFG_BASE)86 87#define DMA_CORE_OFFSET (mmDMA1_CORE_BASE - mmDMA0_CORE_BASE)88 89#define QMAN_LDMA_SRC_OFFSET (mmDMA0_CORE_SRC_BASE_LO - mmDMA0_CORE_CFG_0)90#define QMAN_LDMA_DST_OFFSET (mmDMA0_CORE_DST_BASE_LO - mmDMA0_CORE_CFG_0)91#define QMAN_LDMA_SIZE_OFFSET (mmDMA0_CORE_DST_TSIZE_0 - mmDMA0_CORE_CFG_0)92 93#define QMAN_CPDMA_SRC_OFFSET (mmDMA0_QM_CQ_PTR_LO_4 - mmDMA0_CORE_CFG_0)94#define QMAN_CPDMA_DST_OFFSET (mmDMA0_CORE_DST_BASE_LO - mmDMA0_CORE_CFG_0)95#define QMAN_CPDMA_SIZE_OFFSET (mmDMA0_QM_CQ_TSIZE_4 - mmDMA0_CORE_CFG_0)96 97#define SIF_RTR_CTRL_OFFSET (mmSIF_RTR_CTRL_1_BASE - mmSIF_RTR_CTRL_0_BASE)98 99#define NIF_RTR_CTRL_OFFSET (mmNIF_RTR_CTRL_1_BASE - mmNIF_RTR_CTRL_0_BASE)100 101#define MME_ACC_OFFSET (mmMME1_ACC_BASE - mmMME0_ACC_BASE)102#define SRAM_BANK_OFFSET (mmSRAM_Y0_X1_RTR_BASE - mmSRAM_Y0_X0_RTR_BASE)103 104#define NUM_OF_SOB_IN_BLOCK \105 (((mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_2047 - \106 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0) + 4) >> 2)107 108#define NUM_OF_MONITORS_IN_BLOCK \109 (((mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_511 - \110 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_STATUS_0) + 4) >> 2)111 112#define MONITOR_MAX_SOBS 8113 114/* DRAM Memory Map */115 116#define CPU_FW_IMAGE_SIZE 0x10000000 /* 256MB */117#define MMU_PAGE_TABLES_SIZE 0x0BF00000 /* 191MB */118#define MMU_CACHE_MNG_SIZE 0x00100000 /* 1MB */119#define RESERVED 0x04000000 /* 64MB */120 121#define CPU_FW_IMAGE_ADDR DRAM_PHYS_BASE122#define MMU_PAGE_TABLES_ADDR (CPU_FW_IMAGE_ADDR + CPU_FW_IMAGE_SIZE)123#define MMU_CACHE_MNG_ADDR (MMU_PAGE_TABLES_ADDR + MMU_PAGE_TABLES_SIZE)124 125#define DRAM_DRIVER_END_ADDR (MMU_CACHE_MNG_ADDR + MMU_CACHE_MNG_SIZE +\126 RESERVED)127 128#define DRAM_BASE_ADDR_USER 0x20000000129 130#if (DRAM_DRIVER_END_ADDR > DRAM_BASE_ADDR_USER)131#error "Driver must reserve no more than 512MB"132#endif133 134/* Internal QMANs PQ sizes */135 136#define MME_QMAN_LENGTH 1024137#define MME_QMAN_SIZE_IN_BYTES (MME_QMAN_LENGTH * QMAN_PQ_ENTRY_SIZE)138 139#define HBM_DMA_QMAN_LENGTH 4096140#define HBM_DMA_QMAN_SIZE_IN_BYTES \141 (HBM_DMA_QMAN_LENGTH * QMAN_PQ_ENTRY_SIZE)142 143#define TPC_QMAN_LENGTH 1024144#define TPC_QMAN_SIZE_IN_BYTES (TPC_QMAN_LENGTH * QMAN_PQ_ENTRY_SIZE)145 146#define NIC_QMAN_LENGTH 4096147#define NIC_QMAN_SIZE_IN_BYTES (NIC_QMAN_LENGTH * QMAN_PQ_ENTRY_SIZE)148 149 150#define SRAM_USER_BASE_OFFSET GAUDI_DRIVER_SRAM_RESERVED_SIZE_FROM_START151 152/* Virtual address space */153#define VA_HOST_SPACE_START 0x1000000000000ull /* 256TB */154#define VA_HOST_SPACE_END 0x3FF8000000000ull /* 1PB - 512GB */155#define VA_HOST_SPACE_SIZE (VA_HOST_SPACE_END - \156 VA_HOST_SPACE_START) /* 767TB */157#define HOST_SPACE_INTERNAL_CB_SZ SZ_2M158 159#define HW_CAP_PLL BIT(0)160#define HW_CAP_HBM BIT(1)161#define HW_CAP_MMU BIT(2)162#define HW_CAP_MME BIT(3)163#define HW_CAP_CPU BIT(4)164#define HW_CAP_PCI_DMA BIT(5)165#define HW_CAP_MSI BIT(6)166#define HW_CAP_CPU_Q BIT(7)167#define HW_CAP_HBM_DMA BIT(8)168#define HW_CAP_SRAM_SCRAMBLER BIT(10)169#define HW_CAP_HBM_SCRAMBLER BIT(11)170 171#define HW_CAP_NIC0 BIT(14)172#define HW_CAP_NIC1 BIT(15)173#define HW_CAP_NIC2 BIT(16)174#define HW_CAP_NIC3 BIT(17)175#define HW_CAP_NIC4 BIT(18)176#define HW_CAP_NIC5 BIT(19)177#define HW_CAP_NIC6 BIT(20)178#define HW_CAP_NIC7 BIT(21)179#define HW_CAP_NIC8 BIT(22)180#define HW_CAP_NIC9 BIT(23)181#define HW_CAP_NIC_MASK GENMASK(23, 14)182#define HW_CAP_NIC_SHIFT 14183 184#define HW_CAP_TPC0 BIT(24)185#define HW_CAP_TPC1 BIT(25)186#define HW_CAP_TPC2 BIT(26)187#define HW_CAP_TPC3 BIT(27)188#define HW_CAP_TPC4 BIT(28)189#define HW_CAP_TPC5 BIT(29)190#define HW_CAP_TPC6 BIT(30)191#define HW_CAP_TPC7 BIT(31)192#define HW_CAP_TPC_MASK GENMASK(31, 24)193#define HW_CAP_TPC_SHIFT 24194 195#define NEXT_SYNC_OBJ_ADDR_INTERVAL \196 (mmSYNC_MNGR_W_N_SYNC_MNGR_OBJS_SOB_OBJ_0 - \197 mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0)198#define NUM_OF_MME_ENGINES 2199#define NUM_OF_MME_SUB_ENGINES 2200#define NUM_OF_TPC_ENGINES 8201#define NUM_OF_DMA_ENGINES 8202#define NUM_OF_QUEUES 5203#define NUM_OF_STREAMS 4204#define NUM_OF_FENCES 4205 206 207#define GAUDI_CPU_PCI_MSB_ADDR(addr) (((addr) & GENMASK_ULL(49, 39)) >> 39)208#define GAUDI_PCI_TO_CPU_ADDR(addr) \209 do { \210 (addr) &= ~GENMASK_ULL(49, 39); \211 (addr) |= BIT_ULL(39); \212 } while (0)213#define GAUDI_CPU_TO_PCI_ADDR(addr, extension) \214 do { \215 (addr) &= ~GENMASK_ULL(49, 39); \216 (addr) |= (u64) (extension) << 39; \217 } while (0)218 219enum gaudi_dma_channels {220 GAUDI_PCI_DMA_1,221 GAUDI_PCI_DMA_2,222 GAUDI_HBM_DMA_1,223 GAUDI_HBM_DMA_2,224 GAUDI_HBM_DMA_3,225 GAUDI_HBM_DMA_4,226 GAUDI_HBM_DMA_5,227 GAUDI_HBM_DMA_6,228 GAUDI_DMA_MAX229};230 231enum gaudi_tpc_mask {232 GAUDI_TPC_MASK_TPC0 = 0x01,233 GAUDI_TPC_MASK_TPC1 = 0x02,234 GAUDI_TPC_MASK_TPC2 = 0x04,235 GAUDI_TPC_MASK_TPC3 = 0x08,236 GAUDI_TPC_MASK_TPC4 = 0x10,237 GAUDI_TPC_MASK_TPC5 = 0x20,238 GAUDI_TPC_MASK_TPC6 = 0x40,239 GAUDI_TPC_MASK_TPC7 = 0x80,240 GAUDI_TPC_MASK_ALL = 0xFF241};242 243enum gaudi_nic_mask {244 GAUDI_NIC_MASK_NIC0 = 0x01,245 GAUDI_NIC_MASK_NIC1 = 0x02,246 GAUDI_NIC_MASK_NIC2 = 0x04,247 GAUDI_NIC_MASK_NIC3 = 0x08,248 GAUDI_NIC_MASK_NIC4 = 0x10,249 GAUDI_NIC_MASK_NIC5 = 0x20,250 GAUDI_NIC_MASK_NIC6 = 0x40,251 GAUDI_NIC_MASK_NIC7 = 0x80,252 GAUDI_NIC_MASK_NIC8 = 0x100,253 GAUDI_NIC_MASK_NIC9 = 0x200,254 GAUDI_NIC_MASK_ALL = 0x3FF255};256 257/*258 * struct gaudi_hw_sob_group - H/W SOB group info.259 * @hdev: habanalabs device structure.260 * @kref: refcount of this SOB group. group will reset once refcount is zero.261 * @base_sob_id: base sob id of this SOB group.262 * @queue_id: id of the queue that waits on this sob group263 */264struct gaudi_hw_sob_group {265 struct hl_device *hdev;266 struct kref kref;267 u32 base_sob_id;268 u32 queue_id;269};270 271#define NUM_SOB_GROUPS (HL_RSVD_SOBS * QMAN_STREAMS)272/**273 * struct gaudi_collective_properties -274 * holds all SOB groups and queues info reserved for the collective275 * @hw_sob_group: H/W SOB groups.276 * @next_sob_group_val: the next value to use for the currently used SOB group.277 * @curr_sob_group_idx: the index of the currently used SOB group.278 * @mstr_sob_mask: pre-defined masks for collective master monitors279 */280struct gaudi_collective_properties {281 struct gaudi_hw_sob_group hw_sob_group[NUM_SOB_GROUPS];282 u16 next_sob_group_val[QMAN_STREAMS];283 u8 curr_sob_group_idx[QMAN_STREAMS];284 u8 mstr_sob_mask[HL_COLLECTIVE_RSVD_MSTR_MONS];285};286 287/**288 * struct gaudi_internal_qman_info - Internal QMAN information.289 * @pq_kernel_addr: Kernel address of the PQ memory area in the host.290 * @pq_dma_addr: DMA address of the PQ memory area in the host.291 * @pq_size: Size of allocated host memory for PQ.292 */293struct gaudi_internal_qman_info {294 void *pq_kernel_addr;295 dma_addr_t pq_dma_addr;296 size_t pq_size;297};298 299/**300 * struct gaudi_device - ASIC specific manage structure.301 * @cpucp_info_get: get information on device from CPU-CP302 * @hw_queues_lock: protects the H/W queues from concurrent access.303 * @internal_qmans: Internal QMANs information. The array size is larger than304 * the actual number of internal queues because they are not in305 * consecutive order.306 * @hbm_bar_cur_addr: current address of HBM PCI bar.307 * @events: array that holds all event id's308 * @events_stat: array that holds histogram of all received events.309 * @events_stat_aggregate: same as events_stat but doesn't get cleared on reset310 * @hw_cap_initialized: This field contains a bit per H/W engine. When that311 * engine is initialized, that bit is set by the driver to312 * signal we can use this engine in later code paths.313 * Each bit is cleared upon reset of its corresponding H/W314 * engine.315 * @mmu_cache_inv_pi: PI for MMU cache invalidation flow. The H/W expects an316 * 8-bit value so use u8.317 */318struct gaudi_device {319 int (*cpucp_info_get)(struct hl_device *hdev);320 321 /* TODO: remove hw_queues_lock after moving to scheduler code */322 spinlock_t hw_queues_lock;323 324 struct gaudi_internal_qman_info internal_qmans[GAUDI_QUEUE_ID_SIZE];325 326 struct gaudi_collective_properties collective_props;327 328 u64 hbm_bar_cur_addr;329 330 u32 events[GAUDI_EVENT_SIZE];331 u32 events_stat[GAUDI_EVENT_SIZE];332 u32 events_stat_aggregate[GAUDI_EVENT_SIZE];333 u32 hw_cap_initialized;334 u8 mmu_cache_inv_pi;335};336 337void gaudi_init_security(struct hl_device *hdev);338void gaudi_ack_protection_bits_errors(struct hl_device *hdev);339int gaudi_debug_coresight(struct hl_device *hdev, struct hl_ctx *ctx, void *data);340void gaudi_halt_coresight(struct hl_device *hdev, struct hl_ctx *ctx);341void gaudi_mmu_prepare_reg(struct hl_device *hdev, u64 reg, u32 asid);342 343#endif /* GAUDIP_H_ */344