978 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2020-2024 Intel Corporation4 */5 6#include <linux/circ_buf.h>7#include <linux/highmem.h>8 9#include "ivpu_drv.h"10#include "ivpu_hw.h"11#include "ivpu_hw_reg_io.h"12#include "ivpu_mmu.h"13#include "ivpu_mmu_context.h"14#include "ivpu_pm.h"15 16#define IVPU_MMU_REG_IDR0 0x00200000u17#define IVPU_MMU_REG_IDR1 0x00200004u18#define IVPU_MMU_REG_IDR3 0x0020000cu19#define IVPU_MMU_REG_IDR5 0x00200014u20#define IVPU_MMU_REG_CR0 0x00200020u21#define IVPU_MMU_REG_CR0ACK 0x00200024u22#define IVPU_MMU_REG_CR0ACK_VAL_MASK GENMASK(31, 0)23#define IVPU_MMU_REG_CR1 0x00200028u24#define IVPU_MMU_REG_CR2 0x0020002cu25#define IVPU_MMU_REG_IRQ_CTRL 0x00200050u26#define IVPU_MMU_REG_IRQ_CTRLACK 0x00200054u27#define IVPU_MMU_REG_IRQ_CTRLACK_VAL_MASK GENMASK(31, 0)28 29#define IVPU_MMU_REG_GERROR 0x00200060u30#define IVPU_MMU_REG_GERROR_CMDQ_MASK BIT_MASK(0)31#define IVPU_MMU_REG_GERROR_EVTQ_ABT_MASK BIT_MASK(2)32#define IVPU_MMU_REG_GERROR_PRIQ_ABT_MASK BIT_MASK(3)33#define IVPU_MMU_REG_GERROR_MSI_CMDQ_ABT_MASK BIT_MASK(4)34#define IVPU_MMU_REG_GERROR_MSI_EVTQ_ABT_MASK BIT_MASK(5)35#define IVPU_MMU_REG_GERROR_MSI_PRIQ_ABT_MASK BIT_MASK(6)36#define IVPU_MMU_REG_GERROR_MSI_ABT_MASK BIT_MASK(7)37 38#define IVPU_MMU_REG_GERRORN 0x00200064u39 40#define IVPU_MMU_REG_STRTAB_BASE 0x00200080u41#define IVPU_MMU_REG_STRTAB_BASE_CFG 0x00200088u42#define IVPU_MMU_REG_CMDQ_BASE 0x00200090u43#define IVPU_MMU_REG_CMDQ_PROD 0x00200098u44#define IVPU_MMU_REG_CMDQ_CONS 0x0020009cu45#define IVPU_MMU_REG_CMDQ_CONS_VAL_MASK GENMASK(23, 0)46#define IVPU_MMU_REG_CMDQ_CONS_ERR_MASK GENMASK(30, 24)47#define IVPU_MMU_REG_EVTQ_BASE 0x002000a0u48#define IVPU_MMU_REG_EVTQ_PROD 0x002000a8u49#define IVPU_MMU_REG_EVTQ_CONS 0x002000acu50#define IVPU_MMU_REG_EVTQ_PROD_SEC (0x002000a8u + SZ_64K)51#define IVPU_MMU_REG_EVTQ_CONS_SEC (0x002000acu + SZ_64K)52 53#define IVPU_MMU_IDR0_REF 0x080f3e0f54#define IVPU_MMU_IDR0_REF_SIMICS 0x080f3e1f55#define IVPU_MMU_IDR1_REF 0x0e739d1856#define IVPU_MMU_IDR3_REF 0x0000003c57#define IVPU_MMU_IDR5_REF 0x0004007058#define IVPU_MMU_IDR5_REF_SIMICS 0x0000007559#define IVPU_MMU_IDR5_REF_FPGA 0x0080007560 61#define IVPU_MMU_CDTAB_ENT_SIZE 6462#define IVPU_MMU_CDTAB_ENT_COUNT_LOG2 8 /* 256 entries */63#define IVPU_MMU_CDTAB_ENT_COUNT ((u32)1 << IVPU_MMU_CDTAB_ENT_COUNT_LOG2)64 65#define IVPU_MMU_STREAM_ID0 066#define IVPU_MMU_STREAM_ID3 367 68#define IVPU_MMU_STRTAB_ENT_SIZE 6469#define IVPU_MMU_STRTAB_ENT_COUNT 470#define IVPU_MMU_STRTAB_CFG_LOG2SIZE 271#define IVPU_MMU_STRTAB_CFG IVPU_MMU_STRTAB_CFG_LOG2SIZE72 73#define IVPU_MMU_Q_COUNT_LOG2 4 /* 16 entries */74#define IVPU_MMU_Q_COUNT ((u32)1 << IVPU_MMU_Q_COUNT_LOG2)75#define IVPU_MMU_Q_WRAP_MASK GENMASK(IVPU_MMU_Q_COUNT_LOG2, 0)76#define IVPU_MMU_Q_IDX_MASK (IVPU_MMU_Q_COUNT - 1)77#define IVPU_MMU_Q_IDX(val) ((val) & IVPU_MMU_Q_IDX_MASK)78#define IVPU_MMU_Q_WRP(val) ((val) & IVPU_MMU_Q_COUNT)79 80#define IVPU_MMU_CMDQ_CMD_SIZE 1681#define IVPU_MMU_CMDQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_CMDQ_CMD_SIZE)82 83#define IVPU_MMU_EVTQ_CMD_SIZE 3284#define IVPU_MMU_EVTQ_SIZE (IVPU_MMU_Q_COUNT * IVPU_MMU_EVTQ_CMD_SIZE)85 86#define IVPU_MMU_CMD_OPCODE GENMASK(7, 0)87 88#define IVPU_MMU_CMD_SYNC_0_CS GENMASK(13, 12)89#define IVPU_MMU_CMD_SYNC_0_MSH GENMASK(23, 22)90#define IVPU_MMU_CMD_SYNC_0_MSI_ATTR GENMASK(27, 24)91#define IVPU_MMU_CMD_SYNC_0_MSI_ATTR GENMASK(27, 24)92#define IVPU_MMU_CMD_SYNC_0_MSI_DATA GENMASK(63, 32)93 94#define IVPU_MMU_CMD_CFGI_0_SSEC BIT(10)95#define IVPU_MMU_CMD_CFGI_0_SSV BIT(11)96#define IVPU_MMU_CMD_CFGI_0_SSID GENMASK(31, 12)97#define IVPU_MMU_CMD_CFGI_0_SID GENMASK(63, 32)98#define IVPU_MMU_CMD_CFGI_1_RANGE GENMASK(4, 0)99 100#define IVPU_MMU_CMD_TLBI_0_ASID GENMASK(63, 48)101#define IVPU_MMU_CMD_TLBI_0_VMID GENMASK(47, 32)102 103#define CMD_PREFETCH_CFG 0x1104#define CMD_CFGI_STE 0x3105#define CMD_CFGI_ALL 0x4106#define CMD_CFGI_CD 0x5107#define CMD_CFGI_CD_ALL 0x6108#define CMD_TLBI_NH_ASID 0x11109#define CMD_TLBI_EL2_ALL 0x20110#define CMD_TLBI_NSNH_ALL 0x30111#define CMD_SYNC 0x46112 113#define IVPU_MMU_EVT_F_UUT 0x01114#define IVPU_MMU_EVT_C_BAD_STREAMID 0x02115#define IVPU_MMU_EVT_F_STE_FETCH 0x03116#define IVPU_MMU_EVT_C_BAD_STE 0x04117#define IVPU_MMU_EVT_F_BAD_ATS_TREQ 0x05118#define IVPU_MMU_EVT_F_STREAM_DISABLED 0x06119#define IVPU_MMU_EVT_F_TRANSL_FORBIDDEN 0x07120#define IVPU_MMU_EVT_C_BAD_SUBSTREAMID 0x08121#define IVPU_MMU_EVT_F_CD_FETCH 0x09122#define IVPU_MMU_EVT_C_BAD_CD 0x0a123#define IVPU_MMU_EVT_F_WALK_EABT 0x0b124#define IVPU_MMU_EVT_F_TRANSLATION 0x10125#define IVPU_MMU_EVT_F_ADDR_SIZE 0x11126#define IVPU_MMU_EVT_F_ACCESS 0x12127#define IVPU_MMU_EVT_F_PERMISSION 0x13128#define IVPU_MMU_EVT_F_TLB_CONFLICT 0x20129#define IVPU_MMU_EVT_F_CFG_CONFLICT 0x21130#define IVPU_MMU_EVT_E_PAGE_REQUEST 0x24131#define IVPU_MMU_EVT_F_VMS_FETCH 0x25132 133#define IVPU_MMU_EVT_OP_MASK GENMASK_ULL(7, 0)134#define IVPU_MMU_EVT_SSID_MASK GENMASK_ULL(31, 12)135 136#define IVPU_MMU_Q_BASE_RWA BIT(62)137#define IVPU_MMU_Q_BASE_ADDR_MASK GENMASK_ULL(51, 5)138#define IVPU_MMU_STRTAB_BASE_RA BIT(62)139#define IVPU_MMU_STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6)140 141#define IVPU_MMU_IRQ_EVTQ_EN BIT(2)142#define IVPU_MMU_IRQ_GERROR_EN BIT(0)143 144#define IVPU_MMU_CR0_ATSCHK BIT(4)145#define IVPU_MMU_CR0_CMDQEN BIT(3)146#define IVPU_MMU_CR0_EVTQEN BIT(2)147#define IVPU_MMU_CR0_PRIQEN BIT(1)148#define IVPU_MMU_CR0_SMMUEN BIT(0)149 150#define IVPU_MMU_CR1_TABLE_SH GENMASK(11, 10)151#define IVPU_MMU_CR1_TABLE_OC GENMASK(9, 8)152#define IVPU_MMU_CR1_TABLE_IC GENMASK(7, 6)153#define IVPU_MMU_CR1_QUEUE_SH GENMASK(5, 4)154#define IVPU_MMU_CR1_QUEUE_OC GENMASK(3, 2)155#define IVPU_MMU_CR1_QUEUE_IC GENMASK(1, 0)156#define IVPU_MMU_CACHE_NC 0157#define IVPU_MMU_CACHE_WB 1158#define IVPU_MMU_CACHE_WT 2159#define IVPU_MMU_SH_NSH 0160#define IVPU_MMU_SH_OSH 2161#define IVPU_MMU_SH_ISH 3162 163#define IVPU_MMU_CMDQ_OP GENMASK_ULL(7, 0)164 165#define IVPU_MMU_CD_0_TCR_T0SZ GENMASK_ULL(5, 0)166#define IVPU_MMU_CD_0_TCR_TG0 GENMASK_ULL(7, 6)167#define IVPU_MMU_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8)168#define IVPU_MMU_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10)169#define IVPU_MMU_CD_0_TCR_SH0 GENMASK_ULL(13, 12)170#define IVPU_MMU_CD_0_TCR_EPD0 BIT_ULL(14)171#define IVPU_MMU_CD_0_TCR_EPD1 BIT_ULL(30)172#define IVPU_MMU_CD_0_ENDI BIT(15)173#define IVPU_MMU_CD_0_V BIT(31)174#define IVPU_MMU_CD_0_TCR_IPS GENMASK_ULL(34, 32)175#define IVPU_MMU_CD_0_TCR_TBI0 BIT_ULL(38)176#define IVPU_MMU_CD_0_AA64 BIT(41)177#define IVPU_MMU_CD_0_S BIT(44)178#define IVPU_MMU_CD_0_R BIT(45)179#define IVPU_MMU_CD_0_A BIT(46)180#define IVPU_MMU_CD_0_ASET BIT(47)181#define IVPU_MMU_CD_0_ASID GENMASK_ULL(63, 48)182 183#define IVPU_MMU_T0SZ_48BIT 16184#define IVPU_MMU_T0SZ_38BIT 26185 186#define IVPU_MMU_IPS_48BIT 5187#define IVPU_MMU_IPS_44BIT 4188#define IVPU_MMU_IPS_42BIT 3189#define IVPU_MMU_IPS_40BIT 2190#define IVPU_MMU_IPS_36BIT 1191#define IVPU_MMU_IPS_32BIT 0192 193#define IVPU_MMU_CD_1_TTB0_MASK GENMASK_ULL(51, 4)194 195#define IVPU_MMU_STE_0_S1CDMAX GENMASK_ULL(63, 59)196#define IVPU_MMU_STE_0_S1FMT GENMASK_ULL(5, 4)197#define IVPU_MMU_STE_0_S1FMT_LINEAR 0198#define IVPU_MMU_STE_DWORDS 8199#define IVPU_MMU_STE_0_CFG_S1_TRANS 5200#define IVPU_MMU_STE_0_CFG GENMASK_ULL(3, 1)201#define IVPU_MMU_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6)202#define IVPU_MMU_STE_0_V BIT(0)203 204#define IVPU_MMU_STE_1_STRW_NSEL1 0ul205#define IVPU_MMU_STE_1_CONT GENMASK_ULL(16, 13)206#define IVPU_MMU_STE_1_STRW GENMASK_ULL(31, 30)207#define IVPU_MMU_STE_1_PRIVCFG GENMASK_ULL(49, 48)208#define IVPU_MMU_STE_1_PRIVCFG_UNPRIV 2ul209#define IVPU_MMU_STE_1_INSTCFG GENMASK_ULL(51, 50)210#define IVPU_MMU_STE_1_INSTCFG_DATA 2ul211#define IVPU_MMU_STE_1_MEV BIT(19)212#define IVPU_MMU_STE_1_S1STALLD BIT(27)213#define IVPU_MMU_STE_1_S1C_CACHE_NC 0ul214#define IVPU_MMU_STE_1_S1C_CACHE_WBRA 1ul215#define IVPU_MMU_STE_1_S1C_CACHE_WT 2ul216#define IVPU_MMU_STE_1_S1C_CACHE_WB 3ul217#define IVPU_MMU_STE_1_S1CIR GENMASK_ULL(3, 2)218#define IVPU_MMU_STE_1_S1COR GENMASK_ULL(5, 4)219#define IVPU_MMU_STE_1_S1CSH GENMASK_ULL(7, 6)220#define IVPU_MMU_STE_1_S1DSS GENMASK_ULL(1, 0)221#define IVPU_MMU_STE_1_S1DSS_TERMINATE 0x0222 223#define IVPU_MMU_REG_TIMEOUT_US (10 * USEC_PER_MSEC)224#define IVPU_MMU_QUEUE_TIMEOUT_US (100 * USEC_PER_MSEC)225 226#define IVPU_MMU_GERROR_ERR_MASK ((REG_FLD(IVPU_MMU_REG_GERROR, CMDQ)) | \227 (REG_FLD(IVPU_MMU_REG_GERROR, EVTQ_ABT)) | \228 (REG_FLD(IVPU_MMU_REG_GERROR, PRIQ_ABT)) | \229 (REG_FLD(IVPU_MMU_REG_GERROR, MSI_CMDQ_ABT)) | \230 (REG_FLD(IVPU_MMU_REG_GERROR, MSI_EVTQ_ABT)) | \231 (REG_FLD(IVPU_MMU_REG_GERROR, MSI_PRIQ_ABT)) | \232 (REG_FLD(IVPU_MMU_REG_GERROR, MSI_ABT)))233 234#define IVPU_MMU_CERROR_NONE 0x0235#define IVPU_MMU_CERROR_ILL 0x1236#define IVPU_MMU_CERROR_ABT 0x2237#define IVPU_MMU_CERROR_ATC_INV_SYNC 0x3238 239static const char *ivpu_mmu_event_to_str(u32 cmd)240{241 switch (cmd) {242 case IVPU_MMU_EVT_F_UUT:243 return "Unsupported Upstream Transaction";244 case IVPU_MMU_EVT_C_BAD_STREAMID:245 return "Transaction StreamID out of range";246 case IVPU_MMU_EVT_F_STE_FETCH:247 return "Fetch of STE caused external abort";248 case IVPU_MMU_EVT_C_BAD_STE:249 return "Used STE invalid";250 case IVPU_MMU_EVT_F_BAD_ATS_TREQ:251 return "Address Request disallowed for a StreamID";252 case IVPU_MMU_EVT_F_STREAM_DISABLED:253 return "Transaction marks non-substream disabled";254 case IVPU_MMU_EVT_F_TRANSL_FORBIDDEN:255 return "MMU bypass is disallowed for this StreamID";256 case IVPU_MMU_EVT_C_BAD_SUBSTREAMID:257 return "Invalid StreamID";258 case IVPU_MMU_EVT_F_CD_FETCH:259 return "Fetch of CD caused external abort";260 case IVPU_MMU_EVT_C_BAD_CD:261 return "Fetched CD invalid";262 case IVPU_MMU_EVT_F_WALK_EABT:263 return " An external abort occurred fetching a TLB";264 case IVPU_MMU_EVT_F_TRANSLATION:265 return "Translation fault";266 case IVPU_MMU_EVT_F_ADDR_SIZE:267 return " Output address caused address size fault";268 case IVPU_MMU_EVT_F_ACCESS:269 return "Access flag fault";270 case IVPU_MMU_EVT_F_PERMISSION:271 return "Permission fault occurred on page access";272 case IVPU_MMU_EVT_F_TLB_CONFLICT:273 return "A TLB conflict";274 case IVPU_MMU_EVT_F_CFG_CONFLICT:275 return "A configuration cache conflict";276 case IVPU_MMU_EVT_E_PAGE_REQUEST:277 return "Page request hint from a client device";278 case IVPU_MMU_EVT_F_VMS_FETCH:279 return "Fetch of VMS caused external abort";280 default:281 return "Unknown event";282 }283}284 285static const char *ivpu_mmu_cmdq_err_to_str(u32 err)286{287 switch (err) {288 case IVPU_MMU_CERROR_NONE:289 return "No error";290 case IVPU_MMU_CERROR_ILL:291 return "Illegal command";292 case IVPU_MMU_CERROR_ABT:293 return "External abort on command queue read";294 case IVPU_MMU_CERROR_ATC_INV_SYNC:295 return "Sync failed to complete ATS invalidation";296 default:297 return "Unknown error";298 }299}300 301static void ivpu_mmu_config_check(struct ivpu_device *vdev)302{303 u32 val_ref;304 u32 val;305 306 if (ivpu_is_simics(vdev))307 val_ref = IVPU_MMU_IDR0_REF_SIMICS;308 else309 val_ref = IVPU_MMU_IDR0_REF;310 311 val = REGV_RD32(IVPU_MMU_REG_IDR0);312 if (val != val_ref)313 ivpu_dbg(vdev, MMU, "IDR0 0x%x != IDR0_REF 0x%x\n", val, val_ref);314 315 val = REGV_RD32(IVPU_MMU_REG_IDR1);316 if (val != IVPU_MMU_IDR1_REF)317 ivpu_dbg(vdev, MMU, "IDR1 0x%x != IDR1_REF 0x%x\n", val, IVPU_MMU_IDR1_REF);318 319 val = REGV_RD32(IVPU_MMU_REG_IDR3);320 if (val != IVPU_MMU_IDR3_REF)321 ivpu_dbg(vdev, MMU, "IDR3 0x%x != IDR3_REF 0x%x\n", val, IVPU_MMU_IDR3_REF);322 323 if (ivpu_is_simics(vdev))324 val_ref = IVPU_MMU_IDR5_REF_SIMICS;325 else if (ivpu_is_fpga(vdev))326 val_ref = IVPU_MMU_IDR5_REF_FPGA;327 else328 val_ref = IVPU_MMU_IDR5_REF;329 330 val = REGV_RD32(IVPU_MMU_REG_IDR5);331 if (val != val_ref)332 ivpu_dbg(vdev, MMU, "IDR5 0x%x != IDR5_REF 0x%x\n", val, val_ref);333}334 335static int ivpu_mmu_cdtab_alloc(struct ivpu_device *vdev)336{337 struct ivpu_mmu_info *mmu = vdev->mmu;338 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;339 size_t size = IVPU_MMU_CDTAB_ENT_COUNT * IVPU_MMU_CDTAB_ENT_SIZE;340 341 cdtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &cdtab->dma, GFP_KERNEL);342 if (!cdtab->base)343 return -ENOMEM;344 345 ivpu_dbg(vdev, MMU, "CDTAB alloc: dma=%pad size=%zu\n", &cdtab->dma, size);346 347 return 0;348}349 350static int ivpu_mmu_strtab_alloc(struct ivpu_device *vdev)351{352 struct ivpu_mmu_info *mmu = vdev->mmu;353 struct ivpu_mmu_strtab *strtab = &mmu->strtab;354 size_t size = IVPU_MMU_STRTAB_ENT_COUNT * IVPU_MMU_STRTAB_ENT_SIZE;355 356 strtab->base = dmam_alloc_coherent(vdev->drm.dev, size, &strtab->dma, GFP_KERNEL);357 if (!strtab->base)358 return -ENOMEM;359 360 strtab->base_cfg = IVPU_MMU_STRTAB_CFG;361 strtab->dma_q = IVPU_MMU_STRTAB_BASE_RA;362 strtab->dma_q |= strtab->dma & IVPU_MMU_STRTAB_BASE_ADDR_MASK;363 364 ivpu_dbg(vdev, MMU, "STRTAB alloc: dma=%pad dma_q=%pad size=%zu\n",365 &strtab->dma, &strtab->dma_q, size);366 367 return 0;368}369 370static int ivpu_mmu_cmdq_alloc(struct ivpu_device *vdev)371{372 struct ivpu_mmu_info *mmu = vdev->mmu;373 struct ivpu_mmu_queue *q = &mmu->cmdq;374 375 q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_CMDQ_SIZE, &q->dma, GFP_KERNEL);376 if (!q->base)377 return -ENOMEM;378 379 q->dma_q = IVPU_MMU_Q_BASE_RWA;380 q->dma_q |= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK;381 q->dma_q |= IVPU_MMU_Q_COUNT_LOG2;382 383 ivpu_dbg(vdev, MMU, "CMDQ alloc: dma=%pad dma_q=%pad size=%u\n",384 &q->dma, &q->dma_q, IVPU_MMU_CMDQ_SIZE);385 386 return 0;387}388 389static int ivpu_mmu_evtq_alloc(struct ivpu_device *vdev)390{391 struct ivpu_mmu_info *mmu = vdev->mmu;392 struct ivpu_mmu_queue *q = &mmu->evtq;393 394 q->base = dmam_alloc_coherent(vdev->drm.dev, IVPU_MMU_EVTQ_SIZE, &q->dma, GFP_KERNEL);395 if (!q->base)396 return -ENOMEM;397 398 q->dma_q = IVPU_MMU_Q_BASE_RWA;399 q->dma_q |= q->dma & IVPU_MMU_Q_BASE_ADDR_MASK;400 q->dma_q |= IVPU_MMU_Q_COUNT_LOG2;401 402 ivpu_dbg(vdev, MMU, "EVTQ alloc: dma=%pad dma_q=%pad size=%u\n",403 &q->dma, &q->dma_q, IVPU_MMU_EVTQ_SIZE);404 405 return 0;406}407 408static int ivpu_mmu_structs_alloc(struct ivpu_device *vdev)409{410 int ret;411 412 ret = ivpu_mmu_cdtab_alloc(vdev);413 if (ret) {414 ivpu_err(vdev, "Failed to allocate cdtab: %d\n", ret);415 return ret;416 }417 418 ret = ivpu_mmu_strtab_alloc(vdev);419 if (ret) {420 ivpu_err(vdev, "Failed to allocate strtab: %d\n", ret);421 return ret;422 }423 424 ret = ivpu_mmu_cmdq_alloc(vdev);425 if (ret) {426 ivpu_err(vdev, "Failed to allocate cmdq: %d\n", ret);427 return ret;428 }429 430 ret = ivpu_mmu_evtq_alloc(vdev);431 if (ret)432 ivpu_err(vdev, "Failed to allocate evtq: %d\n", ret);433 434 return ret;435}436 437static int ivpu_mmu_reg_write_cr0(struct ivpu_device *vdev, u32 val)438{439 REGV_WR32(IVPU_MMU_REG_CR0, val);440 441 return REGV_POLL_FLD(IVPU_MMU_REG_CR0ACK, VAL, val, IVPU_MMU_REG_TIMEOUT_US);442}443 444static int ivpu_mmu_reg_write_irq_ctrl(struct ivpu_device *vdev, u32 val)445{446 REGV_WR32(IVPU_MMU_REG_IRQ_CTRL, val);447 448 return REGV_POLL_FLD(IVPU_MMU_REG_IRQ_CTRLACK, VAL, val, IVPU_MMU_REG_TIMEOUT_US);449}450 451static int ivpu_mmu_irqs_setup(struct ivpu_device *vdev)452{453 u32 irq_ctrl = IVPU_MMU_IRQ_EVTQ_EN | IVPU_MMU_IRQ_GERROR_EN;454 int ret;455 456 ret = ivpu_mmu_reg_write_irq_ctrl(vdev, 0);457 if (ret)458 return ret;459 460 return ivpu_mmu_reg_write_irq_ctrl(vdev, irq_ctrl);461}462 463static int ivpu_mmu_cmdq_wait_for_cons(struct ivpu_device *vdev)464{465 struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq;466 int ret;467 468 ret = REGV_POLL_FLD(IVPU_MMU_REG_CMDQ_CONS, VAL, cmdq->prod,469 IVPU_MMU_QUEUE_TIMEOUT_US);470 if (ret)471 return ret;472 473 cmdq->cons = cmdq->prod;474 475 return 0;476}477 478static bool ivpu_mmu_queue_is_full(struct ivpu_mmu_queue *q)479{480 return ((IVPU_MMU_Q_IDX(q->prod) == IVPU_MMU_Q_IDX(q->cons)) &&481 (IVPU_MMU_Q_WRP(q->prod) != IVPU_MMU_Q_WRP(q->cons)));482}483 484static bool ivpu_mmu_queue_is_empty(struct ivpu_mmu_queue *q)485{486 return ((IVPU_MMU_Q_IDX(q->prod) == IVPU_MMU_Q_IDX(q->cons)) &&487 (IVPU_MMU_Q_WRP(q->prod) == IVPU_MMU_Q_WRP(q->cons)));488}489 490static int ivpu_mmu_cmdq_cmd_write(struct ivpu_device *vdev, const char *name, u64 data0, u64 data1)491{492 struct ivpu_mmu_queue *cmdq = &vdev->mmu->cmdq;493 u64 *queue_buffer = cmdq->base;494 int idx = IVPU_MMU_Q_IDX(cmdq->prod) * (IVPU_MMU_CMDQ_CMD_SIZE / sizeof(*queue_buffer));495 496 if (ivpu_mmu_queue_is_full(cmdq)) {497 ivpu_err(vdev, "Failed to write MMU CMD %s\n", name);498 return -EBUSY;499 }500 501 queue_buffer[idx] = data0;502 queue_buffer[idx + 1] = data1;503 cmdq->prod = (cmdq->prod + 1) & IVPU_MMU_Q_WRAP_MASK;504 505 ivpu_dbg(vdev, MMU, "CMD write: %s data: 0x%llx 0x%llx\n", name, data0, data1);506 507 return 0;508}509 510static int ivpu_mmu_cmdq_sync(struct ivpu_device *vdev)511{512 struct ivpu_mmu_queue *q = &vdev->mmu->cmdq;513 u64 val;514 int ret;515 516 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_SYNC);517 518 ret = ivpu_mmu_cmdq_cmd_write(vdev, "SYNC", val, 0);519 if (ret)520 return ret;521 522 if (!ivpu_is_force_snoop_enabled(vdev))523 clflush_cache_range(q->base, IVPU_MMU_CMDQ_SIZE);524 REGV_WR32(IVPU_MMU_REG_CMDQ_PROD, q->prod);525 526 ret = ivpu_mmu_cmdq_wait_for_cons(vdev);527 if (ret) {528 u32 err;529 530 val = REGV_RD32(IVPU_MMU_REG_CMDQ_CONS);531 err = REG_GET_FLD(IVPU_MMU_REG_CMDQ_CONS, ERR, val);532 533 ivpu_err(vdev, "Timed out waiting for MMU consumer: %d, error: %s\n", ret,534 ivpu_mmu_cmdq_err_to_str(err));535 ivpu_hw_diagnose_failure(vdev);536 }537 538 return ret;539}540 541static int ivpu_mmu_cmdq_write_cfgi_all(struct ivpu_device *vdev)542{543 u64 data0 = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_CFGI_ALL);544 u64 data1 = FIELD_PREP(IVPU_MMU_CMD_CFGI_1_RANGE, 0x1f);545 546 return ivpu_mmu_cmdq_cmd_write(vdev, "CFGI_ALL", data0, data1);547}548 549static int ivpu_mmu_cmdq_write_tlbi_nh_asid(struct ivpu_device *vdev, u16 ssid)550{551 u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NH_ASID) |552 FIELD_PREP(IVPU_MMU_CMD_TLBI_0_ASID, ssid);553 554 return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NH_ASID", val, 0);555}556 557static int ivpu_mmu_cmdq_write_tlbi_nsnh_all(struct ivpu_device *vdev)558{559 u64 val = FIELD_PREP(IVPU_MMU_CMD_OPCODE, CMD_TLBI_NSNH_ALL);560 561 return ivpu_mmu_cmdq_cmd_write(vdev, "TLBI_NSNH_ALL", val, 0);562}563 564static int ivpu_mmu_reset(struct ivpu_device *vdev)565{566 struct ivpu_mmu_info *mmu = vdev->mmu;567 u32 val;568 int ret;569 570 memset(mmu->cmdq.base, 0, IVPU_MMU_CMDQ_SIZE);571 if (!ivpu_is_force_snoop_enabled(vdev))572 clflush_cache_range(mmu->cmdq.base, IVPU_MMU_CMDQ_SIZE);573 mmu->cmdq.prod = 0;574 mmu->cmdq.cons = 0;575 576 memset(mmu->evtq.base, 0, IVPU_MMU_EVTQ_SIZE);577 mmu->evtq.prod = 0;578 mmu->evtq.cons = 0;579 580 ret = ivpu_mmu_reg_write_cr0(vdev, 0);581 if (ret)582 return ret;583 584 val = FIELD_PREP(IVPU_MMU_CR1_TABLE_SH, IVPU_MMU_SH_ISH) |585 FIELD_PREP(IVPU_MMU_CR1_TABLE_OC, IVPU_MMU_CACHE_WB) |586 FIELD_PREP(IVPU_MMU_CR1_TABLE_IC, IVPU_MMU_CACHE_WB) |587 FIELD_PREP(IVPU_MMU_CR1_QUEUE_SH, IVPU_MMU_SH_ISH) |588 FIELD_PREP(IVPU_MMU_CR1_QUEUE_OC, IVPU_MMU_CACHE_WB) |589 FIELD_PREP(IVPU_MMU_CR1_QUEUE_IC, IVPU_MMU_CACHE_WB);590 REGV_WR32(IVPU_MMU_REG_CR1, val);591 592 REGV_WR64(IVPU_MMU_REG_STRTAB_BASE, mmu->strtab.dma_q);593 REGV_WR32(IVPU_MMU_REG_STRTAB_BASE_CFG, mmu->strtab.base_cfg);594 595 REGV_WR64(IVPU_MMU_REG_CMDQ_BASE, mmu->cmdq.dma_q);596 REGV_WR32(IVPU_MMU_REG_CMDQ_PROD, 0);597 REGV_WR32(IVPU_MMU_REG_CMDQ_CONS, 0);598 599 val = IVPU_MMU_CR0_CMDQEN;600 ret = ivpu_mmu_reg_write_cr0(vdev, val);601 if (ret)602 return ret;603 604 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);605 if (ret)606 return ret;607 608 ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev);609 if (ret)610 return ret;611 612 ret = ivpu_mmu_cmdq_sync(vdev);613 if (ret)614 return ret;615 616 REGV_WR64(IVPU_MMU_REG_EVTQ_BASE, mmu->evtq.dma_q);617 REGV_WR32(IVPU_MMU_REG_EVTQ_PROD_SEC, 0);618 REGV_WR32(IVPU_MMU_REG_EVTQ_CONS_SEC, 0);619 620 val |= IVPU_MMU_CR0_EVTQEN;621 ret = ivpu_mmu_reg_write_cr0(vdev, val);622 if (ret)623 return ret;624 625 val |= IVPU_MMU_CR0_ATSCHK;626 ret = ivpu_mmu_reg_write_cr0(vdev, val);627 if (ret)628 return ret;629 630 ret = ivpu_mmu_irqs_setup(vdev);631 if (ret)632 return ret;633 634 val |= IVPU_MMU_CR0_SMMUEN;635 return ivpu_mmu_reg_write_cr0(vdev, val);636}637 638static void ivpu_mmu_strtab_link_cd(struct ivpu_device *vdev, u32 sid)639{640 struct ivpu_mmu_info *mmu = vdev->mmu;641 struct ivpu_mmu_strtab *strtab = &mmu->strtab;642 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;643 u64 *entry = strtab->base + (sid * IVPU_MMU_STRTAB_ENT_SIZE);644 u64 str[2];645 646 str[0] = FIELD_PREP(IVPU_MMU_STE_0_CFG, IVPU_MMU_STE_0_CFG_S1_TRANS) |647 FIELD_PREP(IVPU_MMU_STE_0_S1CDMAX, IVPU_MMU_CDTAB_ENT_COUNT_LOG2) |648 FIELD_PREP(IVPU_MMU_STE_0_S1FMT, IVPU_MMU_STE_0_S1FMT_LINEAR) |649 IVPU_MMU_STE_0_V |650 (cdtab->dma & IVPU_MMU_STE_0_S1CTXPTR_MASK);651 652 str[1] = FIELD_PREP(IVPU_MMU_STE_1_S1DSS, IVPU_MMU_STE_1_S1DSS_TERMINATE) |653 FIELD_PREP(IVPU_MMU_STE_1_S1CIR, IVPU_MMU_STE_1_S1C_CACHE_NC) |654 FIELD_PREP(IVPU_MMU_STE_1_S1COR, IVPU_MMU_STE_1_S1C_CACHE_NC) |655 FIELD_PREP(IVPU_MMU_STE_1_S1CSH, IVPU_MMU_SH_NSH) |656 FIELD_PREP(IVPU_MMU_STE_1_PRIVCFG, IVPU_MMU_STE_1_PRIVCFG_UNPRIV) |657 FIELD_PREP(IVPU_MMU_STE_1_INSTCFG, IVPU_MMU_STE_1_INSTCFG_DATA) |658 FIELD_PREP(IVPU_MMU_STE_1_STRW, IVPU_MMU_STE_1_STRW_NSEL1) |659 FIELD_PREP(IVPU_MMU_STE_1_CONT, IVPU_MMU_STRTAB_CFG_LOG2SIZE) |660 IVPU_MMU_STE_1_MEV |661 IVPU_MMU_STE_1_S1STALLD;662 663 WRITE_ONCE(entry[1], str[1]);664 WRITE_ONCE(entry[0], str[0]);665 666 if (!ivpu_is_force_snoop_enabled(vdev))667 clflush_cache_range(entry, IVPU_MMU_STRTAB_ENT_SIZE);668 669 ivpu_dbg(vdev, MMU, "STRTAB write entry (SSID=%u): 0x%llx, 0x%llx\n", sid, str[0], str[1]);670}671 672static int ivpu_mmu_strtab_init(struct ivpu_device *vdev)673{674 ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID0);675 ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID3);676 677 return 0;678}679 680int ivpu_mmu_invalidate_tlb(struct ivpu_device *vdev, u16 ssid)681{682 struct ivpu_mmu_info *mmu = vdev->mmu;683 int ret = 0;684 685 mutex_lock(&mmu->lock);686 if (!mmu->on)687 goto unlock;688 689 ret = ivpu_mmu_cmdq_write_tlbi_nh_asid(vdev, ssid);690 if (ret)691 goto unlock;692 693 ret = ivpu_mmu_cmdq_sync(vdev);694unlock:695 mutex_unlock(&mmu->lock);696 return ret;697}698 699static int ivpu_mmu_cd_add(struct ivpu_device *vdev, u32 ssid, u64 cd_dma)700{701 struct ivpu_mmu_info *mmu = vdev->mmu;702 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;703 u64 *entry;704 u64 cd[4];705 int ret = 0;706 707 if (ssid > IVPU_MMU_CDTAB_ENT_COUNT)708 return -EINVAL;709 710 entry = cdtab->base + (ssid * IVPU_MMU_CDTAB_ENT_SIZE);711 712 if (cd_dma != 0) {713 cd[0] = FIELD_PREP(IVPU_MMU_CD_0_TCR_T0SZ, IVPU_MMU_T0SZ_48BIT) |714 FIELD_PREP(IVPU_MMU_CD_0_TCR_TG0, 0) |715 FIELD_PREP(IVPU_MMU_CD_0_TCR_IRGN0, 0) |716 FIELD_PREP(IVPU_MMU_CD_0_TCR_ORGN0, 0) |717 FIELD_PREP(IVPU_MMU_CD_0_TCR_SH0, 0) |718 FIELD_PREP(IVPU_MMU_CD_0_TCR_IPS, IVPU_MMU_IPS_48BIT) |719 FIELD_PREP(IVPU_MMU_CD_0_ASID, ssid) |720 IVPU_MMU_CD_0_TCR_EPD1 |721 IVPU_MMU_CD_0_AA64 |722 IVPU_MMU_CD_0_R |723 IVPU_MMU_CD_0_ASET |724 IVPU_MMU_CD_0_V;725 cd[1] = cd_dma & IVPU_MMU_CD_1_TTB0_MASK;726 cd[2] = 0;727 cd[3] = 0x0000000000007444;728 729 /* For global context generate memory fault on VPU */730 if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID)731 cd[0] |= IVPU_MMU_CD_0_A;732 } else {733 memset(cd, 0, sizeof(cd));734 }735 736 WRITE_ONCE(entry[1], cd[1]);737 WRITE_ONCE(entry[2], cd[2]);738 WRITE_ONCE(entry[3], cd[3]);739 WRITE_ONCE(entry[0], cd[0]);740 741 if (!ivpu_is_force_snoop_enabled(vdev))742 clflush_cache_range(entry, IVPU_MMU_CDTAB_ENT_SIZE);743 744 ivpu_dbg(vdev, MMU, "CDTAB %s entry (SSID=%u, dma=%pad): 0x%llx, 0x%llx, 0x%llx, 0x%llx\n",745 cd_dma ? "write" : "clear", ssid, &cd_dma, cd[0], cd[1], cd[2], cd[3]);746 747 mutex_lock(&mmu->lock);748 if (!mmu->on)749 goto unlock;750 751 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);752 if (ret)753 goto unlock;754 755 ret = ivpu_mmu_cmdq_sync(vdev);756unlock:757 mutex_unlock(&mmu->lock);758 return ret;759}760 761static int ivpu_mmu_cd_add_gbl(struct ivpu_device *vdev)762{763 int ret;764 765 ret = ivpu_mmu_cd_add(vdev, 0, vdev->gctx.pgtable.pgd_dma);766 if (ret)767 ivpu_err(vdev, "Failed to add global CD entry: %d\n", ret);768 769 return ret;770}771 772static int ivpu_mmu_cd_add_user(struct ivpu_device *vdev, u32 ssid, dma_addr_t cd_dma)773{774 int ret;775 776 if (ssid == 0) {777 ivpu_err(vdev, "Invalid SSID: %u\n", ssid);778 return -EINVAL;779 }780 781 ret = ivpu_mmu_cd_add(vdev, ssid, cd_dma);782 if (ret)783 ivpu_err(vdev, "Failed to add CD entry SSID=%u: %d\n", ssid, ret);784 785 return ret;786}787 788int ivpu_mmu_init(struct ivpu_device *vdev)789{790 struct ivpu_mmu_info *mmu = vdev->mmu;791 int ret;792 793 ivpu_dbg(vdev, MMU, "Init..\n");794 795 ivpu_mmu_config_check(vdev);796 797 ret = drmm_mutex_init(&vdev->drm, &mmu->lock);798 if (ret)799 return ret;800 801 ret = ivpu_mmu_structs_alloc(vdev);802 if (ret)803 return ret;804 805 ret = ivpu_mmu_strtab_init(vdev);806 if (ret) {807 ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret);808 return ret;809 }810 811 ret = ivpu_mmu_cd_add_gbl(vdev);812 if (ret) {813 ivpu_err(vdev, "Failed to initialize strtab: %d\n", ret);814 return ret;815 }816 817 ret = ivpu_mmu_enable(vdev);818 if (ret) {819 ivpu_err(vdev, "Failed to resume MMU: %d\n", ret);820 return ret;821 }822 823 ivpu_dbg(vdev, MMU, "Init done\n");824 825 return 0;826}827 828int ivpu_mmu_enable(struct ivpu_device *vdev)829{830 struct ivpu_mmu_info *mmu = vdev->mmu;831 int ret;832 833 mutex_lock(&mmu->lock);834 835 mmu->on = true;836 837 ret = ivpu_mmu_reset(vdev);838 if (ret) {839 ivpu_err(vdev, "Failed to reset MMU: %d\n", ret);840 goto err;841 }842 843 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);844 if (ret)845 goto err;846 847 ret = ivpu_mmu_cmdq_write_tlbi_nsnh_all(vdev);848 if (ret)849 goto err;850 851 ret = ivpu_mmu_cmdq_sync(vdev);852 if (ret)853 goto err;854 855 mutex_unlock(&mmu->lock);856 857 return 0;858err:859 mmu->on = false;860 mutex_unlock(&mmu->lock);861 return ret;862}863 864void ivpu_mmu_disable(struct ivpu_device *vdev)865{866 struct ivpu_mmu_info *mmu = vdev->mmu;867 868 mutex_lock(&mmu->lock);869 mmu->on = false;870 mutex_unlock(&mmu->lock);871}872 873static void ivpu_mmu_dump_event(struct ivpu_device *vdev, u32 *event)874{875 u32 ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]);876 u32 op = FIELD_GET(IVPU_MMU_EVT_OP_MASK, event[0]);877 u64 fetch_addr = ((u64)event[7]) << 32 | event[6];878 u64 in_addr = ((u64)event[5]) << 32 | event[4];879 u32 sid = event[1];880 881 ivpu_err_ratelimited(vdev, "MMU EVTQ: 0x%x (%s) SSID: %d SID: %d, e[2] %08x, e[3] %08x, in addr: 0x%llx, fetch addr: 0x%llx\n",882 op, ivpu_mmu_event_to_str(op), ssid, sid,883 event[2], event[3], in_addr, fetch_addr);884}885 886static u32 *ivpu_mmu_get_event(struct ivpu_device *vdev)887{888 struct ivpu_mmu_queue *evtq = &vdev->mmu->evtq;889 u32 idx = IVPU_MMU_Q_IDX(evtq->cons);890 u32 *evt = evtq->base + (idx * IVPU_MMU_EVTQ_CMD_SIZE);891 892 evtq->prod = REGV_RD32(IVPU_MMU_REG_EVTQ_PROD_SEC);893 if (ivpu_mmu_queue_is_empty(evtq))894 return NULL;895 896 evtq->cons = (evtq->cons + 1) & IVPU_MMU_Q_WRAP_MASK;897 return evt;898}899 900void ivpu_mmu_irq_evtq_handler(struct ivpu_device *vdev)901{902 u32 *event;903 u32 ssid;904 905 ivpu_dbg(vdev, IRQ, "MMU event queue\n");906 907 while ((event = ivpu_mmu_get_event(vdev)) != NULL) {908 ivpu_mmu_dump_event(vdev, event);909 910 ssid = FIELD_GET(IVPU_MMU_EVT_SSID_MASK, event[0]);911 if (ssid == IVPU_GLOBAL_CONTEXT_MMU_SSID) {912 ivpu_pm_trigger_recovery(vdev, "MMU event");913 return;914 }915 916 ivpu_mmu_user_context_mark_invalid(vdev, ssid);917 REGV_WR32(IVPU_MMU_REG_EVTQ_CONS_SEC, vdev->mmu->evtq.cons);918 }919 920 if (!kfifo_put(&vdev->hw->irq.fifo, IVPU_HW_IRQ_SRC_MMU_EVTQ))921 ivpu_err_ratelimited(vdev, "IRQ FIFO full\n");922}923 924void ivpu_mmu_evtq_dump(struct ivpu_device *vdev)925{926 u32 *event;927 928 while ((event = ivpu_mmu_get_event(vdev)) != NULL)929 ivpu_mmu_dump_event(vdev, event);930}931 932void ivpu_mmu_irq_gerr_handler(struct ivpu_device *vdev)933{934 u32 gerror_val, gerrorn_val, active;935 936 ivpu_dbg(vdev, IRQ, "MMU error\n");937 938 gerror_val = REGV_RD32(IVPU_MMU_REG_GERROR);939 gerrorn_val = REGV_RD32(IVPU_MMU_REG_GERRORN);940 941 active = gerror_val ^ gerrorn_val;942 if (!(active & IVPU_MMU_GERROR_ERR_MASK))943 return;944 945 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_ABT, active))946 ivpu_warn_ratelimited(vdev, "MMU MSI ABT write aborted\n");947 948 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_PRIQ_ABT, active))949 ivpu_warn_ratelimited(vdev, "MMU PRIQ MSI ABT write aborted\n");950 951 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_EVTQ_ABT, active))952 ivpu_warn_ratelimited(vdev, "MMU EVTQ MSI ABT write aborted\n");953 954 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, MSI_CMDQ_ABT, active))955 ivpu_warn_ratelimited(vdev, "MMU CMDQ MSI ABT write aborted\n");956 957 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, PRIQ_ABT, active))958 ivpu_err_ratelimited(vdev, "MMU PRIQ write aborted\n");959 960 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, EVTQ_ABT, active))961 ivpu_err_ratelimited(vdev, "MMU EVTQ write aborted\n");962 963 if (REG_TEST_FLD(IVPU_MMU_REG_GERROR, CMDQ, active))964 ivpu_err_ratelimited(vdev, "MMU CMDQ write aborted\n");965 966 REGV_WR32(IVPU_MMU_REG_GERRORN, gerror_val);967}968 969int ivpu_mmu_set_pgtable(struct ivpu_device *vdev, int ssid, struct ivpu_mmu_pgtable *pgtable)970{971 return ivpu_mmu_cd_add_user(vdev, ssid, pgtable->pgd_dma);972}973 974void ivpu_mmu_clear_pgtable(struct ivpu_device *vdev, int ssid)975{976 ivpu_mmu_cd_add_user(vdev, ssid, 0); /* 0 will clear CD entry */977}978