249 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Guest ITS library, generously donated by drivers/irqchip/irq-gic-v3-its.c4 * over in the kernel tree.5 */6 7#include <linux/kvm.h>8#include <linux/sizes.h>9#include <asm/kvm_para.h>10#include <asm/kvm.h>11 12#include "kvm_util.h"13#include "vgic.h"14#include "gic.h"15#include "gic_v3.h"16#include "processor.h"17 18static u64 its_read_u64(unsigned long offset)19{20 return readq_relaxed(GITS_BASE_GVA + offset);21}22 23static void its_write_u64(unsigned long offset, u64 val)24{25 writeq_relaxed(val, GITS_BASE_GVA + offset);26}27 28static u32 its_read_u32(unsigned long offset)29{30 return readl_relaxed(GITS_BASE_GVA + offset);31}32 33static void its_write_u32(unsigned long offset, u32 val)34{35 writel_relaxed(val, GITS_BASE_GVA + offset);36}37 38static unsigned long its_find_baser(unsigned int type)39{40 int i;41 42 for (i = 0; i < GITS_BASER_NR_REGS; i++) {43 u64 baser;44 unsigned long offset = GITS_BASER + (i * sizeof(baser));45 46 baser = its_read_u64(offset);47 if (GITS_BASER_TYPE(baser) == type)48 return offset;49 }50 51 GUEST_FAIL("Couldn't find an ITS BASER of type %u", type);52 return -1;53}54 55static void its_install_table(unsigned int type, vm_paddr_t base, size_t size)56{57 unsigned long offset = its_find_baser(type);58 u64 baser;59 60 baser = ((size / SZ_64K) - 1) |61 GITS_BASER_PAGE_SIZE_64K |62 GITS_BASER_InnerShareable |63 base |64 GITS_BASER_RaWaWb |65 GITS_BASER_VALID;66 67 its_write_u64(offset, baser);68}69 70static void its_install_cmdq(vm_paddr_t base, size_t size)71{72 u64 cbaser;73 74 cbaser = ((size / SZ_4K) - 1) |75 GITS_CBASER_InnerShareable |76 base |77 GITS_CBASER_RaWaWb |78 GITS_CBASER_VALID;79 80 its_write_u64(GITS_CBASER, cbaser);81}82 83void its_init(vm_paddr_t coll_tbl, size_t coll_tbl_sz,84 vm_paddr_t device_tbl, size_t device_tbl_sz,85 vm_paddr_t cmdq, size_t cmdq_size)86{87 u32 ctlr;88 89 its_install_table(GITS_BASER_TYPE_COLLECTION, coll_tbl, coll_tbl_sz);90 its_install_table(GITS_BASER_TYPE_DEVICE, device_tbl, device_tbl_sz);91 its_install_cmdq(cmdq, cmdq_size);92 93 ctlr = its_read_u32(GITS_CTLR);94 ctlr |= GITS_CTLR_ENABLE;95 its_write_u32(GITS_CTLR, ctlr);96}97 98struct its_cmd_block {99 union {100 u64 raw_cmd[4];101 __le64 raw_cmd_le[4];102 };103};104 105static inline void its_fixup_cmd(struct its_cmd_block *cmd)106{107 /* Let's fixup BE commands */108 cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]);109 cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]);110 cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]);111 cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]);112}113 114static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l)115{116 u64 mask = GENMASK_ULL(h, l);117 *raw_cmd &= ~mask;118 *raw_cmd |= (val << l) & mask;119}120 121static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr)122{123 its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0);124}125 126static void its_encode_devid(struct its_cmd_block *cmd, u32 devid)127{128 its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32);129}130 131static void its_encode_event_id(struct its_cmd_block *cmd, u32 id)132{133 its_mask_encode(&cmd->raw_cmd[1], id, 31, 0);134}135 136static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id)137{138 its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32);139}140 141static void its_encode_size(struct its_cmd_block *cmd, u8 size)142{143 its_mask_encode(&cmd->raw_cmd[1], size, 4, 0);144}145 146static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr)147{148 its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8);149}150 151static void its_encode_valid(struct its_cmd_block *cmd, int valid)152{153 its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63);154}155 156static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr)157{158 its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16);159}160 161static void its_encode_collection(struct its_cmd_block *cmd, u16 col)162{163 its_mask_encode(&cmd->raw_cmd[2], col, 15, 0);164}165 166#define GITS_CMDQ_POLL_ITERATIONS 0167 168static void its_send_cmd(void *cmdq_base, struct its_cmd_block *cmd)169{170 u64 cwriter = its_read_u64(GITS_CWRITER);171 struct its_cmd_block *dst = cmdq_base + cwriter;172 u64 cbaser = its_read_u64(GITS_CBASER);173 size_t cmdq_size;174 u64 next;175 int i;176 177 cmdq_size = ((cbaser & 0xFF) + 1) * SZ_4K;178 179 its_fixup_cmd(cmd);180 181 WRITE_ONCE(*dst, *cmd);182 dsb(ishst);183 next = (cwriter + sizeof(*cmd)) % cmdq_size;184 its_write_u64(GITS_CWRITER, next);185 186 /*187 * Polling isn't necessary considering KVM's ITS emulation at the time188 * of writing this, as the CMDQ is processed synchronously after a write189 * to CWRITER.190 */191 for (i = 0; its_read_u64(GITS_CREADR) != next; i++) {192 __GUEST_ASSERT(i < GITS_CMDQ_POLL_ITERATIONS,193 "ITS didn't process command at offset %lu after %d iterations\n",194 cwriter, i);195 196 cpu_relax();197 }198}199 200void its_send_mapd_cmd(void *cmdq_base, u32 device_id, vm_paddr_t itt_base,201 size_t itt_size, bool valid)202{203 struct its_cmd_block cmd = {};204 205 its_encode_cmd(&cmd, GITS_CMD_MAPD);206 its_encode_devid(&cmd, device_id);207 its_encode_size(&cmd, ilog2(itt_size) - 1);208 its_encode_itt(&cmd, itt_base);209 its_encode_valid(&cmd, valid);210 211 its_send_cmd(cmdq_base, &cmd);212}213 214void its_send_mapc_cmd(void *cmdq_base, u32 vcpu_id, u32 collection_id, bool valid)215{216 struct its_cmd_block cmd = {};217 218 its_encode_cmd(&cmd, GITS_CMD_MAPC);219 its_encode_collection(&cmd, collection_id);220 its_encode_target(&cmd, vcpu_id);221 its_encode_valid(&cmd, valid);222 223 its_send_cmd(cmdq_base, &cmd);224}225 226void its_send_mapti_cmd(void *cmdq_base, u32 device_id, u32 event_id,227 u32 collection_id, u32 intid)228{229 struct its_cmd_block cmd = {};230 231 its_encode_cmd(&cmd, GITS_CMD_MAPTI);232 its_encode_devid(&cmd, device_id);233 its_encode_event_id(&cmd, event_id);234 its_encode_phys_id(&cmd, intid);235 its_encode_collection(&cmd, collection_id);236 237 its_send_cmd(cmdq_base, &cmd);238}239 240void its_send_invall_cmd(void *cmdq_base, u32 collection_id)241{242 struct its_cmd_block cmd = {};243 244 its_encode_cmd(&cmd, GITS_CMD_INVALL);245 its_encode_collection(&cmd, collection_id);246 247 its_send_cmd(cmdq_base, &cmd);248}249