428 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * ARM Generic Interrupt Controller (GIC) v3 support4 */5 6#include <linux/sizes.h>7 8#include "kvm_util.h"9#include "processor.h"10#include "delay.h"11 12#include "gic.h"13#include "gic_v3.h"14#include "gic_private.h"15 16#define GICV3_MAX_CPUS 51217 18#define GICD_INT_DEF_PRI 0xa019#define GICD_INT_DEF_PRI_X4 ((GICD_INT_DEF_PRI << 24) |\20 (GICD_INT_DEF_PRI << 16) |\21 (GICD_INT_DEF_PRI << 8) |\22 GICD_INT_DEF_PRI)23 24#define ICC_PMR_DEF_PRIO 0xf025 26struct gicv3_data {27 unsigned int nr_cpus;28 unsigned int nr_spis;29};30 31#define sgi_base_from_redist(redist_base) (redist_base + SZ_64K)32#define DIST_BIT (1U << 31)33 34enum gicv3_intid_range {35 SGI_RANGE,36 PPI_RANGE,37 SPI_RANGE,38 INVALID_RANGE,39};40 41static struct gicv3_data gicv3_data;42 43static void gicv3_gicd_wait_for_rwp(void)44{45 unsigned int count = 100000; /* 1s */46 47 while (readl(GICD_BASE_GVA + GICD_CTLR) & GICD_CTLR_RWP) {48 GUEST_ASSERT(count--);49 udelay(10);50 }51}52 53static inline volatile void *gicr_base_cpu(uint32_t cpu)54{55 /* Align all the redistributors sequentially */56 return GICR_BASE_GVA + cpu * SZ_64K * 2;57}58 59static void gicv3_gicr_wait_for_rwp(uint32_t cpu)60{61 unsigned int count = 100000; /* 1s */62 63 while (readl(gicr_base_cpu(cpu) + GICR_CTLR) & GICR_CTLR_RWP) {64 GUEST_ASSERT(count--);65 udelay(10);66 }67}68 69static void gicv3_wait_for_rwp(uint32_t cpu_or_dist)70{71 if (cpu_or_dist & DIST_BIT)72 gicv3_gicd_wait_for_rwp();73 else74 gicv3_gicr_wait_for_rwp(cpu_or_dist);75}76 77static enum gicv3_intid_range get_intid_range(unsigned int intid)78{79 switch (intid) {80 case 0 ... 15:81 return SGI_RANGE;82 case 16 ... 31:83 return PPI_RANGE;84 case 32 ... 1019:85 return SPI_RANGE;86 }87 88 /* We should not be reaching here */89 GUEST_ASSERT(0);90 91 return INVALID_RANGE;92}93 94static uint64_t gicv3_read_iar(void)95{96 uint64_t irqstat = read_sysreg_s(SYS_ICC_IAR1_EL1);97 98 dsb(sy);99 return irqstat;100}101 102static void gicv3_write_eoir(uint32_t irq)103{104 write_sysreg_s(irq, SYS_ICC_EOIR1_EL1);105 isb();106}107 108static void gicv3_write_dir(uint32_t irq)109{110 write_sysreg_s(irq, SYS_ICC_DIR_EL1);111 isb();112}113 114static void gicv3_set_priority_mask(uint64_t mask)115{116 write_sysreg_s(mask, SYS_ICC_PMR_EL1);117}118 119static void gicv3_set_eoi_split(bool split)120{121 uint32_t val;122 123 /*124 * All other fields are read-only, so no need to read CTLR first. In125 * fact, the kernel does the same.126 */127 val = split ? (1U << 1) : 0;128 write_sysreg_s(val, SYS_ICC_CTLR_EL1);129 isb();130}131 132uint32_t gicv3_reg_readl(uint32_t cpu_or_dist, uint64_t offset)133{134 volatile void *base = cpu_or_dist & DIST_BIT ? GICD_BASE_GVA135 : sgi_base_from_redist(gicr_base_cpu(cpu_or_dist));136 return readl(base + offset);137}138 139void gicv3_reg_writel(uint32_t cpu_or_dist, uint64_t offset, uint32_t reg_val)140{141 volatile void *base = cpu_or_dist & DIST_BIT ? GICD_BASE_GVA142 : sgi_base_from_redist(gicr_base_cpu(cpu_or_dist));143 writel(reg_val, base + offset);144}145 146uint32_t gicv3_getl_fields(uint32_t cpu_or_dist, uint64_t offset, uint32_t mask)147{148 return gicv3_reg_readl(cpu_or_dist, offset) & mask;149}150 151void gicv3_setl_fields(uint32_t cpu_or_dist, uint64_t offset,152 uint32_t mask, uint32_t reg_val)153{154 uint32_t tmp = gicv3_reg_readl(cpu_or_dist, offset) & ~mask;155 156 tmp |= (reg_val & mask);157 gicv3_reg_writel(cpu_or_dist, offset, tmp);158}159 160/*161 * We use a single offset for the distributor and redistributor maps as they162 * have the same value in both. The only exceptions are registers that only163 * exist in one and not the other, like GICR_WAKER that doesn't exist in the164 * distributor map. Such registers are conveniently marked as reserved in the165 * map that doesn't implement it; like GICR_WAKER's offset of 0x0014 being166 * marked as "Reserved" in the Distributor map.167 */168static void gicv3_access_reg(uint32_t intid, uint64_t offset,169 uint32_t reg_bits, uint32_t bits_per_field,170 bool write, uint32_t *val)171{172 uint32_t cpu = guest_get_vcpuid();173 enum gicv3_intid_range intid_range = get_intid_range(intid);174 uint32_t fields_per_reg, index, mask, shift;175 uint32_t cpu_or_dist;176 177 GUEST_ASSERT(bits_per_field <= reg_bits);178 GUEST_ASSERT(!write || *val < (1U << bits_per_field));179 /*180 * This function does not support 64 bit accesses. Just asserting here181 * until we implement readq/writeq.182 */183 GUEST_ASSERT(reg_bits == 32);184 185 fields_per_reg = reg_bits / bits_per_field;186 index = intid % fields_per_reg;187 shift = index * bits_per_field;188 mask = ((1U << bits_per_field) - 1) << shift;189 190 /* Set offset to the actual register holding intid's config. */191 offset += (intid / fields_per_reg) * (reg_bits / 8);192 193 cpu_or_dist = (intid_range == SPI_RANGE) ? DIST_BIT : cpu;194 195 if (write)196 gicv3_setl_fields(cpu_or_dist, offset, mask, *val << shift);197 *val = gicv3_getl_fields(cpu_or_dist, offset, mask) >> shift;198}199 200static void gicv3_write_reg(uint32_t intid, uint64_t offset,201 uint32_t reg_bits, uint32_t bits_per_field, uint32_t val)202{203 gicv3_access_reg(intid, offset, reg_bits,204 bits_per_field, true, &val);205}206 207static uint32_t gicv3_read_reg(uint32_t intid, uint64_t offset,208 uint32_t reg_bits, uint32_t bits_per_field)209{210 uint32_t val;211 212 gicv3_access_reg(intid, offset, reg_bits,213 bits_per_field, false, &val);214 return val;215}216 217static void gicv3_set_priority(uint32_t intid, uint32_t prio)218{219 gicv3_write_reg(intid, GICD_IPRIORITYR, 32, 8, prio);220}221 222/* Sets the intid to be level-sensitive or edge-triggered. */223static void gicv3_irq_set_config(uint32_t intid, bool is_edge)224{225 uint32_t val;226 227 /* N/A for private interrupts. */228 GUEST_ASSERT(get_intid_range(intid) == SPI_RANGE);229 val = is_edge ? 2 : 0;230 gicv3_write_reg(intid, GICD_ICFGR, 32, 2, val);231}232 233static void gicv3_irq_enable(uint32_t intid)234{235 bool is_spi = get_intid_range(intid) == SPI_RANGE;236 uint32_t cpu = guest_get_vcpuid();237 238 gicv3_write_reg(intid, GICD_ISENABLER, 32, 1, 1);239 gicv3_wait_for_rwp(is_spi ? DIST_BIT : cpu);240}241 242static void gicv3_irq_disable(uint32_t intid)243{244 bool is_spi = get_intid_range(intid) == SPI_RANGE;245 uint32_t cpu = guest_get_vcpuid();246 247 gicv3_write_reg(intid, GICD_ICENABLER, 32, 1, 1);248 gicv3_wait_for_rwp(is_spi ? DIST_BIT : cpu);249}250 251static void gicv3_irq_set_active(uint32_t intid)252{253 gicv3_write_reg(intid, GICD_ISACTIVER, 32, 1, 1);254}255 256static void gicv3_irq_clear_active(uint32_t intid)257{258 gicv3_write_reg(intid, GICD_ICACTIVER, 32, 1, 1);259}260 261static bool gicv3_irq_get_active(uint32_t intid)262{263 return gicv3_read_reg(intid, GICD_ISACTIVER, 32, 1);264}265 266static void gicv3_irq_set_pending(uint32_t intid)267{268 gicv3_write_reg(intid, GICD_ISPENDR, 32, 1, 1);269}270 271static void gicv3_irq_clear_pending(uint32_t intid)272{273 gicv3_write_reg(intid, GICD_ICPENDR, 32, 1, 1);274}275 276static bool gicv3_irq_get_pending(uint32_t intid)277{278 return gicv3_read_reg(intid, GICD_ISPENDR, 32, 1);279}280 281static void gicv3_enable_redist(volatile void *redist_base)282{283 uint32_t val = readl(redist_base + GICR_WAKER);284 unsigned int count = 100000; /* 1s */285 286 val &= ~GICR_WAKER_ProcessorSleep;287 writel(val, redist_base + GICR_WAKER);288 289 /* Wait until the processor is 'active' */290 while (readl(redist_base + GICR_WAKER) & GICR_WAKER_ChildrenAsleep) {291 GUEST_ASSERT(count--);292 udelay(10);293 }294}295 296static void gicv3_cpu_init(unsigned int cpu)297{298 volatile void *sgi_base;299 unsigned int i;300 volatile void *redist_base_cpu;301 302 GUEST_ASSERT(cpu < gicv3_data.nr_cpus);303 304 redist_base_cpu = gicr_base_cpu(cpu);305 sgi_base = sgi_base_from_redist(redist_base_cpu);306 307 gicv3_enable_redist(redist_base_cpu);308 309 /*310 * Mark all the SGI and PPI interrupts as non-secure Group-1.311 * Also, deactivate and disable them.312 */313 writel(~0, sgi_base + GICR_IGROUPR0);314 writel(~0, sgi_base + GICR_ICACTIVER0);315 writel(~0, sgi_base + GICR_ICENABLER0);316 317 /* Set a default priority for all the SGIs and PPIs */318 for (i = 0; i < 32; i += 4)319 writel(GICD_INT_DEF_PRI_X4,320 sgi_base + GICR_IPRIORITYR0 + i);321 322 gicv3_gicr_wait_for_rwp(cpu);323 324 /* Enable the GIC system register (ICC_*) access */325 write_sysreg_s(read_sysreg_s(SYS_ICC_SRE_EL1) | ICC_SRE_EL1_SRE,326 SYS_ICC_SRE_EL1);327 328 /* Set a default priority threshold */329 write_sysreg_s(ICC_PMR_DEF_PRIO, SYS_ICC_PMR_EL1);330 331 /* Enable non-secure Group-1 interrupts */332 write_sysreg_s(ICC_IGRPEN1_EL1_MASK, SYS_ICC_IGRPEN1_EL1);333}334 335static void gicv3_dist_init(void)336{337 unsigned int i;338 339 /* Disable the distributor until we set things up */340 writel(0, GICD_BASE_GVA + GICD_CTLR);341 gicv3_gicd_wait_for_rwp();342 343 /*344 * Mark all the SPI interrupts as non-secure Group-1.345 * Also, deactivate and disable them.346 */347 for (i = 32; i < gicv3_data.nr_spis; i += 32) {348 writel(~0, GICD_BASE_GVA + GICD_IGROUPR + i / 8);349 writel(~0, GICD_BASE_GVA + GICD_ICACTIVER + i / 8);350 writel(~0, GICD_BASE_GVA + GICD_ICENABLER + i / 8);351 }352 353 /* Set a default priority for all the SPIs */354 for (i = 32; i < gicv3_data.nr_spis; i += 4)355 writel(GICD_INT_DEF_PRI_X4,356 GICD_BASE_GVA + GICD_IPRIORITYR + i);357 358 /* Wait for the settings to sync-in */359 gicv3_gicd_wait_for_rwp();360 361 /* Finally, enable the distributor globally with ARE */362 writel(GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A |363 GICD_CTLR_ENABLE_G1, GICD_BASE_GVA + GICD_CTLR);364 gicv3_gicd_wait_for_rwp();365}366 367static void gicv3_init(unsigned int nr_cpus)368{369 GUEST_ASSERT(nr_cpus <= GICV3_MAX_CPUS);370 371 gicv3_data.nr_cpus = nr_cpus;372 gicv3_data.nr_spis = GICD_TYPER_SPIS(373 readl(GICD_BASE_GVA + GICD_TYPER));374 if (gicv3_data.nr_spis > 1020)375 gicv3_data.nr_spis = 1020;376 377 /*378 * Initialize only the distributor for now.379 * The redistributor and CPU interfaces are initialized380 * later for every PE.381 */382 gicv3_dist_init();383}384 385const struct gic_common_ops gicv3_ops = {386 .gic_init = gicv3_init,387 .gic_cpu_init = gicv3_cpu_init,388 .gic_irq_enable = gicv3_irq_enable,389 .gic_irq_disable = gicv3_irq_disable,390 .gic_read_iar = gicv3_read_iar,391 .gic_write_eoir = gicv3_write_eoir,392 .gic_write_dir = gicv3_write_dir,393 .gic_set_priority_mask = gicv3_set_priority_mask,394 .gic_set_eoi_split = gicv3_set_eoi_split,395 .gic_set_priority = gicv3_set_priority,396 .gic_irq_set_active = gicv3_irq_set_active,397 .gic_irq_clear_active = gicv3_irq_clear_active,398 .gic_irq_get_active = gicv3_irq_get_active,399 .gic_irq_set_pending = gicv3_irq_set_pending,400 .gic_irq_clear_pending = gicv3_irq_clear_pending,401 .gic_irq_get_pending = gicv3_irq_get_pending,402 .gic_irq_set_config = gicv3_irq_set_config,403};404 405void gic_rdist_enable_lpis(vm_paddr_t cfg_table, size_t cfg_table_size,406 vm_paddr_t pend_table)407{408 volatile void *rdist_base = gicr_base_cpu(guest_get_vcpuid());409 410 u32 ctlr;411 u64 val;412 413 val = (cfg_table |414 GICR_PROPBASER_InnerShareable |415 GICR_PROPBASER_RaWaWb |416 ((ilog2(cfg_table_size) - 1) & GICR_PROPBASER_IDBITS_MASK));417 writeq_relaxed(val, rdist_base + GICR_PROPBASER);418 419 val = (pend_table |420 GICR_PENDBASER_InnerShareable |421 GICR_PENDBASER_RaWaWb);422 writeq_relaxed(val, rdist_base + GICR_PENDBASER);423 424 ctlr = readl_relaxed(rdist_base + GICR_CTLR);425 ctlr |= GICR_CTLR_ENABLE_LPIS;426 writel_relaxed(ctlr, rdist_base + GICR_CTLR);427}428