285 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright © 2022 Intel Corporation4 */5 6#include "xe_reg_sr.h"7 8#include <kunit/visibility.h>9#include <linux/align.h>10#include <linux/string_helpers.h>11#include <linux/xarray.h>12 13#include <drm/drm_managed.h>14#include <drm/drm_print.h>15 16#include "regs/xe_engine_regs.h"17#include "regs/xe_gt_regs.h"18#include "xe_device_types.h"19#include "xe_force_wake.h"20#include "xe_gt.h"21#include "xe_gt_mcr.h"22#include "xe_gt_printk.h"23#include "xe_hw_engine_types.h"24#include "xe_macros.h"25#include "xe_mmio.h"26#include "xe_reg_whitelist.h"27#include "xe_rtp_types.h"28 29#define XE_REG_SR_GROW_STEP_DEFAULT 1630 31static void reg_sr_fini(struct drm_device *drm, void *arg)32{33 struct xe_reg_sr *sr = arg;34 35 xa_destroy(&sr->xa);36 kfree(sr->pool.arr);37 memset(&sr->pool, 0, sizeof(sr->pool));38}39 40int xe_reg_sr_init(struct xe_reg_sr *sr, const char *name, struct xe_device *xe)41{42 xa_init(&sr->xa);43 memset(&sr->pool, 0, sizeof(sr->pool));44 sr->pool.grow_step = XE_REG_SR_GROW_STEP_DEFAULT;45 sr->name = name;46 47 return drmm_add_action_or_reset(&xe->drm, reg_sr_fini, sr);48}49EXPORT_SYMBOL_IF_KUNIT(xe_reg_sr_init);50 51static struct xe_reg_sr_entry *alloc_entry(struct xe_reg_sr *sr)52{53 if (sr->pool.used == sr->pool.allocated) {54 struct xe_reg_sr_entry *arr;55 56 arr = krealloc_array(sr->pool.arr,57 ALIGN(sr->pool.allocated + 1, sr->pool.grow_step),58 sizeof(*arr), GFP_KERNEL);59 if (!arr)60 return NULL;61 62 sr->pool.arr = arr;63 sr->pool.allocated += sr->pool.grow_step;64 }65 66 return &sr->pool.arr[sr->pool.used++];67}68 69static bool compatible_entries(const struct xe_reg_sr_entry *e1,70 const struct xe_reg_sr_entry *e2)71{72 /*73 * Don't allow overwriting values: clr_bits/set_bits should be disjoint74 * when operating in the same register75 */76 if (e1->clr_bits & e2->clr_bits || e1->set_bits & e2->set_bits ||77 e1->clr_bits & e2->set_bits || e1->set_bits & e2->clr_bits)78 return false;79 80 if (e1->reg.raw != e2->reg.raw)81 return false;82 83 return true;84}85 86static void reg_sr_inc_error(struct xe_reg_sr *sr)87{88#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)89 sr->errors++;90#endif91}92 93int xe_reg_sr_add(struct xe_reg_sr *sr,94 const struct xe_reg_sr_entry *e,95 struct xe_gt *gt)96{97 unsigned long idx = e->reg.addr;98 struct xe_reg_sr_entry *pentry = xa_load(&sr->xa, idx);99 int ret;100 101 if (pentry) {102 if (!compatible_entries(pentry, e)) {103 ret = -EINVAL;104 goto fail;105 }106 107 pentry->clr_bits |= e->clr_bits;108 pentry->set_bits |= e->set_bits;109 pentry->read_mask |= e->read_mask;110 111 return 0;112 }113 114 pentry = alloc_entry(sr);115 if (!pentry) {116 ret = -ENOMEM;117 goto fail;118 }119 120 *pentry = *e;121 ret = xa_err(xa_store(&sr->xa, idx, pentry, GFP_KERNEL));122 if (ret)123 goto fail;124 125 return 0;126 127fail:128 xe_gt_err(gt,129 "discarding save-restore reg %04lx (clear: %08x, set: %08x, masked: %s, mcr: %s): ret=%d\n",130 idx, e->clr_bits, e->set_bits,131 str_yes_no(e->reg.masked),132 str_yes_no(e->reg.mcr),133 ret);134 reg_sr_inc_error(sr);135 136 return ret;137}138 139/*140 * Convert back from encoded value to type-safe, only to be used when reg.mcr141 * is true142 */143static struct xe_reg_mcr to_xe_reg_mcr(const struct xe_reg reg)144{145 return (const struct xe_reg_mcr){.__reg.raw = reg.raw };146}147 148static void apply_one_mmio(struct xe_gt *gt, struct xe_reg_sr_entry *entry)149{150 struct xe_reg reg = entry->reg;151 struct xe_reg_mcr reg_mcr = to_xe_reg_mcr(reg);152 u32 val;153 154 /*155 * If this is a masked register, need to set the upper 16 bits.156 * Set them to clr_bits since that is always a superset of the bits157 * being modified.158 *159 * When it's not masked, we have to read it from hardware, unless we are160 * supposed to set all bits.161 */162 if (reg.masked)163 val = entry->clr_bits << 16;164 else if (entry->clr_bits + 1)165 val = (reg.mcr ?166 xe_gt_mcr_unicast_read_any(gt, reg_mcr) :167 xe_mmio_read32(gt, reg)) & (~entry->clr_bits);168 else169 val = 0;170 171 /*172 * TODO: add selftest to validate all tables, regardless of platform:173 * - Masked registers can't have set_bits with upper bits set174 * - set_bits must be contained in clr_bits175 */176 val |= entry->set_bits;177 178 xe_gt_dbg(gt, "REG[0x%x] = 0x%08x", reg.addr, val);179 180 if (entry->reg.mcr)181 xe_gt_mcr_multicast_write(gt, reg_mcr, val);182 else183 xe_mmio_write32(gt, reg, val);184}185 186void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt)187{188 struct xe_reg_sr_entry *entry;189 unsigned long reg;190 int err;191 192 if (xa_empty(&sr->xa))193 return;194 195 xe_gt_dbg(gt, "Applying %s save-restore MMIOs\n", sr->name);196 197 err = xe_force_wake_get(>->mmio.fw, XE_FORCEWAKE_ALL);198 if (err)199 goto err_force_wake;200 201 xa_for_each(&sr->xa, reg, entry)202 apply_one_mmio(gt, entry);203 204 err = xe_force_wake_put(>->mmio.fw, XE_FORCEWAKE_ALL);205 XE_WARN_ON(err);206 207 return;208 209err_force_wake:210 xe_gt_err(gt, "Failed to apply, err=%d\n", err);211}212 213void xe_reg_sr_apply_whitelist(struct xe_hw_engine *hwe)214{215 struct xe_reg_sr *sr = &hwe->reg_whitelist;216 struct xe_gt *gt = hwe->gt;217 struct xe_device *xe = gt_to_xe(gt);218 struct xe_reg_sr_entry *entry;219 struct drm_printer p;220 u32 mmio_base = hwe->mmio_base;221 unsigned long reg;222 unsigned int slot = 0;223 int err;224 225 if (xa_empty(&sr->xa))226 return;227 228 drm_dbg(&xe->drm, "Whitelisting %s registers\n", sr->name);229 230 err = xe_force_wake_get(>->mmio.fw, XE_FORCEWAKE_ALL);231 if (err)232 goto err_force_wake;233 234 p = drm_dbg_printer(&xe->drm, DRM_UT_DRIVER, NULL);235 xa_for_each(&sr->xa, reg, entry) {236 if (slot == RING_MAX_NONPRIV_SLOTS) {237 xe_gt_err(gt,238 "hwe %s: maximum register whitelist slots (%d) reached, refusing to add more\n",239 hwe->name, RING_MAX_NONPRIV_SLOTS);240 break;241 }242 243 xe_reg_whitelist_print_entry(&p, 0, reg, entry);244 xe_mmio_write32(gt, RING_FORCE_TO_NONPRIV(mmio_base, slot),245 reg | entry->set_bits);246 slot++;247 }248 249 /* And clear the rest just in case of garbage */250 for (; slot < RING_MAX_NONPRIV_SLOTS; slot++) {251 u32 addr = RING_NOPID(mmio_base).addr;252 253 xe_mmio_write32(gt, RING_FORCE_TO_NONPRIV(mmio_base, slot), addr);254 }255 256 err = xe_force_wake_put(>->mmio.fw, XE_FORCEWAKE_ALL);257 XE_WARN_ON(err);258 259 return;260 261err_force_wake:262 drm_err(&xe->drm, "Failed to apply, err=%d\n", err);263}264 265/**266 * xe_reg_sr_dump - print all save/restore entries267 * @sr: Save/restore entries268 * @p: DRM printer269 */270void xe_reg_sr_dump(struct xe_reg_sr *sr, struct drm_printer *p)271{272 struct xe_reg_sr_entry *entry;273 unsigned long reg;274 275 if (!sr->name || xa_empty(&sr->xa))276 return;277 278 drm_printf(p, "%s\n", sr->name);279 xa_for_each(&sr->xa, reg, entry)280 drm_printf(p, "\tREG[0x%lx] clr=0x%08x set=0x%08x masked=%s mcr=%s\n",281 reg, entry->clr_bits, entry->set_bits,282 str_yes_no(entry->reg.masked),283 str_yes_no(entry->reg.mcr));284}285