365 lines · c
1/* SPDX-License-Identifier: GPL-2.0 */2/*3 * AMD Address Translation Library4 *5 * internal.h : Helper functions and common defines6 *7 * Copyright (c) 2023, Advanced Micro Devices, Inc.8 * All Rights Reserved.9 *10 * Author: Yazen Ghannam <Yazen.Ghannam@amd.com>11 */12 13#ifndef __AMD_ATL_INTERNAL_H__14#define __AMD_ATL_INTERNAL_H__15 16#include <linux/bitfield.h>17#include <linux/bitops.h>18#include <linux/ras.h>19 20#include <asm/amd_nb.h>21 22#include "reg_fields.h"23 24#undef pr_fmt25#define pr_fmt(fmt) "amd_atl: " fmt26 27/* Maximum possible number of Coherent Stations within a single Data Fabric. */28#define MAX_COH_ST_CHANNELS 3229 30/* PCI ID for Zen4 Server DF Function 0. */31#define DF_FUNC0_ID_ZEN4_SERVER 0x14AD102232 33/* PCI IDs for MI300 DF Function 0. */34#define DF_FUNC0_ID_MI300 0x1528102235 36/* Shift needed for adjusting register values to true values. */37#define DF_DRAM_BASE_LIMIT_LSB 2838#define MI300_DRAM_LIMIT_LSB 2039 40#define INVALID_SPA ~0ULL41 42enum df_revisions {43 UNKNOWN,44 DF2,45 DF3,46 DF3p5,47 DF4,48 DF4p5,49};50 51/* These are mapped 1:1 to the hardware values. Special cases are set at > 0x20. */52enum intlv_modes {53 NONE = 0x00,54 NOHASH_2CHAN = 0x01,55 NOHASH_4CHAN = 0x03,56 NOHASH_8CHAN = 0x05,57 DF3_6CHAN = 0x06,58 NOHASH_16CHAN = 0x07,59 NOHASH_32CHAN = 0x08,60 DF3_COD4_2CHAN_HASH = 0x0C,61 DF3_COD2_4CHAN_HASH = 0x0D,62 DF3_COD1_8CHAN_HASH = 0x0E,63 DF4_NPS4_2CHAN_HASH = 0x10,64 DF4_NPS2_4CHAN_HASH = 0x11,65 DF4_NPS1_8CHAN_HASH = 0x12,66 DF4_NPS4_3CHAN_HASH = 0x13,67 DF4_NPS2_6CHAN_HASH = 0x14,68 DF4_NPS1_12CHAN_HASH = 0x15,69 DF4_NPS2_5CHAN_HASH = 0x16,70 DF4_NPS1_10CHAN_HASH = 0x17,71 MI3_HASH_8CHAN = 0x18,72 MI3_HASH_16CHAN = 0x19,73 MI3_HASH_32CHAN = 0x1A,74 DF2_2CHAN_HASH = 0x21,75 /* DF4.5 modes are all IntLvNumChan + 0x20 */76 DF4p5_NPS1_16CHAN_1K_HASH = 0x2C,77 DF4p5_NPS0_24CHAN_1K_HASH = 0x2E,78 DF4p5_NPS4_2CHAN_1K_HASH = 0x30,79 DF4p5_NPS2_4CHAN_1K_HASH = 0x31,80 DF4p5_NPS1_8CHAN_1K_HASH = 0x32,81 DF4p5_NPS4_3CHAN_1K_HASH = 0x33,82 DF4p5_NPS2_6CHAN_1K_HASH = 0x34,83 DF4p5_NPS1_12CHAN_1K_HASH = 0x35,84 DF4p5_NPS2_5CHAN_1K_HASH = 0x36,85 DF4p5_NPS1_10CHAN_1K_HASH = 0x37,86 DF4p5_NPS4_2CHAN_2K_HASH = 0x40,87 DF4p5_NPS2_4CHAN_2K_HASH = 0x41,88 DF4p5_NPS1_8CHAN_2K_HASH = 0x42,89 DF4p5_NPS1_16CHAN_2K_HASH = 0x43,90 DF4p5_NPS4_3CHAN_2K_HASH = 0x44,91 DF4p5_NPS2_6CHAN_2K_HASH = 0x45,92 DF4p5_NPS1_12CHAN_2K_HASH = 0x46,93 DF4p5_NPS0_24CHAN_2K_HASH = 0x47,94 DF4p5_NPS2_5CHAN_2K_HASH = 0x48,95 DF4p5_NPS1_10CHAN_2K_HASH = 0x49,96};97 98struct df4p5_denorm_ctx {99 /* Indicates the number of "lost" bits. This will be 1, 2, or 3. */100 u8 perm_shift;101 102 /* A mask indicating the bits that need to be rehashed. */103 u16 rehash_vector;104 105 /*106 * Represents the value that the high bits of the normalized address107 * are divided by during normalization. This value will be 3 for108 * interleave modes with a number of channels divisible by 3 or the109 * value will be 5 for interleave modes with a number of channels110 * divisible by 5. Power-of-two interleave modes are handled111 * separately.112 */113 u8 mod_value;114 115 /*116 * Represents the bits that can be directly pulled from the normalized117 * address. In each case, pass through bits [7:0] of the normalized118 * address. The other bits depend on the interleave bit position which119 * will be bit 10 for 1K interleave stripe cases and bit 11 for 2K120 * interleave stripe cases.121 */122 u64 base_denorm_addr;123 124 /*125 * Represents the high bits of the physical address that have been126 * divided by the mod_value.127 */128 u64 div_addr;129 130 u64 current_spa;131 u64 resolved_spa;132 133 u16 coh_st_fabric_id;134};135 136struct df_flags {137 __u8 legacy_ficaa : 1,138 socket_id_shift_quirk : 1,139 heterogeneous : 1,140 __reserved_0 : 5;141};142 143struct df_config {144 enum df_revisions rev;145 146 /*147 * These masks operate on the 16-bit Coherent Station IDs,148 * e.g. Instance, Fabric, Destination, etc.149 */150 u16 component_id_mask;151 u16 die_id_mask;152 u16 node_id_mask;153 u16 socket_id_mask;154 155 /*156 * Least-significant bit of Node ID portion of the157 * system-wide Coherent Station Fabric ID.158 */159 u8 node_id_shift;160 161 /*162 * Least-significant bit of Die portion of the Node ID.163 * Adjusted to include the Node ID shift in order to apply164 * to the Coherent Station Fabric ID.165 */166 u8 die_id_shift;167 168 /*169 * Least-significant bit of Socket portion of the Node ID.170 * Adjusted to include the Node ID shift in order to apply171 * to the Coherent Station Fabric ID.172 */173 u8 socket_id_shift;174 175 /* Number of DRAM Address maps visible in a Coherent Station. */176 u8 num_coh_st_maps;177 178 u32 dram_hole_base;179 180 /* Global flags to handle special cases. */181 struct df_flags flags;182};183 184extern struct df_config df_cfg;185 186struct dram_addr_map {187 /*188 * Each DRAM Address Map can operate independently189 * in different interleaving modes.190 */191 enum intlv_modes intlv_mode;192 193 /* System-wide number for this address map. */194 u8 num;195 196 /* Raw register values */197 u32 base;198 u32 limit;199 u32 ctl;200 u32 intlv;201 202 /*203 * Logical to Physical Coherent Station Remapping array204 *205 * Index: Logical Coherent Station Instance ID206 * Value: Physical Coherent Station Instance ID207 *208 * phys_coh_st_inst_id = remap_array[log_coh_st_inst_id]209 */210 u8 remap_array[MAX_COH_ST_CHANNELS];211 212 /*213 * Number of bits covering DRAM Address map 0214 * when interleaving is non-power-of-2.215 *216 * Used only for DF3_6CHAN.217 */218 u8 np2_bits;219 220 /* Position of the 'interleave bit'. */221 u8 intlv_bit_pos;222 /* Number of channels interleaved in this map. */223 u8 num_intlv_chan;224 /* Number of dies interleaved in this map. */225 u8 num_intlv_dies;226 /* Number of sockets interleaved in this map. */227 u8 num_intlv_sockets;228 /*229 * Total number of channels interleaved accounting230 * for die and socket interleaving.231 */232 u8 total_intlv_chan;233 /* Total bits needed to cover 'total_intlv_chan'. */234 u8 total_intlv_bits;235};236 237/* Original input values cached for debug printing. */238struct addr_ctx_inputs {239 u64 norm_addr;240 u8 socket_id;241 u8 die_id;242 u8 coh_st_inst_id;243};244 245struct addr_ctx {246 u64 ret_addr;247 248 struct addr_ctx_inputs inputs;249 struct dram_addr_map map;250 251 /* AMD Node ID calculated from Socket and Die IDs. */252 u8 node_id;253 254 /*255 * Coherent Station Instance ID256 * Local ID used within a 'node'.257 */258 u16 inst_id;259 260 /*261 * Coherent Station Fabric ID262 * System-wide ID that includes 'node' bits.263 */264 u16 coh_st_fabric_id;265};266 267int df_indirect_read_instance(u16 node, u8 func, u16 reg, u8 instance_id, u32 *lo);268int df_indirect_read_broadcast(u16 node, u8 func, u16 reg, u32 *lo);269 270int get_df_system_info(void);271int determine_node_id(struct addr_ctx *ctx, u8 socket_num, u8 die_num);272int get_umc_info_mi300(void);273 274int get_address_map(struct addr_ctx *ctx);275 276int denormalize_address(struct addr_ctx *ctx);277int dehash_address(struct addr_ctx *ctx);278 279unsigned long norm_to_sys_addr(u8 socket_id, u8 die_id, u8 coh_st_inst_id, unsigned long addr);280unsigned long convert_umc_mca_addr_to_sys_addr(struct atl_err *err);281 282u64 add_base_and_hole(struct addr_ctx *ctx, u64 addr);283u64 remove_base_and_hole(struct addr_ctx *ctx, u64 addr);284 285#ifdef CONFIG_AMD_ATL_PRM286unsigned long prm_umc_norm_to_sys_addr(u8 socket_id, u64 umc_bank_inst_id, unsigned long addr);287#else288static inline unsigned long prm_umc_norm_to_sys_addr(u8 socket_id, u64 umc_bank_inst_id,289 unsigned long addr)290{291 return -ENODEV;292}293#endif294 295/*296 * Make a gap in @data that is @num_bits long starting at @bit_num.297 * e.g. data = 11111111'b298 * bit_num = 3299 * num_bits = 2300 * result = 1111100111'b301 */302static inline u64 expand_bits(u8 bit_num, u8 num_bits, u64 data)303{304 u64 temp1, temp2;305 306 if (!num_bits)307 return data;308 309 if (!bit_num) {310 WARN_ON_ONCE(num_bits >= BITS_PER_LONG);311 return data << num_bits;312 }313 314 WARN_ON_ONCE(bit_num >= BITS_PER_LONG);315 316 temp1 = data & GENMASK_ULL(bit_num - 1, 0);317 318 temp2 = data & GENMASK_ULL(63, bit_num);319 temp2 <<= num_bits;320 321 return temp1 | temp2;322}323 324/*325 * Remove bits in @data between @low_bit and @high_bit inclusive.326 * e.g. data = XXXYYZZZ'b327 * low_bit = 3328 * high_bit = 4329 * result = XXXZZZ'b330 */331static inline u64 remove_bits(u8 low_bit, u8 high_bit, u64 data)332{333 u64 temp1, temp2;334 335 WARN_ON_ONCE(high_bit >= BITS_PER_LONG);336 WARN_ON_ONCE(low_bit >= BITS_PER_LONG);337 WARN_ON_ONCE(low_bit > high_bit);338 339 if (!low_bit)340 return data >> (high_bit++);341 342 temp1 = GENMASK_ULL(low_bit - 1, 0) & data;343 temp2 = GENMASK_ULL(63, high_bit + 1) & data;344 temp2 >>= high_bit - low_bit + 1;345 346 return temp1 | temp2;347}348 349#define atl_debug(ctx, fmt, arg...) \350 pr_debug("socket_id=%u die_id=%u coh_st_inst_id=%u norm_addr=0x%016llx: " fmt,\351 (ctx)->inputs.socket_id, (ctx)->inputs.die_id,\352 (ctx)->inputs.coh_st_inst_id, (ctx)->inputs.norm_addr, ##arg)353 354static inline void atl_debug_on_bad_df_rev(void)355{356 pr_debug("Unrecognized DF rev: %u", df_cfg.rev);357}358 359static inline void atl_debug_on_bad_intlv_mode(struct addr_ctx *ctx)360{361 atl_debug(ctx, "Unrecognized interleave mode: %u", ctx->map.intlv_mode);362}363 364#endif /* __AMD_ATL_INTERNAL_H__ */365