914 lines · c
1/* SPDX-License-Identifier: GPL-2.0-only */2/* Copyright(c) 2020 Intel Corporation. */3 4#ifndef __CXL_H__5#define __CXL_H__6 7#include <linux/libnvdimm.h>8#include <linux/bitfield.h>9#include <linux/notifier.h>10#include <linux/bitops.h>11#include <linux/log2.h>12#include <linux/node.h>13#include <linux/io.h>14 15extern const struct nvdimm_security_ops *cxl_security_ops;16 17/**18 * DOC: cxl objects19 *20 * The CXL core objects like ports, decoders, and regions are shared21 * between the subsystem drivers cxl_acpi, cxl_pci, and core drivers22 * (port-driver, region-driver, nvdimm object-drivers... etc).23 */24 25/* CXL 2.0 8.2.4 CXL Component Register Layout and Definition */26#define CXL_COMPONENT_REG_BLOCK_SIZE SZ_64K27 28/* CXL 2.0 8.2.5 CXL.cache and CXL.mem Registers*/29#define CXL_CM_OFFSET 0x100030#define CXL_CM_CAP_HDR_OFFSET 0x031#define CXL_CM_CAP_HDR_ID_MASK GENMASK(15, 0)32#define CM_CAP_HDR_CAP_ID 133#define CXL_CM_CAP_HDR_VERSION_MASK GENMASK(19, 16)34#define CM_CAP_HDR_CAP_VERSION 135#define CXL_CM_CAP_HDR_CACHE_MEM_VERSION_MASK GENMASK(23, 20)36#define CM_CAP_HDR_CACHE_MEM_VERSION 137#define CXL_CM_CAP_HDR_ARRAY_SIZE_MASK GENMASK(31, 24)38#define CXL_CM_CAP_PTR_MASK GENMASK(31, 20)39 40#define CXL_CM_CAP_CAP_ID_RAS 0x241#define CXL_CM_CAP_CAP_ID_HDM 0x542#define CXL_CM_CAP_CAP_HDM_VERSION 143 44/* HDM decoders CXL 2.0 8.2.5.12 CXL HDM Decoder Capability Structure */45#define CXL_HDM_DECODER_CAP_OFFSET 0x046#define CXL_HDM_DECODER_COUNT_MASK GENMASK(3, 0)47#define CXL_HDM_DECODER_TARGET_COUNT_MASK GENMASK(7, 4)48#define CXL_HDM_DECODER_INTERLEAVE_11_8 BIT(8)49#define CXL_HDM_DECODER_INTERLEAVE_14_12 BIT(9)50#define CXL_HDM_DECODER_INTERLEAVE_3_6_12_WAY BIT(11)51#define CXL_HDM_DECODER_INTERLEAVE_16_WAY BIT(12)52#define CXL_HDM_DECODER_CTRL_OFFSET 0x453#define CXL_HDM_DECODER_ENABLE BIT(1)54#define CXL_HDM_DECODER0_BASE_LOW_OFFSET(i) (0x20 * (i) + 0x10)55#define CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i) (0x20 * (i) + 0x14)56#define CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i) (0x20 * (i) + 0x18)57#define CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i) (0x20 * (i) + 0x1c)58#define CXL_HDM_DECODER0_CTRL_OFFSET(i) (0x20 * (i) + 0x20)59#define CXL_HDM_DECODER0_CTRL_IG_MASK GENMASK(3, 0)60#define CXL_HDM_DECODER0_CTRL_IW_MASK GENMASK(7, 4)61#define CXL_HDM_DECODER0_CTRL_LOCK BIT(8)62#define CXL_HDM_DECODER0_CTRL_COMMIT BIT(9)63#define CXL_HDM_DECODER0_CTRL_COMMITTED BIT(10)64#define CXL_HDM_DECODER0_CTRL_COMMIT_ERROR BIT(11)65#define CXL_HDM_DECODER0_CTRL_HOSTONLY BIT(12)66#define CXL_HDM_DECODER0_TL_LOW(i) (0x20 * (i) + 0x24)67#define CXL_HDM_DECODER0_TL_HIGH(i) (0x20 * (i) + 0x28)68#define CXL_HDM_DECODER0_SKIP_LOW(i) CXL_HDM_DECODER0_TL_LOW(i)69#define CXL_HDM_DECODER0_SKIP_HIGH(i) CXL_HDM_DECODER0_TL_HIGH(i)70 71/* HDM decoder control register constants CXL 3.0 8.2.5.19.7 */72#define CXL_DECODER_MIN_GRANULARITY 25673#define CXL_DECODER_MAX_ENCODED_IG 674 75static inline int cxl_hdm_decoder_count(u32 cap_hdr)76{77 int val = FIELD_GET(CXL_HDM_DECODER_COUNT_MASK, cap_hdr);78 79 return val ? val * 2 : 1;80}81 82/* Encode defined in CXL 2.0 8.2.5.12.7 HDM Decoder Control Register */83static inline int eig_to_granularity(u16 eig, unsigned int *granularity)84{85 if (eig > CXL_DECODER_MAX_ENCODED_IG)86 return -EINVAL;87 *granularity = CXL_DECODER_MIN_GRANULARITY << eig;88 return 0;89}90 91/* Encode defined in CXL ECN "3, 6, 12 and 16-way memory Interleaving" */92static inline int eiw_to_ways(u8 eiw, unsigned int *ways)93{94 switch (eiw) {95 case 0 ... 4:96 *ways = 1 << eiw;97 break;98 case 8 ... 10:99 *ways = 3 << (eiw - 8);100 break;101 default:102 return -EINVAL;103 }104 105 return 0;106}107 108static inline int granularity_to_eig(int granularity, u16 *eig)109{110 if (granularity > SZ_16K || granularity < CXL_DECODER_MIN_GRANULARITY ||111 !is_power_of_2(granularity))112 return -EINVAL;113 *eig = ilog2(granularity) - 8;114 return 0;115}116 117static inline int ways_to_eiw(unsigned int ways, u8 *eiw)118{119 if (ways > 16)120 return -EINVAL;121 if (is_power_of_2(ways)) {122 *eiw = ilog2(ways);123 return 0;124 }125 if (ways % 3)126 return -EINVAL;127 ways /= 3;128 if (!is_power_of_2(ways))129 return -EINVAL;130 *eiw = ilog2(ways) + 8;131 return 0;132}133 134/* RAS Registers CXL 2.0 8.2.5.9 CXL RAS Capability Structure */135#define CXL_RAS_UNCORRECTABLE_STATUS_OFFSET 0x0136#define CXL_RAS_UNCORRECTABLE_STATUS_MASK (GENMASK(16, 14) | GENMASK(11, 0))137#define CXL_RAS_UNCORRECTABLE_MASK_OFFSET 0x4138#define CXL_RAS_UNCORRECTABLE_MASK_MASK (GENMASK(16, 14) | GENMASK(11, 0))139#define CXL_RAS_UNCORRECTABLE_MASK_F256B_MASK BIT(8)140#define CXL_RAS_UNCORRECTABLE_SEVERITY_OFFSET 0x8141#define CXL_RAS_UNCORRECTABLE_SEVERITY_MASK (GENMASK(16, 14) | GENMASK(11, 0))142#define CXL_RAS_CORRECTABLE_STATUS_OFFSET 0xC143#define CXL_RAS_CORRECTABLE_STATUS_MASK GENMASK(6, 0)144#define CXL_RAS_CORRECTABLE_MASK_OFFSET 0x10145#define CXL_RAS_CORRECTABLE_MASK_MASK GENMASK(6, 0)146#define CXL_RAS_CAP_CONTROL_OFFSET 0x14147#define CXL_RAS_CAP_CONTROL_FE_MASK GENMASK(5, 0)148#define CXL_RAS_HEADER_LOG_OFFSET 0x18149#define CXL_RAS_CAPABILITY_LENGTH 0x58150#define CXL_HEADERLOG_SIZE SZ_512151#define CXL_HEADERLOG_SIZE_U32 SZ_512 / sizeof(u32)152 153/* CXL 2.0 8.2.8.1 Device Capabilities Array Register */154#define CXLDEV_CAP_ARRAY_OFFSET 0x0155#define CXLDEV_CAP_ARRAY_CAP_ID 0156#define CXLDEV_CAP_ARRAY_ID_MASK GENMASK_ULL(15, 0)157#define CXLDEV_CAP_ARRAY_COUNT_MASK GENMASK_ULL(47, 32)158/* CXL 2.0 8.2.8.2 CXL Device Capability Header Register */159#define CXLDEV_CAP_HDR_CAP_ID_MASK GENMASK(15, 0)160/* CXL 2.0 8.2.8.2.1 CXL Device Capabilities */161#define CXLDEV_CAP_CAP_ID_DEVICE_STATUS 0x1162#define CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX 0x2163#define CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX 0x3164#define CXLDEV_CAP_CAP_ID_MEMDEV 0x4000165 166/* CXL 3.0 8.2.8.3.1 Event Status Register */167#define CXLDEV_DEV_EVENT_STATUS_OFFSET 0x00168#define CXLDEV_EVENT_STATUS_INFO BIT(0)169#define CXLDEV_EVENT_STATUS_WARN BIT(1)170#define CXLDEV_EVENT_STATUS_FAIL BIT(2)171#define CXLDEV_EVENT_STATUS_FATAL BIT(3)172 173#define CXLDEV_EVENT_STATUS_ALL (CXLDEV_EVENT_STATUS_INFO | \174 CXLDEV_EVENT_STATUS_WARN | \175 CXLDEV_EVENT_STATUS_FAIL | \176 CXLDEV_EVENT_STATUS_FATAL)177 178/* CXL rev 3.0 section 8.2.9.2.4; Table 8-52 */179#define CXLDEV_EVENT_INT_MODE_MASK GENMASK(1, 0)180#define CXLDEV_EVENT_INT_MSGNUM_MASK GENMASK(7, 4)181 182/* CXL 2.0 8.2.8.4 Mailbox Registers */183#define CXLDEV_MBOX_CAPS_OFFSET 0x00184#define CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK GENMASK(4, 0)185#define CXLDEV_MBOX_CAP_BG_CMD_IRQ BIT(6)186#define CXLDEV_MBOX_CAP_IRQ_MSGNUM_MASK GENMASK(10, 7)187#define CXLDEV_MBOX_CTRL_OFFSET 0x04188#define CXLDEV_MBOX_CTRL_DOORBELL BIT(0)189#define CXLDEV_MBOX_CTRL_BG_CMD_IRQ BIT(2)190#define CXLDEV_MBOX_CMD_OFFSET 0x08191#define CXLDEV_MBOX_CMD_COMMAND_OPCODE_MASK GENMASK_ULL(15, 0)192#define CXLDEV_MBOX_CMD_PAYLOAD_LENGTH_MASK GENMASK_ULL(36, 16)193#define CXLDEV_MBOX_STATUS_OFFSET 0x10194#define CXLDEV_MBOX_STATUS_BG_CMD BIT(0)195#define CXLDEV_MBOX_STATUS_RET_CODE_MASK GENMASK_ULL(47, 32)196#define CXLDEV_MBOX_BG_CMD_STATUS_OFFSET 0x18197#define CXLDEV_MBOX_BG_CMD_COMMAND_OPCODE_MASK GENMASK_ULL(15, 0)198#define CXLDEV_MBOX_BG_CMD_COMMAND_PCT_MASK GENMASK_ULL(22, 16)199#define CXLDEV_MBOX_BG_CMD_COMMAND_RC_MASK GENMASK_ULL(47, 32)200#define CXLDEV_MBOX_BG_CMD_COMMAND_VENDOR_MASK GENMASK_ULL(63, 48)201#define CXLDEV_MBOX_PAYLOAD_OFFSET 0x20202 203/*204 * Using struct_group() allows for per register-block-type helper routines,205 * without requiring block-type agnostic code to include the prefix.206 */207struct cxl_regs {208 /*209 * Common set of CXL Component register block base pointers210 * @hdm_decoder: CXL 2.0 8.2.5.12 CXL HDM Decoder Capability Structure211 * @ras: CXL 2.0 8.2.5.9 CXL RAS Capability Structure212 */213 struct_group_tagged(cxl_component_regs, component,214 void __iomem *hdm_decoder;215 void __iomem *ras;216 );217 /*218 * Common set of CXL Device register block base pointers219 * @status: CXL 2.0 8.2.8.3 Device Status Registers220 * @mbox: CXL 2.0 8.2.8.4 Mailbox Registers221 * @memdev: CXL 2.0 8.2.8.5 Memory Device Registers222 */223 struct_group_tagged(cxl_device_regs, device_regs,224 void __iomem *status, *mbox, *memdev;225 );226 227 struct_group_tagged(cxl_pmu_regs, pmu_regs,228 void __iomem *pmu;229 );230 231 /*232 * RCH downstream port specific RAS register233 * @aer: CXL 3.0 8.2.1.1 RCH Downstream Port RCRB234 */235 struct_group_tagged(cxl_rch_regs, rch_regs,236 void __iomem *dport_aer;237 );238};239 240struct cxl_reg_map {241 bool valid;242 int id;243 unsigned long offset;244 unsigned long size;245};246 247struct cxl_component_reg_map {248 struct cxl_reg_map hdm_decoder;249 struct cxl_reg_map ras;250};251 252struct cxl_device_reg_map {253 struct cxl_reg_map status;254 struct cxl_reg_map mbox;255 struct cxl_reg_map memdev;256};257 258struct cxl_pmu_reg_map {259 struct cxl_reg_map pmu;260};261 262/**263 * struct cxl_register_map - DVSEC harvested register block mapping parameters264 * @host: device for devm operations and logging265 * @base: virtual base of the register-block-BAR + @block_offset266 * @resource: physical resource base of the register block267 * @max_size: maximum mapping size to perform register search268 * @reg_type: see enum cxl_regloc_type269 * @component_map: cxl_reg_map for component registers270 * @device_map: cxl_reg_maps for device registers271 * @pmu_map: cxl_reg_maps for CXL Performance Monitoring Units272 */273struct cxl_register_map {274 struct device *host;275 void __iomem *base;276 resource_size_t resource;277 resource_size_t max_size;278 u8 reg_type;279 union {280 struct cxl_component_reg_map component_map;281 struct cxl_device_reg_map device_map;282 struct cxl_pmu_reg_map pmu_map;283 };284};285 286void cxl_probe_component_regs(struct device *dev, void __iomem *base,287 struct cxl_component_reg_map *map);288void cxl_probe_device_regs(struct device *dev, void __iomem *base,289 struct cxl_device_reg_map *map);290int cxl_map_component_regs(const struct cxl_register_map *map,291 struct cxl_component_regs *regs,292 unsigned long map_mask);293int cxl_map_device_regs(const struct cxl_register_map *map,294 struct cxl_device_regs *regs);295int cxl_map_pmu_regs(struct cxl_register_map *map, struct cxl_pmu_regs *regs);296 297enum cxl_regloc_type;298int cxl_count_regblock(struct pci_dev *pdev, enum cxl_regloc_type type);299int cxl_find_regblock_instance(struct pci_dev *pdev, enum cxl_regloc_type type,300 struct cxl_register_map *map, int index);301int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type,302 struct cxl_register_map *map);303int cxl_setup_regs(struct cxl_register_map *map);304struct cxl_dport;305resource_size_t cxl_rcd_component_reg_phys(struct device *dev,306 struct cxl_dport *dport);307 308#define CXL_RESOURCE_NONE ((resource_size_t) -1)309#define CXL_TARGET_STRLEN 20310 311/*312 * cxl_decoder flags that define the type of memory / devices this313 * decoder supports as well as configuration lock status See "CXL 2.0314 * 8.2.5.12.7 CXL HDM Decoder 0 Control Register" for details.315 * Additionally indicate whether decoder settings were autodetected,316 * user customized.317 */318#define CXL_DECODER_F_RAM BIT(0)319#define CXL_DECODER_F_PMEM BIT(1)320#define CXL_DECODER_F_TYPE2 BIT(2)321#define CXL_DECODER_F_TYPE3 BIT(3)322#define CXL_DECODER_F_LOCK BIT(4)323#define CXL_DECODER_F_ENABLE BIT(5)324#define CXL_DECODER_F_MASK GENMASK(5, 0)325 326enum cxl_decoder_type {327 CXL_DECODER_DEVMEM = 2,328 CXL_DECODER_HOSTONLYMEM = 3,329};330 331/*332 * Current specification goes up to 8, double that seems a reasonable333 * software max for the foreseeable future334 */335#define CXL_DECODER_MAX_INTERLEAVE 16336 337#define CXL_QOS_CLASS_INVALID -1338 339/**340 * struct cxl_decoder - Common CXL HDM Decoder Attributes341 * @dev: this decoder's device342 * @id: kernel device name id343 * @hpa_range: Host physical address range mapped by this decoder344 * @interleave_ways: number of cxl_dports in this decode345 * @interleave_granularity: data stride per dport346 * @target_type: accelerator vs expander (type2 vs type3) selector347 * @region: currently assigned region for this decoder348 * @flags: memory type capabilities and locking349 * @commit: device/decoder-type specific callback to commit settings to hw350 * @reset: device/decoder-type specific callback to reset hw settings351*/352struct cxl_decoder {353 struct device dev;354 int id;355 struct range hpa_range;356 int interleave_ways;357 int interleave_granularity;358 enum cxl_decoder_type target_type;359 struct cxl_region *region;360 unsigned long flags;361 int (*commit)(struct cxl_decoder *cxld);362 void (*reset)(struct cxl_decoder *cxld);363};364 365/*366 * CXL_DECODER_DEAD prevents endpoints from being reattached to regions367 * while cxld_unregister() is running368 */369enum cxl_decoder_mode {370 CXL_DECODER_NONE,371 CXL_DECODER_RAM,372 CXL_DECODER_PMEM,373 CXL_DECODER_MIXED,374 CXL_DECODER_DEAD,375};376 377static inline const char *cxl_decoder_mode_name(enum cxl_decoder_mode mode)378{379 static const char * const names[] = {380 [CXL_DECODER_NONE] = "none",381 [CXL_DECODER_RAM] = "ram",382 [CXL_DECODER_PMEM] = "pmem",383 [CXL_DECODER_MIXED] = "mixed",384 };385 386 if (mode >= CXL_DECODER_NONE && mode <= CXL_DECODER_MIXED)387 return names[mode];388 return "mixed";389}390 391/*392 * Track whether this decoder is reserved for region autodiscovery, or393 * free for userspace provisioning.394 */395enum cxl_decoder_state {396 CXL_DECODER_STATE_MANUAL,397 CXL_DECODER_STATE_AUTO,398};399 400/**401 * struct cxl_endpoint_decoder - Endpoint / SPA to DPA decoder402 * @cxld: base cxl_decoder_object403 * @dpa_res: actively claimed DPA span of this decoder404 * @skip: offset into @dpa_res where @cxld.hpa_range maps405 * @mode: which memory type / access-mode-partition this decoder targets406 * @state: autodiscovery state407 * @pos: interleave position in @cxld.region408 */409struct cxl_endpoint_decoder {410 struct cxl_decoder cxld;411 struct resource *dpa_res;412 resource_size_t skip;413 enum cxl_decoder_mode mode;414 enum cxl_decoder_state state;415 int pos;416};417 418/**419 * struct cxl_switch_decoder - Switch specific CXL HDM Decoder420 * @cxld: base cxl_decoder object421 * @nr_targets: number of elements in @target422 * @target: active ordered target list in current decoder configuration423 *424 * The 'switch' decoder type represents the decoder instances of cxl_port's that425 * route from the root of a CXL memory decode topology to the endpoints. They426 * come in two flavors, root-level decoders, statically defined by platform427 * firmware, and mid-level decoders, where interleave-granularity,428 * interleave-width, and the target list are mutable.429 */430struct cxl_switch_decoder {431 struct cxl_decoder cxld;432 int nr_targets;433 struct cxl_dport *target[];434};435 436struct cxl_root_decoder;437typedef u64 (*cxl_hpa_to_spa_fn)(struct cxl_root_decoder *cxlrd, u64 hpa);438 439/**440 * struct cxl_root_decoder - Static platform CXL address decoder441 * @res: host / parent resource for region allocations442 * @region_id: region id for next region provisioning event443 * @hpa_to_spa: translate CXL host-physical-address to Platform system-physical-address444 * @platform_data: platform specific configuration data445 * @range_lock: sync region autodiscovery by address range446 * @qos_class: QoS performance class cookie447 * @cxlsd: base cxl switch decoder448 */449struct cxl_root_decoder {450 struct resource *res;451 atomic_t region_id;452 cxl_hpa_to_spa_fn hpa_to_spa;453 void *platform_data;454 struct mutex range_lock;455 int qos_class;456 struct cxl_switch_decoder cxlsd;457};458 459/*460 * enum cxl_config_state - State machine for region configuration461 * @CXL_CONFIG_IDLE: Any sysfs attribute can be written freely462 * @CXL_CONFIG_INTERLEAVE_ACTIVE: region size has been set, no more463 * changes to interleave_ways or interleave_granularity464 * @CXL_CONFIG_ACTIVE: All targets have been added the region is now465 * active466 * @CXL_CONFIG_RESET_PENDING: see commit_store()467 * @CXL_CONFIG_COMMIT: Soft-config has been committed to hardware468 */469enum cxl_config_state {470 CXL_CONFIG_IDLE,471 CXL_CONFIG_INTERLEAVE_ACTIVE,472 CXL_CONFIG_ACTIVE,473 CXL_CONFIG_RESET_PENDING,474 CXL_CONFIG_COMMIT,475};476 477/**478 * struct cxl_region_params - region settings479 * @state: allow the driver to lockdown further parameter changes480 * @uuid: unique id for persistent regions481 * @interleave_ways: number of endpoints in the region482 * @interleave_granularity: capacity each endpoint contributes to a stripe483 * @res: allocated iomem capacity for this region484 * @targets: active ordered targets in current decoder configuration485 * @nr_targets: number of targets486 *487 * State transitions are protected by the cxl_region_rwsem488 */489struct cxl_region_params {490 enum cxl_config_state state;491 uuid_t uuid;492 int interleave_ways;493 int interleave_granularity;494 struct resource *res;495 struct cxl_endpoint_decoder *targets[CXL_DECODER_MAX_INTERLEAVE];496 int nr_targets;497};498 499/*500 * Indicate whether this region has been assembled by autodetection or501 * userspace assembly. Prevent endpoint decoders outside of automatic502 * detection from being added to the region.503 */504#define CXL_REGION_F_AUTO 0505 506/*507 * Require that a committed region successfully complete a teardown once508 * any of its associated decoders have been torn down. This maintains509 * the commit state for the region since there are committed decoders,510 * but blocks cxl_region_probe().511 */512#define CXL_REGION_F_NEEDS_RESET 1513 514/**515 * struct cxl_region - CXL region516 * @dev: This region's device517 * @id: This region's id. Id is globally unique across all regions518 * @mode: Endpoint decoder allocation / access mode519 * @type: Endpoint decoder target type520 * @cxl_nvb: nvdimm bridge for coordinating @cxlr_pmem setup / shutdown521 * @cxlr_pmem: (for pmem regions) cached copy of the nvdimm bridge522 * @flags: Region state flags523 * @params: active + config params for the region524 * @coord: QoS access coordinates for the region525 * @memory_notifier: notifier for setting the access coordinates to node526 * @adist_notifier: notifier for calculating the abstract distance of node527 */528struct cxl_region {529 struct device dev;530 int id;531 enum cxl_decoder_mode mode;532 enum cxl_decoder_type type;533 struct cxl_nvdimm_bridge *cxl_nvb;534 struct cxl_pmem_region *cxlr_pmem;535 unsigned long flags;536 struct cxl_region_params params;537 struct access_coordinate coord[ACCESS_COORDINATE_MAX];538 struct notifier_block memory_notifier;539 struct notifier_block adist_notifier;540};541 542struct cxl_nvdimm_bridge {543 int id;544 struct device dev;545 struct cxl_port *port;546 struct nvdimm_bus *nvdimm_bus;547 struct nvdimm_bus_descriptor nd_desc;548};549 550#define CXL_DEV_ID_LEN 19551 552struct cxl_nvdimm {553 struct device dev;554 struct cxl_memdev *cxlmd;555 u8 dev_id[CXL_DEV_ID_LEN]; /* for nvdimm, string of 'serial' */556};557 558struct cxl_pmem_region_mapping {559 struct cxl_memdev *cxlmd;560 struct cxl_nvdimm *cxl_nvd;561 u64 start;562 u64 size;563 int position;564};565 566struct cxl_pmem_region {567 struct device dev;568 struct cxl_region *cxlr;569 struct nd_region *nd_region;570 struct range hpa_range;571 int nr_mappings;572 struct cxl_pmem_region_mapping mapping[];573};574 575struct cxl_dax_region {576 struct device dev;577 struct cxl_region *cxlr;578 struct range hpa_range;579};580 581/**582 * struct cxl_port - logical collection of upstream port devices and583 * downstream port devices to construct a CXL memory584 * decode hierarchy.585 * @dev: this port's device586 * @uport_dev: PCI or platform device implementing the upstream port capability587 * @host_bridge: Shortcut to the platform attach point for this port588 * @id: id for port device-name589 * @dports: cxl_dport instances referenced by decoders590 * @endpoints: cxl_ep instances, endpoints that are a descendant of this port591 * @regions: cxl_region_ref instances, regions mapped by this port592 * @parent_dport: dport that points to this port in the parent593 * @decoder_ida: allocator for decoder ids594 * @reg_map: component and ras register mapping parameters595 * @nr_dports: number of entries in @dports596 * @hdm_end: track last allocated HDM decoder instance for allocation ordering597 * @commit_end: cursor to track highest committed decoder for commit ordering598 * @dead: last ep has been removed, force port re-creation599 * @depth: How deep this port is relative to the root. depth 0 is the root.600 * @cdat: Cached CDAT data601 * @cdat_available: Should a CDAT attribute be available in sysfs602 * @pci_latency: Upstream latency in picoseconds603 */604struct cxl_port {605 struct device dev;606 struct device *uport_dev;607 struct device *host_bridge;608 int id;609 struct xarray dports;610 struct xarray endpoints;611 struct xarray regions;612 struct cxl_dport *parent_dport;613 struct ida decoder_ida;614 struct cxl_register_map reg_map;615 int nr_dports;616 int hdm_end;617 int commit_end;618 bool dead;619 unsigned int depth;620 struct cxl_cdat {621 void *table;622 size_t length;623 } cdat;624 bool cdat_available;625 long pci_latency;626};627 628/**629 * struct cxl_root - logical collection of root cxl_port items630 *631 * @port: cxl_port member632 * @ops: cxl root operations633 */634struct cxl_root {635 struct cxl_port port;636 const struct cxl_root_ops *ops;637};638 639static inline struct cxl_root *640to_cxl_root(const struct cxl_port *port)641{642 return container_of(port, struct cxl_root, port);643}644 645struct cxl_root_ops {646 int (*qos_class)(struct cxl_root *cxl_root,647 struct access_coordinate *coord, int entries,648 int *qos_class);649};650 651static inline struct cxl_dport *652cxl_find_dport_by_dev(struct cxl_port *port, const struct device *dport_dev)653{654 return xa_load(&port->dports, (unsigned long)dport_dev);655}656 657struct cxl_rcrb_info {658 resource_size_t base;659 u16 aer_cap;660};661 662/**663 * struct cxl_dport - CXL downstream port664 * @dport_dev: PCI bridge or firmware device representing the downstream link665 * @reg_map: component and ras register mapping parameters666 * @port_id: unique hardware identifier for dport in decoder target list667 * @rcrb: Data about the Root Complex Register Block layout668 * @rch: Indicate whether this dport was enumerated in RCH or VH mode669 * @port: reference to cxl_port that contains this downstream port670 * @regs: Dport parsed register blocks671 * @coord: access coordinates (bandwidth and latency performance attributes)672 * @link_latency: calculated PCIe downstream latency673 */674struct cxl_dport {675 struct device *dport_dev;676 struct cxl_register_map reg_map;677 int port_id;678 struct cxl_rcrb_info rcrb;679 bool rch;680 struct cxl_port *port;681 struct cxl_regs regs;682 struct access_coordinate coord[ACCESS_COORDINATE_MAX];683 long link_latency;684};685 686/**687 * struct cxl_ep - track an endpoint's interest in a port688 * @ep: device that hosts a generic CXL endpoint (expander or accelerator)689 * @dport: which dport routes to this endpoint on @port690 * @next: cxl switch port across the link attached to @dport NULL if691 * attached to an endpoint692 */693struct cxl_ep {694 struct device *ep;695 struct cxl_dport *dport;696 struct cxl_port *next;697};698 699/**700 * struct cxl_region_ref - track a region's interest in a port701 * @port: point in topology to install this reference702 * @decoder: decoder assigned for @region in @port703 * @region: region for this reference704 * @endpoints: cxl_ep references for region members beneath @port705 * @nr_targets_set: track how many targets have been programmed during setup706 * @nr_eps: number of endpoints beneath @port707 * @nr_targets: number of distinct targets needed to reach @nr_eps708 */709struct cxl_region_ref {710 struct cxl_port *port;711 struct cxl_decoder *decoder;712 struct cxl_region *region;713 struct xarray endpoints;714 int nr_targets_set;715 int nr_eps;716 int nr_targets;717};718 719/*720 * The platform firmware device hosting the root is also the top of the721 * CXL port topology. All other CXL ports have another CXL port as their722 * parent and their ->uport_dev / host device is out-of-line of the port723 * ancestry.724 */725static inline bool is_cxl_root(struct cxl_port *port)726{727 return port->uport_dev == port->dev.parent;728}729 730int cxl_num_decoders_committed(struct cxl_port *port);731bool is_cxl_port(const struct device *dev);732struct cxl_port *to_cxl_port(const struct device *dev);733void cxl_port_commit_reap(struct cxl_decoder *cxld);734struct pci_bus;735int devm_cxl_register_pci_bus(struct device *host, struct device *uport_dev,736 struct pci_bus *bus);737struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port);738struct cxl_port *devm_cxl_add_port(struct device *host,739 struct device *uport_dev,740 resource_size_t component_reg_phys,741 struct cxl_dport *parent_dport);742struct cxl_root *devm_cxl_add_root(struct device *host,743 const struct cxl_root_ops *ops);744struct cxl_root *find_cxl_root(struct cxl_port *port);745void put_cxl_root(struct cxl_root *cxl_root);746DEFINE_FREE(put_cxl_root, struct cxl_root *, if (_T) put_cxl_root(_T))747 748DEFINE_FREE(put_cxl_port, struct cxl_port *, if (!IS_ERR_OR_NULL(_T)) put_device(&_T->dev))749int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd);750void cxl_bus_rescan(void);751void cxl_bus_drain(void);752struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev,753 struct cxl_dport **dport);754struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd,755 struct cxl_dport **dport);756bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd);757 758struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port,759 struct device *dport, int port_id,760 resource_size_t component_reg_phys);761struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port,762 struct device *dport_dev, int port_id,763 resource_size_t rcrb);764 765#ifdef CONFIG_PCIEAER_CXL766void cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport);767void cxl_dport_init_ras_reporting(struct cxl_dport *dport, struct device *host);768#else769static inline void cxl_dport_init_ras_reporting(struct cxl_dport *dport,770 struct device *host) { }771#endif772 773struct cxl_decoder *to_cxl_decoder(struct device *dev);774struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev);775struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev);776struct cxl_endpoint_decoder *to_cxl_endpoint_decoder(struct device *dev);777bool is_root_decoder(struct device *dev);778bool is_switch_decoder(struct device *dev);779bool is_endpoint_decoder(struct device *dev);780struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port,781 unsigned int nr_targets);782struct cxl_switch_decoder *cxl_switch_decoder_alloc(struct cxl_port *port,783 unsigned int nr_targets);784int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map);785struct cxl_endpoint_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port);786int cxl_decoder_add_locked(struct cxl_decoder *cxld, int *target_map);787int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld);788static inline int cxl_root_decoder_autoremove(struct device *host,789 struct cxl_root_decoder *cxlrd)790{791 return cxl_decoder_autoremove(host, &cxlrd->cxlsd.cxld);792}793int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint);794 795/**796 * struct cxl_endpoint_dvsec_info - Cached DVSEC info797 * @mem_enabled: cached value of mem_enabled in the DVSEC at init time798 * @ranges: Number of active HDM ranges this device uses.799 * @port: endpoint port associated with this info instance800 * @dvsec_range: cached attributes of the ranges in the DVSEC, PCIE_DEVICE801 */802struct cxl_endpoint_dvsec_info {803 bool mem_enabled;804 int ranges;805 struct cxl_port *port;806 struct range dvsec_range[2];807};808 809struct cxl_hdm;810struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port,811 struct cxl_endpoint_dvsec_info *info);812int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm,813 struct cxl_endpoint_dvsec_info *info);814int devm_cxl_add_passthrough_decoder(struct cxl_port *port);815int cxl_dvsec_rr_decode(struct device *dev, struct cxl_port *port,816 struct cxl_endpoint_dvsec_info *info);817 818bool is_cxl_region(struct device *dev);819 820extern struct bus_type cxl_bus_type;821 822struct cxl_driver {823 const char *name;824 int (*probe)(struct device *dev);825 void (*remove)(struct device *dev);826 struct device_driver drv;827 int id;828};829 830#define to_cxl_drv(__drv) container_of_const(__drv, struct cxl_driver, drv)831 832int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner,833 const char *modname);834#define cxl_driver_register(x) __cxl_driver_register(x, THIS_MODULE, KBUILD_MODNAME)835void cxl_driver_unregister(struct cxl_driver *cxl_drv);836 837#define module_cxl_driver(__cxl_driver) \838 module_driver(__cxl_driver, cxl_driver_register, cxl_driver_unregister)839 840#define CXL_DEVICE_NVDIMM_BRIDGE 1841#define CXL_DEVICE_NVDIMM 2842#define CXL_DEVICE_PORT 3843#define CXL_DEVICE_ROOT 4844#define CXL_DEVICE_MEMORY_EXPANDER 5845#define CXL_DEVICE_REGION 6846#define CXL_DEVICE_PMEM_REGION 7847#define CXL_DEVICE_DAX_REGION 8848#define CXL_DEVICE_PMU 9849 850#define MODULE_ALIAS_CXL(type) MODULE_ALIAS("cxl:t" __stringify(type) "*")851#define CXL_MODALIAS_FMT "cxl:t%d"852 853struct cxl_nvdimm_bridge *to_cxl_nvdimm_bridge(struct device *dev);854struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host,855 struct cxl_port *port);856struct cxl_nvdimm *to_cxl_nvdimm(struct device *dev);857bool is_cxl_nvdimm(struct device *dev);858bool is_cxl_nvdimm_bridge(struct device *dev);859int devm_cxl_add_nvdimm(struct cxl_port *parent_port, struct cxl_memdev *cxlmd);860struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(struct cxl_port *port);861 862#ifdef CONFIG_CXL_REGION863bool is_cxl_pmem_region(struct device *dev);864struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev);865int cxl_add_to_region(struct cxl_port *root,866 struct cxl_endpoint_decoder *cxled);867struct cxl_dax_region *to_cxl_dax_region(struct device *dev);868#else869static inline bool is_cxl_pmem_region(struct device *dev)870{871 return false;872}873static inline struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)874{875 return NULL;876}877static inline int cxl_add_to_region(struct cxl_port *root,878 struct cxl_endpoint_decoder *cxled)879{880 return 0;881}882static inline struct cxl_dax_region *to_cxl_dax_region(struct device *dev)883{884 return NULL;885}886#endif887 888void cxl_endpoint_parse_cdat(struct cxl_port *port);889void cxl_switch_parse_cdat(struct cxl_port *port);890 891int cxl_endpoint_get_perf_coordinates(struct cxl_port *port,892 struct access_coordinate *coord);893void cxl_region_perf_data_calculate(struct cxl_region *cxlr,894 struct cxl_endpoint_decoder *cxled);895void cxl_region_shared_upstream_bandwidth_update(struct cxl_region *cxlr);896 897void cxl_memdev_update_perf(struct cxl_memdev *cxlmd);898 899void cxl_coordinates_combine(struct access_coordinate *out,900 struct access_coordinate *c1,901 struct access_coordinate *c2);902 903bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port);904 905/*906 * Unit test builds overrides this to __weak, find the 'strong' version907 * of these symbols in tools/testing/cxl/.908 */909#ifndef __mock910#define __mock static911#endif912 913#endif /* __CXL_H__ */914