1077 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/* Copyright(c) 2022 Intel Corporation. All rights reserved. */3#include <linux/seq_file.h>4#include <linux/device.h>5#include <linux/delay.h>6 7#include "cxlmem.h"8#include "core.h"9 10/**11 * DOC: cxl core hdm12 *13 * Compute Express Link Host Managed Device Memory, starting with the14 * CXL 2.0 specification, is managed by an array of HDM Decoder register15 * instances per CXL port and per CXL endpoint. Define common helpers16 * for enumerating these registers and capabilities.17 */18 19DECLARE_RWSEM(cxl_dpa_rwsem);20 21static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,22 int *target_map)23{24 int rc;25 26 rc = cxl_decoder_add_locked(cxld, target_map);27 if (rc) {28 put_device(&cxld->dev);29 dev_err(&port->dev, "Failed to add decoder\n");30 return rc;31 }32 33 rc = cxl_decoder_autoremove(&port->dev, cxld);34 if (rc)35 return rc;36 37 dev_dbg(&cxld->dev, "Added to port %s\n", dev_name(&port->dev));38 39 return 0;40}41 42/*43 * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure)44 * single ported host-bridges need not publish a decoder capability when a45 * passthrough decode can be assumed, i.e. all transactions that the uport sees46 * are claimed and passed to the single dport. Disable the range until the first47 * CXL region is enumerated / activated.48 */49int devm_cxl_add_passthrough_decoder(struct cxl_port *port)50{51 struct cxl_switch_decoder *cxlsd;52 struct cxl_dport *dport = NULL;53 int single_port_map[1];54 unsigned long index;55 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);56 57 /*58 * Capability checks are moot for passthrough decoders, support59 * any and all possibilities.60 */61 cxlhdm->interleave_mask = ~0U;62 cxlhdm->iw_cap_mask = ~0UL;63 64 cxlsd = cxl_switch_decoder_alloc(port, 1);65 if (IS_ERR(cxlsd))66 return PTR_ERR(cxlsd);67 68 device_lock_assert(&port->dev);69 70 xa_for_each(&port->dports, index, dport)71 break;72 single_port_map[0] = dport->port_id;73 74 return add_hdm_decoder(port, &cxlsd->cxld, single_port_map);75}76EXPORT_SYMBOL_NS_GPL(devm_cxl_add_passthrough_decoder, CXL);77 78static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm)79{80 u32 hdm_cap;81 82 hdm_cap = readl(cxlhdm->regs.hdm_decoder + CXL_HDM_DECODER_CAP_OFFSET);83 cxlhdm->decoder_count = cxl_hdm_decoder_count(hdm_cap);84 cxlhdm->target_count =85 FIELD_GET(CXL_HDM_DECODER_TARGET_COUNT_MASK, hdm_cap);86 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_11_8, hdm_cap))87 cxlhdm->interleave_mask |= GENMASK(11, 8);88 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_14_12, hdm_cap))89 cxlhdm->interleave_mask |= GENMASK(14, 12);90 cxlhdm->iw_cap_mask = BIT(1) | BIT(2) | BIT(4) | BIT(8);91 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_3_6_12_WAY, hdm_cap))92 cxlhdm->iw_cap_mask |= BIT(3) | BIT(6) | BIT(12);93 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_16_WAY, hdm_cap))94 cxlhdm->iw_cap_mask |= BIT(16);95}96 97static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info)98{99 struct cxl_hdm *cxlhdm;100 void __iomem *hdm;101 u32 ctrl;102 int i;103 104 if (!info)105 return false;106 107 cxlhdm = dev_get_drvdata(&info->port->dev);108 hdm = cxlhdm->regs.hdm_decoder;109 110 if (!hdm)111 return true;112 113 /*114 * If HDM decoders are present and the driver is in control of115 * Mem_Enable skip DVSEC based emulation116 */117 if (!info->mem_enabled)118 return false;119 120 /*121 * If any decoders are committed already, there should not be any122 * emulated DVSEC decoders.123 */124 for (i = 0; i < cxlhdm->decoder_count; i++) {125 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));126 dev_dbg(&info->port->dev,127 "decoder%d.%d: committed: %ld base: %#x_%.8x size: %#x_%.8x\n",128 info->port->id, i,129 FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl),130 readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i)),131 readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(i)),132 readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i)),133 readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i)));134 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))135 return false;136 }137 138 return true;139}140 141/**142 * devm_cxl_setup_hdm - map HDM decoder component registers143 * @port: cxl_port to map144 * @info: cached DVSEC range register info145 */146struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port,147 struct cxl_endpoint_dvsec_info *info)148{149 struct cxl_register_map *reg_map = &port->reg_map;150 struct device *dev = &port->dev;151 struct cxl_hdm *cxlhdm;152 int rc;153 154 cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL);155 if (!cxlhdm)156 return ERR_PTR(-ENOMEM);157 cxlhdm->port = port;158 dev_set_drvdata(dev, cxlhdm);159 160 /* Memory devices can configure device HDM using DVSEC range regs. */161 if (reg_map->resource == CXL_RESOURCE_NONE) {162 if (!info || !info->mem_enabled) {163 dev_err(dev, "No component registers mapped\n");164 return ERR_PTR(-ENXIO);165 }166 167 cxlhdm->decoder_count = info->ranges;168 return cxlhdm;169 }170 171 if (!reg_map->component_map.hdm_decoder.valid) {172 dev_dbg(&port->dev, "HDM decoder registers not implemented\n");173 /* unique error code to indicate no HDM decoder capability */174 return ERR_PTR(-ENODEV);175 }176 177 rc = cxl_map_component_regs(reg_map, &cxlhdm->regs,178 BIT(CXL_CM_CAP_CAP_ID_HDM));179 if (rc) {180 dev_err(dev, "Failed to map HDM capability.\n");181 return ERR_PTR(rc);182 }183 184 parse_hdm_decoder_caps(cxlhdm);185 if (cxlhdm->decoder_count == 0) {186 dev_err(dev, "Spec violation. Caps invalid\n");187 return ERR_PTR(-ENXIO);188 }189 190 /*191 * Now that the hdm capability is parsed, decide if range192 * register emulation is needed and fixup cxlhdm accordingly.193 */194 if (should_emulate_decoders(info)) {195 dev_dbg(dev, "Fallback map %d range register%s\n", info->ranges,196 info->ranges > 1 ? "s" : "");197 cxlhdm->decoder_count = info->ranges;198 }199 200 return cxlhdm;201}202EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_hdm, CXL);203 204static void __cxl_dpa_debug(struct seq_file *file, struct resource *r, int depth)205{206 unsigned long long start = r->start, end = r->end;207 208 seq_printf(file, "%*s%08llx-%08llx : %s\n", depth * 2, "", start, end,209 r->name);210}211 212void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds)213{214 struct resource *p1, *p2;215 216 down_read(&cxl_dpa_rwsem);217 for (p1 = cxlds->dpa_res.child; p1; p1 = p1->sibling) {218 __cxl_dpa_debug(file, p1, 0);219 for (p2 = p1->child; p2; p2 = p2->sibling)220 __cxl_dpa_debug(file, p2, 1);221 }222 up_read(&cxl_dpa_rwsem);223}224EXPORT_SYMBOL_NS_GPL(cxl_dpa_debug, CXL);225 226/*227 * Must be called in a context that synchronizes against this decoder's228 * port ->remove() callback (like an endpoint decoder sysfs attribute)229 */230static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled)231{232 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);233 struct cxl_port *port = cxled_to_port(cxled);234 struct cxl_dev_state *cxlds = cxlmd->cxlds;235 struct resource *res = cxled->dpa_res;236 resource_size_t skip_start;237 238 lockdep_assert_held_write(&cxl_dpa_rwsem);239 240 /* save @skip_start, before @res is released */241 skip_start = res->start - cxled->skip;242 __release_region(&cxlds->dpa_res, res->start, resource_size(res));243 if (cxled->skip)244 __release_region(&cxlds->dpa_res, skip_start, cxled->skip);245 cxled->skip = 0;246 cxled->dpa_res = NULL;247 put_device(&cxled->cxld.dev);248 port->hdm_end--;249}250 251static void cxl_dpa_release(void *cxled)252{253 down_write(&cxl_dpa_rwsem);254 __cxl_dpa_release(cxled);255 up_write(&cxl_dpa_rwsem);256}257 258/*259 * Must be called from context that will not race port device260 * unregistration, like decoder sysfs attribute methods261 */262static void devm_cxl_dpa_release(struct cxl_endpoint_decoder *cxled)263{264 struct cxl_port *port = cxled_to_port(cxled);265 266 lockdep_assert_held_write(&cxl_dpa_rwsem);267 devm_remove_action(&port->dev, cxl_dpa_release, cxled);268 __cxl_dpa_release(cxled);269}270 271static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,272 resource_size_t base, resource_size_t len,273 resource_size_t skipped)274{275 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);276 struct cxl_port *port = cxled_to_port(cxled);277 struct cxl_dev_state *cxlds = cxlmd->cxlds;278 struct device *dev = &port->dev;279 struct resource *res;280 281 lockdep_assert_held_write(&cxl_dpa_rwsem);282 283 if (!len) {284 dev_warn(dev, "decoder%d.%d: empty reservation attempted\n",285 port->id, cxled->cxld.id);286 return -EINVAL;287 }288 289 if (cxled->dpa_res) {290 dev_dbg(dev, "decoder%d.%d: existing allocation %pr assigned\n",291 port->id, cxled->cxld.id, cxled->dpa_res);292 return -EBUSY;293 }294 295 if (port->hdm_end + 1 != cxled->cxld.id) {296 /*297 * Assumes alloc and commit order is always in hardware instance298 * order per expectations from 8.2.5.12.20 Committing Decoder299 * Programming that enforce decoder[m] committed before300 * decoder[m+1] commit start.301 */302 dev_dbg(dev, "decoder%d.%d: expected decoder%d.%d\n", port->id,303 cxled->cxld.id, port->id, port->hdm_end + 1);304 return -EBUSY;305 }306 307 if (skipped) {308 res = __request_region(&cxlds->dpa_res, base - skipped, skipped,309 dev_name(&cxled->cxld.dev), 0);310 if (!res) {311 dev_dbg(dev,312 "decoder%d.%d: failed to reserve skipped space\n",313 port->id, cxled->cxld.id);314 return -EBUSY;315 }316 }317 res = __request_region(&cxlds->dpa_res, base, len,318 dev_name(&cxled->cxld.dev), 0);319 if (!res) {320 dev_dbg(dev, "decoder%d.%d: failed to reserve allocation\n",321 port->id, cxled->cxld.id);322 if (skipped)323 __release_region(&cxlds->dpa_res, base - skipped,324 skipped);325 return -EBUSY;326 }327 cxled->dpa_res = res;328 cxled->skip = skipped;329 330 if (resource_contains(&cxlds->pmem_res, res))331 cxled->mode = CXL_DECODER_PMEM;332 else if (resource_contains(&cxlds->ram_res, res))333 cxled->mode = CXL_DECODER_RAM;334 else {335 dev_warn(dev, "decoder%d.%d: %pr mixed mode not supported\n",336 port->id, cxled->cxld.id, cxled->dpa_res);337 cxled->mode = CXL_DECODER_MIXED;338 }339 340 port->hdm_end++;341 get_device(&cxled->cxld.dev);342 return 0;343}344 345int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,346 resource_size_t base, resource_size_t len,347 resource_size_t skipped)348{349 struct cxl_port *port = cxled_to_port(cxled);350 int rc;351 352 down_write(&cxl_dpa_rwsem);353 rc = __cxl_dpa_reserve(cxled, base, len, skipped);354 up_write(&cxl_dpa_rwsem);355 356 if (rc)357 return rc;358 359 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);360}361EXPORT_SYMBOL_NS_GPL(devm_cxl_dpa_reserve, CXL);362 363resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled)364{365 resource_size_t size = 0;366 367 down_read(&cxl_dpa_rwsem);368 if (cxled->dpa_res)369 size = resource_size(cxled->dpa_res);370 up_read(&cxl_dpa_rwsem);371 372 return size;373}374 375resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled)376{377 resource_size_t base = -1;378 379 lockdep_assert_held(&cxl_dpa_rwsem);380 if (cxled->dpa_res)381 base = cxled->dpa_res->start;382 383 return base;384}385 386int cxl_dpa_free(struct cxl_endpoint_decoder *cxled)387{388 struct cxl_port *port = cxled_to_port(cxled);389 struct device *dev = &cxled->cxld.dev;390 int rc;391 392 down_write(&cxl_dpa_rwsem);393 if (!cxled->dpa_res) {394 rc = 0;395 goto out;396 }397 if (cxled->cxld.region) {398 dev_dbg(dev, "decoder assigned to: %s\n",399 dev_name(&cxled->cxld.region->dev));400 rc = -EBUSY;401 goto out;402 }403 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {404 dev_dbg(dev, "decoder enabled\n");405 rc = -EBUSY;406 goto out;407 }408 if (cxled->cxld.id != port->hdm_end) {409 dev_dbg(dev, "expected decoder%d.%d\n", port->id,410 port->hdm_end);411 rc = -EBUSY;412 goto out;413 }414 devm_cxl_dpa_release(cxled);415 rc = 0;416out:417 up_write(&cxl_dpa_rwsem);418 return rc;419}420 421int cxl_dpa_set_mode(struct cxl_endpoint_decoder *cxled,422 enum cxl_decoder_mode mode)423{424 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);425 struct cxl_dev_state *cxlds = cxlmd->cxlds;426 struct device *dev = &cxled->cxld.dev;427 int rc;428 429 switch (mode) {430 case CXL_DECODER_RAM:431 case CXL_DECODER_PMEM:432 break;433 default:434 dev_dbg(dev, "unsupported mode: %d\n", mode);435 return -EINVAL;436 }437 438 down_write(&cxl_dpa_rwsem);439 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {440 rc = -EBUSY;441 goto out;442 }443 444 /*445 * Only allow modes that are supported by the current partition446 * configuration447 */448 if (mode == CXL_DECODER_PMEM && !resource_size(&cxlds->pmem_res)) {449 dev_dbg(dev, "no available pmem capacity\n");450 rc = -ENXIO;451 goto out;452 }453 if (mode == CXL_DECODER_RAM && !resource_size(&cxlds->ram_res)) {454 dev_dbg(dev, "no available ram capacity\n");455 rc = -ENXIO;456 goto out;457 }458 459 cxled->mode = mode;460 rc = 0;461out:462 up_write(&cxl_dpa_rwsem);463 464 return rc;465}466 467int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, unsigned long long size)468{469 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);470 resource_size_t free_ram_start, free_pmem_start;471 struct cxl_port *port = cxled_to_port(cxled);472 struct cxl_dev_state *cxlds = cxlmd->cxlds;473 struct device *dev = &cxled->cxld.dev;474 resource_size_t start, avail, skip;475 struct resource *p, *last;476 int rc;477 478 down_write(&cxl_dpa_rwsem);479 if (cxled->cxld.region) {480 dev_dbg(dev, "decoder attached to %s\n",481 dev_name(&cxled->cxld.region->dev));482 rc = -EBUSY;483 goto out;484 }485 486 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) {487 dev_dbg(dev, "decoder enabled\n");488 rc = -EBUSY;489 goto out;490 }491 492 for (p = cxlds->ram_res.child, last = NULL; p; p = p->sibling)493 last = p;494 if (last)495 free_ram_start = last->end + 1;496 else497 free_ram_start = cxlds->ram_res.start;498 499 for (p = cxlds->pmem_res.child, last = NULL; p; p = p->sibling)500 last = p;501 if (last)502 free_pmem_start = last->end + 1;503 else504 free_pmem_start = cxlds->pmem_res.start;505 506 if (cxled->mode == CXL_DECODER_RAM) {507 start = free_ram_start;508 avail = cxlds->ram_res.end - start + 1;509 skip = 0;510 } else if (cxled->mode == CXL_DECODER_PMEM) {511 resource_size_t skip_start, skip_end;512 513 start = free_pmem_start;514 avail = cxlds->pmem_res.end - start + 1;515 skip_start = free_ram_start;516 517 /*518 * If some pmem is already allocated, then that allocation519 * already handled the skip.520 */521 if (cxlds->pmem_res.child &&522 skip_start == cxlds->pmem_res.child->start)523 skip_end = skip_start - 1;524 else525 skip_end = start - 1;526 skip = skip_end - skip_start + 1;527 } else {528 dev_dbg(dev, "mode not set\n");529 rc = -EINVAL;530 goto out;531 }532 533 if (size > avail) {534 dev_dbg(dev, "%pa exceeds available %s capacity: %pa\n", &size,535 cxl_decoder_mode_name(cxled->mode), &avail);536 rc = -ENOSPC;537 goto out;538 }539 540 rc = __cxl_dpa_reserve(cxled, start, size, skip);541out:542 up_write(&cxl_dpa_rwsem);543 544 if (rc)545 return rc;546 547 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled);548}549 550static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl)551{552 u16 eig;553 u8 eiw;554 555 /*556 * Input validation ensures these warns never fire, but otherwise557 * suppress unititalized variable usage warnings.558 */559 if (WARN_ONCE(ways_to_eiw(cxld->interleave_ways, &eiw),560 "invalid interleave_ways: %d\n", cxld->interleave_ways))561 return;562 if (WARN_ONCE(granularity_to_eig(cxld->interleave_granularity, &eig),563 "invalid interleave_granularity: %d\n",564 cxld->interleave_granularity))565 return;566 567 u32p_replace_bits(ctrl, eig, CXL_HDM_DECODER0_CTRL_IG_MASK);568 u32p_replace_bits(ctrl, eiw, CXL_HDM_DECODER0_CTRL_IW_MASK);569 *ctrl |= CXL_HDM_DECODER0_CTRL_COMMIT;570}571 572static void cxld_set_type(struct cxl_decoder *cxld, u32 *ctrl)573{574 u32p_replace_bits(ctrl,575 !!(cxld->target_type == CXL_DECODER_HOSTONLYMEM),576 CXL_HDM_DECODER0_CTRL_HOSTONLY);577}578 579static void cxlsd_set_targets(struct cxl_switch_decoder *cxlsd, u64 *tgt)580{581 struct cxl_dport **t = &cxlsd->target[0];582 int ways = cxlsd->cxld.interleave_ways;583 584 *tgt = FIELD_PREP(GENMASK(7, 0), t[0]->port_id);585 if (ways > 1)586 *tgt |= FIELD_PREP(GENMASK(15, 8), t[1]->port_id);587 if (ways > 2)588 *tgt |= FIELD_PREP(GENMASK(23, 16), t[2]->port_id);589 if (ways > 3)590 *tgt |= FIELD_PREP(GENMASK(31, 24), t[3]->port_id);591 if (ways > 4)592 *tgt |= FIELD_PREP(GENMASK_ULL(39, 32), t[4]->port_id);593 if (ways > 5)594 *tgt |= FIELD_PREP(GENMASK_ULL(47, 40), t[5]->port_id);595 if (ways > 6)596 *tgt |= FIELD_PREP(GENMASK_ULL(55, 48), t[6]->port_id);597 if (ways > 7)598 *tgt |= FIELD_PREP(GENMASK_ULL(63, 56), t[7]->port_id);599}600 601/*602 * Per CXL 2.0 8.2.5.12.20 Committing Decoder Programming, hardware must set603 * committed or error within 10ms, but just be generous with 20ms to account for604 * clock skew and other marginal behavior605 */606#define COMMIT_TIMEOUT_MS 20607static int cxld_await_commit(void __iomem *hdm, int id)608{609 u32 ctrl;610 int i;611 612 for (i = 0; i < COMMIT_TIMEOUT_MS; i++) {613 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));614 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMIT_ERROR, ctrl)) {615 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;616 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));617 return -EIO;618 }619 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl))620 return 0;621 fsleep(1000);622 }623 624 return -ETIMEDOUT;625}626 627static int cxl_decoder_commit(struct cxl_decoder *cxld)628{629 struct cxl_port *port = to_cxl_port(cxld->dev.parent);630 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);631 void __iomem *hdm = cxlhdm->regs.hdm_decoder;632 int id = cxld->id, rc;633 u64 base, size;634 u32 ctrl;635 636 if (cxld->flags & CXL_DECODER_F_ENABLE)637 return 0;638 639 if (cxl_num_decoders_committed(port) != id) {640 dev_dbg(&port->dev,641 "%s: out of order commit, expected decoder%d.%d\n",642 dev_name(&cxld->dev), port->id,643 cxl_num_decoders_committed(port));644 return -EBUSY;645 }646 647 /*648 * For endpoint decoders hosted on CXL memory devices that649 * support the sanitize operation, make sure sanitize is not in-flight.650 */651 if (is_endpoint_decoder(&cxld->dev)) {652 struct cxl_endpoint_decoder *cxled =653 to_cxl_endpoint_decoder(&cxld->dev);654 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);655 struct cxl_memdev_state *mds =656 to_cxl_memdev_state(cxlmd->cxlds);657 658 if (mds && mds->security.sanitize_active) {659 dev_dbg(&cxlmd->dev,660 "attempted to commit %s during sanitize\n",661 dev_name(&cxld->dev));662 return -EBUSY;663 }664 }665 666 down_read(&cxl_dpa_rwsem);667 /* common decoder settings */668 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id));669 cxld_set_interleave(cxld, &ctrl);670 cxld_set_type(cxld, &ctrl);671 base = cxld->hpa_range.start;672 size = range_len(&cxld->hpa_range);673 674 writel(upper_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));675 writel(lower_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));676 writel(upper_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));677 writel(lower_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));678 679 if (is_switch_decoder(&cxld->dev)) {680 struct cxl_switch_decoder *cxlsd =681 to_cxl_switch_decoder(&cxld->dev);682 void __iomem *tl_hi = hdm + CXL_HDM_DECODER0_TL_HIGH(id);683 void __iomem *tl_lo = hdm + CXL_HDM_DECODER0_TL_LOW(id);684 u64 targets;685 686 cxlsd_set_targets(cxlsd, &targets);687 writel(upper_32_bits(targets), tl_hi);688 writel(lower_32_bits(targets), tl_lo);689 } else {690 struct cxl_endpoint_decoder *cxled =691 to_cxl_endpoint_decoder(&cxld->dev);692 void __iomem *sk_hi = hdm + CXL_HDM_DECODER0_SKIP_HIGH(id);693 void __iomem *sk_lo = hdm + CXL_HDM_DECODER0_SKIP_LOW(id);694 695 writel(upper_32_bits(cxled->skip), sk_hi);696 writel(lower_32_bits(cxled->skip), sk_lo);697 }698 699 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));700 up_read(&cxl_dpa_rwsem);701 702 port->commit_end++;703 rc = cxld_await_commit(hdm, cxld->id);704 if (rc) {705 dev_dbg(&port->dev, "%s: error %d committing decoder\n",706 dev_name(&cxld->dev), rc);707 cxld->reset(cxld);708 return rc;709 }710 cxld->flags |= CXL_DECODER_F_ENABLE;711 712 return 0;713}714 715static int commit_reap(struct device *dev, const void *data)716{717 struct cxl_port *port = to_cxl_port(dev->parent);718 struct cxl_decoder *cxld;719 720 if (!is_switch_decoder(dev) && !is_endpoint_decoder(dev))721 return 0;722 723 cxld = to_cxl_decoder(dev);724 if (port->commit_end == cxld->id &&725 ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {726 port->commit_end--;727 dev_dbg(&port->dev, "reap: %s commit_end: %d\n",728 dev_name(&cxld->dev), port->commit_end);729 }730 731 return 0;732}733 734void cxl_port_commit_reap(struct cxl_decoder *cxld)735{736 struct cxl_port *port = to_cxl_port(cxld->dev.parent);737 738 lockdep_assert_held_write(&cxl_region_rwsem);739 740 /*741 * Once the highest committed decoder is disabled, free any other742 * decoders that were pinned allocated by out-of-order release.743 */744 port->commit_end--;745 dev_dbg(&port->dev, "reap: %s commit_end: %d\n", dev_name(&cxld->dev),746 port->commit_end);747 device_for_each_child_reverse_from(&port->dev, &cxld->dev, NULL,748 commit_reap);749}750EXPORT_SYMBOL_NS_GPL(cxl_port_commit_reap, CXL);751 752static void cxl_decoder_reset(struct cxl_decoder *cxld)753{754 struct cxl_port *port = to_cxl_port(cxld->dev.parent);755 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev);756 void __iomem *hdm = cxlhdm->regs.hdm_decoder;757 int id = cxld->id;758 u32 ctrl;759 760 if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)761 return;762 763 if (port->commit_end == id)764 cxl_port_commit_reap(cxld);765 else766 dev_dbg(&port->dev,767 "%s: out of order reset, expected decoder%d.%d\n",768 dev_name(&cxld->dev), port->id, port->commit_end);769 770 down_read(&cxl_dpa_rwsem);771 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));772 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT;773 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id));774 775 writel(0, hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id));776 writel(0, hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id));777 writel(0, hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id));778 writel(0, hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id));779 up_read(&cxl_dpa_rwsem);780 781 cxld->flags &= ~CXL_DECODER_F_ENABLE;782 783 /* Userspace is now responsible for reconfiguring this decoder */784 if (is_endpoint_decoder(&cxld->dev)) {785 struct cxl_endpoint_decoder *cxled;786 787 cxled = to_cxl_endpoint_decoder(&cxld->dev);788 cxled->state = CXL_DECODER_STATE_MANUAL;789 }790}791 792static int cxl_setup_hdm_decoder_from_dvsec(793 struct cxl_port *port, struct cxl_decoder *cxld, u64 *dpa_base,794 int which, struct cxl_endpoint_dvsec_info *info)795{796 struct cxl_endpoint_decoder *cxled;797 u64 len;798 int rc;799 800 if (!is_cxl_endpoint(port))801 return -EOPNOTSUPP;802 803 cxled = to_cxl_endpoint_decoder(&cxld->dev);804 len = range_len(&info->dvsec_range[which]);805 if (!len)806 return -ENOENT;807 808 cxld->target_type = CXL_DECODER_HOSTONLYMEM;809 cxld->commit = NULL;810 cxld->reset = NULL;811 cxld->hpa_range = info->dvsec_range[which];812 813 /*814 * Set the emulated decoder as locked pending additional support to815 * change the range registers at run time.816 */817 cxld->flags |= CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK;818 port->commit_end = cxld->id;819 820 rc = devm_cxl_dpa_reserve(cxled, *dpa_base, len, 0);821 if (rc) {822 dev_err(&port->dev,823 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",824 port->id, cxld->id, *dpa_base, *dpa_base + len - 1, rc);825 return rc;826 }827 *dpa_base += len;828 cxled->state = CXL_DECODER_STATE_AUTO;829 830 return 0;831}832 833static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,834 int *target_map, void __iomem *hdm, int which,835 u64 *dpa_base, struct cxl_endpoint_dvsec_info *info)836{837 struct cxl_endpoint_decoder *cxled = NULL;838 u64 size, base, skip, dpa_size, lo, hi;839 bool committed;840 u32 remainder;841 int i, rc;842 u32 ctrl;843 union {844 u64 value;845 unsigned char target_id[8];846 } target_list;847 848 if (should_emulate_decoders(info))849 return cxl_setup_hdm_decoder_from_dvsec(port, cxld, dpa_base,850 which, info);851 852 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));853 lo = readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(which));854 hi = readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(which));855 base = (hi << 32) + lo;856 lo = readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(which));857 hi = readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(which));858 size = (hi << 32) + lo;859 committed = !!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED);860 cxld->commit = cxl_decoder_commit;861 cxld->reset = cxl_decoder_reset;862 863 if (!committed)864 size = 0;865 if (base == U64_MAX || size == U64_MAX) {866 dev_warn(&port->dev, "decoder%d.%d: Invalid resource range\n",867 port->id, cxld->id);868 return -ENXIO;869 }870 871 if (info)872 cxled = to_cxl_endpoint_decoder(&cxld->dev);873 cxld->hpa_range = (struct range) {874 .start = base,875 .end = base + size - 1,876 };877 878 /* decoders are enabled if committed */879 if (committed) {880 cxld->flags |= CXL_DECODER_F_ENABLE;881 if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK)882 cxld->flags |= CXL_DECODER_F_LOCK;883 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl))884 cxld->target_type = CXL_DECODER_HOSTONLYMEM;885 else886 cxld->target_type = CXL_DECODER_DEVMEM;887 888 guard(rwsem_write)(&cxl_region_rwsem);889 if (cxld->id != cxl_num_decoders_committed(port)) {890 dev_warn(&port->dev,891 "decoder%d.%d: Committed out of order\n",892 port->id, cxld->id);893 return -ENXIO;894 }895 896 if (size == 0) {897 dev_warn(&port->dev,898 "decoder%d.%d: Committed with zero size\n",899 port->id, cxld->id);900 return -ENXIO;901 }902 port->commit_end = cxld->id;903 } else {904 if (cxled) {905 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);906 struct cxl_dev_state *cxlds = cxlmd->cxlds;907 908 /*909 * Default by devtype until a device arrives that needs910 * more precision.911 */912 if (cxlds->type == CXL_DEVTYPE_CLASSMEM)913 cxld->target_type = CXL_DECODER_HOSTONLYMEM;914 else915 cxld->target_type = CXL_DECODER_DEVMEM;916 } else {917 /* To be overridden by region type at commit time */918 cxld->target_type = CXL_DECODER_HOSTONLYMEM;919 }920 921 if (!FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl) &&922 cxld->target_type == CXL_DECODER_HOSTONLYMEM) {923 ctrl |= CXL_HDM_DECODER0_CTRL_HOSTONLY;924 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which));925 }926 }927 rc = eiw_to_ways(FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl),928 &cxld->interleave_ways);929 if (rc) {930 dev_warn(&port->dev,931 "decoder%d.%d: Invalid interleave ways (ctrl: %#x)\n",932 port->id, cxld->id, ctrl);933 return rc;934 }935 rc = eig_to_granularity(FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl),936 &cxld->interleave_granularity);937 if (rc) {938 dev_warn(&port->dev,939 "decoder%d.%d: Invalid interleave granularity (ctrl: %#x)\n",940 port->id, cxld->id, ctrl);941 return rc;942 }943 944 dev_dbg(&port->dev, "decoder%d.%d: range: %#llx-%#llx iw: %d ig: %d\n",945 port->id, cxld->id, cxld->hpa_range.start, cxld->hpa_range.end,946 cxld->interleave_ways, cxld->interleave_granularity);947 948 if (!cxled) {949 lo = readl(hdm + CXL_HDM_DECODER0_TL_LOW(which));950 hi = readl(hdm + CXL_HDM_DECODER0_TL_HIGH(which));951 target_list.value = (hi << 32) + lo;952 for (i = 0; i < cxld->interleave_ways; i++)953 target_map[i] = target_list.target_id[i];954 955 return 0;956 }957 958 if (!committed)959 return 0;960 961 dpa_size = div_u64_rem(size, cxld->interleave_ways, &remainder);962 if (remainder) {963 dev_err(&port->dev,964 "decoder%d.%d: invalid committed configuration size: %#llx ways: %d\n",965 port->id, cxld->id, size, cxld->interleave_ways);966 return -ENXIO;967 }968 lo = readl(hdm + CXL_HDM_DECODER0_SKIP_LOW(which));969 hi = readl(hdm + CXL_HDM_DECODER0_SKIP_HIGH(which));970 skip = (hi << 32) + lo;971 rc = devm_cxl_dpa_reserve(cxled, *dpa_base + skip, dpa_size, skip);972 if (rc) {973 dev_err(&port->dev,974 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)",975 port->id, cxld->id, *dpa_base,976 *dpa_base + dpa_size + skip - 1, rc);977 return rc;978 }979 *dpa_base += dpa_size + skip;980 981 cxled->state = CXL_DECODER_STATE_AUTO;982 983 return 0;984}985 986static void cxl_settle_decoders(struct cxl_hdm *cxlhdm)987{988 void __iomem *hdm = cxlhdm->regs.hdm_decoder;989 int committed, i;990 u32 ctrl;991 992 if (!hdm)993 return;994 995 /*996 * Since the register resource was recently claimed via request_region()997 * be careful about trusting the "not-committed" status until the commit998 * timeout has elapsed. The commit timeout is 10ms (CXL 2.0999 * 8.2.5.12.20), but double it to be tolerant of any clock skew between1000 * host and target.1001 */1002 for (i = 0, committed = 0; i < cxlhdm->decoder_count; i++) {1003 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i));1004 if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED)1005 committed++;1006 }1007 1008 /* ensure that future checks of committed can be trusted */1009 if (committed != cxlhdm->decoder_count)1010 msleep(20);1011}1012 1013/**1014 * devm_cxl_enumerate_decoders - add decoder objects per HDM register set1015 * @cxlhdm: Structure to populate with HDM capabilities1016 * @info: cached DVSEC range register info1017 */1018int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm,1019 struct cxl_endpoint_dvsec_info *info)1020{1021 void __iomem *hdm = cxlhdm->regs.hdm_decoder;1022 struct cxl_port *port = cxlhdm->port;1023 int i;1024 u64 dpa_base = 0;1025 1026 cxl_settle_decoders(cxlhdm);1027 1028 for (i = 0; i < cxlhdm->decoder_count; i++) {1029 int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 };1030 int rc, target_count = cxlhdm->target_count;1031 struct cxl_decoder *cxld;1032 1033 if (is_cxl_endpoint(port)) {1034 struct cxl_endpoint_decoder *cxled;1035 1036 cxled = cxl_endpoint_decoder_alloc(port);1037 if (IS_ERR(cxled)) {1038 dev_warn(&port->dev,1039 "Failed to allocate decoder%d.%d\n",1040 port->id, i);1041 return PTR_ERR(cxled);1042 }1043 cxld = &cxled->cxld;1044 } else {1045 struct cxl_switch_decoder *cxlsd;1046 1047 cxlsd = cxl_switch_decoder_alloc(port, target_count);1048 if (IS_ERR(cxlsd)) {1049 dev_warn(&port->dev,1050 "Failed to allocate decoder%d.%d\n",1051 port->id, i);1052 return PTR_ERR(cxlsd);1053 }1054 cxld = &cxlsd->cxld;1055 }1056 1057 rc = init_hdm_decoder(port, cxld, target_map, hdm, i,1058 &dpa_base, info);1059 if (rc) {1060 dev_warn(&port->dev,1061 "Failed to initialize decoder%d.%d\n",1062 port->id, i);1063 put_device(&cxld->dev);1064 return rc;1065 }1066 rc = add_hdm_decoder(port, cxld, target_map);1067 if (rc) {1068 dev_warn(&port->dev,1069 "Failed to add decoder%d.%d\n", port->id, i);1070 return rc;1071 }1072 }1073 1074 return 0;1075}1076EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_decoders, CXL);1077