1083 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/* Copyright(c) 2023 Intel Corporation. All rights reserved. */3#include <linux/acpi.h>4#include <linux/xarray.h>5#include <linux/fw_table.h>6#include <linux/node.h>7#include <linux/overflow.h>8#include "cxlpci.h"9#include "cxlmem.h"10#include "core.h"11#include "cxl.h"12 13struct dsmas_entry {14 struct range dpa_range;15 u8 handle;16 struct access_coordinate coord[ACCESS_COORDINATE_MAX];17 struct access_coordinate cdat_coord[ACCESS_COORDINATE_MAX];18 int entries;19 int qos_class;20};21 22static u32 cdat_normalize(u16 entry, u64 base, u8 type)23{24 u32 value;25 26 /*27 * Check for invalid and overflow values28 */29 if (entry == 0xffff || !entry)30 return 0;31 else if (base > (UINT_MAX / (entry)))32 return 0;33 34 /*35 * CDAT fields follow the format of HMAT fields. See table 5 Device36 * Scoped Latency and Bandwidth Information Structure in Coherent Device37 * Attribute Table (CDAT) Specification v1.01.38 */39 value = entry * base;40 switch (type) {41 case ACPI_HMAT_ACCESS_LATENCY:42 case ACPI_HMAT_READ_LATENCY:43 case ACPI_HMAT_WRITE_LATENCY:44 value = DIV_ROUND_UP(value, 1000);45 break;46 default:47 break;48 }49 return value;50}51 52static int cdat_dsmas_handler(union acpi_subtable_headers *header, void *arg,53 const unsigned long end)54{55 struct acpi_cdat_header *hdr = &header->cdat;56 struct acpi_cdat_dsmas *dsmas;57 int size = sizeof(*hdr) + sizeof(*dsmas);58 struct xarray *dsmas_xa = arg;59 struct dsmas_entry *dent;60 u16 len;61 int rc;62 63 len = le16_to_cpu((__force __le16)hdr->length);64 if (len != size || (unsigned long)hdr + len > end) {65 pr_warn("Malformed DSMAS table length: (%u:%u)\n", size, len);66 return -EINVAL;67 }68 69 /* Skip common header */70 dsmas = (struct acpi_cdat_dsmas *)(hdr + 1);71 72 dent = kzalloc(sizeof(*dent), GFP_KERNEL);73 if (!dent)74 return -ENOMEM;75 76 dent->handle = dsmas->dsmad_handle;77 dent->dpa_range.start = le64_to_cpu((__force __le64)dsmas->dpa_base_address);78 dent->dpa_range.end = le64_to_cpu((__force __le64)dsmas->dpa_base_address) +79 le64_to_cpu((__force __le64)dsmas->dpa_length) - 1;80 81 rc = xa_insert(dsmas_xa, dent->handle, dent, GFP_KERNEL);82 if (rc) {83 kfree(dent);84 return rc;85 }86 87 return 0;88}89 90static void __cxl_access_coordinate_set(struct access_coordinate *coord,91 int access, unsigned int val)92{93 switch (access) {94 case ACPI_HMAT_ACCESS_LATENCY:95 coord->read_latency = val;96 coord->write_latency = val;97 break;98 case ACPI_HMAT_READ_LATENCY:99 coord->read_latency = val;100 break;101 case ACPI_HMAT_WRITE_LATENCY:102 coord->write_latency = val;103 break;104 case ACPI_HMAT_ACCESS_BANDWIDTH:105 coord->read_bandwidth = val;106 coord->write_bandwidth = val;107 break;108 case ACPI_HMAT_READ_BANDWIDTH:109 coord->read_bandwidth = val;110 break;111 case ACPI_HMAT_WRITE_BANDWIDTH:112 coord->write_bandwidth = val;113 break;114 }115}116 117static void cxl_access_coordinate_set(struct access_coordinate *coord,118 int access, unsigned int val)119{120 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++)121 __cxl_access_coordinate_set(&coord[i], access, val);122}123 124static int cdat_dslbis_handler(union acpi_subtable_headers *header, void *arg,125 const unsigned long end)126{127 struct acpi_cdat_header *hdr = &header->cdat;128 struct acpi_cdat_dslbis *dslbis;129 int size = sizeof(*hdr) + sizeof(*dslbis);130 struct xarray *dsmas_xa = arg;131 struct dsmas_entry *dent;132 __le64 le_base;133 __le16 le_val;134 u64 val;135 u16 len;136 137 len = le16_to_cpu((__force __le16)hdr->length);138 if (len != size || (unsigned long)hdr + len > end) {139 pr_warn("Malformed DSLBIS table length: (%u:%u)\n", size, len);140 return -EINVAL;141 }142 143 /* Skip common header */144 dslbis = (struct acpi_cdat_dslbis *)(hdr + 1);145 146 /* Skip unrecognized data type */147 if (dslbis->data_type > ACPI_HMAT_WRITE_BANDWIDTH)148 return 0;149 150 /* Not a memory type, skip */151 if ((dslbis->flags & ACPI_HMAT_MEMORY_HIERARCHY) != ACPI_HMAT_MEMORY)152 return 0;153 154 dent = xa_load(dsmas_xa, dslbis->handle);155 if (!dent) {156 pr_warn("No matching DSMAS entry for DSLBIS entry.\n");157 return 0;158 }159 160 le_base = (__force __le64)dslbis->entry_base_unit;161 le_val = (__force __le16)dslbis->entry[0];162 val = cdat_normalize(le16_to_cpu(le_val), le64_to_cpu(le_base),163 dslbis->data_type);164 165 cxl_access_coordinate_set(dent->cdat_coord, dslbis->data_type, val);166 167 return 0;168}169 170static int cdat_table_parse_output(int rc)171{172 if (rc < 0)173 return rc;174 if (rc == 0)175 return -ENOENT;176 177 return 0;178}179 180static int cxl_cdat_endpoint_process(struct cxl_port *port,181 struct xarray *dsmas_xa)182{183 int rc;184 185 rc = cdat_table_parse(ACPI_CDAT_TYPE_DSMAS, cdat_dsmas_handler,186 dsmas_xa, port->cdat.table, port->cdat.length);187 rc = cdat_table_parse_output(rc);188 if (rc)189 return rc;190 191 rc = cdat_table_parse(ACPI_CDAT_TYPE_DSLBIS, cdat_dslbis_handler,192 dsmas_xa, port->cdat.table, port->cdat.length);193 return cdat_table_parse_output(rc);194}195 196static int cxl_port_perf_data_calculate(struct cxl_port *port,197 struct xarray *dsmas_xa)198{199 struct access_coordinate ep_c[ACCESS_COORDINATE_MAX];200 struct dsmas_entry *dent;201 int valid_entries = 0;202 unsigned long index;203 int rc;204 205 rc = cxl_endpoint_get_perf_coordinates(port, ep_c);206 if (rc) {207 dev_dbg(&port->dev, "Failed to retrieve ep perf coordinates.\n");208 return rc;209 }210 211 struct cxl_root *cxl_root __free(put_cxl_root) = find_cxl_root(port);212 213 if (!cxl_root)214 return -ENODEV;215 216 if (!cxl_root->ops || !cxl_root->ops->qos_class)217 return -EOPNOTSUPP;218 219 xa_for_each(dsmas_xa, index, dent) {220 int qos_class;221 222 cxl_coordinates_combine(dent->coord, dent->cdat_coord, ep_c);223 dent->entries = 1;224 rc = cxl_root->ops->qos_class(cxl_root,225 &dent->coord[ACCESS_COORDINATE_CPU],226 1, &qos_class);227 if (rc != 1)228 continue;229 230 valid_entries++;231 dent->qos_class = qos_class;232 }233 234 if (!valid_entries)235 return -ENOENT;236 237 return 0;238}239 240static void update_perf_entry(struct device *dev, struct dsmas_entry *dent,241 struct cxl_dpa_perf *dpa_perf)242{243 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {244 dpa_perf->coord[i] = dent->coord[i];245 dpa_perf->cdat_coord[i] = dent->cdat_coord[i];246 }247 dpa_perf->dpa_range = dent->dpa_range;248 dpa_perf->qos_class = dent->qos_class;249 dev_dbg(dev,250 "DSMAS: dpa: %#llx qos: %d read_bw: %d write_bw %d read_lat: %d write_lat: %d\n",251 dent->dpa_range.start, dpa_perf->qos_class,252 dent->coord[ACCESS_COORDINATE_CPU].read_bandwidth,253 dent->coord[ACCESS_COORDINATE_CPU].write_bandwidth,254 dent->coord[ACCESS_COORDINATE_CPU].read_latency,255 dent->coord[ACCESS_COORDINATE_CPU].write_latency);256}257 258static void cxl_memdev_set_qos_class(struct cxl_dev_state *cxlds,259 struct xarray *dsmas_xa)260{261 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);262 struct device *dev = cxlds->dev;263 struct range pmem_range = {264 .start = cxlds->pmem_res.start,265 .end = cxlds->pmem_res.end,266 };267 struct range ram_range = {268 .start = cxlds->ram_res.start,269 .end = cxlds->ram_res.end,270 };271 struct dsmas_entry *dent;272 unsigned long index;273 274 xa_for_each(dsmas_xa, index, dent) {275 if (resource_size(&cxlds->ram_res) &&276 range_contains(&ram_range, &dent->dpa_range))277 update_perf_entry(dev, dent, &mds->ram_perf);278 else if (resource_size(&cxlds->pmem_res) &&279 range_contains(&pmem_range, &dent->dpa_range))280 update_perf_entry(dev, dent, &mds->pmem_perf);281 else282 dev_dbg(dev, "no partition for dsmas dpa: %#llx\n",283 dent->dpa_range.start);284 }285}286 287static int match_cxlrd_qos_class(struct device *dev, void *data)288{289 int dev_qos_class = *(int *)data;290 struct cxl_root_decoder *cxlrd;291 292 if (!is_root_decoder(dev))293 return 0;294 295 cxlrd = to_cxl_root_decoder(dev);296 if (cxlrd->qos_class == CXL_QOS_CLASS_INVALID)297 return 0;298 299 if (cxlrd->qos_class == dev_qos_class)300 return 1;301 302 return 0;303}304 305static void reset_dpa_perf(struct cxl_dpa_perf *dpa_perf)306{307 *dpa_perf = (struct cxl_dpa_perf) {308 .qos_class = CXL_QOS_CLASS_INVALID,309 };310}311 312static bool cxl_qos_match(struct cxl_port *root_port,313 struct cxl_dpa_perf *dpa_perf)314{315 if (dpa_perf->qos_class == CXL_QOS_CLASS_INVALID)316 return false;317 318 if (!device_for_each_child(&root_port->dev, &dpa_perf->qos_class,319 match_cxlrd_qos_class))320 return false;321 322 return true;323}324 325static int match_cxlrd_hb(struct device *dev, void *data)326{327 struct device *host_bridge = data;328 struct cxl_switch_decoder *cxlsd;329 struct cxl_root_decoder *cxlrd;330 331 if (!is_root_decoder(dev))332 return 0;333 334 cxlrd = to_cxl_root_decoder(dev);335 cxlsd = &cxlrd->cxlsd;336 337 guard(rwsem_read)(&cxl_region_rwsem);338 for (int i = 0; i < cxlsd->nr_targets; i++) {339 if (host_bridge == cxlsd->target[i]->dport_dev)340 return 1;341 }342 343 return 0;344}345 346static int cxl_qos_class_verify(struct cxl_memdev *cxlmd)347{348 struct cxl_dev_state *cxlds = cxlmd->cxlds;349 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);350 struct cxl_port *root_port;351 int rc;352 353 struct cxl_root *cxl_root __free(put_cxl_root) =354 find_cxl_root(cxlmd->endpoint);355 356 if (!cxl_root)357 return -ENODEV;358 359 root_port = &cxl_root->port;360 361 /* Check that the QTG IDs are all sane between end device and root decoders */362 if (!cxl_qos_match(root_port, &mds->ram_perf))363 reset_dpa_perf(&mds->ram_perf);364 if (!cxl_qos_match(root_port, &mds->pmem_perf))365 reset_dpa_perf(&mds->pmem_perf);366 367 /* Check to make sure that the device's host bridge is under a root decoder */368 rc = device_for_each_child(&root_port->dev,369 cxlmd->endpoint->host_bridge, match_cxlrd_hb);370 if (!rc) {371 reset_dpa_perf(&mds->ram_perf);372 reset_dpa_perf(&mds->pmem_perf);373 }374 375 return rc;376}377 378static void discard_dsmas(struct xarray *xa)379{380 unsigned long index;381 void *ent;382 383 xa_for_each(xa, index, ent) {384 xa_erase(xa, index);385 kfree(ent);386 }387 xa_destroy(xa);388}389DEFINE_FREE(dsmas, struct xarray *, if (_T) discard_dsmas(_T))390 391void cxl_endpoint_parse_cdat(struct cxl_port *port)392{393 struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev);394 struct cxl_dev_state *cxlds = cxlmd->cxlds;395 struct xarray __dsmas_xa;396 struct xarray *dsmas_xa __free(dsmas) = &__dsmas_xa;397 int rc;398 399 xa_init(&__dsmas_xa);400 if (!port->cdat.table)401 return;402 403 rc = cxl_cdat_endpoint_process(port, dsmas_xa);404 if (rc < 0) {405 dev_dbg(&port->dev, "Failed to parse CDAT: %d\n", rc);406 return;407 }408 409 rc = cxl_port_perf_data_calculate(port, dsmas_xa);410 if (rc) {411 dev_dbg(&port->dev, "Failed to do perf coord calculations.\n");412 return;413 }414 415 cxl_memdev_set_qos_class(cxlds, dsmas_xa);416 cxl_qos_class_verify(cxlmd);417 cxl_memdev_update_perf(cxlmd);418}419EXPORT_SYMBOL_NS_GPL(cxl_endpoint_parse_cdat, CXL);420 421static int cdat_sslbis_handler(union acpi_subtable_headers *header, void *arg,422 const unsigned long end)423{424 struct acpi_cdat_sslbis_table {425 struct acpi_cdat_header header;426 struct acpi_cdat_sslbis sslbis_header;427 struct acpi_cdat_sslbe entries[];428 } *tbl = (struct acpi_cdat_sslbis_table *)header;429 int size = sizeof(header->cdat) + sizeof(tbl->sslbis_header);430 struct acpi_cdat_sslbis *sslbis;431 struct cxl_port *port = arg;432 struct device *dev = &port->dev;433 int remain, entries, i;434 u16 len;435 436 len = le16_to_cpu((__force __le16)header->cdat.length);437 remain = len - size;438 if (!remain || remain % sizeof(tbl->entries[0]) ||439 (unsigned long)header + len > end) {440 dev_warn(dev, "Malformed SSLBIS table length: (%u)\n", len);441 return -EINVAL;442 }443 444 sslbis = &tbl->sslbis_header;445 /* Unrecognized data type, we can skip */446 if (sslbis->data_type > ACPI_HMAT_WRITE_BANDWIDTH)447 return 0;448 449 entries = remain / sizeof(tbl->entries[0]);450 if (struct_size(tbl, entries, entries) != len)451 return -EINVAL;452 453 for (i = 0; i < entries; i++) {454 u16 x = le16_to_cpu((__force __le16)tbl->entries[i].portx_id);455 u16 y = le16_to_cpu((__force __le16)tbl->entries[i].porty_id);456 __le64 le_base;457 __le16 le_val;458 struct cxl_dport *dport;459 unsigned long index;460 u16 dsp_id;461 u64 val;462 463 switch (x) {464 case ACPI_CDAT_SSLBIS_US_PORT:465 dsp_id = y;466 break;467 case ACPI_CDAT_SSLBIS_ANY_PORT:468 switch (y) {469 case ACPI_CDAT_SSLBIS_US_PORT:470 dsp_id = x;471 break;472 case ACPI_CDAT_SSLBIS_ANY_PORT:473 dsp_id = ACPI_CDAT_SSLBIS_ANY_PORT;474 break;475 default:476 dsp_id = y;477 break;478 }479 break;480 default:481 dsp_id = x;482 break;483 }484 485 le_base = (__force __le64)tbl->sslbis_header.entry_base_unit;486 le_val = (__force __le16)tbl->entries[i].latency_or_bandwidth;487 val = cdat_normalize(le16_to_cpu(le_val), le64_to_cpu(le_base),488 sslbis->data_type);489 490 xa_for_each(&port->dports, index, dport) {491 if (dsp_id == ACPI_CDAT_SSLBIS_ANY_PORT ||492 dsp_id == dport->port_id) {493 cxl_access_coordinate_set(dport->coord,494 sslbis->data_type,495 val);496 }497 }498 }499 500 return 0;501}502 503void cxl_switch_parse_cdat(struct cxl_port *port)504{505 int rc;506 507 if (!port->cdat.table)508 return;509 510 rc = cdat_table_parse(ACPI_CDAT_TYPE_SSLBIS, cdat_sslbis_handler,511 port, port->cdat.table, port->cdat.length);512 rc = cdat_table_parse_output(rc);513 if (rc)514 dev_dbg(&port->dev, "Failed to parse SSLBIS: %d\n", rc);515}516EXPORT_SYMBOL_NS_GPL(cxl_switch_parse_cdat, CXL);517 518static void __cxl_coordinates_combine(struct access_coordinate *out,519 struct access_coordinate *c1,520 struct access_coordinate *c2)521{522 if (c1->write_bandwidth && c2->write_bandwidth)523 out->write_bandwidth = min(c1->write_bandwidth,524 c2->write_bandwidth);525 out->write_latency = c1->write_latency + c2->write_latency;526 527 if (c1->read_bandwidth && c2->read_bandwidth)528 out->read_bandwidth = min(c1->read_bandwidth,529 c2->read_bandwidth);530 out->read_latency = c1->read_latency + c2->read_latency;531}532 533/**534 * cxl_coordinates_combine - Combine the two input coordinates535 *536 * @out: Output coordinate of c1 and c2 combined537 * @c1: input coordinates538 * @c2: input coordinates539 */540void cxl_coordinates_combine(struct access_coordinate *out,541 struct access_coordinate *c1,542 struct access_coordinate *c2)543{544 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++)545 __cxl_coordinates_combine(&out[i], &c1[i], &c2[i]);546}547 548MODULE_IMPORT_NS(CXL);549 550static void cxl_bandwidth_add(struct access_coordinate *coord,551 struct access_coordinate *c1,552 struct access_coordinate *c2)553{554 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {555 coord[i].read_bandwidth = c1[i].read_bandwidth +556 c2[i].read_bandwidth;557 coord[i].write_bandwidth = c1[i].write_bandwidth +558 c2[i].write_bandwidth;559 }560}561 562static bool dpa_perf_contains(struct cxl_dpa_perf *perf,563 struct resource *dpa_res)564{565 struct range dpa = {566 .start = dpa_res->start,567 .end = dpa_res->end,568 };569 570 return range_contains(&perf->dpa_range, &dpa);571}572 573static struct cxl_dpa_perf *cxled_get_dpa_perf(struct cxl_endpoint_decoder *cxled,574 enum cxl_decoder_mode mode)575{576 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);577 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);578 struct cxl_dpa_perf *perf;579 580 switch (mode) {581 case CXL_DECODER_RAM:582 perf = &mds->ram_perf;583 break;584 case CXL_DECODER_PMEM:585 perf = &mds->pmem_perf;586 break;587 default:588 return ERR_PTR(-EINVAL);589 }590 591 if (!dpa_perf_contains(perf, cxled->dpa_res))592 return ERR_PTR(-EINVAL);593 594 return perf;595}596 597/*598 * Transient context for containing the current calculation of bandwidth when599 * doing walking the port hierarchy to deal with shared upstream link.600 */601struct cxl_perf_ctx {602 struct access_coordinate coord[ACCESS_COORDINATE_MAX];603 struct cxl_port *port;604};605 606/**607 * cxl_endpoint_gather_bandwidth - collect all the endpoint bandwidth in an xarray608 * @cxlr: CXL region for the bandwidth calculation609 * @cxled: endpoint decoder to start on610 * @usp_xa: (output) the xarray that collects all the bandwidth coordinates611 * indexed by the upstream device with data of 'struct cxl_perf_ctx'.612 * @gp_is_root: (output) bool of whether the grandparent is cxl root.613 *614 * Return: 0 for success or -errno615 *616 * Collects aggregated endpoint bandwidth and store the bandwidth in617 * an xarray indexed by the upstream device of the switch or the RP618 * device. Each endpoint consists the minimum of the bandwidth from DSLBIS619 * from the endpoint CDAT, the endpoint upstream link bandwidth, and the620 * bandwidth from the SSLBIS of the switch CDAT for the switch upstream port to621 * the downstream port that's associated with the endpoint. If the622 * device is directly connected to a RP, then no SSLBIS is involved.623 */624static int cxl_endpoint_gather_bandwidth(struct cxl_region *cxlr,625 struct cxl_endpoint_decoder *cxled,626 struct xarray *usp_xa,627 bool *gp_is_root)628{629 struct cxl_port *endpoint = to_cxl_port(cxled->cxld.dev.parent);630 struct cxl_port *parent_port = to_cxl_port(endpoint->dev.parent);631 struct cxl_port *gp_port = to_cxl_port(parent_port->dev.parent);632 struct access_coordinate pci_coord[ACCESS_COORDINATE_MAX];633 struct access_coordinate sw_coord[ACCESS_COORDINATE_MAX];634 struct access_coordinate ep_coord[ACCESS_COORDINATE_MAX];635 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);636 struct cxl_dev_state *cxlds = cxlmd->cxlds;637 struct pci_dev *pdev = to_pci_dev(cxlds->dev);638 struct cxl_perf_ctx *perf_ctx;639 struct cxl_dpa_perf *perf;640 unsigned long index;641 void *ptr;642 int rc;643 644 if (!dev_is_pci(cxlds->dev))645 return -ENODEV;646 647 if (cxlds->rcd)648 return -ENODEV;649 650 perf = cxled_get_dpa_perf(cxled, cxlr->mode);651 if (IS_ERR(perf))652 return PTR_ERR(perf);653 654 gp_port = to_cxl_port(parent_port->dev.parent);655 *gp_is_root = is_cxl_root(gp_port);656 657 /*658 * If the grandparent is cxl root, then index is the root port,659 * otherwise it's the parent switch upstream device.660 */661 if (*gp_is_root)662 index = (unsigned long)endpoint->parent_dport->dport_dev;663 else664 index = (unsigned long)parent_port->uport_dev;665 666 perf_ctx = xa_load(usp_xa, index);667 if (!perf_ctx) {668 struct cxl_perf_ctx *c __free(kfree) =669 kzalloc(sizeof(*perf_ctx), GFP_KERNEL);670 671 if (!c)672 return -ENOMEM;673 ptr = xa_store(usp_xa, index, c, GFP_KERNEL);674 if (xa_is_err(ptr))675 return xa_err(ptr);676 perf_ctx = no_free_ptr(c);677 perf_ctx->port = parent_port;678 }679 680 /* Direct upstream link from EP bandwidth */681 rc = cxl_pci_get_bandwidth(pdev, pci_coord);682 if (rc < 0)683 return rc;684 685 /*686 * Min of upstream link bandwidth and Endpoint CDAT bandwidth from687 * DSLBIS.688 */689 cxl_coordinates_combine(ep_coord, pci_coord, perf->cdat_coord);690 691 /*692 * If grandparent port is root, then there's no switch involved and693 * the endpoint is connected to a root port.694 */695 if (!*gp_is_root) {696 /*697 * Retrieve the switch SSLBIS for switch downstream port698 * associated with the endpoint bandwidth.699 */700 rc = cxl_port_get_switch_dport_bandwidth(endpoint, sw_coord);701 if (rc)702 return rc;703 704 /*705 * Min of the earlier coordinates with the switch SSLBIS706 * bandwidth707 */708 cxl_coordinates_combine(ep_coord, ep_coord, sw_coord);709 }710 711 /*712 * Aggregate the computed bandwidth with the current aggregated bandwidth713 * of the endpoints with the same switch upstream device or RP.714 */715 cxl_bandwidth_add(perf_ctx->coord, perf_ctx->coord, ep_coord);716 717 return 0;718}719 720static void free_perf_xa(struct xarray *xa)721{722 struct cxl_perf_ctx *ctx;723 unsigned long index;724 725 if (!xa)726 return;727 728 xa_for_each(xa, index, ctx)729 kfree(ctx);730 xa_destroy(xa);731 kfree(xa);732}733DEFINE_FREE(free_perf_xa, struct xarray *, if (_T) free_perf_xa(_T))734 735/**736 * cxl_switch_gather_bandwidth - collect all the bandwidth at switch level in an xarray737 * @cxlr: The region being operated on738 * @input_xa: xarray indexed by upstream device of a switch with data of 'struct739 * cxl_perf_ctx'740 * @gp_is_root: (output) bool of whether the grandparent is cxl root.741 *742 * Return: a xarray of resulting cxl_perf_ctx per parent switch or root port743 * or ERR_PTR(-errno)744 *745 * Iterate through the xarray. Take the minimum of the downstream calculated746 * bandwidth, the upstream link bandwidth, and the SSLBIS of the upstream747 * switch if exists. Sum the resulting bandwidth under the switch upstream748 * device or a RP device. The function can be iterated over multiple switches749 * if the switches are present.750 */751static struct xarray *cxl_switch_gather_bandwidth(struct cxl_region *cxlr,752 struct xarray *input_xa,753 bool *gp_is_root)754{755 struct xarray *res_xa __free(free_perf_xa) =756 kzalloc(sizeof(*res_xa), GFP_KERNEL);757 struct access_coordinate coords[ACCESS_COORDINATE_MAX];758 struct cxl_perf_ctx *ctx, *us_ctx;759 unsigned long index, us_index;760 int dev_count = 0;761 int gp_count = 0;762 void *ptr;763 int rc;764 765 if (!res_xa)766 return ERR_PTR(-ENOMEM);767 xa_init(res_xa);768 769 xa_for_each(input_xa, index, ctx) {770 struct device *dev = (struct device *)index;771 struct cxl_port *port = ctx->port;772 struct cxl_port *parent_port = to_cxl_port(port->dev.parent);773 struct cxl_port *gp_port = to_cxl_port(parent_port->dev.parent);774 struct cxl_dport *dport = port->parent_dport;775 bool is_root = false;776 777 dev_count++;778 if (is_cxl_root(gp_port)) {779 is_root = true;780 gp_count++;781 }782 783 /*784 * If the grandparent is cxl root, then index is the root port,785 * otherwise it's the parent switch upstream device.786 */787 if (is_root)788 us_index = (unsigned long)port->parent_dport->dport_dev;789 else790 us_index = (unsigned long)parent_port->uport_dev;791 792 us_ctx = xa_load(res_xa, us_index);793 if (!us_ctx) {794 struct cxl_perf_ctx *n __free(kfree) =795 kzalloc(sizeof(*n), GFP_KERNEL);796 797 if (!n)798 return ERR_PTR(-ENOMEM);799 800 ptr = xa_store(res_xa, us_index, n, GFP_KERNEL);801 if (xa_is_err(ptr))802 return ERR_PTR(xa_err(ptr));803 us_ctx = no_free_ptr(n);804 us_ctx->port = parent_port;805 }806 807 /*808 * If the device isn't an upstream PCIe port, there's something809 * wrong with the topology.810 */811 if (!dev_is_pci(dev))812 return ERR_PTR(-EINVAL);813 814 /* Retrieve the upstream link bandwidth */815 rc = cxl_pci_get_bandwidth(to_pci_dev(dev), coords);816 if (rc)817 return ERR_PTR(-ENXIO);818 819 /*820 * Take the min of downstream bandwidth and the upstream link821 * bandwidth.822 */823 cxl_coordinates_combine(coords, coords, ctx->coord);824 825 /*826 * Take the min of the calculated bandwdith and the upstream827 * switch SSLBIS bandwidth if there's a parent switch828 */829 if (!is_root)830 cxl_coordinates_combine(coords, coords, dport->coord);831 832 /*833 * Aggregate the calculated bandwidth common to an upstream834 * switch.835 */836 cxl_bandwidth_add(us_ctx->coord, us_ctx->coord, coords);837 }838 839 /* Asymmetric topology detected. */840 if (gp_count) {841 if (gp_count != dev_count) {842 dev_dbg(&cxlr->dev,843 "Asymmetric hierarchy detected, bandwidth not updated\n");844 return ERR_PTR(-EOPNOTSUPP);845 }846 *gp_is_root = true;847 }848 849 return no_free_ptr(res_xa);850}851 852/**853 * cxl_rp_gather_bandwidth - handle the root port level bandwidth collection854 * @xa: the xarray that holds the cxl_perf_ctx that has the bandwidth calculated855 * below each root port device.856 *857 * Return: xarray that holds cxl_perf_ctx per host bridge or ERR_PTR(-errno)858 */859static struct xarray *cxl_rp_gather_bandwidth(struct xarray *xa)860{861 struct xarray *hb_xa __free(free_perf_xa) =862 kzalloc(sizeof(*hb_xa), GFP_KERNEL);863 struct cxl_perf_ctx *ctx;864 unsigned long index;865 866 if (!hb_xa)867 return ERR_PTR(-ENOMEM);868 xa_init(hb_xa);869 870 xa_for_each(xa, index, ctx) {871 struct cxl_port *port = ctx->port;872 unsigned long hb_index = (unsigned long)port->uport_dev;873 struct cxl_perf_ctx *hb_ctx;874 void *ptr;875 876 hb_ctx = xa_load(hb_xa, hb_index);877 if (!hb_ctx) {878 struct cxl_perf_ctx *n __free(kfree) =879 kzalloc(sizeof(*n), GFP_KERNEL);880 881 if (!n)882 return ERR_PTR(-ENOMEM);883 ptr = xa_store(hb_xa, hb_index, n, GFP_KERNEL);884 if (xa_is_err(ptr))885 return ERR_PTR(xa_err(ptr));886 hb_ctx = no_free_ptr(n);887 hb_ctx->port = port;888 }889 890 cxl_bandwidth_add(hb_ctx->coord, hb_ctx->coord, ctx->coord);891 }892 893 return no_free_ptr(hb_xa);894}895 896/**897 * cxl_hb_gather_bandwidth - handle the host bridge level bandwidth collection898 * @xa: the xarray that holds the cxl_perf_ctx that has the bandwidth calculated899 * below each host bridge.900 *901 * Return: xarray that holds cxl_perf_ctx per ACPI0017 device or ERR_PTR(-errno)902 */903static struct xarray *cxl_hb_gather_bandwidth(struct xarray *xa)904{905 struct xarray *mw_xa __free(free_perf_xa) =906 kzalloc(sizeof(*mw_xa), GFP_KERNEL);907 struct cxl_perf_ctx *ctx;908 unsigned long index;909 910 if (!mw_xa)911 return ERR_PTR(-ENOMEM);912 xa_init(mw_xa);913 914 xa_for_each(xa, index, ctx) {915 struct cxl_port *port = ctx->port;916 struct cxl_port *parent_port;917 struct cxl_perf_ctx *mw_ctx;918 struct cxl_dport *dport;919 unsigned long mw_index;920 void *ptr;921 922 parent_port = to_cxl_port(port->dev.parent);923 mw_index = (unsigned long)parent_port->uport_dev;924 925 mw_ctx = xa_load(mw_xa, mw_index);926 if (!mw_ctx) {927 struct cxl_perf_ctx *n __free(kfree) =928 kzalloc(sizeof(*n), GFP_KERNEL);929 930 if (!n)931 return ERR_PTR(-ENOMEM);932 ptr = xa_store(mw_xa, mw_index, n, GFP_KERNEL);933 if (xa_is_err(ptr))934 return ERR_PTR(xa_err(ptr));935 mw_ctx = no_free_ptr(n);936 }937 938 dport = port->parent_dport;939 cxl_coordinates_combine(ctx->coord, ctx->coord, dport->coord);940 cxl_bandwidth_add(mw_ctx->coord, mw_ctx->coord, ctx->coord);941 }942 943 return no_free_ptr(mw_xa);944}945 946/**947 * cxl_region_update_bandwidth - Update the bandwidth access coordinates of a region948 * @cxlr: The region being operated on949 * @input_xa: xarray holds cxl_perf_ctx wht calculated bandwidth per ACPI0017 instance950 */951static void cxl_region_update_bandwidth(struct cxl_region *cxlr,952 struct xarray *input_xa)953{954 struct access_coordinate coord[ACCESS_COORDINATE_MAX];955 struct cxl_perf_ctx *ctx;956 unsigned long index;957 958 memset(coord, 0, sizeof(coord));959 xa_for_each(input_xa, index, ctx)960 cxl_bandwidth_add(coord, coord, ctx->coord);961 962 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {963 cxlr->coord[i].read_bandwidth = coord[i].read_bandwidth;964 cxlr->coord[i].write_bandwidth = coord[i].write_bandwidth;965 }966}967 968/**969 * cxl_region_shared_upstream_bandwidth_update - Recalculate the bandwidth for970 * the region971 * @cxlr: the cxl region to recalculate972 *973 * The function walks the topology from bottom up and calculates the bandwidth. It974 * starts at the endpoints, processes at the switches if any, processes at the rootport975 * level, at the host bridge level, and finally aggregates at the region.976 */977void cxl_region_shared_upstream_bandwidth_update(struct cxl_region *cxlr)978{979 struct xarray *working_xa;980 int root_count = 0;981 bool is_root;982 int rc;983 984 lockdep_assert_held(&cxl_dpa_rwsem);985 986 struct xarray *usp_xa __free(free_perf_xa) =987 kzalloc(sizeof(*usp_xa), GFP_KERNEL);988 989 if (!usp_xa)990 return;991 992 xa_init(usp_xa);993 994 /* Collect bandwidth data from all the endpoints. */995 for (int i = 0; i < cxlr->params.nr_targets; i++) {996 struct cxl_endpoint_decoder *cxled = cxlr->params.targets[i];997 998 is_root = false;999 rc = cxl_endpoint_gather_bandwidth(cxlr, cxled, usp_xa, &is_root);1000 if (rc)1001 return;1002 root_count += is_root;1003 }1004 1005 /* Detect asymmetric hierarchy with some direct attached endpoints. */1006 if (root_count && root_count != cxlr->params.nr_targets) {1007 dev_dbg(&cxlr->dev,1008 "Asymmetric hierarchy detected, bandwidth not updated\n");1009 return;1010 }1011 1012 /*1013 * Walk up one or more switches to deal with the bandwidth of the1014 * switches if they exist. Endpoints directly attached to RPs skip1015 * over this part.1016 */1017 if (!root_count) {1018 do {1019 working_xa = cxl_switch_gather_bandwidth(cxlr, usp_xa,1020 &is_root);1021 if (IS_ERR(working_xa))1022 return;1023 free_perf_xa(usp_xa);1024 usp_xa = working_xa;1025 } while (!is_root);1026 }1027 1028 /* Handle the bandwidth at the root port of the hierarchy */1029 working_xa = cxl_rp_gather_bandwidth(usp_xa);1030 if (IS_ERR(working_xa))1031 return;1032 free_perf_xa(usp_xa);1033 usp_xa = working_xa;1034 1035 /* Handle the bandwidth at the host bridge of the hierarchy */1036 working_xa = cxl_hb_gather_bandwidth(usp_xa);1037 if (IS_ERR(working_xa))1038 return;1039 free_perf_xa(usp_xa);1040 usp_xa = working_xa;1041 1042 /*1043 * Aggregate all the bandwidth collected per CFMWS (ACPI0017) and1044 * update the region bandwidth with the final calculated values.1045 */1046 cxl_region_update_bandwidth(cxlr, usp_xa);1047}1048 1049void cxl_region_perf_data_calculate(struct cxl_region *cxlr,1050 struct cxl_endpoint_decoder *cxled)1051{1052 struct cxl_dpa_perf *perf;1053 1054 lockdep_assert_held(&cxl_dpa_rwsem);1055 1056 perf = cxled_get_dpa_perf(cxled, cxlr->mode);1057 if (IS_ERR(perf))1058 return;1059 1060 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {1061 /* Get total bandwidth and the worst latency for the cxl region */1062 cxlr->coord[i].read_latency = max_t(unsigned int,1063 cxlr->coord[i].read_latency,1064 perf->coord[i].read_latency);1065 cxlr->coord[i].write_latency = max_t(unsigned int,1066 cxlr->coord[i].write_latency,1067 perf->coord[i].write_latency);1068 cxlr->coord[i].read_bandwidth += perf->coord[i].read_bandwidth;1069 cxlr->coord[i].write_bandwidth += perf->coord[i].write_bandwidth;1070 }1071}1072 1073int cxl_update_hmat_access_coordinates(int nid, struct cxl_region *cxlr,1074 enum access_coordinate_class access)1075{1076 return hmat_update_target_coordinates(nid, &cxlr->coord[access], access);1077}1078 1079bool cxl_need_node_perf_attrs_update(int nid)1080{1081 return !acpi_node_backed_by_real_pxm(nid);1082}1083