1090 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright (c) 2019, Intel Corporation.4 *5 * Heterogeneous Memory Attributes Table (HMAT) representation6 *7 * This program parses and reports the platform's HMAT tables, and registers8 * the applicable attributes with the node's interfaces.9 */10 11#define pr_fmt(fmt) "acpi/hmat: " fmt12 13#include <linux/acpi.h>14#include <linux/bitops.h>15#include <linux/device.h>16#include <linux/init.h>17#include <linux/list.h>18#include <linux/mm.h>19#include <linux/platform_device.h>20#include <linux/list_sort.h>21#include <linux/memregion.h>22#include <linux/memory.h>23#include <linux/mutex.h>24#include <linux/node.h>25#include <linux/sysfs.h>26#include <linux/dax.h>27#include <linux/memory-tiers.h>28 29static u8 hmat_revision;30static int hmat_disable __initdata;31 32void __init disable_hmat(void)33{34 hmat_disable = 1;35}36 37static LIST_HEAD(targets);38static LIST_HEAD(initiators);39static LIST_HEAD(localities);40 41static DEFINE_MUTEX(target_lock);42 43/*44 * The defined enum order is used to prioritize attributes to break ties when45 * selecting the best performing node.46 */47enum locality_types {48 WRITE_LATENCY,49 READ_LATENCY,50 WRITE_BANDWIDTH,51 READ_BANDWIDTH,52};53 54static struct memory_locality *localities_types[4];55 56struct target_cache {57 struct list_head node;58 struct node_cache_attrs cache_attrs;59};60 61enum {62 NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL = ACCESS_COORDINATE_MAX,63 NODE_ACCESS_CLASS_GENPORT_SINK_CPU,64 NODE_ACCESS_CLASS_MAX,65};66 67struct memory_target {68 struct list_head node;69 unsigned int memory_pxm;70 unsigned int processor_pxm;71 struct resource memregions;72 struct access_coordinate coord[NODE_ACCESS_CLASS_MAX];73 struct list_head caches;74 struct node_cache_attrs cache_attrs;75 u8 gen_port_device_handle[ACPI_SRAT_DEVICE_HANDLE_SIZE];76 bool registered;77 bool ext_updated; /* externally updated */78};79 80struct memory_initiator {81 struct list_head node;82 unsigned int processor_pxm;83 bool has_cpu;84};85 86struct memory_locality {87 struct list_head node;88 struct acpi_hmat_locality *hmat_loc;89};90 91static struct memory_initiator *find_mem_initiator(unsigned int cpu_pxm)92{93 struct memory_initiator *initiator;94 95 list_for_each_entry(initiator, &initiators, node)96 if (initiator->processor_pxm == cpu_pxm)97 return initiator;98 return NULL;99}100 101static struct memory_target *find_mem_target(unsigned int mem_pxm)102{103 struct memory_target *target;104 105 list_for_each_entry(target, &targets, node)106 if (target->memory_pxm == mem_pxm)107 return target;108 return NULL;109}110 111static struct memory_target *acpi_find_genport_target(u32 uid)112{113 struct memory_target *target;114 u32 target_uid;115 u8 *uid_ptr;116 117 list_for_each_entry(target, &targets, node) {118 uid_ptr = target->gen_port_device_handle + 8;119 target_uid = *(u32 *)uid_ptr;120 if (uid == target_uid)121 return target;122 }123 124 return NULL;125}126 127/**128 * acpi_get_genport_coordinates - Retrieve the access coordinates for a generic port129 * @uid: ACPI unique id130 * @coord: The access coordinates written back out for the generic port.131 * Expect 2 levels array.132 *133 * Return: 0 on success. Errno on failure.134 *135 * Only supports device handles that are ACPI. Assume ACPI0016 HID for CXL.136 */137int acpi_get_genport_coordinates(u32 uid,138 struct access_coordinate *coord)139{140 struct memory_target *target;141 142 guard(mutex)(&target_lock);143 target = acpi_find_genport_target(uid);144 if (!target)145 return -ENOENT;146 147 coord[ACCESS_COORDINATE_LOCAL] =148 target->coord[NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL];149 coord[ACCESS_COORDINATE_CPU] =150 target->coord[NODE_ACCESS_CLASS_GENPORT_SINK_CPU];151 152 return 0;153}154EXPORT_SYMBOL_NS_GPL(acpi_get_genport_coordinates, CXL);155 156static __init void alloc_memory_initiator(unsigned int cpu_pxm)157{158 struct memory_initiator *initiator;159 160 if (pxm_to_node(cpu_pxm) == NUMA_NO_NODE)161 return;162 163 initiator = find_mem_initiator(cpu_pxm);164 if (initiator)165 return;166 167 initiator = kzalloc(sizeof(*initiator), GFP_KERNEL);168 if (!initiator)169 return;170 171 initiator->processor_pxm = cpu_pxm;172 initiator->has_cpu = node_state(pxm_to_node(cpu_pxm), N_CPU);173 list_add_tail(&initiator->node, &initiators);174}175 176static __init struct memory_target *alloc_target(unsigned int mem_pxm)177{178 struct memory_target *target;179 180 target = find_mem_target(mem_pxm);181 if (!target) {182 target = kzalloc(sizeof(*target), GFP_KERNEL);183 if (!target)184 return NULL;185 target->memory_pxm = mem_pxm;186 target->processor_pxm = PXM_INVAL;187 target->memregions = (struct resource) {188 .name = "ACPI mem",189 .start = 0,190 .end = -1,191 .flags = IORESOURCE_MEM,192 };193 list_add_tail(&target->node, &targets);194 INIT_LIST_HEAD(&target->caches);195 }196 197 return target;198}199 200static __init void alloc_memory_target(unsigned int mem_pxm,201 resource_size_t start,202 resource_size_t len)203{204 struct memory_target *target;205 206 target = alloc_target(mem_pxm);207 if (!target)208 return;209 210 /*211 * There are potentially multiple ranges per PXM, so record each212 * in the per-target memregions resource tree.213 */214 if (!__request_region(&target->memregions, start, len, "memory target",215 IORESOURCE_MEM))216 pr_warn("failed to reserve %#llx - %#llx in pxm: %d\n",217 start, start + len, mem_pxm);218}219 220static __init void alloc_genport_target(unsigned int mem_pxm, u8 *handle)221{222 struct memory_target *target;223 224 target = alloc_target(mem_pxm);225 if (!target)226 return;227 228 memcpy(target->gen_port_device_handle, handle,229 ACPI_SRAT_DEVICE_HANDLE_SIZE);230}231 232static __init const char *hmat_data_type(u8 type)233{234 switch (type) {235 case ACPI_HMAT_ACCESS_LATENCY:236 return "Access Latency";237 case ACPI_HMAT_READ_LATENCY:238 return "Read Latency";239 case ACPI_HMAT_WRITE_LATENCY:240 return "Write Latency";241 case ACPI_HMAT_ACCESS_BANDWIDTH:242 return "Access Bandwidth";243 case ACPI_HMAT_READ_BANDWIDTH:244 return "Read Bandwidth";245 case ACPI_HMAT_WRITE_BANDWIDTH:246 return "Write Bandwidth";247 default:248 return "Reserved";249 }250}251 252static __init const char *hmat_data_type_suffix(u8 type)253{254 switch (type) {255 case ACPI_HMAT_ACCESS_LATENCY:256 case ACPI_HMAT_READ_LATENCY:257 case ACPI_HMAT_WRITE_LATENCY:258 return " nsec";259 case ACPI_HMAT_ACCESS_BANDWIDTH:260 case ACPI_HMAT_READ_BANDWIDTH:261 case ACPI_HMAT_WRITE_BANDWIDTH:262 return " MB/s";263 default:264 return "";265 }266}267 268static u32 hmat_normalize(u16 entry, u64 base, u8 type)269{270 u32 value;271 272 /*273 * Check for invalid and overflow values274 */275 if (entry == 0xffff || !entry)276 return 0;277 else if (base > (UINT_MAX / (entry)))278 return 0;279 280 /*281 * Divide by the base unit for version 1, convert latency from282 * picosenonds to nanoseconds if revision 2.283 */284 value = entry * base;285 if (hmat_revision == 1) {286 if (value < 10)287 return 0;288 value = DIV_ROUND_UP(value, 10);289 } else if (hmat_revision == 2) {290 switch (type) {291 case ACPI_HMAT_ACCESS_LATENCY:292 case ACPI_HMAT_READ_LATENCY:293 case ACPI_HMAT_WRITE_LATENCY:294 value = DIV_ROUND_UP(value, 1000);295 break;296 default:297 break;298 }299 }300 return value;301}302 303static void hmat_update_target_access(struct memory_target *target,304 u8 type, u32 value, int access)305{306 switch (type) {307 case ACPI_HMAT_ACCESS_LATENCY:308 target->coord[access].read_latency = value;309 target->coord[access].write_latency = value;310 break;311 case ACPI_HMAT_READ_LATENCY:312 target->coord[access].read_latency = value;313 break;314 case ACPI_HMAT_WRITE_LATENCY:315 target->coord[access].write_latency = value;316 break;317 case ACPI_HMAT_ACCESS_BANDWIDTH:318 target->coord[access].read_bandwidth = value;319 target->coord[access].write_bandwidth = value;320 break;321 case ACPI_HMAT_READ_BANDWIDTH:322 target->coord[access].read_bandwidth = value;323 break;324 case ACPI_HMAT_WRITE_BANDWIDTH:325 target->coord[access].write_bandwidth = value;326 break;327 default:328 break;329 }330}331 332int hmat_update_target_coordinates(int nid, struct access_coordinate *coord,333 enum access_coordinate_class access)334{335 struct memory_target *target;336 int pxm;337 338 if (nid == NUMA_NO_NODE)339 return -EINVAL;340 341 pxm = node_to_pxm(nid);342 guard(mutex)(&target_lock);343 target = find_mem_target(pxm);344 if (!target)345 return -ENODEV;346 347 hmat_update_target_access(target, ACPI_HMAT_READ_LATENCY,348 coord->read_latency, access);349 hmat_update_target_access(target, ACPI_HMAT_WRITE_LATENCY,350 coord->write_latency, access);351 hmat_update_target_access(target, ACPI_HMAT_READ_BANDWIDTH,352 coord->read_bandwidth, access);353 hmat_update_target_access(target, ACPI_HMAT_WRITE_BANDWIDTH,354 coord->write_bandwidth, access);355 target->ext_updated = true;356 357 return 0;358}359EXPORT_SYMBOL_GPL(hmat_update_target_coordinates);360 361static __init void hmat_add_locality(struct acpi_hmat_locality *hmat_loc)362{363 struct memory_locality *loc;364 365 loc = kzalloc(sizeof(*loc), GFP_KERNEL);366 if (!loc) {367 pr_notice_once("Failed to allocate HMAT locality\n");368 return;369 }370 371 loc->hmat_loc = hmat_loc;372 list_add_tail(&loc->node, &localities);373 374 switch (hmat_loc->data_type) {375 case ACPI_HMAT_ACCESS_LATENCY:376 localities_types[READ_LATENCY] = loc;377 localities_types[WRITE_LATENCY] = loc;378 break;379 case ACPI_HMAT_READ_LATENCY:380 localities_types[READ_LATENCY] = loc;381 break;382 case ACPI_HMAT_WRITE_LATENCY:383 localities_types[WRITE_LATENCY] = loc;384 break;385 case ACPI_HMAT_ACCESS_BANDWIDTH:386 localities_types[READ_BANDWIDTH] = loc;387 localities_types[WRITE_BANDWIDTH] = loc;388 break;389 case ACPI_HMAT_READ_BANDWIDTH:390 localities_types[READ_BANDWIDTH] = loc;391 break;392 case ACPI_HMAT_WRITE_BANDWIDTH:393 localities_types[WRITE_BANDWIDTH] = loc;394 break;395 default:396 break;397 }398}399 400static __init void hmat_update_target(unsigned int tgt_pxm, unsigned int init_pxm,401 u8 mem_hier, u8 type, u32 value)402{403 struct memory_target *target = find_mem_target(tgt_pxm);404 405 if (mem_hier != ACPI_HMAT_MEMORY)406 return;407 408 if (target && target->processor_pxm == init_pxm) {409 hmat_update_target_access(target, type, value,410 ACCESS_COORDINATE_LOCAL);411 /* If the node has a CPU, update access ACCESS_COORDINATE_CPU */412 if (node_state(pxm_to_node(init_pxm), N_CPU))413 hmat_update_target_access(target, type, value,414 ACCESS_COORDINATE_CPU);415 }416}417 418static __init int hmat_parse_locality(union acpi_subtable_headers *header,419 const unsigned long end)420{421 struct acpi_hmat_locality *hmat_loc = (void *)header;422 unsigned int init, targ, total_size, ipds, tpds;423 u32 *inits, *targs, value;424 u16 *entries;425 u8 type, mem_hier;426 427 if (hmat_loc->header.length < sizeof(*hmat_loc)) {428 pr_notice("Unexpected locality header length: %u\n",429 hmat_loc->header.length);430 return -EINVAL;431 }432 433 type = hmat_loc->data_type;434 mem_hier = hmat_loc->flags & ACPI_HMAT_MEMORY_HIERARCHY;435 ipds = hmat_loc->number_of_initiator_Pds;436 tpds = hmat_loc->number_of_target_Pds;437 total_size = sizeof(*hmat_loc) + sizeof(*entries) * ipds * tpds +438 sizeof(*inits) * ipds + sizeof(*targs) * tpds;439 if (hmat_loc->header.length < total_size) {440 pr_notice("Unexpected locality header length:%u, minimum required:%u\n",441 hmat_loc->header.length, total_size);442 return -EINVAL;443 }444 445 pr_info("Locality: Flags:%02x Type:%s Initiator Domains:%u Target Domains:%u Base:%lld\n",446 hmat_loc->flags, hmat_data_type(type), ipds, tpds,447 hmat_loc->entry_base_unit);448 449 inits = (u32 *)(hmat_loc + 1);450 targs = inits + ipds;451 entries = (u16 *)(targs + tpds);452 for (init = 0; init < ipds; init++) {453 alloc_memory_initiator(inits[init]);454 for (targ = 0; targ < tpds; targ++) {455 value = hmat_normalize(entries[init * tpds + targ],456 hmat_loc->entry_base_unit,457 type);458 pr_info(" Initiator-Target[%u-%u]:%u%s\n",459 inits[init], targs[targ], value,460 hmat_data_type_suffix(type));461 462 hmat_update_target(targs[targ], inits[init],463 mem_hier, type, value);464 }465 }466 467 if (mem_hier == ACPI_HMAT_MEMORY)468 hmat_add_locality(hmat_loc);469 470 return 0;471}472 473static __init int hmat_parse_cache(union acpi_subtable_headers *header,474 const unsigned long end)475{476 struct acpi_hmat_cache *cache = (void *)header;477 struct memory_target *target;478 struct target_cache *tcache;479 u32 attrs;480 481 if (cache->header.length < sizeof(*cache)) {482 pr_notice("Unexpected cache header length: %u\n",483 cache->header.length);484 return -EINVAL;485 }486 487 attrs = cache->cache_attributes;488 pr_info("Cache: Domain:%u Size:%llu Attrs:%08x SMBIOS Handles:%d\n",489 cache->memory_PD, cache->cache_size, attrs,490 cache->number_of_SMBIOShandles);491 492 target = find_mem_target(cache->memory_PD);493 if (!target)494 return 0;495 496 tcache = kzalloc(sizeof(*tcache), GFP_KERNEL);497 if (!tcache) {498 pr_notice_once("Failed to allocate HMAT cache info\n");499 return 0;500 }501 502 tcache->cache_attrs.size = cache->cache_size;503 tcache->cache_attrs.level = (attrs & ACPI_HMAT_CACHE_LEVEL) >> 4;504 tcache->cache_attrs.line_size = (attrs & ACPI_HMAT_CACHE_LINE_SIZE) >> 16;505 506 switch ((attrs & ACPI_HMAT_CACHE_ASSOCIATIVITY) >> 8) {507 case ACPI_HMAT_CA_DIRECT_MAPPED:508 tcache->cache_attrs.indexing = NODE_CACHE_DIRECT_MAP;509 break;510 case ACPI_HMAT_CA_COMPLEX_CACHE_INDEXING:511 tcache->cache_attrs.indexing = NODE_CACHE_INDEXED;512 break;513 case ACPI_HMAT_CA_NONE:514 default:515 tcache->cache_attrs.indexing = NODE_CACHE_OTHER;516 break;517 }518 519 switch ((attrs & ACPI_HMAT_WRITE_POLICY) >> 12) {520 case ACPI_HMAT_CP_WB:521 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_BACK;522 break;523 case ACPI_HMAT_CP_WT:524 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_THROUGH;525 break;526 case ACPI_HMAT_CP_NONE:527 default:528 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_OTHER;529 break;530 }531 list_add_tail(&tcache->node, &target->caches);532 533 return 0;534}535 536static int __init hmat_parse_proximity_domain(union acpi_subtable_headers *header,537 const unsigned long end)538{539 struct acpi_hmat_proximity_domain *p = (void *)header;540 struct memory_target *target = NULL;541 542 if (p->header.length != sizeof(*p)) {543 pr_notice("Unexpected address range header length: %u\n",544 p->header.length);545 return -EINVAL;546 }547 548 if (hmat_revision == 1)549 pr_info("Memory (%#llx length %#llx) Flags:%04x Processor Domain:%u Memory Domain:%u\n",550 p->reserved3, p->reserved4, p->flags, p->processor_PD,551 p->memory_PD);552 else553 pr_info("Memory Flags:%04x Processor Domain:%u Memory Domain:%u\n",554 p->flags, p->processor_PD, p->memory_PD);555 556 if ((hmat_revision == 1 && p->flags & ACPI_HMAT_MEMORY_PD_VALID) ||557 hmat_revision > 1) {558 target = find_mem_target(p->memory_PD);559 if (!target) {560 pr_debug("Memory Domain missing from SRAT\n");561 return -EINVAL;562 }563 }564 if (target && p->flags & ACPI_HMAT_PROCESSOR_PD_VALID) {565 int p_node = pxm_to_node(p->processor_PD);566 567 if (p_node == NUMA_NO_NODE) {568 pr_debug("Invalid Processor Domain\n");569 return -EINVAL;570 }571 target->processor_pxm = p->processor_PD;572 }573 574 return 0;575}576 577static int __init hmat_parse_subtable(union acpi_subtable_headers *header,578 const unsigned long end)579{580 struct acpi_hmat_structure *hdr = (void *)header;581 582 if (!hdr)583 return -EINVAL;584 585 switch (hdr->type) {586 case ACPI_HMAT_TYPE_PROXIMITY:587 return hmat_parse_proximity_domain(header, end);588 case ACPI_HMAT_TYPE_LOCALITY:589 return hmat_parse_locality(header, end);590 case ACPI_HMAT_TYPE_CACHE:591 return hmat_parse_cache(header, end);592 default:593 return -EINVAL;594 }595}596 597static __init int srat_parse_mem_affinity(union acpi_subtable_headers *header,598 const unsigned long end)599{600 struct acpi_srat_mem_affinity *ma = (void *)header;601 602 if (!ma)603 return -EINVAL;604 if (!(ma->flags & ACPI_SRAT_MEM_ENABLED))605 return 0;606 alloc_memory_target(ma->proximity_domain, ma->base_address, ma->length);607 return 0;608}609 610static __init int srat_parse_genport_affinity(union acpi_subtable_headers *header,611 const unsigned long end)612{613 struct acpi_srat_generic_affinity *ga = (void *)header;614 615 if (!ga)616 return -EINVAL;617 618 if (!(ga->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED))619 return 0;620 621 /* Skip PCI device_handle for now */622 if (ga->device_handle_type != 0)623 return 0;624 625 alloc_genport_target(ga->proximity_domain,626 (u8 *)ga->device_handle);627 628 return 0;629}630 631static u32 hmat_initiator_perf(struct memory_target *target,632 struct memory_initiator *initiator,633 struct acpi_hmat_locality *hmat_loc)634{635 unsigned int ipds, tpds, i, idx = 0, tdx = 0;636 u32 *inits, *targs;637 u16 *entries;638 639 ipds = hmat_loc->number_of_initiator_Pds;640 tpds = hmat_loc->number_of_target_Pds;641 inits = (u32 *)(hmat_loc + 1);642 targs = inits + ipds;643 entries = (u16 *)(targs + tpds);644 645 for (i = 0; i < ipds; i++) {646 if (inits[i] == initiator->processor_pxm) {647 idx = i;648 break;649 }650 }651 652 if (i == ipds)653 return 0;654 655 for (i = 0; i < tpds; i++) {656 if (targs[i] == target->memory_pxm) {657 tdx = i;658 break;659 }660 }661 if (i == tpds)662 return 0;663 664 return hmat_normalize(entries[idx * tpds + tdx],665 hmat_loc->entry_base_unit,666 hmat_loc->data_type);667}668 669static bool hmat_update_best(u8 type, u32 value, u32 *best)670{671 bool updated = false;672 673 if (!value)674 return false;675 676 switch (type) {677 case ACPI_HMAT_ACCESS_LATENCY:678 case ACPI_HMAT_READ_LATENCY:679 case ACPI_HMAT_WRITE_LATENCY:680 if (!*best || *best > value) {681 *best = value;682 updated = true;683 }684 break;685 case ACPI_HMAT_ACCESS_BANDWIDTH:686 case ACPI_HMAT_READ_BANDWIDTH:687 case ACPI_HMAT_WRITE_BANDWIDTH:688 if (!*best || *best < value) {689 *best = value;690 updated = true;691 }692 break;693 }694 695 return updated;696}697 698static int initiator_cmp(void *priv, const struct list_head *a,699 const struct list_head *b)700{701 struct memory_initiator *ia;702 struct memory_initiator *ib;703 704 ia = list_entry(a, struct memory_initiator, node);705 ib = list_entry(b, struct memory_initiator, node);706 707 return ia->processor_pxm - ib->processor_pxm;708}709 710static int initiators_to_nodemask(unsigned long *p_nodes)711{712 struct memory_initiator *initiator;713 714 if (list_empty(&initiators))715 return -ENXIO;716 717 list_for_each_entry(initiator, &initiators, node)718 set_bit(initiator->processor_pxm, p_nodes);719 720 return 0;721}722 723static void hmat_update_target_attrs(struct memory_target *target,724 unsigned long *p_nodes, int access)725{726 struct memory_initiator *initiator;727 unsigned int cpu_nid;728 struct memory_locality *loc = NULL;729 u32 best = 0;730 int i;731 732 /* Don't update if an external agent has changed the data. */733 if (target->ext_updated)734 return;735 736 /* Don't update for generic port if there's no device handle */737 if ((access == NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL ||738 access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) &&739 !(*(u16 *)target->gen_port_device_handle))740 return;741 742 bitmap_zero(p_nodes, MAX_NUMNODES);743 /*744 * If the Address Range Structure provides a local processor pxm, set745 * only that one. Otherwise, find the best performance attributes and746 * collect all initiators that match.747 */748 if (target->processor_pxm != PXM_INVAL) {749 cpu_nid = pxm_to_node(target->processor_pxm);750 if (access == ACCESS_COORDINATE_LOCAL ||751 node_state(cpu_nid, N_CPU)) {752 set_bit(target->processor_pxm, p_nodes);753 return;754 }755 }756 757 if (list_empty(&localities))758 return;759 760 /*761 * We need the initiator list sorted so we can use bitmap_clear for762 * previously set initiators when we find a better memory accessor.763 * We'll also use the sorting to prime the candidate nodes with known764 * initiators.765 */766 list_sort(NULL, &initiators, initiator_cmp);767 if (initiators_to_nodemask(p_nodes) < 0)768 return;769 770 for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) {771 loc = localities_types[i];772 if (!loc)773 continue;774 775 best = 0;776 list_for_each_entry(initiator, &initiators, node) {777 u32 value;778 779 if ((access == ACCESS_COORDINATE_CPU ||780 access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) &&781 !initiator->has_cpu) {782 clear_bit(initiator->processor_pxm, p_nodes);783 continue;784 }785 if (!test_bit(initiator->processor_pxm, p_nodes))786 continue;787 788 value = hmat_initiator_perf(target, initiator, loc->hmat_loc);789 if (hmat_update_best(loc->hmat_loc->data_type, value, &best))790 bitmap_clear(p_nodes, 0, initiator->processor_pxm);791 if (value != best)792 clear_bit(initiator->processor_pxm, p_nodes);793 }794 if (best)795 hmat_update_target_access(target, loc->hmat_loc->data_type, best, access);796 }797}798 799static void __hmat_register_target_initiators(struct memory_target *target,800 unsigned long *p_nodes,801 int access)802{803 unsigned int mem_nid, cpu_nid;804 int i;805 806 mem_nid = pxm_to_node(target->memory_pxm);807 hmat_update_target_attrs(target, p_nodes, access);808 for_each_set_bit(i, p_nodes, MAX_NUMNODES) {809 cpu_nid = pxm_to_node(i);810 register_memory_node_under_compute_node(mem_nid, cpu_nid, access);811 }812}813 814static void hmat_update_generic_target(struct memory_target *target)815{816 static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);817 818 hmat_update_target_attrs(target, p_nodes,819 NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL);820 hmat_update_target_attrs(target, p_nodes,821 NODE_ACCESS_CLASS_GENPORT_SINK_CPU);822}823 824static void hmat_register_target_initiators(struct memory_target *target)825{826 static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);827 828 __hmat_register_target_initiators(target, p_nodes,829 ACCESS_COORDINATE_LOCAL);830 __hmat_register_target_initiators(target, p_nodes,831 ACCESS_COORDINATE_CPU);832}833 834static void hmat_register_target_cache(struct memory_target *target)835{836 unsigned mem_nid = pxm_to_node(target->memory_pxm);837 struct target_cache *tcache;838 839 list_for_each_entry(tcache, &target->caches, node)840 node_add_cache(mem_nid, &tcache->cache_attrs);841}842 843static void hmat_register_target_perf(struct memory_target *target, int access)844{845 unsigned mem_nid = pxm_to_node(target->memory_pxm);846 node_set_perf_attrs(mem_nid, &target->coord[access], access);847}848 849static void hmat_register_target_devices(struct memory_target *target)850{851 struct resource *res;852 853 /*854 * Do not bother creating devices if no driver is available to855 * consume them.856 */857 if (!IS_ENABLED(CONFIG_DEV_DAX_HMEM))858 return;859 860 for (res = target->memregions.child; res; res = res->sibling) {861 int target_nid = pxm_to_node(target->memory_pxm);862 863 hmem_register_resource(target_nid, res);864 }865}866 867static void hmat_register_target(struct memory_target *target)868{869 int nid = pxm_to_node(target->memory_pxm);870 871 /*872 * Devices may belong to either an offline or online873 * node, so unconditionally add them.874 */875 hmat_register_target_devices(target);876 877 /*878 * Register generic port perf numbers. The nid may not be879 * initialized and is still NUMA_NO_NODE.880 */881 mutex_lock(&target_lock);882 if (*(u16 *)target->gen_port_device_handle) {883 hmat_update_generic_target(target);884 target->registered = true;885 }886 mutex_unlock(&target_lock);887 888 /*889 * Skip offline nodes. This can happen when memory890 * marked EFI_MEMORY_SP, "specific purpose", is applied891 * to all the memory in a proximity domain leading to892 * the node being marked offline / unplugged, or if893 * memory-only "hotplug" node is offline.894 */895 if (nid == NUMA_NO_NODE || !node_online(nid))896 return;897 898 mutex_lock(&target_lock);899 if (!target->registered) {900 hmat_register_target_initiators(target);901 hmat_register_target_cache(target);902 hmat_register_target_perf(target, ACCESS_COORDINATE_LOCAL);903 hmat_register_target_perf(target, ACCESS_COORDINATE_CPU);904 target->registered = true;905 }906 mutex_unlock(&target_lock);907}908 909static void hmat_register_targets(void)910{911 struct memory_target *target;912 913 list_for_each_entry(target, &targets, node)914 hmat_register_target(target);915}916 917static int hmat_callback(struct notifier_block *self,918 unsigned long action, void *arg)919{920 struct memory_target *target;921 struct memory_notify *mnb = arg;922 int pxm, nid = mnb->status_change_nid;923 924 if (nid == NUMA_NO_NODE || action != MEM_ONLINE)925 return NOTIFY_OK;926 927 pxm = node_to_pxm(nid);928 target = find_mem_target(pxm);929 if (!target)930 return NOTIFY_OK;931 932 hmat_register_target(target);933 return NOTIFY_OK;934}935 936static int __init hmat_set_default_dram_perf(void)937{938 int rc;939 int nid, pxm;940 struct memory_target *target;941 struct access_coordinate *attrs;942 943 for_each_node_mask(nid, default_dram_nodes) {944 pxm = node_to_pxm(nid);945 target = find_mem_target(pxm);946 if (!target)947 continue;948 attrs = &target->coord[ACCESS_COORDINATE_CPU];949 rc = mt_set_default_dram_perf(nid, attrs, "ACPI HMAT");950 if (rc)951 return rc;952 }953 954 return 0;955}956 957static int hmat_calculate_adistance(struct notifier_block *self,958 unsigned long nid, void *data)959{960 static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);961 struct memory_target *target;962 struct access_coordinate *perf;963 int *adist = data;964 int pxm;965 966 pxm = node_to_pxm(nid);967 target = find_mem_target(pxm);968 if (!target)969 return NOTIFY_OK;970 971 mutex_lock(&target_lock);972 hmat_update_target_attrs(target, p_nodes, ACCESS_COORDINATE_CPU);973 mutex_unlock(&target_lock);974 975 perf = &target->coord[ACCESS_COORDINATE_CPU];976 977 if (mt_perf_to_adistance(perf, adist))978 return NOTIFY_OK;979 980 return NOTIFY_STOP;981}982 983static struct notifier_block hmat_adist_nb __meminitdata = {984 .notifier_call = hmat_calculate_adistance,985 .priority = 100,986};987 988static __init void hmat_free_structures(void)989{990 struct memory_target *target, *tnext;991 struct memory_locality *loc, *lnext;992 struct memory_initiator *initiator, *inext;993 struct target_cache *tcache, *cnext;994 995 list_for_each_entry_safe(target, tnext, &targets, node) {996 struct resource *res, *res_next;997 998 list_for_each_entry_safe(tcache, cnext, &target->caches, node) {999 list_del(&tcache->node);1000 kfree(tcache);1001 }1002 1003 list_del(&target->node);1004 res = target->memregions.child;1005 while (res) {1006 res_next = res->sibling;1007 __release_region(&target->memregions, res->start,1008 resource_size(res));1009 res = res_next;1010 }1011 kfree(target);1012 }1013 1014 list_for_each_entry_safe(initiator, inext, &initiators, node) {1015 list_del(&initiator->node);1016 kfree(initiator);1017 }1018 1019 list_for_each_entry_safe(loc, lnext, &localities, node) {1020 list_del(&loc->node);1021 kfree(loc);1022 }1023}1024 1025static __init int hmat_init(void)1026{1027 struct acpi_table_header *tbl;1028 enum acpi_hmat_type i;1029 acpi_status status;1030 1031 if (srat_disabled() || hmat_disable)1032 return 0;1033 1034 status = acpi_get_table(ACPI_SIG_SRAT, 0, &tbl);1035 if (ACPI_FAILURE(status))1036 return 0;1037 1038 if (acpi_table_parse_entries(ACPI_SIG_SRAT,1039 sizeof(struct acpi_table_srat),1040 ACPI_SRAT_TYPE_MEMORY_AFFINITY,1041 srat_parse_mem_affinity, 0) < 0)1042 goto out_put;1043 1044 if (acpi_table_parse_entries(ACPI_SIG_SRAT,1045 sizeof(struct acpi_table_srat),1046 ACPI_SRAT_TYPE_GENERIC_PORT_AFFINITY,1047 srat_parse_genport_affinity, 0) < 0)1048 goto out_put;1049 1050 acpi_put_table(tbl);1051 1052 status = acpi_get_table(ACPI_SIG_HMAT, 0, &tbl);1053 if (ACPI_FAILURE(status))1054 goto out_put;1055 1056 hmat_revision = tbl->revision;1057 switch (hmat_revision) {1058 case 1:1059 case 2:1060 break;1061 default:1062 pr_notice("Ignoring: Unknown revision:%d\n", hmat_revision);1063 goto out_put;1064 }1065 1066 for (i = ACPI_HMAT_TYPE_PROXIMITY; i < ACPI_HMAT_TYPE_RESERVED; i++) {1067 if (acpi_table_parse_entries(ACPI_SIG_HMAT,1068 sizeof(struct acpi_table_hmat), i,1069 hmat_parse_subtable, 0) < 0) {1070 pr_notice("Ignoring: Invalid table");1071 goto out_put;1072 }1073 }1074 hmat_register_targets();1075 1076 /* Keep the table and structures if the notifier may use them */1077 if (hotplug_memory_notifier(hmat_callback, HMAT_CALLBACK_PRI))1078 goto out_put;1079 1080 if (!hmat_set_default_dram_perf())1081 register_mt_adistance_algorithm(&hmat_adist_nb);1082 1083 return 0;1084out_put:1085 hmat_free_structures();1086 acpi_put_table(tbl);1087 return 0;1088}1089subsys_initcall(hmat_init);1090