1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2019, Intel Corporation. 4 * 5 * Heterogeneous Memory Attributes Table (HMAT) representation 6 * 7 * This program parses and reports the platform's HMAT tables, and registers 8 * the applicable attributes with the node's interfaces. 9 */ 10 11 #define pr_fmt(fmt) "acpi/hmat: " fmt 12 13 #include <linux/acpi.h> 14 #include <linux/bitops.h> 15 #include <linux/device.h> 16 #include <linux/init.h> 17 #include <linux/list.h> 18 #include <linux/mm.h> 19 #include <linux/platform_device.h> 20 #include <linux/list_sort.h> 21 #include <linux/memregion.h> 22 #include <linux/memory.h> 23 #include <linux/mutex.h> 24 #include <linux/node.h> 25 #include <linux/sysfs.h> 26 #include <linux/dax.h> 27 #include <linux/memory-tiers.h> 28 29 static u8 hmat_revision; 30 static int hmat_disable __initdata; 31 32 void __init disable_hmat(void) 33 { 34 hmat_disable = 1; 35 } 36 37 static LIST_HEAD(targets); 38 static LIST_HEAD(initiators); 39 static LIST_HEAD(localities); 40 41 static DEFINE_MUTEX(target_lock); 42 43 /* 44 * The defined enum order is used to prioritize attributes to break ties when 45 * selecting the best performing node. 46 */ 47 enum locality_types { 48 WRITE_LATENCY, 49 READ_LATENCY, 50 WRITE_BANDWIDTH, 51 READ_BANDWIDTH, 52 }; 53 54 static struct memory_locality *localities_types[4]; 55 56 struct target_cache { 57 struct list_head node; 58 struct node_cache_attrs cache_attrs; 59 }; 60 61 enum { 62 NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL = ACCESS_COORDINATE_MAX, 63 NODE_ACCESS_CLASS_GENPORT_SINK_CPU, 64 NODE_ACCESS_CLASS_MAX, 65 }; 66 67 struct memory_target { 68 struct list_head node; 69 unsigned int memory_pxm; 70 unsigned int processor_pxm; 71 struct resource memregions; 72 struct access_coordinate coord[NODE_ACCESS_CLASS_MAX]; 73 struct list_head caches; 74 struct node_cache_attrs cache_attrs; 75 u8 gen_port_device_handle[ACPI_SRAT_DEVICE_HANDLE_SIZE]; 76 bool registered; 77 bool ext_updated; /* externally updated */ 78 }; 79 80 struct memory_initiator { 81 struct list_head node; 82 unsigned int processor_pxm; 83 bool has_cpu; 84 }; 85 86 struct memory_locality { 87 struct list_head node; 88 struct acpi_hmat_locality *hmat_loc; 89 }; 90 91 static struct memory_initiator *find_mem_initiator(unsigned int cpu_pxm) 92 { 93 struct memory_initiator *initiator; 94 95 list_for_each_entry(initiator, &initiators, node) 96 if (initiator->processor_pxm == cpu_pxm) 97 return initiator; 98 return NULL; 99 } 100 101 static struct memory_target *find_mem_target(unsigned int mem_pxm) 102 { 103 struct memory_target *target; 104 105 list_for_each_entry(target, &targets, node) 106 if (target->memory_pxm == mem_pxm) 107 return target; 108 return NULL; 109 } 110 111 static struct memory_target *acpi_find_genport_target(u32 uid) 112 { 113 struct memory_target *target; 114 u32 target_uid; 115 u8 *uid_ptr; 116 117 list_for_each_entry(target, &targets, node) { 118 uid_ptr = target->gen_port_device_handle + 8; 119 target_uid = *(u32 *)uid_ptr; 120 if (uid == target_uid) 121 return target; 122 } 123 124 return NULL; 125 } 126 127 /** 128 * acpi_get_genport_coordinates - Retrieve the access coordinates for a generic port 129 * @uid: ACPI unique id 130 * @coord: The access coordinates written back out for the generic port. 131 * Expect 2 levels array. 132 * 133 * Return: 0 on success. Errno on failure. 134 * 135 * Only supports device handles that are ACPI. Assume ACPI0016 HID for CXL. 136 */ 137 int acpi_get_genport_coordinates(u32 uid, 138 struct access_coordinate *coord) 139 { 140 struct memory_target *target; 141 142 guard(mutex)(&target_lock); 143 target = acpi_find_genport_target(uid); 144 if (!target) 145 return -ENOENT; 146 147 coord[ACCESS_COORDINATE_LOCAL] = 148 target->coord[NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL]; 149 coord[ACCESS_COORDINATE_CPU] = 150 target->coord[NODE_ACCESS_CLASS_GENPORT_SINK_CPU]; 151 152 return 0; 153 } 154 EXPORT_SYMBOL_NS_GPL(acpi_get_genport_coordinates, CXL); 155 156 static __init void alloc_memory_initiator(unsigned int cpu_pxm) 157 { 158 struct memory_initiator *initiator; 159 160 if (pxm_to_node(cpu_pxm) == NUMA_NO_NODE) 161 return; 162 163 initiator = find_mem_initiator(cpu_pxm); 164 if (initiator) 165 return; 166 167 initiator = kzalloc(sizeof(*initiator), GFP_KERNEL); 168 if (!initiator) 169 return; 170 171 initiator->processor_pxm = cpu_pxm; 172 initiator->has_cpu = node_state(pxm_to_node(cpu_pxm), N_CPU); 173 list_add_tail(&initiator->node, &initiators); 174 } 175 176 static __init struct memory_target *alloc_target(unsigned int mem_pxm) 177 { 178 struct memory_target *target; 179 180 target = find_mem_target(mem_pxm); 181 if (!target) { 182 target = kzalloc(sizeof(*target), GFP_KERNEL); 183 if (!target) 184 return NULL; 185 target->memory_pxm = mem_pxm; 186 target->processor_pxm = PXM_INVAL; 187 target->memregions = (struct resource) { 188 .name = "ACPI mem", 189 .start = 0, 190 .end = -1, 191 .flags = IORESOURCE_MEM, 192 }; 193 list_add_tail(&target->node, &targets); 194 INIT_LIST_HEAD(&target->caches); 195 } 196 197 return target; 198 } 199 200 static __init void alloc_memory_target(unsigned int mem_pxm, 201 resource_size_t start, 202 resource_size_t len) 203 { 204 struct memory_target *target; 205 206 target = alloc_target(mem_pxm); 207 if (!target) 208 return; 209 210 /* 211 * There are potentially multiple ranges per PXM, so record each 212 * in the per-target memregions resource tree. 213 */ 214 if (!__request_region(&target->memregions, start, len, "memory target", 215 IORESOURCE_MEM)) 216 pr_warn("failed to reserve %#llx - %#llx in pxm: %d\n", 217 start, start + len, mem_pxm); 218 } 219 220 static __init void alloc_genport_target(unsigned int mem_pxm, u8 *handle) 221 { 222 struct memory_target *target; 223 224 target = alloc_target(mem_pxm); 225 if (!target) 226 return; 227 228 memcpy(target->gen_port_device_handle, handle, 229 ACPI_SRAT_DEVICE_HANDLE_SIZE); 230 } 231 232 static __init const char *hmat_data_type(u8 type) 233 { 234 switch (type) { 235 case ACPI_HMAT_ACCESS_LATENCY: 236 return "Access Latency"; 237 case ACPI_HMAT_READ_LATENCY: 238 return "Read Latency"; 239 case ACPI_HMAT_WRITE_LATENCY: 240 return "Write Latency"; 241 case ACPI_HMAT_ACCESS_BANDWIDTH: 242 return "Access Bandwidth"; 243 case ACPI_HMAT_READ_BANDWIDTH: 244 return "Read Bandwidth"; 245 case ACPI_HMAT_WRITE_BANDWIDTH: 246 return "Write Bandwidth"; 247 default: 248 return "Reserved"; 249 } 250 } 251 252 static __init const char *hmat_data_type_suffix(u8 type) 253 { 254 switch (type) { 255 case ACPI_HMAT_ACCESS_LATENCY: 256 case ACPI_HMAT_READ_LATENCY: 257 case ACPI_HMAT_WRITE_LATENCY: 258 return " nsec"; 259 case ACPI_HMAT_ACCESS_BANDWIDTH: 260 case ACPI_HMAT_READ_BANDWIDTH: 261 case ACPI_HMAT_WRITE_BANDWIDTH: 262 return " MB/s"; 263 default: 264 return ""; 265 } 266 } 267 268 static u32 hmat_normalize(u16 entry, u64 base, u8 type) 269 { 270 u32 value; 271 272 /* 273 * Check for invalid and overflow values 274 */ 275 if (entry == 0xffff || !entry) 276 return 0; 277 else if (base > (UINT_MAX / (entry))) 278 return 0; 279 280 /* 281 * Divide by the base unit for version 1, convert latency from 282 * picosenonds to nanoseconds if revision 2. 283 */ 284 value = entry * base; 285 if (hmat_revision == 1) { 286 if (value < 10) 287 return 0; 288 value = DIV_ROUND_UP(value, 10); 289 } else if (hmat_revision == 2) { 290 switch (type) { 291 case ACPI_HMAT_ACCESS_LATENCY: 292 case ACPI_HMAT_READ_LATENCY: 293 case ACPI_HMAT_WRITE_LATENCY: 294 value = DIV_ROUND_UP(value, 1000); 295 break; 296 default: 297 break; 298 } 299 } 300 return value; 301 } 302 303 static void hmat_update_target_access(struct memory_target *target, 304 u8 type, u32 value, int access) 305 { 306 switch (type) { 307 case ACPI_HMAT_ACCESS_LATENCY: 308 target->coord[access].read_latency = value; 309 target->coord[access].write_latency = value; 310 break; 311 case ACPI_HMAT_READ_LATENCY: 312 target->coord[access].read_latency = value; 313 break; 314 case ACPI_HMAT_WRITE_LATENCY: 315 target->coord[access].write_latency = value; 316 break; 317 case ACPI_HMAT_ACCESS_BANDWIDTH: 318 target->coord[access].read_bandwidth = value; 319 target->coord[access].write_bandwidth = value; 320 break; 321 case ACPI_HMAT_READ_BANDWIDTH: 322 target->coord[access].read_bandwidth = value; 323 break; 324 case ACPI_HMAT_WRITE_BANDWIDTH: 325 target->coord[access].write_bandwidth = value; 326 break; 327 default: 328 break; 329 } 330 } 331 332 int hmat_update_target_coordinates(int nid, struct access_coordinate *coord, 333 enum access_coordinate_class access) 334 { 335 struct memory_target *target; 336 int pxm; 337 338 if (nid == NUMA_NO_NODE) 339 return -EINVAL; 340 341 pxm = node_to_pxm(nid); 342 guard(mutex)(&target_lock); 343 target = find_mem_target(pxm); 344 if (!target) 345 return -ENODEV; 346 347 hmat_update_target_access(target, ACPI_HMAT_READ_LATENCY, 348 coord->read_latency, access); 349 hmat_update_target_access(target, ACPI_HMAT_WRITE_LATENCY, 350 coord->write_latency, access); 351 hmat_update_target_access(target, ACPI_HMAT_READ_BANDWIDTH, 352 coord->read_bandwidth, access); 353 hmat_update_target_access(target, ACPI_HMAT_WRITE_BANDWIDTH, 354 coord->write_bandwidth, access); 355 target->ext_updated = true; 356 357 return 0; 358 } 359 EXPORT_SYMBOL_GPL(hmat_update_target_coordinates); 360 361 static __init void hmat_add_locality(struct acpi_hmat_locality *hmat_loc) 362 { 363 struct memory_locality *loc; 364 365 loc = kzalloc(sizeof(*loc), GFP_KERNEL); 366 if (!loc) { 367 pr_notice_once("Failed to allocate HMAT locality\n"); 368 return; 369 } 370 371 loc->hmat_loc = hmat_loc; 372 list_add_tail(&loc->node, &localities); 373 374 switch (hmat_loc->data_type) { 375 case ACPI_HMAT_ACCESS_LATENCY: 376 localities_types[READ_LATENCY] = loc; 377 localities_types[WRITE_LATENCY] = loc; 378 break; 379 case ACPI_HMAT_READ_LATENCY: 380 localities_types[READ_LATENCY] = loc; 381 break; 382 case ACPI_HMAT_WRITE_LATENCY: 383 localities_types[WRITE_LATENCY] = loc; 384 break; 385 case ACPI_HMAT_ACCESS_BANDWIDTH: 386 localities_types[READ_BANDWIDTH] = loc; 387 localities_types[WRITE_BANDWIDTH] = loc; 388 break; 389 case ACPI_HMAT_READ_BANDWIDTH: 390 localities_types[READ_BANDWIDTH] = loc; 391 break; 392 case ACPI_HMAT_WRITE_BANDWIDTH: 393 localities_types[WRITE_BANDWIDTH] = loc; 394 break; 395 default: 396 break; 397 } 398 } 399 400 static __init void hmat_update_target(unsigned int tgt_pxm, unsigned int init_pxm, 401 u8 mem_hier, u8 type, u32 value) 402 { 403 struct memory_target *target = find_mem_target(tgt_pxm); 404 405 if (mem_hier != ACPI_HMAT_MEMORY) 406 return; 407 408 if (target && target->processor_pxm == init_pxm) { 409 hmat_update_target_access(target, type, value, 410 ACCESS_COORDINATE_LOCAL); 411 /* If the node has a CPU, update access 1 */ 412 if (node_state(pxm_to_node(init_pxm), N_CPU)) 413 hmat_update_target_access(target, type, value, 414 ACCESS_COORDINATE_CPU); 415 } 416 } 417 418 static __init int hmat_parse_locality(union acpi_subtable_headers *header, 419 const unsigned long end) 420 { 421 struct acpi_hmat_locality *hmat_loc = (void *)header; 422 unsigned int init, targ, total_size, ipds, tpds; 423 u32 *inits, *targs, value; 424 u16 *entries; 425 u8 type, mem_hier; 426 427 if (hmat_loc->header.length < sizeof(*hmat_loc)) { 428 pr_notice("Unexpected locality header length: %u\n", 429 hmat_loc->header.length); 430 return -EINVAL; 431 } 432 433 type = hmat_loc->data_type; 434 mem_hier = hmat_loc->flags & ACPI_HMAT_MEMORY_HIERARCHY; 435 ipds = hmat_loc->number_of_initiator_Pds; 436 tpds = hmat_loc->number_of_target_Pds; 437 total_size = sizeof(*hmat_loc) + sizeof(*entries) * ipds * tpds + 438 sizeof(*inits) * ipds + sizeof(*targs) * tpds; 439 if (hmat_loc->header.length < total_size) { 440 pr_notice("Unexpected locality header length:%u, minimum required:%u\n", 441 hmat_loc->header.length, total_size); 442 return -EINVAL; 443 } 444 445 pr_info("Locality: Flags:%02x Type:%s Initiator Domains:%u Target Domains:%u Base:%lld\n", 446 hmat_loc->flags, hmat_data_type(type), ipds, tpds, 447 hmat_loc->entry_base_unit); 448 449 inits = (u32 *)(hmat_loc + 1); 450 targs = inits + ipds; 451 entries = (u16 *)(targs + tpds); 452 for (init = 0; init < ipds; init++) { 453 alloc_memory_initiator(inits[init]); 454 for (targ = 0; targ < tpds; targ++) { 455 value = hmat_normalize(entries[init * tpds + targ], 456 hmat_loc->entry_base_unit, 457 type); 458 pr_info(" Initiator-Target[%u-%u]:%u%s\n", 459 inits[init], targs[targ], value, 460 hmat_data_type_suffix(type)); 461 462 hmat_update_target(targs[targ], inits[init], 463 mem_hier, type, value); 464 } 465 } 466 467 if (mem_hier == ACPI_HMAT_MEMORY) 468 hmat_add_locality(hmat_loc); 469 470 return 0; 471 } 472 473 static __init int hmat_parse_cache(union acpi_subtable_headers *header, 474 const unsigned long end) 475 { 476 struct acpi_hmat_cache *cache = (void *)header; 477 struct memory_target *target; 478 struct target_cache *tcache; 479 u32 attrs; 480 481 if (cache->header.length < sizeof(*cache)) { 482 pr_notice("Unexpected cache header length: %u\n", 483 cache->header.length); 484 return -EINVAL; 485 } 486 487 attrs = cache->cache_attributes; 488 pr_info("Cache: Domain:%u Size:%llu Attrs:%08x SMBIOS Handles:%d\n", 489 cache->memory_PD, cache->cache_size, attrs, 490 cache->number_of_SMBIOShandles); 491 492 target = find_mem_target(cache->memory_PD); 493 if (!target) 494 return 0; 495 496 tcache = kzalloc(sizeof(*tcache), GFP_KERNEL); 497 if (!tcache) { 498 pr_notice_once("Failed to allocate HMAT cache info\n"); 499 return 0; 500 } 501 502 tcache->cache_attrs.size = cache->cache_size; 503 tcache->cache_attrs.level = (attrs & ACPI_HMAT_CACHE_LEVEL) >> 4; 504 tcache->cache_attrs.line_size = (attrs & ACPI_HMAT_CACHE_LINE_SIZE) >> 16; 505 506 switch ((attrs & ACPI_HMAT_CACHE_ASSOCIATIVITY) >> 8) { 507 case ACPI_HMAT_CA_DIRECT_MAPPED: 508 tcache->cache_attrs.indexing = NODE_CACHE_DIRECT_MAP; 509 break; 510 case ACPI_HMAT_CA_COMPLEX_CACHE_INDEXING: 511 tcache->cache_attrs.indexing = NODE_CACHE_INDEXED; 512 break; 513 case ACPI_HMAT_CA_NONE: 514 default: 515 tcache->cache_attrs.indexing = NODE_CACHE_OTHER; 516 break; 517 } 518 519 switch ((attrs & ACPI_HMAT_WRITE_POLICY) >> 12) { 520 case ACPI_HMAT_CP_WB: 521 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_BACK; 522 break; 523 case ACPI_HMAT_CP_WT: 524 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_THROUGH; 525 break; 526 case ACPI_HMAT_CP_NONE: 527 default: 528 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_OTHER; 529 break; 530 } 531 list_add_tail(&tcache->node, &target->caches); 532 533 return 0; 534 } 535 536 static int __init hmat_parse_proximity_domain(union acpi_subtable_headers *header, 537 const unsigned long end) 538 { 539 struct acpi_hmat_proximity_domain *p = (void *)header; 540 struct memory_target *target = NULL; 541 542 if (p->header.length != sizeof(*p)) { 543 pr_notice("Unexpected address range header length: %u\n", 544 p->header.length); 545 return -EINVAL; 546 } 547 548 if (hmat_revision == 1) 549 pr_info("Memory (%#llx length %#llx) Flags:%04x Processor Domain:%u Memory Domain:%u\n", 550 p->reserved3, p->reserved4, p->flags, p->processor_PD, 551 p->memory_PD); 552 else 553 pr_info("Memory Flags:%04x Processor Domain:%u Memory Domain:%u\n", 554 p->flags, p->processor_PD, p->memory_PD); 555 556 if ((hmat_revision == 1 && p->flags & ACPI_HMAT_MEMORY_PD_VALID) || 557 hmat_revision > 1) { 558 target = find_mem_target(p->memory_PD); 559 if (!target) { 560 pr_debug("Memory Domain missing from SRAT\n"); 561 return -EINVAL; 562 } 563 } 564 if (target && p->flags & ACPI_HMAT_PROCESSOR_PD_VALID) { 565 int p_node = pxm_to_node(p->processor_PD); 566 567 if (p_node == NUMA_NO_NODE) { 568 pr_debug("Invalid Processor Domain\n"); 569 return -EINVAL; 570 } 571 target->processor_pxm = p->processor_PD; 572 } 573 574 return 0; 575 } 576 577 static int __init hmat_parse_subtable(union acpi_subtable_headers *header, 578 const unsigned long end) 579 { 580 struct acpi_hmat_structure *hdr = (void *)header; 581 582 if (!hdr) 583 return -EINVAL; 584 585 switch (hdr->type) { 586 case ACPI_HMAT_TYPE_PROXIMITY: 587 return hmat_parse_proximity_domain(header, end); 588 case ACPI_HMAT_TYPE_LOCALITY: 589 return hmat_parse_locality(header, end); 590 case ACPI_HMAT_TYPE_CACHE: 591 return hmat_parse_cache(header, end); 592 default: 593 return -EINVAL; 594 } 595 } 596 597 static __init int srat_parse_mem_affinity(union acpi_subtable_headers *header, 598 const unsigned long end) 599 { 600 struct acpi_srat_mem_affinity *ma = (void *)header; 601 602 if (!ma) 603 return -EINVAL; 604 if (!(ma->flags & ACPI_SRAT_MEM_ENABLED)) 605 return 0; 606 alloc_memory_target(ma->proximity_domain, ma->base_address, ma->length); 607 return 0; 608 } 609 610 static __init int srat_parse_genport_affinity(union acpi_subtable_headers *header, 611 const unsigned long end) 612 { 613 struct acpi_srat_generic_affinity *ga = (void *)header; 614 615 if (!ga) 616 return -EINVAL; 617 618 if (!(ga->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED)) 619 return 0; 620 621 /* Skip PCI device_handle for now */ 622 if (ga->device_handle_type != 0) 623 return 0; 624 625 alloc_genport_target(ga->proximity_domain, 626 (u8 *)ga->device_handle); 627 628 return 0; 629 } 630 631 static u32 hmat_initiator_perf(struct memory_target *target, 632 struct memory_initiator *initiator, 633 struct acpi_hmat_locality *hmat_loc) 634 { 635 unsigned int ipds, tpds, i, idx = 0, tdx = 0; 636 u32 *inits, *targs; 637 u16 *entries; 638 639 ipds = hmat_loc->number_of_initiator_Pds; 640 tpds = hmat_loc->number_of_target_Pds; 641 inits = (u32 *)(hmat_loc + 1); 642 targs = inits + ipds; 643 entries = (u16 *)(targs + tpds); 644 645 for (i = 0; i < ipds; i++) { 646 if (inits[i] == initiator->processor_pxm) { 647 idx = i; 648 break; 649 } 650 } 651 652 if (i == ipds) 653 return 0; 654 655 for (i = 0; i < tpds; i++) { 656 if (targs[i] == target->memory_pxm) { 657 tdx = i; 658 break; 659 } 660 } 661 if (i == tpds) 662 return 0; 663 664 return hmat_normalize(entries[idx * tpds + tdx], 665 hmat_loc->entry_base_unit, 666 hmat_loc->data_type); 667 } 668 669 static bool hmat_update_best(u8 type, u32 value, u32 *best) 670 { 671 bool updated = false; 672 673 if (!value) 674 return false; 675 676 switch (type) { 677 case ACPI_HMAT_ACCESS_LATENCY: 678 case ACPI_HMAT_READ_LATENCY: 679 case ACPI_HMAT_WRITE_LATENCY: 680 if (!*best || *best > value) { 681 *best = value; 682 updated = true; 683 } 684 break; 685 case ACPI_HMAT_ACCESS_BANDWIDTH: 686 case ACPI_HMAT_READ_BANDWIDTH: 687 case ACPI_HMAT_WRITE_BANDWIDTH: 688 if (!*best || *best < value) { 689 *best = value; 690 updated = true; 691 } 692 break; 693 } 694 695 return updated; 696 } 697 698 static int initiator_cmp(void *priv, const struct list_head *a, 699 const struct list_head *b) 700 { 701 struct memory_initiator *ia; 702 struct memory_initiator *ib; 703 704 ia = list_entry(a, struct memory_initiator, node); 705 ib = list_entry(b, struct memory_initiator, node); 706 707 return ia->processor_pxm - ib->processor_pxm; 708 } 709 710 static int initiators_to_nodemask(unsigned long *p_nodes) 711 { 712 struct memory_initiator *initiator; 713 714 if (list_empty(&initiators)) 715 return -ENXIO; 716 717 list_for_each_entry(initiator, &initiators, node) 718 set_bit(initiator->processor_pxm, p_nodes); 719 720 return 0; 721 } 722 723 static void hmat_update_target_attrs(struct memory_target *target, 724 unsigned long *p_nodes, int access) 725 { 726 struct memory_initiator *initiator; 727 unsigned int cpu_nid; 728 struct memory_locality *loc = NULL; 729 u32 best = 0; 730 int i; 731 732 /* Don't update if an external agent has changed the data. */ 733 if (target->ext_updated) 734 return; 735 736 /* Don't update for generic port if there's no device handle */ 737 if ((access == NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL || 738 access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) && 739 !(*(u16 *)target->gen_port_device_handle)) 740 return; 741 742 bitmap_zero(p_nodes, MAX_NUMNODES); 743 /* 744 * If the Address Range Structure provides a local processor pxm, set 745 * only that one. Otherwise, find the best performance attributes and 746 * collect all initiators that match. 747 */ 748 if (target->processor_pxm != PXM_INVAL) { 749 cpu_nid = pxm_to_node(target->processor_pxm); 750 if (access == ACCESS_COORDINATE_LOCAL || 751 node_state(cpu_nid, N_CPU)) { 752 set_bit(target->processor_pxm, p_nodes); 753 return; 754 } 755 } 756 757 if (list_empty(&localities)) 758 return; 759 760 /* 761 * We need the initiator list sorted so we can use bitmap_clear for 762 * previously set initiators when we find a better memory accessor. 763 * We'll also use the sorting to prime the candidate nodes with known 764 * initiators. 765 */ 766 list_sort(NULL, &initiators, initiator_cmp); 767 if (initiators_to_nodemask(p_nodes) < 0) 768 return; 769 770 for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) { 771 loc = localities_types[i]; 772 if (!loc) 773 continue; 774 775 best = 0; 776 list_for_each_entry(initiator, &initiators, node) { 777 u32 value; 778 779 if ((access == ACCESS_COORDINATE_CPU || 780 access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) && 781 !initiator->has_cpu) { 782 clear_bit(initiator->processor_pxm, p_nodes); 783 continue; 784 } 785 if (!test_bit(initiator->processor_pxm, p_nodes)) 786 continue; 787 788 value = hmat_initiator_perf(target, initiator, loc->hmat_loc); 789 if (hmat_update_best(loc->hmat_loc->data_type, value, &best)) 790 bitmap_clear(p_nodes, 0, initiator->processor_pxm); 791 if (value != best) 792 clear_bit(initiator->processor_pxm, p_nodes); 793 } 794 if (best) 795 hmat_update_target_access(target, loc->hmat_loc->data_type, best, access); 796 } 797 } 798 799 static void __hmat_register_target_initiators(struct memory_target *target, 800 unsigned long *p_nodes, 801 int access) 802 { 803 unsigned int mem_nid, cpu_nid; 804 int i; 805 806 mem_nid = pxm_to_node(target->memory_pxm); 807 hmat_update_target_attrs(target, p_nodes, access); 808 for_each_set_bit(i, p_nodes, MAX_NUMNODES) { 809 cpu_nid = pxm_to_node(i); 810 register_memory_node_under_compute_node(mem_nid, cpu_nid, access); 811 } 812 } 813 814 static void hmat_update_generic_target(struct memory_target *target) 815 { 816 static DECLARE_BITMAP(p_nodes, MAX_NUMNODES); 817 818 hmat_update_target_attrs(target, p_nodes, 819 NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL); 820 hmat_update_target_attrs(target, p_nodes, 821 NODE_ACCESS_CLASS_GENPORT_SINK_CPU); 822 } 823 824 static void hmat_register_target_initiators(struct memory_target *target) 825 { 826 static DECLARE_BITMAP(p_nodes, MAX_NUMNODES); 827 828 __hmat_register_target_initiators(target, p_nodes, 829 ACCESS_COORDINATE_LOCAL); 830 __hmat_register_target_initiators(target, p_nodes, 831 ACCESS_COORDINATE_CPU); 832 } 833 834 static void hmat_register_target_cache(struct memory_target *target) 835 { 836 unsigned mem_nid = pxm_to_node(target->memory_pxm); 837 struct target_cache *tcache; 838 839 list_for_each_entry(tcache, &target->caches, node) 840 node_add_cache(mem_nid, &tcache->cache_attrs); 841 } 842 843 static void hmat_register_target_perf(struct memory_target *target, int access) 844 { 845 unsigned mem_nid = pxm_to_node(target->memory_pxm); 846 node_set_perf_attrs(mem_nid, &target->coord[access], access); 847 } 848 849 static void hmat_register_target_devices(struct memory_target *target) 850 { 851 struct resource *res; 852 853 /* 854 * Do not bother creating devices if no driver is available to 855 * consume them. 856 */ 857 if (!IS_ENABLED(CONFIG_DEV_DAX_HMEM)) 858 return; 859 860 for (res = target->memregions.child; res; res = res->sibling) { 861 int target_nid = pxm_to_node(target->memory_pxm); 862 863 hmem_register_resource(target_nid, res); 864 } 865 } 866 867 static void hmat_register_target(struct memory_target *target) 868 { 869 int nid = pxm_to_node(target->memory_pxm); 870 871 /* 872 * Devices may belong to either an offline or online 873 * node, so unconditionally add them. 874 */ 875 hmat_register_target_devices(target); 876 877 /* 878 * Register generic port perf numbers. The nid may not be 879 * initialized and is still NUMA_NO_NODE. 880 */ 881 mutex_lock(&target_lock); 882 if (*(u16 *)target->gen_port_device_handle) { 883 hmat_update_generic_target(target); 884 target->registered = true; 885 } 886 mutex_unlock(&target_lock); 887 888 /* 889 * Skip offline nodes. This can happen when memory 890 * marked EFI_MEMORY_SP, "specific purpose", is applied 891 * to all the memory in a proximity domain leading to 892 * the node being marked offline / unplugged, or if 893 * memory-only "hotplug" node is offline. 894 */ 895 if (nid == NUMA_NO_NODE || !node_online(nid)) 896 return; 897 898 mutex_lock(&target_lock); 899 if (!target->registered) { 900 hmat_register_target_initiators(target); 901 hmat_register_target_cache(target); 902 hmat_register_target_perf(target, ACCESS_COORDINATE_LOCAL); 903 hmat_register_target_perf(target, ACCESS_COORDINATE_CPU); 904 target->registered = true; 905 } 906 mutex_unlock(&target_lock); 907 } 908 909 static void hmat_register_targets(void) 910 { 911 struct memory_target *target; 912 913 list_for_each_entry(target, &targets, node) 914 hmat_register_target(target); 915 } 916 917 static int hmat_callback(struct notifier_block *self, 918 unsigned long action, void *arg) 919 { 920 struct memory_target *target; 921 struct memory_notify *mnb = arg; 922 int pxm, nid = mnb->status_change_nid; 923 924 if (nid == NUMA_NO_NODE || action != MEM_ONLINE) 925 return NOTIFY_OK; 926 927 pxm = node_to_pxm(nid); 928 target = find_mem_target(pxm); 929 if (!target) 930 return NOTIFY_OK; 931 932 hmat_register_target(target); 933 return NOTIFY_OK; 934 } 935 936 static int hmat_set_default_dram_perf(void) 937 { 938 int rc; 939 int nid, pxm; 940 struct memory_target *target; 941 struct access_coordinate *attrs; 942 943 if (!default_dram_type) 944 return -EIO; 945 946 for_each_node_mask(nid, default_dram_type->nodes) { 947 pxm = node_to_pxm(nid); 948 target = find_mem_target(pxm); 949 if (!target) 950 continue; 951 attrs = &target->coord[1]; 952 rc = mt_set_default_dram_perf(nid, attrs, "ACPI HMAT"); 953 if (rc) 954 return rc; 955 } 956 957 return 0; 958 } 959 960 static int hmat_calculate_adistance(struct notifier_block *self, 961 unsigned long nid, void *data) 962 { 963 static DECLARE_BITMAP(p_nodes, MAX_NUMNODES); 964 struct memory_target *target; 965 struct access_coordinate *perf; 966 int *adist = data; 967 int pxm; 968 969 pxm = node_to_pxm(nid); 970 target = find_mem_target(pxm); 971 if (!target) 972 return NOTIFY_OK; 973 974 mutex_lock(&target_lock); 975 hmat_update_target_attrs(target, p_nodes, ACCESS_COORDINATE_CPU); 976 mutex_unlock(&target_lock); 977 978 perf = &target->coord[1]; 979 980 if (mt_perf_to_adistance(perf, adist)) 981 return NOTIFY_OK; 982 983 return NOTIFY_STOP; 984 } 985 986 static struct notifier_block hmat_adist_nb __meminitdata = { 987 .notifier_call = hmat_calculate_adistance, 988 .priority = 100, 989 }; 990 991 static __init void hmat_free_structures(void) 992 { 993 struct memory_target *target, *tnext; 994 struct memory_locality *loc, *lnext; 995 struct memory_initiator *initiator, *inext; 996 struct target_cache *tcache, *cnext; 997 998 list_for_each_entry_safe(target, tnext, &targets, node) { 999 struct resource *res, *res_next; 1000 1001 list_for_each_entry_safe(tcache, cnext, &target->caches, node) { 1002 list_del(&tcache->node); 1003 kfree(tcache); 1004 } 1005 1006 list_del(&target->node); 1007 res = target->memregions.child; 1008 while (res) { 1009 res_next = res->sibling; 1010 __release_region(&target->memregions, res->start, 1011 resource_size(res)); 1012 res = res_next; 1013 } 1014 kfree(target); 1015 } 1016 1017 list_for_each_entry_safe(initiator, inext, &initiators, node) { 1018 list_del(&initiator->node); 1019 kfree(initiator); 1020 } 1021 1022 list_for_each_entry_safe(loc, lnext, &localities, node) { 1023 list_del(&loc->node); 1024 kfree(loc); 1025 } 1026 } 1027 1028 static __init int hmat_init(void) 1029 { 1030 struct acpi_table_header *tbl; 1031 enum acpi_hmat_type i; 1032 acpi_status status; 1033 1034 if (srat_disabled() || hmat_disable) 1035 return 0; 1036 1037 status = acpi_get_table(ACPI_SIG_SRAT, 0, &tbl); 1038 if (ACPI_FAILURE(status)) 1039 return 0; 1040 1041 if (acpi_table_parse_entries(ACPI_SIG_SRAT, 1042 sizeof(struct acpi_table_srat), 1043 ACPI_SRAT_TYPE_MEMORY_AFFINITY, 1044 srat_parse_mem_affinity, 0) < 0) 1045 goto out_put; 1046 1047 if (acpi_table_parse_entries(ACPI_SIG_SRAT, 1048 sizeof(struct acpi_table_srat), 1049 ACPI_SRAT_TYPE_GENERIC_PORT_AFFINITY, 1050 srat_parse_genport_affinity, 0) < 0) 1051 goto out_put; 1052 1053 acpi_put_table(tbl); 1054 1055 status = acpi_get_table(ACPI_SIG_HMAT, 0, &tbl); 1056 if (ACPI_FAILURE(status)) 1057 goto out_put; 1058 1059 hmat_revision = tbl->revision; 1060 switch (hmat_revision) { 1061 case 1: 1062 case 2: 1063 break; 1064 default: 1065 pr_notice("Ignoring: Unknown revision:%d\n", hmat_revision); 1066 goto out_put; 1067 } 1068 1069 for (i = ACPI_HMAT_TYPE_PROXIMITY; i < ACPI_HMAT_TYPE_RESERVED; i++) { 1070 if (acpi_table_parse_entries(ACPI_SIG_HMAT, 1071 sizeof(struct acpi_table_hmat), i, 1072 hmat_parse_subtable, 0) < 0) { 1073 pr_notice("Ignoring: Invalid table"); 1074 goto out_put; 1075 } 1076 } 1077 hmat_register_targets(); 1078 1079 /* Keep the table and structures if the notifier may use them */ 1080 if (hotplug_memory_notifier(hmat_callback, HMAT_CALLBACK_PRI)) 1081 goto out_put; 1082 1083 if (!hmat_set_default_dram_perf()) 1084 register_mt_adistance_algorithm(&hmat_adist_nb); 1085 1086 return 0; 1087 out_put: 1088 hmat_free_structures(); 1089 acpi_put_table(tbl); 1090 return 0; 1091 } 1092 subsys_initcall(hmat_init); 1093