1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/debugfs.h> 3 #include <linux/mm.h> 4 #include <linux/slab.h> 5 #include <linux/uaccess.h> 6 #include <linux/memblock.h> 7 #include <linux/stacktrace.h> 8 #include <linux/page_owner.h> 9 #include <linux/jump_label.h> 10 #include <linux/migrate.h> 11 #include <linux/stackdepot.h> 12 #include <linux/seq_file.h> 13 14 #include "internal.h" 15 16 /* 17 * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack) 18 * to use off stack temporal storage 19 */ 20 #define PAGE_OWNER_STACK_DEPTH (16) 21 22 struct page_owner { 23 unsigned short order; 24 short last_migrate_reason; 25 gfp_t gfp_mask; 26 depot_stack_handle_t handle; 27 depot_stack_handle_t free_handle; 28 }; 29 30 static bool page_owner_enabled = false; 31 DEFINE_STATIC_KEY_FALSE(page_owner_inited); 32 33 static depot_stack_handle_t dummy_handle; 34 static depot_stack_handle_t failure_handle; 35 static depot_stack_handle_t early_handle; 36 37 static void init_early_allocated_pages(void); 38 39 static int __init early_page_owner_param(char *buf) 40 { 41 if (!buf) 42 return -EINVAL; 43 44 if (strcmp(buf, "on") == 0) 45 page_owner_enabled = true; 46 47 return 0; 48 } 49 early_param("page_owner", early_page_owner_param); 50 51 static bool need_page_owner(void) 52 { 53 return page_owner_enabled; 54 } 55 56 static __always_inline depot_stack_handle_t create_dummy_stack(void) 57 { 58 unsigned long entries[4]; 59 unsigned int nr_entries; 60 61 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0); 62 return stack_depot_save(entries, nr_entries, GFP_KERNEL); 63 } 64 65 static noinline void register_dummy_stack(void) 66 { 67 dummy_handle = create_dummy_stack(); 68 } 69 70 static noinline void register_failure_stack(void) 71 { 72 failure_handle = create_dummy_stack(); 73 } 74 75 static noinline void register_early_stack(void) 76 { 77 early_handle = create_dummy_stack(); 78 } 79 80 static void init_page_owner(void) 81 { 82 if (!page_owner_enabled) 83 return; 84 85 register_dummy_stack(); 86 register_failure_stack(); 87 register_early_stack(); 88 static_branch_enable(&page_owner_inited); 89 init_early_allocated_pages(); 90 } 91 92 struct page_ext_operations page_owner_ops = { 93 .size = sizeof(struct page_owner), 94 .need = need_page_owner, 95 .init = init_page_owner, 96 }; 97 98 static inline struct page_owner *get_page_owner(struct page_ext *page_ext) 99 { 100 return (void *)page_ext + page_owner_ops.offset; 101 } 102 103 static inline bool check_recursive_alloc(unsigned long *entries, 104 unsigned int nr_entries, 105 unsigned long ip) 106 { 107 unsigned int i; 108 109 for (i = 0; i < nr_entries; i++) { 110 if (entries[i] == ip) 111 return true; 112 } 113 return false; 114 } 115 116 static noinline depot_stack_handle_t save_stack(gfp_t flags) 117 { 118 unsigned long entries[PAGE_OWNER_STACK_DEPTH]; 119 depot_stack_handle_t handle; 120 unsigned int nr_entries; 121 122 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 2); 123 124 /* 125 * We need to check recursion here because our request to 126 * stackdepot could trigger memory allocation to save new 127 * entry. New memory allocation would reach here and call 128 * stack_depot_save_entries() again if we don't catch it. There is 129 * still not enough memory in stackdepot so it would try to 130 * allocate memory again and loop forever. 131 */ 132 if (check_recursive_alloc(entries, nr_entries, _RET_IP_)) 133 return dummy_handle; 134 135 handle = stack_depot_save(entries, nr_entries, flags); 136 if (!handle) 137 handle = failure_handle; 138 139 return handle; 140 } 141 142 void __reset_page_owner(struct page *page, unsigned int order) 143 { 144 int i; 145 struct page_ext *page_ext; 146 depot_stack_handle_t handle = 0; 147 struct page_owner *page_owner; 148 149 handle = save_stack(GFP_NOWAIT | __GFP_NOWARN); 150 151 page_ext = lookup_page_ext(page); 152 if (unlikely(!page_ext)) 153 return; 154 for (i = 0; i < (1 << order); i++) { 155 __clear_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags); 156 page_owner = get_page_owner(page_ext); 157 page_owner->free_handle = handle; 158 page_ext = page_ext_next(page_ext); 159 } 160 } 161 162 static inline void __set_page_owner_handle(struct page *page, 163 struct page_ext *page_ext, depot_stack_handle_t handle, 164 unsigned int order, gfp_t gfp_mask) 165 { 166 struct page_owner *page_owner; 167 int i; 168 169 for (i = 0; i < (1 << order); i++) { 170 page_owner = get_page_owner(page_ext); 171 page_owner->handle = handle; 172 page_owner->order = order; 173 page_owner->gfp_mask = gfp_mask; 174 page_owner->last_migrate_reason = -1; 175 __set_bit(PAGE_EXT_OWNER, &page_ext->flags); 176 __set_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags); 177 178 page_ext = page_ext_next(page_ext); 179 } 180 } 181 182 noinline void __set_page_owner(struct page *page, unsigned int order, 183 gfp_t gfp_mask) 184 { 185 struct page_ext *page_ext = lookup_page_ext(page); 186 depot_stack_handle_t handle; 187 188 if (unlikely(!page_ext)) 189 return; 190 191 handle = save_stack(gfp_mask); 192 __set_page_owner_handle(page, page_ext, handle, order, gfp_mask); 193 } 194 195 void __set_page_owner_migrate_reason(struct page *page, int reason) 196 { 197 struct page_ext *page_ext = lookup_page_ext(page); 198 struct page_owner *page_owner; 199 200 if (unlikely(!page_ext)) 201 return; 202 203 page_owner = get_page_owner(page_ext); 204 page_owner->last_migrate_reason = reason; 205 } 206 207 void __split_page_owner(struct page *page, unsigned int order) 208 { 209 int i; 210 struct page_ext *page_ext = lookup_page_ext(page); 211 struct page_owner *page_owner; 212 213 if (unlikely(!page_ext)) 214 return; 215 216 for (i = 0; i < (1 << order); i++) { 217 page_owner = get_page_owner(page_ext); 218 page_owner->order = 0; 219 page_ext = page_ext_next(page_ext); 220 } 221 } 222 223 void __copy_page_owner(struct page *oldpage, struct page *newpage) 224 { 225 struct page_ext *old_ext = lookup_page_ext(oldpage); 226 struct page_ext *new_ext = lookup_page_ext(newpage); 227 struct page_owner *old_page_owner, *new_page_owner; 228 229 if (unlikely(!old_ext || !new_ext)) 230 return; 231 232 old_page_owner = get_page_owner(old_ext); 233 new_page_owner = get_page_owner(new_ext); 234 new_page_owner->order = old_page_owner->order; 235 new_page_owner->gfp_mask = old_page_owner->gfp_mask; 236 new_page_owner->last_migrate_reason = 237 old_page_owner->last_migrate_reason; 238 new_page_owner->handle = old_page_owner->handle; 239 240 /* 241 * We don't clear the bit on the oldpage as it's going to be freed 242 * after migration. Until then, the info can be useful in case of 243 * a bug, and the overal stats will be off a bit only temporarily. 244 * Also, migrate_misplaced_transhuge_page() can still fail the 245 * migration and then we want the oldpage to retain the info. But 246 * in that case we also don't need to explicitly clear the info from 247 * the new page, which will be freed. 248 */ 249 __set_bit(PAGE_EXT_OWNER, &new_ext->flags); 250 __set_bit(PAGE_EXT_OWNER_ALLOCATED, &new_ext->flags); 251 } 252 253 void pagetypeinfo_showmixedcount_print(struct seq_file *m, 254 pg_data_t *pgdat, struct zone *zone) 255 { 256 struct page *page; 257 struct page_ext *page_ext; 258 struct page_owner *page_owner; 259 unsigned long pfn = zone->zone_start_pfn, block_end_pfn; 260 unsigned long end_pfn = pfn + zone->spanned_pages; 261 unsigned long count[MIGRATE_TYPES] = { 0, }; 262 int pageblock_mt, page_mt; 263 int i; 264 265 /* Scan block by block. First and last block may be incomplete */ 266 pfn = zone->zone_start_pfn; 267 268 /* 269 * Walk the zone in pageblock_nr_pages steps. If a page block spans 270 * a zone boundary, it will be double counted between zones. This does 271 * not matter as the mixed block count will still be correct 272 */ 273 for (; pfn < end_pfn; ) { 274 page = pfn_to_online_page(pfn); 275 if (!page) { 276 pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES); 277 continue; 278 } 279 280 block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); 281 block_end_pfn = min(block_end_pfn, end_pfn); 282 283 pageblock_mt = get_pageblock_migratetype(page); 284 285 for (; pfn < block_end_pfn; pfn++) { 286 if (!pfn_valid_within(pfn)) 287 continue; 288 289 /* The pageblock is online, no need to recheck. */ 290 page = pfn_to_page(pfn); 291 292 if (page_zone(page) != zone) 293 continue; 294 295 if (PageBuddy(page)) { 296 unsigned long freepage_order; 297 298 freepage_order = page_order_unsafe(page); 299 if (freepage_order < MAX_ORDER) 300 pfn += (1UL << freepage_order) - 1; 301 continue; 302 } 303 304 if (PageReserved(page)) 305 continue; 306 307 page_ext = lookup_page_ext(page); 308 if (unlikely(!page_ext)) 309 continue; 310 311 if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) 312 continue; 313 314 page_owner = get_page_owner(page_ext); 315 page_mt = gfp_migratetype(page_owner->gfp_mask); 316 if (pageblock_mt != page_mt) { 317 if (is_migrate_cma(pageblock_mt)) 318 count[MIGRATE_MOVABLE]++; 319 else 320 count[pageblock_mt]++; 321 322 pfn = block_end_pfn; 323 break; 324 } 325 pfn += (1UL << page_owner->order) - 1; 326 } 327 } 328 329 /* Print counts */ 330 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); 331 for (i = 0; i < MIGRATE_TYPES; i++) 332 seq_printf(m, "%12lu ", count[i]); 333 seq_putc(m, '\n'); 334 } 335 336 static ssize_t 337 print_page_owner(char __user *buf, size_t count, unsigned long pfn, 338 struct page *page, struct page_owner *page_owner, 339 depot_stack_handle_t handle) 340 { 341 int ret, pageblock_mt, page_mt; 342 unsigned long *entries; 343 unsigned int nr_entries; 344 char *kbuf; 345 346 count = min_t(size_t, count, PAGE_SIZE); 347 kbuf = kmalloc(count, GFP_KERNEL); 348 if (!kbuf) 349 return -ENOMEM; 350 351 ret = snprintf(kbuf, count, 352 "Page allocated via order %u, mask %#x(%pGg)\n", 353 page_owner->order, page_owner->gfp_mask, 354 &page_owner->gfp_mask); 355 356 if (ret >= count) 357 goto err; 358 359 /* Print information relevant to grouping pages by mobility */ 360 pageblock_mt = get_pageblock_migratetype(page); 361 page_mt = gfp_migratetype(page_owner->gfp_mask); 362 ret += snprintf(kbuf + ret, count - ret, 363 "PFN %lu type %s Block %lu type %s Flags %#lx(%pGp)\n", 364 pfn, 365 migratetype_names[page_mt], 366 pfn >> pageblock_order, 367 migratetype_names[pageblock_mt], 368 page->flags, &page->flags); 369 370 if (ret >= count) 371 goto err; 372 373 nr_entries = stack_depot_fetch(handle, &entries); 374 ret += stack_trace_snprint(kbuf + ret, count - ret, entries, nr_entries, 0); 375 if (ret >= count) 376 goto err; 377 378 if (page_owner->last_migrate_reason != -1) { 379 ret += snprintf(kbuf + ret, count - ret, 380 "Page has been migrated, last migrate reason: %s\n", 381 migrate_reason_names[page_owner->last_migrate_reason]); 382 if (ret >= count) 383 goto err; 384 } 385 386 ret += snprintf(kbuf + ret, count - ret, "\n"); 387 if (ret >= count) 388 goto err; 389 390 if (copy_to_user(buf, kbuf, ret)) 391 ret = -EFAULT; 392 393 kfree(kbuf); 394 return ret; 395 396 err: 397 kfree(kbuf); 398 return -ENOMEM; 399 } 400 401 void __dump_page_owner(struct page *page) 402 { 403 struct page_ext *page_ext = lookup_page_ext(page); 404 struct page_owner *page_owner; 405 depot_stack_handle_t handle; 406 unsigned long *entries; 407 unsigned int nr_entries; 408 gfp_t gfp_mask; 409 int mt; 410 411 if (unlikely(!page_ext)) { 412 pr_alert("There is not page extension available.\n"); 413 return; 414 } 415 416 page_owner = get_page_owner(page_ext); 417 gfp_mask = page_owner->gfp_mask; 418 mt = gfp_migratetype(gfp_mask); 419 420 if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) { 421 pr_alert("page_owner info is not present (never set?)\n"); 422 return; 423 } 424 425 if (test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) 426 pr_alert("page_owner tracks the page as allocated\n"); 427 else 428 pr_alert("page_owner tracks the page as freed\n"); 429 430 pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg)\n", 431 page_owner->order, migratetype_names[mt], gfp_mask, &gfp_mask); 432 433 handle = READ_ONCE(page_owner->handle); 434 if (!handle) { 435 pr_alert("page_owner allocation stack trace missing\n"); 436 } else { 437 nr_entries = stack_depot_fetch(handle, &entries); 438 stack_trace_print(entries, nr_entries, 0); 439 } 440 441 handle = READ_ONCE(page_owner->free_handle); 442 if (!handle) { 443 pr_alert("page_owner free stack trace missing\n"); 444 } else { 445 nr_entries = stack_depot_fetch(handle, &entries); 446 pr_alert("page last free stack trace:\n"); 447 stack_trace_print(entries, nr_entries, 0); 448 } 449 450 if (page_owner->last_migrate_reason != -1) 451 pr_alert("page has been migrated, last migrate reason: %s\n", 452 migrate_reason_names[page_owner->last_migrate_reason]); 453 } 454 455 static ssize_t 456 read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) 457 { 458 unsigned long pfn; 459 struct page *page; 460 struct page_ext *page_ext; 461 struct page_owner *page_owner; 462 depot_stack_handle_t handle; 463 464 if (!static_branch_unlikely(&page_owner_inited)) 465 return -EINVAL; 466 467 page = NULL; 468 pfn = min_low_pfn + *ppos; 469 470 /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */ 471 while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0) 472 pfn++; 473 474 drain_all_pages(NULL); 475 476 /* Find an allocated page */ 477 for (; pfn < max_pfn; pfn++) { 478 /* 479 * If the new page is in a new MAX_ORDER_NR_PAGES area, 480 * validate the area as existing, skip it if not 481 */ 482 if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) { 483 pfn += MAX_ORDER_NR_PAGES - 1; 484 continue; 485 } 486 487 /* Check for holes within a MAX_ORDER area */ 488 if (!pfn_valid_within(pfn)) 489 continue; 490 491 page = pfn_to_page(pfn); 492 if (PageBuddy(page)) { 493 unsigned long freepage_order = page_order_unsafe(page); 494 495 if (freepage_order < MAX_ORDER) 496 pfn += (1UL << freepage_order) - 1; 497 continue; 498 } 499 500 page_ext = lookup_page_ext(page); 501 if (unlikely(!page_ext)) 502 continue; 503 504 /* 505 * Some pages could be missed by concurrent allocation or free, 506 * because we don't hold the zone lock. 507 */ 508 if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) 509 continue; 510 511 /* 512 * Although we do have the info about past allocation of free 513 * pages, it's not relevant for current memory usage. 514 */ 515 if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags)) 516 continue; 517 518 page_owner = get_page_owner(page_ext); 519 520 /* 521 * Don't print "tail" pages of high-order allocations as that 522 * would inflate the stats. 523 */ 524 if (!IS_ALIGNED(pfn, 1 << page_owner->order)) 525 continue; 526 527 /* 528 * Access to page_ext->handle isn't synchronous so we should 529 * be careful to access it. 530 */ 531 handle = READ_ONCE(page_owner->handle); 532 if (!handle) 533 continue; 534 535 /* Record the next PFN to read in the file offset */ 536 *ppos = (pfn - min_low_pfn) + 1; 537 538 return print_page_owner(buf, count, pfn, page, 539 page_owner, handle); 540 } 541 542 return 0; 543 } 544 545 static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone) 546 { 547 unsigned long pfn = zone->zone_start_pfn; 548 unsigned long end_pfn = zone_end_pfn(zone); 549 unsigned long count = 0; 550 551 /* 552 * Walk the zone in pageblock_nr_pages steps. If a page block spans 553 * a zone boundary, it will be double counted between zones. This does 554 * not matter as the mixed block count will still be correct 555 */ 556 for (; pfn < end_pfn; ) { 557 unsigned long block_end_pfn; 558 559 if (!pfn_valid(pfn)) { 560 pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES); 561 continue; 562 } 563 564 block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); 565 block_end_pfn = min(block_end_pfn, end_pfn); 566 567 for (; pfn < block_end_pfn; pfn++) { 568 struct page *page; 569 struct page_ext *page_ext; 570 571 if (!pfn_valid_within(pfn)) 572 continue; 573 574 page = pfn_to_page(pfn); 575 576 if (page_zone(page) != zone) 577 continue; 578 579 /* 580 * To avoid having to grab zone->lock, be a little 581 * careful when reading buddy page order. The only 582 * danger is that we skip too much and potentially miss 583 * some early allocated pages, which is better than 584 * heavy lock contention. 585 */ 586 if (PageBuddy(page)) { 587 unsigned long order = page_order_unsafe(page); 588 589 if (order > 0 && order < MAX_ORDER) 590 pfn += (1UL << order) - 1; 591 continue; 592 } 593 594 if (PageReserved(page)) 595 continue; 596 597 page_ext = lookup_page_ext(page); 598 if (unlikely(!page_ext)) 599 continue; 600 601 /* Maybe overlapping zone */ 602 if (test_bit(PAGE_EXT_OWNER, &page_ext->flags)) 603 continue; 604 605 /* Found early allocated page */ 606 __set_page_owner_handle(page, page_ext, early_handle, 607 0, 0); 608 count++; 609 } 610 cond_resched(); 611 } 612 613 pr_info("Node %d, zone %8s: page owner found early allocated %lu pages\n", 614 pgdat->node_id, zone->name, count); 615 } 616 617 static void init_zones_in_node(pg_data_t *pgdat) 618 { 619 struct zone *zone; 620 struct zone *node_zones = pgdat->node_zones; 621 622 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { 623 if (!populated_zone(zone)) 624 continue; 625 626 init_pages_in_zone(pgdat, zone); 627 } 628 } 629 630 static void init_early_allocated_pages(void) 631 { 632 pg_data_t *pgdat; 633 634 for_each_online_pgdat(pgdat) 635 init_zones_in_node(pgdat); 636 } 637 638 static const struct file_operations proc_page_owner_operations = { 639 .read = read_page_owner, 640 }; 641 642 static int __init pageowner_init(void) 643 { 644 if (!static_branch_unlikely(&page_owner_inited)) { 645 pr_info("page_owner is disabled\n"); 646 return 0; 647 } 648 649 debugfs_create_file("page_owner", 0400, NULL, NULL, 650 &proc_page_owner_operations); 651 652 return 0; 653 } 654 late_initcall(pageowner_init) 655