1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * linux/mm/swap_state.c 4 * 5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 6 * Swap reorganised 29.12.95, Stephen Tweedie 7 * 8 * Rewritten to use page cache, (C) 1998 Stephen Tweedie 9 */ 10 #include <linux/mm.h> 11 #include <linux/gfp.h> 12 #include <linux/kernel_stat.h> 13 #include <linux/swap.h> 14 #include <linux/swapops.h> 15 #include <linux/init.h> 16 #include <linux/pagemap.h> 17 #include <linux/backing-dev.h> 18 #include <linux/blkdev.h> 19 #include <linux/pagevec.h> 20 #include <linux/migrate.h> 21 #include <linux/vmalloc.h> 22 #include <linux/swap_slots.h> 23 #include <linux/huge_mm.h> 24 #include "internal.h" 25 26 /* 27 * swapper_space is a fiction, retained to simplify the path through 28 * vmscan's shrink_page_list. 29 */ 30 static const struct address_space_operations swap_aops = { 31 .writepage = swap_writepage, 32 .set_page_dirty = swap_set_page_dirty, 33 #ifdef CONFIG_MIGRATION 34 .migratepage = migrate_page, 35 #endif 36 }; 37 38 struct address_space *swapper_spaces[MAX_SWAPFILES] __read_mostly; 39 static unsigned int nr_swapper_spaces[MAX_SWAPFILES] __read_mostly; 40 static bool enable_vma_readahead __read_mostly = true; 41 42 #define SWAP_RA_WIN_SHIFT (PAGE_SHIFT / 2) 43 #define SWAP_RA_HITS_MASK ((1UL << SWAP_RA_WIN_SHIFT) - 1) 44 #define SWAP_RA_HITS_MAX SWAP_RA_HITS_MASK 45 #define SWAP_RA_WIN_MASK (~PAGE_MASK & ~SWAP_RA_HITS_MASK) 46 47 #define SWAP_RA_HITS(v) ((v) & SWAP_RA_HITS_MASK) 48 #define SWAP_RA_WIN(v) (((v) & SWAP_RA_WIN_MASK) >> SWAP_RA_WIN_SHIFT) 49 #define SWAP_RA_ADDR(v) ((v) & PAGE_MASK) 50 51 #define SWAP_RA_VAL(addr, win, hits) \ 52 (((addr) & PAGE_MASK) | \ 53 (((win) << SWAP_RA_WIN_SHIFT) & SWAP_RA_WIN_MASK) | \ 54 ((hits) & SWAP_RA_HITS_MASK)) 55 56 /* Initial readahead hits is 4 to start up with a small window */ 57 #define GET_SWAP_RA_VAL(vma) \ 58 (atomic_long_read(&(vma)->swap_readahead_info) ? : 4) 59 60 #define INC_CACHE_INFO(x) do { swap_cache_info.x++; } while (0) 61 #define ADD_CACHE_INFO(x, nr) do { swap_cache_info.x += (nr); } while (0) 62 63 static struct { 64 unsigned long add_total; 65 unsigned long del_total; 66 unsigned long find_success; 67 unsigned long find_total; 68 } swap_cache_info; 69 70 unsigned long total_swapcache_pages(void) 71 { 72 unsigned int i, j, nr; 73 unsigned long ret = 0; 74 struct address_space *spaces; 75 struct swap_info_struct *si; 76 77 for (i = 0; i < MAX_SWAPFILES; i++) { 78 swp_entry_t entry = swp_entry(i, 1); 79 80 /* Avoid get_swap_device() to warn for bad swap entry */ 81 if (!swp_swap_info(entry)) 82 continue; 83 /* Prevent swapoff to free swapper_spaces */ 84 si = get_swap_device(entry); 85 if (!si) 86 continue; 87 nr = nr_swapper_spaces[i]; 88 spaces = swapper_spaces[i]; 89 for (j = 0; j < nr; j++) 90 ret += spaces[j].nrpages; 91 put_swap_device(si); 92 } 93 return ret; 94 } 95 96 static atomic_t swapin_readahead_hits = ATOMIC_INIT(4); 97 98 void show_swap_cache_info(void) 99 { 100 printk("%lu pages in swap cache\n", total_swapcache_pages()); 101 printk("Swap cache stats: add %lu, delete %lu, find %lu/%lu\n", 102 swap_cache_info.add_total, swap_cache_info.del_total, 103 swap_cache_info.find_success, swap_cache_info.find_total); 104 printk("Free swap = %ldkB\n", 105 get_nr_swap_pages() << (PAGE_SHIFT - 10)); 106 printk("Total swap = %lukB\n", total_swap_pages << (PAGE_SHIFT - 10)); 107 } 108 109 /* 110 * add_to_swap_cache resembles add_to_page_cache_locked on swapper_space, 111 * but sets SwapCache flag and private instead of mapping and index. 112 */ 113 int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp) 114 { 115 struct address_space *address_space = swap_address_space(entry); 116 pgoff_t idx = swp_offset(entry); 117 XA_STATE_ORDER(xas, &address_space->i_pages, idx, compound_order(page)); 118 unsigned long i, nr = hpage_nr_pages(page); 119 120 VM_BUG_ON_PAGE(!PageLocked(page), page); 121 VM_BUG_ON_PAGE(PageSwapCache(page), page); 122 VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 123 124 page_ref_add(page, nr); 125 SetPageSwapCache(page); 126 127 do { 128 xas_lock_irq(&xas); 129 xas_create_range(&xas); 130 if (xas_error(&xas)) 131 goto unlock; 132 for (i = 0; i < nr; i++) { 133 VM_BUG_ON_PAGE(xas.xa_index != idx + i, page); 134 set_page_private(page + i, entry.val + i); 135 xas_store(&xas, page); 136 xas_next(&xas); 137 } 138 address_space->nrpages += nr; 139 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr); 140 ADD_CACHE_INFO(add_total, nr); 141 unlock: 142 xas_unlock_irq(&xas); 143 } while (xas_nomem(&xas, gfp)); 144 145 if (!xas_error(&xas)) 146 return 0; 147 148 ClearPageSwapCache(page); 149 page_ref_sub(page, nr); 150 return xas_error(&xas); 151 } 152 153 /* 154 * This must be called only on pages that have 155 * been verified to be in the swap cache. 156 */ 157 void __delete_from_swap_cache(struct page *page, swp_entry_t entry) 158 { 159 struct address_space *address_space = swap_address_space(entry); 160 int i, nr = hpage_nr_pages(page); 161 pgoff_t idx = swp_offset(entry); 162 XA_STATE(xas, &address_space->i_pages, idx); 163 164 VM_BUG_ON_PAGE(!PageLocked(page), page); 165 VM_BUG_ON_PAGE(!PageSwapCache(page), page); 166 VM_BUG_ON_PAGE(PageWriteback(page), page); 167 168 for (i = 0; i < nr; i++) { 169 void *entry = xas_store(&xas, NULL); 170 VM_BUG_ON_PAGE(entry != page, entry); 171 set_page_private(page + i, 0); 172 xas_next(&xas); 173 } 174 ClearPageSwapCache(page); 175 address_space->nrpages -= nr; 176 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, -nr); 177 ADD_CACHE_INFO(del_total, nr); 178 } 179 180 /** 181 * add_to_swap - allocate swap space for a page 182 * @page: page we want to move to swap 183 * 184 * Allocate swap space for the page and add the page to the 185 * swap cache. Caller needs to hold the page lock. 186 */ 187 int add_to_swap(struct page *page) 188 { 189 swp_entry_t entry; 190 int err; 191 192 VM_BUG_ON_PAGE(!PageLocked(page), page); 193 VM_BUG_ON_PAGE(!PageUptodate(page), page); 194 195 entry = get_swap_page(page); 196 if (!entry.val) 197 return 0; 198 199 /* 200 * XArray node allocations from PF_MEMALLOC contexts could 201 * completely exhaust the page allocator. __GFP_NOMEMALLOC 202 * stops emergency reserves from being allocated. 203 * 204 * TODO: this could cause a theoretical memory reclaim 205 * deadlock in the swap out path. 206 */ 207 /* 208 * Add it to the swap cache. 209 */ 210 err = add_to_swap_cache(page, entry, 211 __GFP_HIGH|__GFP_NOMEMALLOC|__GFP_NOWARN); 212 if (err) 213 /* 214 * add_to_swap_cache() doesn't return -EEXIST, so we can safely 215 * clear SWAP_HAS_CACHE flag. 216 */ 217 goto fail; 218 /* 219 * Normally the page will be dirtied in unmap because its pte should be 220 * dirty. A special case is MADV_FREE page. The page'e pte could have 221 * dirty bit cleared but the page's SwapBacked bit is still set because 222 * clearing the dirty bit and SwapBacked bit has no lock protected. For 223 * such page, unmap will not set dirty bit for it, so page reclaim will 224 * not write the page out. This can cause data corruption when the page 225 * is swap in later. Always setting the dirty bit for the page solves 226 * the problem. 227 */ 228 set_page_dirty(page); 229 230 return 1; 231 232 fail: 233 put_swap_page(page, entry); 234 return 0; 235 } 236 237 /* 238 * This must be called only on pages that have 239 * been verified to be in the swap cache and locked. 240 * It will never put the page into the free list, 241 * the caller has a reference on the page. 242 */ 243 void delete_from_swap_cache(struct page *page) 244 { 245 swp_entry_t entry = { .val = page_private(page) }; 246 struct address_space *address_space = swap_address_space(entry); 247 248 xa_lock_irq(&address_space->i_pages); 249 __delete_from_swap_cache(page, entry); 250 xa_unlock_irq(&address_space->i_pages); 251 252 put_swap_page(page, entry); 253 page_ref_sub(page, hpage_nr_pages(page)); 254 } 255 256 /* 257 * If we are the only user, then try to free up the swap cache. 258 * 259 * Its ok to check for PageSwapCache without the page lock 260 * here because we are going to recheck again inside 261 * try_to_free_swap() _with_ the lock. 262 * - Marcelo 263 */ 264 static inline void free_swap_cache(struct page *page) 265 { 266 if (PageSwapCache(page) && !page_mapped(page) && trylock_page(page)) { 267 try_to_free_swap(page); 268 unlock_page(page); 269 } 270 } 271 272 /* 273 * Perform a free_page(), also freeing any swap cache associated with 274 * this page if it is the last user of the page. 275 */ 276 void free_page_and_swap_cache(struct page *page) 277 { 278 free_swap_cache(page); 279 if (!is_huge_zero_page(page)) 280 put_page(page); 281 } 282 283 /* 284 * Passed an array of pages, drop them all from swapcache and then release 285 * them. They are removed from the LRU and freed if this is their last use. 286 */ 287 void free_pages_and_swap_cache(struct page **pages, int nr) 288 { 289 struct page **pagep = pages; 290 int i; 291 292 lru_add_drain(); 293 for (i = 0; i < nr; i++) 294 free_swap_cache(pagep[i]); 295 release_pages(pagep, nr); 296 } 297 298 static inline bool swap_use_vma_readahead(void) 299 { 300 return READ_ONCE(enable_vma_readahead) && !atomic_read(&nr_rotate_swap); 301 } 302 303 /* 304 * Lookup a swap entry in the swap cache. A found page will be returned 305 * unlocked and with its refcount incremented - we rely on the kernel 306 * lock getting page table operations atomic even if we drop the page 307 * lock before returning. 308 */ 309 struct page *lookup_swap_cache(swp_entry_t entry, struct vm_area_struct *vma, 310 unsigned long addr) 311 { 312 struct page *page; 313 struct swap_info_struct *si; 314 315 si = get_swap_device(entry); 316 if (!si) 317 return NULL; 318 page = find_get_page(swap_address_space(entry), swp_offset(entry)); 319 put_swap_device(si); 320 321 INC_CACHE_INFO(find_total); 322 if (page) { 323 bool vma_ra = swap_use_vma_readahead(); 324 bool readahead; 325 326 INC_CACHE_INFO(find_success); 327 /* 328 * At the moment, we don't support PG_readahead for anon THP 329 * so let's bail out rather than confusing the readahead stat. 330 */ 331 if (unlikely(PageTransCompound(page))) 332 return page; 333 334 readahead = TestClearPageReadahead(page); 335 if (vma && vma_ra) { 336 unsigned long ra_val; 337 int win, hits; 338 339 ra_val = GET_SWAP_RA_VAL(vma); 340 win = SWAP_RA_WIN(ra_val); 341 hits = SWAP_RA_HITS(ra_val); 342 if (readahead) 343 hits = min_t(int, hits + 1, SWAP_RA_HITS_MAX); 344 atomic_long_set(&vma->swap_readahead_info, 345 SWAP_RA_VAL(addr, win, hits)); 346 } 347 348 if (readahead) { 349 count_vm_event(SWAP_RA_HIT); 350 if (!vma || !vma_ra) 351 atomic_inc(&swapin_readahead_hits); 352 } 353 } 354 355 return page; 356 } 357 358 struct page *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, 359 struct vm_area_struct *vma, unsigned long addr, 360 bool *new_page_allocated) 361 { 362 struct swap_info_struct *si; 363 struct page *page; 364 365 *new_page_allocated = false; 366 367 for (;;) { 368 int err; 369 /* 370 * First check the swap cache. Since this is normally 371 * called after lookup_swap_cache() failed, re-calling 372 * that would confuse statistics. 373 */ 374 si = get_swap_device(entry); 375 if (!si) 376 return NULL; 377 page = find_get_page(swap_address_space(entry), 378 swp_offset(entry)); 379 put_swap_device(si); 380 if (page) 381 return page; 382 383 /* 384 * Just skip read ahead for unused swap slot. 385 * During swap_off when swap_slot_cache is disabled, 386 * we have to handle the race between putting 387 * swap entry in swap cache and marking swap slot 388 * as SWAP_HAS_CACHE. That's done in later part of code or 389 * else swap_off will be aborted if we return NULL. 390 */ 391 if (!__swp_swapcount(entry) && swap_slot_cache_enabled) 392 return NULL; 393 394 /* 395 * Get a new page to read into from swap. Allocate it now, 396 * before marking swap_map SWAP_HAS_CACHE, when -EEXIST will 397 * cause any racers to loop around until we add it to cache. 398 */ 399 page = alloc_page_vma(gfp_mask, vma, addr); 400 if (!page) 401 return NULL; 402 403 /* 404 * Swap entry may have been freed since our caller observed it. 405 */ 406 err = swapcache_prepare(entry); 407 if (!err) 408 break; 409 410 put_page(page); 411 if (err != -EEXIST) 412 return NULL; 413 414 /* 415 * We might race against __delete_from_swap_cache(), and 416 * stumble across a swap_map entry whose SWAP_HAS_CACHE 417 * has not yet been cleared. Or race against another 418 * __read_swap_cache_async(), which has set SWAP_HAS_CACHE 419 * in swap_map, but not yet added its page to swap cache. 420 */ 421 cond_resched(); 422 } 423 424 /* 425 * The swap entry is ours to swap in. Prepare the new page. 426 */ 427 428 __SetPageLocked(page); 429 __SetPageSwapBacked(page); 430 431 /* May fail (-ENOMEM) if XArray node allocation failed. */ 432 if (add_to_swap_cache(page, entry, gfp_mask & GFP_RECLAIM_MASK)) { 433 put_swap_page(page, entry); 434 goto fail_unlock; 435 } 436 437 if (mem_cgroup_charge(page, NULL, gfp_mask)) { 438 delete_from_swap_cache(page); 439 goto fail_unlock; 440 } 441 442 /* XXX: Move to lru_cache_add() when it supports new vs putback */ 443 spin_lock_irq(&page_pgdat(page)->lru_lock); 444 lru_note_cost_page(page); 445 spin_unlock_irq(&page_pgdat(page)->lru_lock); 446 447 /* Caller will initiate read into locked page */ 448 SetPageWorkingset(page); 449 lru_cache_add(page); 450 *new_page_allocated = true; 451 return page; 452 453 fail_unlock: 454 unlock_page(page); 455 put_page(page); 456 return NULL; 457 } 458 459 /* 460 * Locate a page of swap in physical memory, reserving swap cache space 461 * and reading the disk if it is not already cached. 462 * A failure return means that either the page allocation failed or that 463 * the swap entry is no longer in use. 464 */ 465 struct page *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, 466 struct vm_area_struct *vma, unsigned long addr, bool do_poll) 467 { 468 bool page_was_allocated; 469 struct page *retpage = __read_swap_cache_async(entry, gfp_mask, 470 vma, addr, &page_was_allocated); 471 472 if (page_was_allocated) 473 swap_readpage(retpage, do_poll); 474 475 return retpage; 476 } 477 478 static unsigned int __swapin_nr_pages(unsigned long prev_offset, 479 unsigned long offset, 480 int hits, 481 int max_pages, 482 int prev_win) 483 { 484 unsigned int pages, last_ra; 485 486 /* 487 * This heuristic has been found to work well on both sequential and 488 * random loads, swapping to hard disk or to SSD: please don't ask 489 * what the "+ 2" means, it just happens to work well, that's all. 490 */ 491 pages = hits + 2; 492 if (pages == 2) { 493 /* 494 * We can have no readahead hits to judge by: but must not get 495 * stuck here forever, so check for an adjacent offset instead 496 * (and don't even bother to check whether swap type is same). 497 */ 498 if (offset != prev_offset + 1 && offset != prev_offset - 1) 499 pages = 1; 500 } else { 501 unsigned int roundup = 4; 502 while (roundup < pages) 503 roundup <<= 1; 504 pages = roundup; 505 } 506 507 if (pages > max_pages) 508 pages = max_pages; 509 510 /* Don't shrink readahead too fast */ 511 last_ra = prev_win / 2; 512 if (pages < last_ra) 513 pages = last_ra; 514 515 return pages; 516 } 517 518 static unsigned long swapin_nr_pages(unsigned long offset) 519 { 520 static unsigned long prev_offset; 521 unsigned int hits, pages, max_pages; 522 static atomic_t last_readahead_pages; 523 524 max_pages = 1 << READ_ONCE(page_cluster); 525 if (max_pages <= 1) 526 return 1; 527 528 hits = atomic_xchg(&swapin_readahead_hits, 0); 529 pages = __swapin_nr_pages(READ_ONCE(prev_offset), offset, hits, 530 max_pages, 531 atomic_read(&last_readahead_pages)); 532 if (!hits) 533 WRITE_ONCE(prev_offset, offset); 534 atomic_set(&last_readahead_pages, pages); 535 536 return pages; 537 } 538 539 /** 540 * swap_cluster_readahead - swap in pages in hope we need them soon 541 * @entry: swap entry of this memory 542 * @gfp_mask: memory allocation flags 543 * @vmf: fault information 544 * 545 * Returns the struct page for entry and addr, after queueing swapin. 546 * 547 * Primitive swap readahead code. We simply read an aligned block of 548 * (1 << page_cluster) entries in the swap area. This method is chosen 549 * because it doesn't cost us any seek time. We also make sure to queue 550 * the 'original' request together with the readahead ones... 551 * 552 * This has been extended to use the NUMA policies from the mm triggering 553 * the readahead. 554 * 555 * Caller must hold read mmap_lock if vmf->vma is not NULL. 556 */ 557 struct page *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask, 558 struct vm_fault *vmf) 559 { 560 struct page *page; 561 unsigned long entry_offset = swp_offset(entry); 562 unsigned long offset = entry_offset; 563 unsigned long start_offset, end_offset; 564 unsigned long mask; 565 struct swap_info_struct *si = swp_swap_info(entry); 566 struct blk_plug plug; 567 bool do_poll = true, page_allocated; 568 struct vm_area_struct *vma = vmf->vma; 569 unsigned long addr = vmf->address; 570 571 mask = swapin_nr_pages(offset) - 1; 572 if (!mask) 573 goto skip; 574 575 /* Test swap type to make sure the dereference is safe */ 576 if (likely(si->flags & (SWP_BLKDEV | SWP_FS))) { 577 struct inode *inode = si->swap_file->f_mapping->host; 578 if (inode_read_congested(inode)) 579 goto skip; 580 } 581 582 do_poll = false; 583 /* Read a page_cluster sized and aligned cluster around offset. */ 584 start_offset = offset & ~mask; 585 end_offset = offset | mask; 586 if (!start_offset) /* First page is swap header. */ 587 start_offset++; 588 if (end_offset >= si->max) 589 end_offset = si->max - 1; 590 591 blk_start_plug(&plug); 592 for (offset = start_offset; offset <= end_offset ; offset++) { 593 /* Ok, do the async read-ahead now */ 594 page = __read_swap_cache_async( 595 swp_entry(swp_type(entry), offset), 596 gfp_mask, vma, addr, &page_allocated); 597 if (!page) 598 continue; 599 if (page_allocated) { 600 swap_readpage(page, false); 601 if (offset != entry_offset) { 602 SetPageReadahead(page); 603 count_vm_event(SWAP_RA); 604 } 605 } 606 put_page(page); 607 } 608 blk_finish_plug(&plug); 609 610 lru_add_drain(); /* Push any new pages onto the LRU now */ 611 skip: 612 return read_swap_cache_async(entry, gfp_mask, vma, addr, do_poll); 613 } 614 615 int init_swap_address_space(unsigned int type, unsigned long nr_pages) 616 { 617 struct address_space *spaces, *space; 618 unsigned int i, nr; 619 620 nr = DIV_ROUND_UP(nr_pages, SWAP_ADDRESS_SPACE_PAGES); 621 spaces = kvcalloc(nr, sizeof(struct address_space), GFP_KERNEL); 622 if (!spaces) 623 return -ENOMEM; 624 for (i = 0; i < nr; i++) { 625 space = spaces + i; 626 xa_init_flags(&space->i_pages, XA_FLAGS_LOCK_IRQ); 627 atomic_set(&space->i_mmap_writable, 0); 628 space->a_ops = &swap_aops; 629 /* swap cache doesn't use writeback related tags */ 630 mapping_set_no_writeback_tags(space); 631 } 632 nr_swapper_spaces[type] = nr; 633 swapper_spaces[type] = spaces; 634 635 return 0; 636 } 637 638 void exit_swap_address_space(unsigned int type) 639 { 640 kvfree(swapper_spaces[type]); 641 nr_swapper_spaces[type] = 0; 642 swapper_spaces[type] = NULL; 643 } 644 645 static inline void swap_ra_clamp_pfn(struct vm_area_struct *vma, 646 unsigned long faddr, 647 unsigned long lpfn, 648 unsigned long rpfn, 649 unsigned long *start, 650 unsigned long *end) 651 { 652 *start = max3(lpfn, PFN_DOWN(vma->vm_start), 653 PFN_DOWN(faddr & PMD_MASK)); 654 *end = min3(rpfn, PFN_DOWN(vma->vm_end), 655 PFN_DOWN((faddr & PMD_MASK) + PMD_SIZE)); 656 } 657 658 static void swap_ra_info(struct vm_fault *vmf, 659 struct vma_swap_readahead *ra_info) 660 { 661 struct vm_area_struct *vma = vmf->vma; 662 unsigned long ra_val; 663 swp_entry_t entry; 664 unsigned long faddr, pfn, fpfn; 665 unsigned long start, end; 666 pte_t *pte, *orig_pte; 667 unsigned int max_win, hits, prev_win, win, left; 668 #ifndef CONFIG_64BIT 669 pte_t *tpte; 670 #endif 671 672 max_win = 1 << min_t(unsigned int, READ_ONCE(page_cluster), 673 SWAP_RA_ORDER_CEILING); 674 if (max_win == 1) { 675 ra_info->win = 1; 676 return; 677 } 678 679 faddr = vmf->address; 680 orig_pte = pte = pte_offset_map(vmf->pmd, faddr); 681 entry = pte_to_swp_entry(*pte); 682 if ((unlikely(non_swap_entry(entry)))) { 683 pte_unmap(orig_pte); 684 return; 685 } 686 687 fpfn = PFN_DOWN(faddr); 688 ra_val = GET_SWAP_RA_VAL(vma); 689 pfn = PFN_DOWN(SWAP_RA_ADDR(ra_val)); 690 prev_win = SWAP_RA_WIN(ra_val); 691 hits = SWAP_RA_HITS(ra_val); 692 ra_info->win = win = __swapin_nr_pages(pfn, fpfn, hits, 693 max_win, prev_win); 694 atomic_long_set(&vma->swap_readahead_info, 695 SWAP_RA_VAL(faddr, win, 0)); 696 697 if (win == 1) { 698 pte_unmap(orig_pte); 699 return; 700 } 701 702 /* Copy the PTEs because the page table may be unmapped */ 703 if (fpfn == pfn + 1) 704 swap_ra_clamp_pfn(vma, faddr, fpfn, fpfn + win, &start, &end); 705 else if (pfn == fpfn + 1) 706 swap_ra_clamp_pfn(vma, faddr, fpfn - win + 1, fpfn + 1, 707 &start, &end); 708 else { 709 left = (win - 1) / 2; 710 swap_ra_clamp_pfn(vma, faddr, fpfn - left, fpfn + win - left, 711 &start, &end); 712 } 713 ra_info->nr_pte = end - start; 714 ra_info->offset = fpfn - start; 715 pte -= ra_info->offset; 716 #ifdef CONFIG_64BIT 717 ra_info->ptes = pte; 718 #else 719 tpte = ra_info->ptes; 720 for (pfn = start; pfn != end; pfn++) 721 *tpte++ = *pte++; 722 #endif 723 pte_unmap(orig_pte); 724 } 725 726 /** 727 * swap_vma_readahead - swap in pages in hope we need them soon 728 * @entry: swap entry of this memory 729 * @gfp_mask: memory allocation flags 730 * @vmf: fault information 731 * 732 * Returns the struct page for entry and addr, after queueing swapin. 733 * 734 * Primitive swap readahead code. We simply read in a few pages whoes 735 * virtual addresses are around the fault address in the same vma. 736 * 737 * Caller must hold read mmap_lock if vmf->vma is not NULL. 738 * 739 */ 740 static struct page *swap_vma_readahead(swp_entry_t fentry, gfp_t gfp_mask, 741 struct vm_fault *vmf) 742 { 743 struct blk_plug plug; 744 struct vm_area_struct *vma = vmf->vma; 745 struct page *page; 746 pte_t *pte, pentry; 747 swp_entry_t entry; 748 unsigned int i; 749 bool page_allocated; 750 struct vma_swap_readahead ra_info = {0,}; 751 752 swap_ra_info(vmf, &ra_info); 753 if (ra_info.win == 1) 754 goto skip; 755 756 blk_start_plug(&plug); 757 for (i = 0, pte = ra_info.ptes; i < ra_info.nr_pte; 758 i++, pte++) { 759 pentry = *pte; 760 if (pte_none(pentry)) 761 continue; 762 if (pte_present(pentry)) 763 continue; 764 entry = pte_to_swp_entry(pentry); 765 if (unlikely(non_swap_entry(entry))) 766 continue; 767 page = __read_swap_cache_async(entry, gfp_mask, vma, 768 vmf->address, &page_allocated); 769 if (!page) 770 continue; 771 if (page_allocated) { 772 swap_readpage(page, false); 773 if (i != ra_info.offset) { 774 SetPageReadahead(page); 775 count_vm_event(SWAP_RA); 776 } 777 } 778 put_page(page); 779 } 780 blk_finish_plug(&plug); 781 lru_add_drain(); 782 skip: 783 return read_swap_cache_async(fentry, gfp_mask, vma, vmf->address, 784 ra_info.win == 1); 785 } 786 787 /** 788 * swapin_readahead - swap in pages in hope we need them soon 789 * @entry: swap entry of this memory 790 * @gfp_mask: memory allocation flags 791 * @vmf: fault information 792 * 793 * Returns the struct page for entry and addr, after queueing swapin. 794 * 795 * It's a main entry function for swap readahead. By the configuration, 796 * it will read ahead blocks by cluster-based(ie, physical disk based) 797 * or vma-based(ie, virtual address based on faulty address) readahead. 798 */ 799 struct page *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask, 800 struct vm_fault *vmf) 801 { 802 return swap_use_vma_readahead() ? 803 swap_vma_readahead(entry, gfp_mask, vmf) : 804 swap_cluster_readahead(entry, gfp_mask, vmf); 805 } 806 807 #ifdef CONFIG_SYSFS 808 static ssize_t vma_ra_enabled_show(struct kobject *kobj, 809 struct kobj_attribute *attr, char *buf) 810 { 811 return sprintf(buf, "%s\n", enable_vma_readahead ? "true" : "false"); 812 } 813 static ssize_t vma_ra_enabled_store(struct kobject *kobj, 814 struct kobj_attribute *attr, 815 const char *buf, size_t count) 816 { 817 if (!strncmp(buf, "true", 4) || !strncmp(buf, "1", 1)) 818 enable_vma_readahead = true; 819 else if (!strncmp(buf, "false", 5) || !strncmp(buf, "0", 1)) 820 enable_vma_readahead = false; 821 else 822 return -EINVAL; 823 824 return count; 825 } 826 static struct kobj_attribute vma_ra_enabled_attr = 827 __ATTR(vma_ra_enabled, 0644, vma_ra_enabled_show, 828 vma_ra_enabled_store); 829 830 static struct attribute *swap_attrs[] = { 831 &vma_ra_enabled_attr.attr, 832 NULL, 833 }; 834 835 static struct attribute_group swap_attr_group = { 836 .attrs = swap_attrs, 837 }; 838 839 static int __init swap_init_sysfs(void) 840 { 841 int err; 842 struct kobject *swap_kobj; 843 844 swap_kobj = kobject_create_and_add("swap", mm_kobj); 845 if (!swap_kobj) { 846 pr_err("failed to create swap kobject\n"); 847 return -ENOMEM; 848 } 849 err = sysfs_create_group(swap_kobj, &swap_attr_group); 850 if (err) { 851 pr_err("failed to register swap group\n"); 852 goto delete_obj; 853 } 854 return 0; 855 856 delete_obj: 857 kobject_put(swap_kobj); 858 return err; 859 } 860 subsys_initcall(swap_init_sysfs); 861 #endif 862