1 /* 2 * mm/mprotect.c 3 * 4 * (C) Copyright 1994 Linus Torvalds 5 * (C) Copyright 2002 Christoph Hellwig 6 * 7 * Address space accounting code <alan@lxorguk.ukuu.org.uk> 8 * (C) Copyright 2002 Red Hat Inc, All Rights Reserved 9 */ 10 11 #include <linux/mm.h> 12 #include <linux/hugetlb.h> 13 #include <linux/shm.h> 14 #include <linux/mman.h> 15 #include <linux/fs.h> 16 #include <linux/highmem.h> 17 #include <linux/security.h> 18 #include <linux/mempolicy.h> 19 #include <linux/personality.h> 20 #include <linux/syscalls.h> 21 #include <linux/swap.h> 22 #include <linux/swapops.h> 23 #include <linux/mmu_notifier.h> 24 #include <linux/migrate.h> 25 #include <linux/perf_event.h> 26 #include <linux/pkeys.h> 27 #include <linux/ksm.h> 28 #include <linux/uaccess.h> 29 #include <asm/pgtable.h> 30 #include <asm/cacheflush.h> 31 #include <asm/mmu_context.h> 32 #include <asm/tlbflush.h> 33 34 #include "internal.h" 35 36 /* 37 * For a prot_numa update we only hold mmap_sem for read so there is a 38 * potential race with faulting where a pmd was temporarily none. This 39 * function checks for a transhuge pmd under the appropriate lock. It 40 * returns a pte if it was successfully locked or NULL if it raced with 41 * a transhuge insertion. 42 */ 43 static pte_t *lock_pte_protection(struct vm_area_struct *vma, pmd_t *pmd, 44 unsigned long addr, int prot_numa, spinlock_t **ptl) 45 { 46 pte_t *pte; 47 spinlock_t *pmdl; 48 49 /* !prot_numa is protected by mmap_sem held for write */ 50 if (!prot_numa) 51 return pte_offset_map_lock(vma->vm_mm, pmd, addr, ptl); 52 53 pmdl = pmd_lock(vma->vm_mm, pmd); 54 if (unlikely(pmd_trans_huge(*pmd) || pmd_none(*pmd))) { 55 spin_unlock(pmdl); 56 return NULL; 57 } 58 59 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, ptl); 60 spin_unlock(pmdl); 61 return pte; 62 } 63 64 static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, 65 unsigned long addr, unsigned long end, pgprot_t newprot, 66 int dirty_accountable, int prot_numa) 67 { 68 struct mm_struct *mm = vma->vm_mm; 69 pte_t *pte, oldpte; 70 spinlock_t *ptl; 71 unsigned long pages = 0; 72 int target_node = NUMA_NO_NODE; 73 74 pte = lock_pte_protection(vma, pmd, addr, prot_numa, &ptl); 75 if (!pte) 76 return 0; 77 78 /* Get target node for single threaded private VMAs */ 79 if (prot_numa && !(vma->vm_flags & VM_SHARED) && 80 atomic_read(&vma->vm_mm->mm_users) == 1) 81 target_node = numa_node_id(); 82 83 arch_enter_lazy_mmu_mode(); 84 do { 85 oldpte = *pte; 86 if (pte_present(oldpte)) { 87 pte_t ptent; 88 bool preserve_write = prot_numa && pte_write(oldpte); 89 90 /* 91 * Avoid trapping faults against the zero or KSM 92 * pages. See similar comment in change_huge_pmd. 93 */ 94 if (prot_numa) { 95 struct page *page; 96 97 page = vm_normal_page(vma, addr, oldpte); 98 if (!page || PageKsm(page)) 99 continue; 100 101 /* Avoid TLB flush if possible */ 102 if (pte_protnone(oldpte)) 103 continue; 104 105 /* 106 * Don't mess with PTEs if page is already on the node 107 * a single-threaded process is running on. 108 */ 109 if (target_node == page_to_nid(page)) 110 continue; 111 } 112 113 ptent = ptep_modify_prot_start(mm, addr, pte); 114 ptent = pte_modify(ptent, newprot); 115 if (preserve_write) 116 ptent = pte_mkwrite(ptent); 117 118 /* Avoid taking write faults for known dirty pages */ 119 if (dirty_accountable && pte_dirty(ptent) && 120 (pte_soft_dirty(ptent) || 121 !(vma->vm_flags & VM_SOFTDIRTY))) { 122 ptent = pte_mkwrite(ptent); 123 } 124 ptep_modify_prot_commit(mm, addr, pte, ptent); 125 pages++; 126 } else if (IS_ENABLED(CONFIG_MIGRATION)) { 127 swp_entry_t entry = pte_to_swp_entry(oldpte); 128 129 if (is_write_migration_entry(entry)) { 130 pte_t newpte; 131 /* 132 * A protection check is difficult so 133 * just be safe and disable write 134 */ 135 make_migration_entry_read(&entry); 136 newpte = swp_entry_to_pte(entry); 137 if (pte_swp_soft_dirty(oldpte)) 138 newpte = pte_swp_mksoft_dirty(newpte); 139 set_pte_at(mm, addr, pte, newpte); 140 141 pages++; 142 } 143 } 144 } while (pte++, addr += PAGE_SIZE, addr != end); 145 arch_leave_lazy_mmu_mode(); 146 pte_unmap_unlock(pte - 1, ptl); 147 148 return pages; 149 } 150 151 static inline unsigned long change_pmd_range(struct vm_area_struct *vma, 152 pud_t *pud, unsigned long addr, unsigned long end, 153 pgprot_t newprot, int dirty_accountable, int prot_numa) 154 { 155 pmd_t *pmd; 156 struct mm_struct *mm = vma->vm_mm; 157 unsigned long next; 158 unsigned long pages = 0; 159 unsigned long nr_huge_updates = 0; 160 unsigned long mni_start = 0; 161 162 pmd = pmd_offset(pud, addr); 163 do { 164 unsigned long this_pages; 165 166 next = pmd_addr_end(addr, end); 167 if (!pmd_trans_huge(*pmd) && !pmd_devmap(*pmd) 168 && pmd_none_or_clear_bad(pmd)) 169 continue; 170 171 /* invoke the mmu notifier if the pmd is populated */ 172 if (!mni_start) { 173 mni_start = addr; 174 mmu_notifier_invalidate_range_start(mm, mni_start, end); 175 } 176 177 if (pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) { 178 if (next - addr != HPAGE_PMD_SIZE) { 179 __split_huge_pmd(vma, pmd, addr, false, NULL); 180 if (pmd_trans_unstable(pmd)) 181 continue; 182 } else { 183 int nr_ptes = change_huge_pmd(vma, pmd, addr, 184 newprot, prot_numa); 185 186 if (nr_ptes) { 187 if (nr_ptes == HPAGE_PMD_NR) { 188 pages += HPAGE_PMD_NR; 189 nr_huge_updates++; 190 } 191 192 /* huge pmd was handled */ 193 continue; 194 } 195 } 196 /* fall through, the trans huge pmd just split */ 197 } 198 this_pages = change_pte_range(vma, pmd, addr, next, newprot, 199 dirty_accountable, prot_numa); 200 pages += this_pages; 201 } while (pmd++, addr = next, addr != end); 202 203 if (mni_start) 204 mmu_notifier_invalidate_range_end(mm, mni_start, end); 205 206 if (nr_huge_updates) 207 count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates); 208 return pages; 209 } 210 211 static inline unsigned long change_pud_range(struct vm_area_struct *vma, 212 pgd_t *pgd, unsigned long addr, unsigned long end, 213 pgprot_t newprot, int dirty_accountable, int prot_numa) 214 { 215 pud_t *pud; 216 unsigned long next; 217 unsigned long pages = 0; 218 219 pud = pud_offset(pgd, addr); 220 do { 221 next = pud_addr_end(addr, end); 222 if (pud_none_or_clear_bad(pud)) 223 continue; 224 pages += change_pmd_range(vma, pud, addr, next, newprot, 225 dirty_accountable, prot_numa); 226 } while (pud++, addr = next, addr != end); 227 228 return pages; 229 } 230 231 static unsigned long change_protection_range(struct vm_area_struct *vma, 232 unsigned long addr, unsigned long end, pgprot_t newprot, 233 int dirty_accountable, int prot_numa) 234 { 235 struct mm_struct *mm = vma->vm_mm; 236 pgd_t *pgd; 237 unsigned long next; 238 unsigned long start = addr; 239 unsigned long pages = 0; 240 241 BUG_ON(addr >= end); 242 pgd = pgd_offset(mm, addr); 243 flush_cache_range(vma, addr, end); 244 set_tlb_flush_pending(mm); 245 do { 246 next = pgd_addr_end(addr, end); 247 if (pgd_none_or_clear_bad(pgd)) 248 continue; 249 pages += change_pud_range(vma, pgd, addr, next, newprot, 250 dirty_accountable, prot_numa); 251 } while (pgd++, addr = next, addr != end); 252 253 /* Only flush the TLB if we actually modified any entries: */ 254 if (pages) 255 flush_tlb_range(vma, start, end); 256 clear_tlb_flush_pending(mm); 257 258 return pages; 259 } 260 261 unsigned long change_protection(struct vm_area_struct *vma, unsigned long start, 262 unsigned long end, pgprot_t newprot, 263 int dirty_accountable, int prot_numa) 264 { 265 unsigned long pages; 266 267 if (is_vm_hugetlb_page(vma)) 268 pages = hugetlb_change_protection(vma, start, end, newprot); 269 else 270 pages = change_protection_range(vma, start, end, newprot, dirty_accountable, prot_numa); 271 272 return pages; 273 } 274 275 int 276 mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev, 277 unsigned long start, unsigned long end, unsigned long newflags) 278 { 279 struct mm_struct *mm = vma->vm_mm; 280 unsigned long oldflags = vma->vm_flags; 281 long nrpages = (end - start) >> PAGE_SHIFT; 282 unsigned long charged = 0; 283 pgoff_t pgoff; 284 int error; 285 int dirty_accountable = 0; 286 287 if (newflags == oldflags) { 288 *pprev = vma; 289 return 0; 290 } 291 292 /* 293 * If we make a private mapping writable we increase our commit; 294 * but (without finer accounting) cannot reduce our commit if we 295 * make it unwritable again. hugetlb mapping were accounted for 296 * even if read-only so there is no need to account for them here 297 */ 298 if (newflags & VM_WRITE) { 299 /* Check space limits when area turns into data. */ 300 if (!may_expand_vm(mm, newflags, nrpages) && 301 may_expand_vm(mm, oldflags, nrpages)) 302 return -ENOMEM; 303 if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB| 304 VM_SHARED|VM_NORESERVE))) { 305 charged = nrpages; 306 if (security_vm_enough_memory_mm(mm, charged)) 307 return -ENOMEM; 308 newflags |= VM_ACCOUNT; 309 } 310 } 311 312 /* 313 * First try to merge with previous and/or next vma. 314 */ 315 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); 316 *pprev = vma_merge(mm, *pprev, start, end, newflags, 317 vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), 318 vma->vm_userfaultfd_ctx); 319 if (*pprev) { 320 vma = *pprev; 321 VM_WARN_ON((vma->vm_flags ^ newflags) & ~VM_SOFTDIRTY); 322 goto success; 323 } 324 325 *pprev = vma; 326 327 if (start != vma->vm_start) { 328 error = split_vma(mm, vma, start, 1); 329 if (error) 330 goto fail; 331 } 332 333 if (end != vma->vm_end) { 334 error = split_vma(mm, vma, end, 0); 335 if (error) 336 goto fail; 337 } 338 339 success: 340 /* 341 * vm_flags and vm_page_prot are protected by the mmap_sem 342 * held in write mode. 343 */ 344 vma->vm_flags = newflags; 345 dirty_accountable = vma_wants_writenotify(vma, vma->vm_page_prot); 346 vma_set_page_prot(vma); 347 348 change_protection(vma, start, end, vma->vm_page_prot, 349 dirty_accountable, 0); 350 351 /* 352 * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major 353 * fault on access. 354 */ 355 if ((oldflags & (VM_WRITE | VM_SHARED | VM_LOCKED)) == VM_LOCKED && 356 (newflags & VM_WRITE)) { 357 populate_vma_page_range(vma, start, end, NULL); 358 } 359 360 vm_stat_account(mm, oldflags, -nrpages); 361 vm_stat_account(mm, newflags, nrpages); 362 perf_event_mmap(vma); 363 return 0; 364 365 fail: 366 vm_unacct_memory(charged); 367 return error; 368 } 369 370 /* 371 * pkey==-1 when doing a legacy mprotect() 372 */ 373 static int do_mprotect_pkey(unsigned long start, size_t len, 374 unsigned long prot, int pkey) 375 { 376 unsigned long nstart, end, tmp, reqprot; 377 struct vm_area_struct *vma, *prev; 378 int error = -EINVAL; 379 const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP); 380 const bool rier = (current->personality & READ_IMPLIES_EXEC) && 381 (prot & PROT_READ); 382 383 prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP); 384 if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */ 385 return -EINVAL; 386 387 if (start & ~PAGE_MASK) 388 return -EINVAL; 389 if (!len) 390 return 0; 391 len = PAGE_ALIGN(len); 392 end = start + len; 393 if (end <= start) 394 return -ENOMEM; 395 if (!arch_validate_prot(prot)) 396 return -EINVAL; 397 398 reqprot = prot; 399 400 if (down_write_killable(¤t->mm->mmap_sem)) 401 return -EINTR; 402 403 /* 404 * If userspace did not allocate the pkey, do not let 405 * them use it here. 406 */ 407 error = -EINVAL; 408 if ((pkey != -1) && !mm_pkey_is_allocated(current->mm, pkey)) 409 goto out; 410 411 vma = find_vma(current->mm, start); 412 error = -ENOMEM; 413 if (!vma) 414 goto out; 415 prev = vma->vm_prev; 416 if (unlikely(grows & PROT_GROWSDOWN)) { 417 if (vma->vm_start >= end) 418 goto out; 419 start = vma->vm_start; 420 error = -EINVAL; 421 if (!(vma->vm_flags & VM_GROWSDOWN)) 422 goto out; 423 } else { 424 if (vma->vm_start > start) 425 goto out; 426 if (unlikely(grows & PROT_GROWSUP)) { 427 end = vma->vm_end; 428 error = -EINVAL; 429 if (!(vma->vm_flags & VM_GROWSUP)) 430 goto out; 431 } 432 } 433 if (start > vma->vm_start) 434 prev = vma; 435 436 for (nstart = start ; ; ) { 437 unsigned long mask_off_old_flags; 438 unsigned long newflags; 439 int new_vma_pkey; 440 441 /* Here we know that vma->vm_start <= nstart < vma->vm_end. */ 442 443 /* Does the application expect PROT_READ to imply PROT_EXEC */ 444 if (rier && (vma->vm_flags & VM_MAYEXEC)) 445 prot |= PROT_EXEC; 446 447 /* 448 * Each mprotect() call explicitly passes r/w/x permissions. 449 * If a permission is not passed to mprotect(), it must be 450 * cleared from the VMA. 451 */ 452 mask_off_old_flags = VM_READ | VM_WRITE | VM_EXEC | 453 ARCH_VM_PKEY_FLAGS; 454 455 new_vma_pkey = arch_override_mprotect_pkey(vma, prot, pkey); 456 newflags = calc_vm_prot_bits(prot, new_vma_pkey); 457 newflags |= (vma->vm_flags & ~mask_off_old_flags); 458 459 /* newflags >> 4 shift VM_MAY% in place of VM_% */ 460 if ((newflags & ~(newflags >> 4)) & (VM_READ | VM_WRITE | VM_EXEC)) { 461 error = -EACCES; 462 goto out; 463 } 464 465 error = security_file_mprotect(vma, reqprot, prot); 466 if (error) 467 goto out; 468 469 tmp = vma->vm_end; 470 if (tmp > end) 471 tmp = end; 472 error = mprotect_fixup(vma, &prev, nstart, tmp, newflags); 473 if (error) 474 goto out; 475 nstart = tmp; 476 477 if (nstart < prev->vm_end) 478 nstart = prev->vm_end; 479 if (nstart >= end) 480 goto out; 481 482 vma = prev->vm_next; 483 if (!vma || vma->vm_start != nstart) { 484 error = -ENOMEM; 485 goto out; 486 } 487 prot = reqprot; 488 } 489 out: 490 up_write(¤t->mm->mmap_sem); 491 return error; 492 } 493 494 SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len, 495 unsigned long, prot) 496 { 497 return do_mprotect_pkey(start, len, prot, -1); 498 } 499 500 #ifdef CONFIG_ARCH_HAS_PKEYS 501 502 SYSCALL_DEFINE4(pkey_mprotect, unsigned long, start, size_t, len, 503 unsigned long, prot, int, pkey) 504 { 505 return do_mprotect_pkey(start, len, prot, pkey); 506 } 507 508 SYSCALL_DEFINE2(pkey_alloc, unsigned long, flags, unsigned long, init_val) 509 { 510 int pkey; 511 int ret; 512 513 /* No flags supported yet. */ 514 if (flags) 515 return -EINVAL; 516 /* check for unsupported init values */ 517 if (init_val & ~PKEY_ACCESS_MASK) 518 return -EINVAL; 519 520 down_write(¤t->mm->mmap_sem); 521 pkey = mm_pkey_alloc(current->mm); 522 523 ret = -ENOSPC; 524 if (pkey == -1) 525 goto out; 526 527 ret = arch_set_user_pkey_access(current, pkey, init_val); 528 if (ret) { 529 mm_pkey_free(current->mm, pkey); 530 goto out; 531 } 532 ret = pkey; 533 out: 534 up_write(¤t->mm->mmap_sem); 535 return ret; 536 } 537 538 SYSCALL_DEFINE1(pkey_free, int, pkey) 539 { 540 int ret; 541 542 down_write(¤t->mm->mmap_sem); 543 ret = mm_pkey_free(current->mm, pkey); 544 up_write(¤t->mm->mmap_sem); 545 546 /* 547 * We could provie warnings or errors if any VMA still 548 * has the pkey set here. 549 */ 550 return ret; 551 } 552 553 #endif /* CONFIG_ARCH_HAS_PKEYS */ 554