1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright (c) 2015, Joyent, Inc. All rights reserved. 24 * Copyright (c) 2016 by Delphix. All rights reserved. 25 */ 26 27 #include <sys/param.h> 28 #include <sys/user.h> 29 #include <sys/mman.h> 30 #include <sys/kmem.h> 31 #include <sys/sysmacros.h> 32 #include <sys/cmn_err.h> 33 #include <sys/systm.h> 34 #include <sys/tuneable.h> 35 #include <vm/hat.h> 36 #include <vm/seg.h> 37 #include <vm/as.h> 38 #include <vm/anon.h> 39 #include <vm/page.h> 40 #include <sys/buf.h> 41 #include <sys/swap.h> 42 #include <sys/atomic.h> 43 #include <vm/seg_spt.h> 44 #include <sys/debug.h> 45 #include <sys/vtrace.h> 46 #include <sys/shm.h> 47 #include <sys/shm_impl.h> 48 #include <sys/lgrp.h> 49 #include <sys/vmsystm.h> 50 #include <sys/policy.h> 51 #include <sys/project.h> 52 #include <sys/tnf_probe.h> 53 #include <sys/zone.h> 54 55 #define SEGSPTADDR (caddr_t)0x0 56 57 /* 58 * # pages used for spt 59 */ 60 size_t spt_used; 61 62 /* 63 * segspt_minfree is the memory left for system after ISM 64 * locked its pages; it is set up to 5% of availrmem in 65 * sptcreate when ISM is created. ISM should not use more 66 * than ~90% of availrmem; if it does, then the performance 67 * of the system may decrease. Machines with large memories may 68 * be able to use up more memory for ISM so we set the default 69 * segspt_minfree to 5% (which gives ISM max 95% of availrmem. 70 * If somebody wants even more memory for ISM (risking hanging 71 * the system) they can patch the segspt_minfree to smaller number. 72 */ 73 pgcnt_t segspt_minfree = 0; 74 75 static int segspt_create(struct seg *seg, caddr_t argsp); 76 static int segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize); 77 static void segspt_free(struct seg *seg); 78 static void segspt_free_pages(struct seg *seg, caddr_t addr, size_t len); 79 static lgrp_mem_policy_info_t *segspt_getpolicy(struct seg *seg, caddr_t addr); 80 81 static void 82 segspt_badop() 83 { 84 panic("segspt_badop called"); 85 /*NOTREACHED*/ 86 } 87 88 #define SEGSPT_BADOP(t) (t(*)())segspt_badop 89 90 struct seg_ops segspt_ops = { 91 SEGSPT_BADOP(int), /* dup */ 92 segspt_unmap, 93 segspt_free, 94 SEGSPT_BADOP(int), /* fault */ 95 SEGSPT_BADOP(faultcode_t), /* faulta */ 96 SEGSPT_BADOP(int), /* setprot */ 97 SEGSPT_BADOP(int), /* checkprot */ 98 SEGSPT_BADOP(int), /* kluster */ 99 SEGSPT_BADOP(size_t), /* swapout */ 100 SEGSPT_BADOP(int), /* sync */ 101 SEGSPT_BADOP(size_t), /* incore */ 102 SEGSPT_BADOP(int), /* lockop */ 103 SEGSPT_BADOP(int), /* getprot */ 104 SEGSPT_BADOP(u_offset_t), /* getoffset */ 105 SEGSPT_BADOP(int), /* gettype */ 106 SEGSPT_BADOP(int), /* getvp */ 107 SEGSPT_BADOP(int), /* advise */ 108 SEGSPT_BADOP(void), /* dump */ 109 SEGSPT_BADOP(int), /* pagelock */ 110 SEGSPT_BADOP(int), /* setpgsz */ 111 SEGSPT_BADOP(int), /* getmemid */ 112 segspt_getpolicy, /* getpolicy */ 113 SEGSPT_BADOP(int), /* capable */ 114 seg_inherit_notsup /* inherit */ 115 }; 116 117 static int segspt_shmdup(struct seg *seg, struct seg *newseg); 118 static int segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize); 119 static void segspt_shmfree(struct seg *seg); 120 static faultcode_t segspt_shmfault(struct hat *hat, struct seg *seg, 121 caddr_t addr, size_t len, enum fault_type type, enum seg_rw rw); 122 static faultcode_t segspt_shmfaulta(struct seg *seg, caddr_t addr); 123 static int segspt_shmsetprot(register struct seg *seg, register caddr_t addr, 124 register size_t len, register uint_t prot); 125 static int segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size, 126 uint_t prot); 127 static int segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta); 128 static size_t segspt_shmswapout(struct seg *seg); 129 static size_t segspt_shmincore(struct seg *seg, caddr_t addr, size_t len, 130 register char *vec); 131 static int segspt_shmsync(struct seg *seg, register caddr_t addr, size_t len, 132 int attr, uint_t flags); 133 static int segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len, 134 int attr, int op, ulong_t *lockmap, size_t pos); 135 static int segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len, 136 uint_t *protv); 137 static u_offset_t segspt_shmgetoffset(struct seg *seg, caddr_t addr); 138 static int segspt_shmgettype(struct seg *seg, caddr_t addr); 139 static int segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp); 140 static int segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len, 141 uint_t behav); 142 static void segspt_shmdump(struct seg *seg); 143 static int segspt_shmpagelock(struct seg *, caddr_t, size_t, 144 struct page ***, enum lock_type, enum seg_rw); 145 static int segspt_shmsetpgsz(struct seg *, caddr_t, size_t, uint_t); 146 static int segspt_shmgetmemid(struct seg *, caddr_t, memid_t *); 147 static lgrp_mem_policy_info_t *segspt_shmgetpolicy(struct seg *, caddr_t); 148 static int segspt_shmcapable(struct seg *, segcapability_t); 149 150 struct seg_ops segspt_shmops = { 151 segspt_shmdup, 152 segspt_shmunmap, 153 segspt_shmfree, 154 segspt_shmfault, 155 segspt_shmfaulta, 156 segspt_shmsetprot, 157 segspt_shmcheckprot, 158 segspt_shmkluster, 159 segspt_shmswapout, 160 segspt_shmsync, 161 segspt_shmincore, 162 segspt_shmlockop, 163 segspt_shmgetprot, 164 segspt_shmgetoffset, 165 segspt_shmgettype, 166 segspt_shmgetvp, 167 segspt_shmadvise, /* advise */ 168 segspt_shmdump, 169 segspt_shmpagelock, 170 segspt_shmsetpgsz, 171 segspt_shmgetmemid, 172 segspt_shmgetpolicy, 173 segspt_shmcapable, 174 seg_inherit_notsup 175 }; 176 177 static void segspt_purge(struct seg *seg); 178 static int segspt_reclaim(void *, caddr_t, size_t, struct page **, 179 enum seg_rw, int); 180 static int spt_anon_getpages(struct seg *seg, caddr_t addr, size_t len, 181 page_t **ppa); 182 183 184 185 /*ARGSUSED*/ 186 int 187 sptcreate(size_t size, struct seg **sptseg, struct anon_map *amp, 188 uint_t prot, uint_t flags, uint_t share_szc) 189 { 190 int err; 191 struct as *newas; 192 struct segspt_crargs sptcargs; 193 194 #ifdef DEBUG 195 TNF_PROBE_1(sptcreate, "spt", /* CSTYLED */, 196 tnf_ulong, size, size ); 197 #endif 198 if (segspt_minfree == 0) /* leave min 5% of availrmem for */ 199 segspt_minfree = availrmem/20; /* for the system */ 200 201 if (!hat_supported(HAT_SHARED_PT, (void *)0)) 202 return (EINVAL); 203 204 /* 205 * get a new as for this shared memory segment 206 */ 207 newas = as_alloc(); 208 newas->a_proc = NULL; 209 sptcargs.amp = amp; 210 sptcargs.prot = prot; 211 sptcargs.flags = flags; 212 sptcargs.szc = share_szc; 213 /* 214 * create a shared page table (spt) segment 215 */ 216 217 if (err = as_map(newas, SEGSPTADDR, size, segspt_create, &sptcargs)) { 218 as_free(newas); 219 return (err); 220 } 221 *sptseg = sptcargs.seg_spt; 222 return (0); 223 } 224 225 void 226 sptdestroy(struct as *as, struct anon_map *amp) 227 { 228 229 #ifdef DEBUG 230 TNF_PROBE_0(sptdestroy, "spt", /* CSTYLED */); 231 #endif 232 (void) as_unmap(as, SEGSPTADDR, amp->size); 233 as_free(as); 234 } 235 236 /* 237 * called from seg_free(). 238 * free (i.e., unlock, unmap, return to free list) 239 * all the pages in the given seg. 240 */ 241 void 242 segspt_free(struct seg *seg) 243 { 244 struct spt_data *sptd = (struct spt_data *)seg->s_data; 245 246 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as)); 247 248 if (sptd != NULL) { 249 if (sptd->spt_realsize) 250 segspt_free_pages(seg, seg->s_base, sptd->spt_realsize); 251 252 if (sptd->spt_ppa_lckcnt) 253 kmem_free(sptd->spt_ppa_lckcnt, 254 sizeof (*sptd->spt_ppa_lckcnt) 255 * btopr(sptd->spt_amp->size)); 256 kmem_free(sptd->spt_vp, sizeof (*sptd->spt_vp)); 257 cv_destroy(&sptd->spt_cv); 258 mutex_destroy(&sptd->spt_lock); 259 kmem_free(sptd, sizeof (*sptd)); 260 } 261 } 262 263 /*ARGSUSED*/ 264 static int 265 segspt_shmsync(struct seg *seg, caddr_t addr, size_t len, int attr, 266 uint_t flags) 267 { 268 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 269 270 return (0); 271 } 272 273 /*ARGSUSED*/ 274 static size_t 275 segspt_shmincore(struct seg *seg, caddr_t addr, size_t len, char *vec) 276 { 277 caddr_t eo_seg; 278 pgcnt_t npages; 279 struct shm_data *shmd = (struct shm_data *)seg->s_data; 280 struct seg *sptseg; 281 struct spt_data *sptd; 282 283 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 284 #ifdef lint 285 seg = seg; 286 #endif 287 sptseg = shmd->shm_sptseg; 288 sptd = sptseg->s_data; 289 290 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 291 eo_seg = addr + len; 292 while (addr < eo_seg) { 293 /* page exists, and it's locked. */ 294 *vec++ = SEG_PAGE_INCORE | SEG_PAGE_LOCKED | 295 SEG_PAGE_ANON; 296 addr += PAGESIZE; 297 } 298 return (len); 299 } else { 300 struct anon_map *amp = shmd->shm_amp; 301 struct anon *ap; 302 page_t *pp; 303 pgcnt_t anon_index; 304 struct vnode *vp; 305 u_offset_t off; 306 ulong_t i; 307 int ret; 308 anon_sync_obj_t cookie; 309 310 addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 311 anon_index = seg_page(seg, addr); 312 npages = btopr(len); 313 if (anon_index + npages > btopr(shmd->shm_amp->size)) { 314 return (EINVAL); 315 } 316 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 317 for (i = 0; i < npages; i++, anon_index++) { 318 ret = 0; 319 anon_array_enter(amp, anon_index, &cookie); 320 ap = anon_get_ptr(amp->ahp, anon_index); 321 if (ap != NULL) { 322 swap_xlate(ap, &vp, &off); 323 anon_array_exit(&cookie); 324 pp = page_lookup_nowait(vp, off, SE_SHARED); 325 if (pp != NULL) { 326 ret |= SEG_PAGE_INCORE | SEG_PAGE_ANON; 327 page_unlock(pp); 328 } 329 } else { 330 anon_array_exit(&cookie); 331 } 332 if (shmd->shm_vpage[anon_index] & DISM_PG_LOCKED) { 333 ret |= SEG_PAGE_LOCKED; 334 } 335 *vec++ = (char)ret; 336 } 337 ANON_LOCK_EXIT(&->a_rwlock); 338 return (len); 339 } 340 } 341 342 static int 343 segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize) 344 { 345 size_t share_size; 346 347 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as)); 348 349 /* 350 * seg.s_size may have been rounded up to the largest page size 351 * in shmat(). 352 * XXX This should be cleanedup. sptdestroy should take a length 353 * argument which should be the same as sptcreate. Then 354 * this rounding would not be needed (or is done in shm.c) 355 * Only the check for full segment will be needed. 356 * 357 * XXX -- shouldn't raddr == 0 always? These tests don't seem 358 * to be useful at all. 359 */ 360 share_size = page_get_pagesize(seg->s_szc); 361 ssize = P2ROUNDUP(ssize, share_size); 362 363 if (raddr == seg->s_base && ssize == seg->s_size) { 364 seg_free(seg); 365 return (0); 366 } else 367 return (EINVAL); 368 } 369 370 int 371 segspt_create(struct seg *seg, caddr_t argsp) 372 { 373 int err; 374 caddr_t addr = seg->s_base; 375 struct spt_data *sptd; 376 struct segspt_crargs *sptcargs = (struct segspt_crargs *)argsp; 377 struct anon_map *amp = sptcargs->amp; 378 struct kshmid *sp = amp->a_sp; 379 struct cred *cred = CRED(); 380 ulong_t i, j, anon_index = 0; 381 pgcnt_t npages = btopr(amp->size); 382 struct vnode *vp; 383 page_t **ppa; 384 uint_t hat_flags; 385 size_t pgsz; 386 pgcnt_t pgcnt; 387 caddr_t a; 388 pgcnt_t pidx; 389 size_t sz; 390 proc_t *procp = curproc; 391 rctl_qty_t lockedbytes = 0; 392 kproject_t *proj; 393 394 /* 395 * We are holding the a_lock on the underlying dummy as, 396 * so we can make calls to the HAT layer. 397 */ 398 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as)); 399 ASSERT(sp != NULL); 400 401 #ifdef DEBUG 402 TNF_PROBE_2(segspt_create, "spt", /* CSTYLED */, 403 tnf_opaque, addr, addr, tnf_ulong, len, seg->s_size); 404 #endif 405 if ((sptcargs->flags & SHM_PAGEABLE) == 0) { 406 if (err = anon_swap_adjust(npages)) 407 return (err); 408 } 409 err = ENOMEM; 410 411 if ((sptd = kmem_zalloc(sizeof (*sptd), KM_NOSLEEP)) == NULL) 412 goto out1; 413 414 if ((sptcargs->flags & SHM_PAGEABLE) == 0) { 415 if ((ppa = kmem_zalloc(((sizeof (page_t *)) * npages), 416 KM_NOSLEEP)) == NULL) 417 goto out2; 418 } 419 420 mutex_init(&sptd->spt_lock, NULL, MUTEX_DEFAULT, NULL); 421 422 if ((vp = kmem_zalloc(sizeof (*vp), KM_NOSLEEP)) == NULL) 423 goto out3; 424 425 seg->s_ops = &segspt_ops; 426 sptd->spt_vp = vp; 427 sptd->spt_amp = amp; 428 sptd->spt_prot = sptcargs->prot; 429 sptd->spt_flags = sptcargs->flags; 430 seg->s_data = (caddr_t)sptd; 431 sptd->spt_ppa = NULL; 432 sptd->spt_ppa_lckcnt = NULL; 433 seg->s_szc = sptcargs->szc; 434 cv_init(&sptd->spt_cv, NULL, CV_DEFAULT, NULL); 435 sptd->spt_gen = 0; 436 437 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 438 if (seg->s_szc > amp->a_szc) { 439 amp->a_szc = seg->s_szc; 440 } 441 ANON_LOCK_EXIT(&->a_rwlock); 442 443 /* 444 * Set policy to affect initial allocation of pages in 445 * anon_map_createpages() 446 */ 447 (void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, amp, anon_index, 448 NULL, 0, ptob(npages)); 449 450 if (sptcargs->flags & SHM_PAGEABLE) { 451 size_t share_sz; 452 pgcnt_t new_npgs, more_pgs; 453 struct anon_hdr *nahp; 454 zone_t *zone; 455 456 share_sz = page_get_pagesize(seg->s_szc); 457 if (!IS_P2ALIGNED(amp->size, share_sz)) { 458 /* 459 * We are rounding up the size of the anon array 460 * on 4 M boundary because we always create 4 M 461 * of page(s) when locking, faulting pages and we 462 * don't have to check for all corner cases e.g. 463 * if there is enough space to allocate 4 M 464 * page. 465 */ 466 new_npgs = btop(P2ROUNDUP(amp->size, share_sz)); 467 more_pgs = new_npgs - npages; 468 469 /* 470 * The zone will never be NULL, as a fully created 471 * shm always has an owning zone. 472 */ 473 zone = sp->shm_perm.ipc_zone_ref.zref_zone; 474 ASSERT(zone != NULL); 475 if (anon_resv_zone(ptob(more_pgs), zone) == 0) { 476 err = ENOMEM; 477 goto out4; 478 } 479 480 nahp = anon_create(new_npgs, ANON_SLEEP); 481 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 482 (void) anon_copy_ptr(amp->ahp, 0, nahp, 0, npages, 483 ANON_SLEEP); 484 anon_release(amp->ahp, npages); 485 amp->ahp = nahp; 486 ASSERT(amp->swresv == ptob(npages)); 487 amp->swresv = amp->size = ptob(new_npgs); 488 ANON_LOCK_EXIT(&->a_rwlock); 489 npages = new_npgs; 490 } 491 492 sptd->spt_ppa_lckcnt = kmem_zalloc(npages * 493 sizeof (*sptd->spt_ppa_lckcnt), KM_SLEEP); 494 sptd->spt_pcachecnt = 0; 495 sptd->spt_realsize = ptob(npages); 496 sptcargs->seg_spt = seg; 497 return (0); 498 } 499 500 /* 501 * get array of pages for each anon slot in amp 502 */ 503 if ((err = anon_map_createpages(amp, anon_index, ptob(npages), ppa, 504 seg, addr, S_CREATE, cred)) != 0) 505 goto out4; 506 507 mutex_enter(&sp->shm_mlock); 508 509 /* May be partially locked, so, count bytes to charge for locking */ 510 for (i = 0; i < npages; i++) 511 if (ppa[i]->p_lckcnt == 0) 512 lockedbytes += PAGESIZE; 513 514 proj = sp->shm_perm.ipc_proj; 515 516 if (lockedbytes > 0) { 517 mutex_enter(&procp->p_lock); 518 if (rctl_incr_locked_mem(procp, proj, lockedbytes, 0)) { 519 mutex_exit(&procp->p_lock); 520 mutex_exit(&sp->shm_mlock); 521 for (i = 0; i < npages; i++) 522 page_unlock(ppa[i]); 523 err = ENOMEM; 524 goto out4; 525 } 526 mutex_exit(&procp->p_lock); 527 } 528 529 /* 530 * addr is initial address corresponding to the first page on ppa list 531 */ 532 for (i = 0; i < npages; i++) { 533 /* attempt to lock all pages */ 534 if (page_pp_lock(ppa[i], 0, 1) == 0) { 535 /* 536 * if unable to lock any page, unlock all 537 * of them and return error 538 */ 539 for (j = 0; j < i; j++) 540 page_pp_unlock(ppa[j], 0, 1); 541 for (i = 0; i < npages; i++) 542 page_unlock(ppa[i]); 543 rctl_decr_locked_mem(NULL, proj, lockedbytes, 0); 544 mutex_exit(&sp->shm_mlock); 545 err = ENOMEM; 546 goto out4; 547 } 548 } 549 mutex_exit(&sp->shm_mlock); 550 551 /* 552 * Some platforms assume that ISM mappings are HAT_LOAD_LOCK 553 * for the entire life of the segment. For example platforms 554 * that do not support Dynamic Reconfiguration. 555 */ 556 hat_flags = HAT_LOAD_SHARE; 557 if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL)) 558 hat_flags |= HAT_LOAD_LOCK; 559 560 /* 561 * Load translations one lare page at a time 562 * to make sure we don't create mappings bigger than 563 * segment's size code in case underlying pages 564 * are shared with segvn's segment that uses bigger 565 * size code than we do. 566 */ 567 pgsz = page_get_pagesize(seg->s_szc); 568 pgcnt = page_get_pagecnt(seg->s_szc); 569 for (a = addr, pidx = 0; pidx < npages; a += pgsz, pidx += pgcnt) { 570 sz = MIN(pgsz, ptob(npages - pidx)); 571 hat_memload_array(seg->s_as->a_hat, a, sz, 572 &ppa[pidx], sptd->spt_prot, hat_flags); 573 } 574 575 /* 576 * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP, 577 * we will leave the pages locked SE_SHARED for the life 578 * of the ISM segment. This will prevent any calls to 579 * hat_pageunload() on this ISM segment for those platforms. 580 */ 581 if (!(hat_flags & HAT_LOAD_LOCK)) { 582 /* 583 * On platforms that support HAT_DYNAMIC_ISM_UNMAP, 584 * we no longer need to hold the SE_SHARED lock on the pages, 585 * since L_PAGELOCK and F_SOFTLOCK calls will grab the 586 * SE_SHARED lock on the pages as necessary. 587 */ 588 for (i = 0; i < npages; i++) 589 page_unlock(ppa[i]); 590 } 591 sptd->spt_pcachecnt = 0; 592 kmem_free(ppa, ((sizeof (page_t *)) * npages)); 593 sptd->spt_realsize = ptob(npages); 594 atomic_add_long(&spt_used, npages); 595 sptcargs->seg_spt = seg; 596 return (0); 597 598 out4: 599 seg->s_data = NULL; 600 kmem_free(vp, sizeof (*vp)); 601 cv_destroy(&sptd->spt_cv); 602 out3: 603 mutex_destroy(&sptd->spt_lock); 604 if ((sptcargs->flags & SHM_PAGEABLE) == 0) 605 kmem_free(ppa, (sizeof (*ppa) * npages)); 606 out2: 607 kmem_free(sptd, sizeof (*sptd)); 608 out1: 609 if ((sptcargs->flags & SHM_PAGEABLE) == 0) 610 anon_swap_restore(npages); 611 return (err); 612 } 613 614 /*ARGSUSED*/ 615 void 616 segspt_free_pages(struct seg *seg, caddr_t addr, size_t len) 617 { 618 struct page *pp; 619 struct spt_data *sptd = (struct spt_data *)seg->s_data; 620 pgcnt_t npages; 621 ulong_t anon_idx; 622 struct anon_map *amp; 623 struct anon *ap; 624 struct vnode *vp; 625 u_offset_t off; 626 uint_t hat_flags; 627 int root = 0; 628 pgcnt_t pgs, curnpgs = 0; 629 page_t *rootpp; 630 rctl_qty_t unlocked_bytes = 0; 631 kproject_t *proj; 632 kshmid_t *sp; 633 634 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as)); 635 636 len = P2ROUNDUP(len, PAGESIZE); 637 638 npages = btop(len); 639 640 hat_flags = HAT_UNLOAD_UNLOCK | HAT_UNLOAD_UNMAP; 641 if ((hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) || 642 (sptd->spt_flags & SHM_PAGEABLE)) { 643 hat_flags = HAT_UNLOAD_UNMAP; 644 } 645 646 hat_unload(seg->s_as->a_hat, addr, len, hat_flags); 647 648 amp = sptd->spt_amp; 649 if (sptd->spt_flags & SHM_PAGEABLE) 650 npages = btop(amp->size); 651 652 ASSERT(amp != NULL); 653 654 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 655 sp = amp->a_sp; 656 proj = sp->shm_perm.ipc_proj; 657 mutex_enter(&sp->shm_mlock); 658 } 659 for (anon_idx = 0; anon_idx < npages; anon_idx++) { 660 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 661 if ((ap = anon_get_ptr(amp->ahp, anon_idx)) == NULL) { 662 panic("segspt_free_pages: null app"); 663 /*NOTREACHED*/ 664 } 665 } else { 666 if ((ap = anon_get_next_ptr(amp->ahp, &anon_idx)) 667 == NULL) 668 continue; 669 } 670 ASSERT(ANON_ISBUSY(anon_get_slot(amp->ahp, anon_idx)) == 0); 671 swap_xlate(ap, &vp, &off); 672 673 /* 674 * If this platform supports HAT_DYNAMIC_ISM_UNMAP, 675 * the pages won't be having SE_SHARED lock at this 676 * point. 677 * 678 * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP, 679 * the pages are still held SE_SHARED locked from the 680 * original segspt_create() 681 * 682 * Our goal is to get SE_EXCL lock on each page, remove 683 * permanent lock on it and invalidate the page. 684 */ 685 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 686 if (hat_flags == HAT_UNLOAD_UNMAP) 687 pp = page_lookup(vp, off, SE_EXCL); 688 else { 689 if ((pp = page_find(vp, off)) == NULL) { 690 panic("segspt_free_pages: " 691 "page not locked"); 692 /*NOTREACHED*/ 693 } 694 if (!page_tryupgrade(pp)) { 695 page_unlock(pp); 696 pp = page_lookup(vp, off, SE_EXCL); 697 } 698 } 699 if (pp == NULL) { 700 panic("segspt_free_pages: " 701 "page not in the system"); 702 /*NOTREACHED*/ 703 } 704 ASSERT(pp->p_lckcnt > 0); 705 page_pp_unlock(pp, 0, 1); 706 if (pp->p_lckcnt == 0) 707 unlocked_bytes += PAGESIZE; 708 } else { 709 if ((pp = page_lookup(vp, off, SE_EXCL)) == NULL) 710 continue; 711 } 712 /* 713 * It's logical to invalidate the pages here as in most cases 714 * these were created by segspt. 715 */ 716 if (pp->p_szc != 0) { 717 if (root == 0) { 718 ASSERT(curnpgs == 0); 719 root = 1; 720 rootpp = pp; 721 pgs = curnpgs = page_get_pagecnt(pp->p_szc); 722 ASSERT(pgs > 1); 723 ASSERT(IS_P2ALIGNED(pgs, pgs)); 724 ASSERT(!(page_pptonum(pp) & (pgs - 1))); 725 curnpgs--; 726 } else if ((page_pptonum(pp) & (pgs - 1)) == pgs - 1) { 727 ASSERT(curnpgs == 1); 728 ASSERT(page_pptonum(pp) == 729 page_pptonum(rootpp) + (pgs - 1)); 730 page_destroy_pages(rootpp); 731 root = 0; 732 curnpgs = 0; 733 } else { 734 ASSERT(curnpgs > 1); 735 ASSERT(page_pptonum(pp) == 736 page_pptonum(rootpp) + (pgs - curnpgs)); 737 curnpgs--; 738 } 739 } else { 740 if (root != 0 || curnpgs != 0) { 741 panic("segspt_free_pages: bad large page"); 742 /*NOTREACHED*/ 743 } 744 /* 745 * Before destroying the pages, we need to take care 746 * of the rctl locked memory accounting. For that 747 * we need to calculte the unlocked_bytes. 748 */ 749 if (pp->p_lckcnt > 0) 750 unlocked_bytes += PAGESIZE; 751 /*LINTED: constant in conditional context */ 752 VN_DISPOSE(pp, B_INVAL, 0, kcred); 753 } 754 } 755 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 756 if (unlocked_bytes > 0) 757 rctl_decr_locked_mem(NULL, proj, unlocked_bytes, 0); 758 mutex_exit(&sp->shm_mlock); 759 } 760 if (root != 0 || curnpgs != 0) { 761 panic("segspt_free_pages: bad large page"); 762 /*NOTREACHED*/ 763 } 764 765 /* 766 * mark that pages have been released 767 */ 768 sptd->spt_realsize = 0; 769 770 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 771 atomic_add_long(&spt_used, -npages); 772 anon_swap_restore(npages); 773 } 774 } 775 776 /* 777 * Get memory allocation policy info for specified address in given segment 778 */ 779 static lgrp_mem_policy_info_t * 780 segspt_getpolicy(struct seg *seg, caddr_t addr) 781 { 782 struct anon_map *amp; 783 ulong_t anon_index; 784 lgrp_mem_policy_info_t *policy_info; 785 struct spt_data *spt_data; 786 787 ASSERT(seg != NULL); 788 789 /* 790 * Get anon_map from segspt 791 * 792 * Assume that no lock needs to be held on anon_map, since 793 * it should be protected by its reference count which must be 794 * nonzero for an existing segment 795 * Need to grab readers lock on policy tree though 796 */ 797 spt_data = (struct spt_data *)seg->s_data; 798 if (spt_data == NULL) 799 return (NULL); 800 amp = spt_data->spt_amp; 801 ASSERT(amp->refcnt != 0); 802 803 /* 804 * Get policy info 805 * 806 * Assume starting anon index of 0 807 */ 808 anon_index = seg_page(seg, addr); 809 policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0); 810 811 return (policy_info); 812 } 813 814 /* 815 * DISM only. 816 * Return locked pages over a given range. 817 * 818 * We will cache all DISM locked pages and save the pplist for the 819 * entire segment in the ppa field of the underlying DISM segment structure. 820 * Later, during a call to segspt_reclaim() we will use this ppa array 821 * to page_unlock() all of the pages and then we will free this ppa list. 822 */ 823 /*ARGSUSED*/ 824 static int 825 segspt_dismpagelock(struct seg *seg, caddr_t addr, size_t len, 826 struct page ***ppp, enum lock_type type, enum seg_rw rw) 827 { 828 struct shm_data *shmd = (struct shm_data *)seg->s_data; 829 struct seg *sptseg = shmd->shm_sptseg; 830 struct spt_data *sptd = sptseg->s_data; 831 pgcnt_t pg_idx, npages, tot_npages, npgs; 832 struct page **pplist, **pl, **ppa, *pp; 833 struct anon_map *amp; 834 spgcnt_t an_idx; 835 int ret = ENOTSUP; 836 uint_t pl_built = 0; 837 struct anon *ap; 838 struct vnode *vp; 839 u_offset_t off; 840 pgcnt_t claim_availrmem = 0; 841 uint_t szc; 842 843 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 844 ASSERT(type == L_PAGELOCK || type == L_PAGEUNLOCK); 845 846 /* 847 * We want to lock/unlock the entire ISM segment. Therefore, 848 * we will be using the underlying sptseg and it's base address 849 * and length for the caching arguments. 850 */ 851 ASSERT(sptseg); 852 ASSERT(sptd); 853 854 pg_idx = seg_page(seg, addr); 855 npages = btopr(len); 856 857 /* 858 * check if the request is larger than number of pages covered 859 * by amp 860 */ 861 if (pg_idx + npages > btopr(sptd->spt_amp->size)) { 862 *ppp = NULL; 863 return (ENOTSUP); 864 } 865 866 if (type == L_PAGEUNLOCK) { 867 ASSERT(sptd->spt_ppa != NULL); 868 869 seg_pinactive(seg, NULL, seg->s_base, sptd->spt_amp->size, 870 sptd->spt_ppa, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim); 871 872 /* 873 * If someone is blocked while unmapping, we purge 874 * segment page cache and thus reclaim pplist synchronously 875 * without waiting for seg_pasync_thread. This speeds up 876 * unmapping in cases where munmap(2) is called, while 877 * raw async i/o is still in progress or where a thread 878 * exits on data fault in a multithreaded application. 879 */ 880 if ((sptd->spt_flags & DISM_PPA_CHANGED) || 881 (AS_ISUNMAPWAIT(seg->s_as) && 882 shmd->shm_softlockcnt > 0)) { 883 segspt_purge(seg); 884 } 885 return (0); 886 } 887 888 /* The L_PAGELOCK case ... */ 889 890 if (sptd->spt_flags & DISM_PPA_CHANGED) { 891 segspt_purge(seg); 892 /* 893 * for DISM ppa needs to be rebuild since 894 * number of locked pages could be changed 895 */ 896 *ppp = NULL; 897 return (ENOTSUP); 898 } 899 900 /* 901 * First try to find pages in segment page cache, without 902 * holding the segment lock. 903 */ 904 pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size, 905 S_WRITE, SEGP_FORCE_WIRED); 906 if (pplist != NULL) { 907 ASSERT(sptd->spt_ppa != NULL); 908 ASSERT(sptd->spt_ppa == pplist); 909 ppa = sptd->spt_ppa; 910 for (an_idx = pg_idx; an_idx < pg_idx + npages; ) { 911 if (ppa[an_idx] == NULL) { 912 seg_pinactive(seg, NULL, seg->s_base, 913 sptd->spt_amp->size, ppa, 914 S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim); 915 *ppp = NULL; 916 return (ENOTSUP); 917 } 918 if ((szc = ppa[an_idx]->p_szc) != 0) { 919 npgs = page_get_pagecnt(szc); 920 an_idx = P2ROUNDUP(an_idx + 1, npgs); 921 } else { 922 an_idx++; 923 } 924 } 925 /* 926 * Since we cache the entire DISM segment, we want to 927 * set ppp to point to the first slot that corresponds 928 * to the requested addr, i.e. pg_idx. 929 */ 930 *ppp = &(sptd->spt_ppa[pg_idx]); 931 return (0); 932 } 933 934 mutex_enter(&sptd->spt_lock); 935 /* 936 * try to find pages in segment page cache with mutex 937 */ 938 pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size, 939 S_WRITE, SEGP_FORCE_WIRED); 940 if (pplist != NULL) { 941 ASSERT(sptd->spt_ppa != NULL); 942 ASSERT(sptd->spt_ppa == pplist); 943 ppa = sptd->spt_ppa; 944 for (an_idx = pg_idx; an_idx < pg_idx + npages; ) { 945 if (ppa[an_idx] == NULL) { 946 mutex_exit(&sptd->spt_lock); 947 seg_pinactive(seg, NULL, seg->s_base, 948 sptd->spt_amp->size, ppa, 949 S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim); 950 *ppp = NULL; 951 return (ENOTSUP); 952 } 953 if ((szc = ppa[an_idx]->p_szc) != 0) { 954 npgs = page_get_pagecnt(szc); 955 an_idx = P2ROUNDUP(an_idx + 1, npgs); 956 } else { 957 an_idx++; 958 } 959 } 960 /* 961 * Since we cache the entire DISM segment, we want to 962 * set ppp to point to the first slot that corresponds 963 * to the requested addr, i.e. pg_idx. 964 */ 965 mutex_exit(&sptd->spt_lock); 966 *ppp = &(sptd->spt_ppa[pg_idx]); 967 return (0); 968 } 969 if (seg_pinsert_check(seg, NULL, seg->s_base, sptd->spt_amp->size, 970 SEGP_FORCE_WIRED) == SEGP_FAIL) { 971 mutex_exit(&sptd->spt_lock); 972 *ppp = NULL; 973 return (ENOTSUP); 974 } 975 976 /* 977 * No need to worry about protections because DISM pages are always rw. 978 */ 979 pl = pplist = NULL; 980 amp = sptd->spt_amp; 981 982 /* 983 * Do we need to build the ppa array? 984 */ 985 if (sptd->spt_ppa == NULL) { 986 pgcnt_t lpg_cnt = 0; 987 988 pl_built = 1; 989 tot_npages = btopr(sptd->spt_amp->size); 990 991 ASSERT(sptd->spt_pcachecnt == 0); 992 pplist = kmem_zalloc(sizeof (page_t *) * tot_npages, KM_SLEEP); 993 pl = pplist; 994 995 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 996 for (an_idx = 0; an_idx < tot_npages; ) { 997 ap = anon_get_ptr(amp->ahp, an_idx); 998 /* 999 * Cache only mlocked pages. For large pages 1000 * if one (constituent) page is mlocked 1001 * all pages for that large page 1002 * are cached also. This is for quick 1003 * lookups of ppa array; 1004 */ 1005 if ((ap != NULL) && (lpg_cnt != 0 || 1006 (sptd->spt_ppa_lckcnt[an_idx] != 0))) { 1007 1008 swap_xlate(ap, &vp, &off); 1009 pp = page_lookup(vp, off, SE_SHARED); 1010 ASSERT(pp != NULL); 1011 if (lpg_cnt == 0) { 1012 lpg_cnt++; 1013 /* 1014 * For a small page, we are done -- 1015 * lpg_count is reset to 0 below. 1016 * 1017 * For a large page, we are guaranteed 1018 * to find the anon structures of all 1019 * constituent pages and a non-zero 1020 * lpg_cnt ensures that we don't test 1021 * for mlock for these. We are done 1022 * when lpg_count reaches (npgs + 1). 1023 * If we are not the first constituent 1024 * page, restart at the first one. 1025 */ 1026 npgs = page_get_pagecnt(pp->p_szc); 1027 if (!IS_P2ALIGNED(an_idx, npgs)) { 1028 an_idx = P2ALIGN(an_idx, npgs); 1029 page_unlock(pp); 1030 continue; 1031 } 1032 } 1033 if (++lpg_cnt > npgs) 1034 lpg_cnt = 0; 1035 1036 /* 1037 * availrmem is decremented only 1038 * for unlocked pages 1039 */ 1040 if (sptd->spt_ppa_lckcnt[an_idx] == 0) 1041 claim_availrmem++; 1042 pplist[an_idx] = pp; 1043 } 1044 an_idx++; 1045 } 1046 ANON_LOCK_EXIT(&->a_rwlock); 1047 1048 if (claim_availrmem) { 1049 mutex_enter(&freemem_lock); 1050 if (availrmem < tune.t_minarmem + claim_availrmem) { 1051 mutex_exit(&freemem_lock); 1052 ret = ENOTSUP; 1053 claim_availrmem = 0; 1054 goto insert_fail; 1055 } else { 1056 availrmem -= claim_availrmem; 1057 } 1058 mutex_exit(&freemem_lock); 1059 } 1060 1061 sptd->spt_ppa = pl; 1062 } else { 1063 /* 1064 * We already have a valid ppa[]. 1065 */ 1066 pl = sptd->spt_ppa; 1067 } 1068 1069 ASSERT(pl != NULL); 1070 1071 ret = seg_pinsert(seg, NULL, seg->s_base, sptd->spt_amp->size, 1072 sptd->spt_amp->size, pl, S_WRITE, SEGP_FORCE_WIRED, 1073 segspt_reclaim); 1074 if (ret == SEGP_FAIL) { 1075 /* 1076 * seg_pinsert failed. We return 1077 * ENOTSUP, so that the as_pagelock() code will 1078 * then try the slower F_SOFTLOCK path. 1079 */ 1080 if (pl_built) { 1081 /* 1082 * No one else has referenced the ppa[]. 1083 * We created it and we need to destroy it. 1084 */ 1085 sptd->spt_ppa = NULL; 1086 } 1087 ret = ENOTSUP; 1088 goto insert_fail; 1089 } 1090 1091 /* 1092 * In either case, we increment softlockcnt on the 'real' segment. 1093 */ 1094 sptd->spt_pcachecnt++; 1095 atomic_inc_ulong((ulong_t *)(&(shmd->shm_softlockcnt))); 1096 1097 ppa = sptd->spt_ppa; 1098 for (an_idx = pg_idx; an_idx < pg_idx + npages; ) { 1099 if (ppa[an_idx] == NULL) { 1100 mutex_exit(&sptd->spt_lock); 1101 seg_pinactive(seg, NULL, seg->s_base, 1102 sptd->spt_amp->size, 1103 pl, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim); 1104 *ppp = NULL; 1105 return (ENOTSUP); 1106 } 1107 if ((szc = ppa[an_idx]->p_szc) != 0) { 1108 npgs = page_get_pagecnt(szc); 1109 an_idx = P2ROUNDUP(an_idx + 1, npgs); 1110 } else { 1111 an_idx++; 1112 } 1113 } 1114 /* 1115 * We can now drop the sptd->spt_lock since the ppa[] 1116 * exists and we have incremented pacachecnt. 1117 */ 1118 mutex_exit(&sptd->spt_lock); 1119 1120 /* 1121 * Since we cache the entire segment, we want to 1122 * set ppp to point to the first slot that corresponds 1123 * to the requested addr, i.e. pg_idx. 1124 */ 1125 *ppp = &(sptd->spt_ppa[pg_idx]); 1126 return (0); 1127 1128 insert_fail: 1129 /* 1130 * We will only reach this code if we tried and failed. 1131 * 1132 * And we can drop the lock on the dummy seg, once we've failed 1133 * to set up a new ppa[]. 1134 */ 1135 mutex_exit(&sptd->spt_lock); 1136 1137 if (pl_built) { 1138 if (claim_availrmem) { 1139 mutex_enter(&freemem_lock); 1140 availrmem += claim_availrmem; 1141 mutex_exit(&freemem_lock); 1142 } 1143 1144 /* 1145 * We created pl and we need to destroy it. 1146 */ 1147 pplist = pl; 1148 for (an_idx = 0; an_idx < tot_npages; an_idx++) { 1149 if (pplist[an_idx] != NULL) 1150 page_unlock(pplist[an_idx]); 1151 } 1152 kmem_free(pl, sizeof (page_t *) * tot_npages); 1153 } 1154 1155 if (shmd->shm_softlockcnt <= 0) { 1156 if (AS_ISUNMAPWAIT(seg->s_as)) { 1157 mutex_enter(&seg->s_as->a_contents); 1158 if (AS_ISUNMAPWAIT(seg->s_as)) { 1159 AS_CLRUNMAPWAIT(seg->s_as); 1160 cv_broadcast(&seg->s_as->a_cv); 1161 } 1162 mutex_exit(&seg->s_as->a_contents); 1163 } 1164 } 1165 *ppp = NULL; 1166 return (ret); 1167 } 1168 1169 1170 1171 /* 1172 * return locked pages over a given range. 1173 * 1174 * We will cache the entire ISM segment and save the pplist for the 1175 * entire segment in the ppa field of the underlying ISM segment structure. 1176 * Later, during a call to segspt_reclaim() we will use this ppa array 1177 * to page_unlock() all of the pages and then we will free this ppa list. 1178 */ 1179 /*ARGSUSED*/ 1180 static int 1181 segspt_shmpagelock(struct seg *seg, caddr_t addr, size_t len, 1182 struct page ***ppp, enum lock_type type, enum seg_rw rw) 1183 { 1184 struct shm_data *shmd = (struct shm_data *)seg->s_data; 1185 struct seg *sptseg = shmd->shm_sptseg; 1186 struct spt_data *sptd = sptseg->s_data; 1187 pgcnt_t np, page_index, npages; 1188 caddr_t a, spt_base; 1189 struct page **pplist, **pl, *pp; 1190 struct anon_map *amp; 1191 ulong_t anon_index; 1192 int ret = ENOTSUP; 1193 uint_t pl_built = 0; 1194 struct anon *ap; 1195 struct vnode *vp; 1196 u_offset_t off; 1197 1198 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 1199 ASSERT(type == L_PAGELOCK || type == L_PAGEUNLOCK); 1200 1201 1202 /* 1203 * We want to lock/unlock the entire ISM segment. Therefore, 1204 * we will be using the underlying sptseg and it's base address 1205 * and length for the caching arguments. 1206 */ 1207 ASSERT(sptseg); 1208 ASSERT(sptd); 1209 1210 if (sptd->spt_flags & SHM_PAGEABLE) { 1211 return (segspt_dismpagelock(seg, addr, len, ppp, type, rw)); 1212 } 1213 1214 page_index = seg_page(seg, addr); 1215 npages = btopr(len); 1216 1217 /* 1218 * check if the request is larger than number of pages covered 1219 * by amp 1220 */ 1221 if (page_index + npages > btopr(sptd->spt_amp->size)) { 1222 *ppp = NULL; 1223 return (ENOTSUP); 1224 } 1225 1226 if (type == L_PAGEUNLOCK) { 1227 1228 ASSERT(sptd->spt_ppa != NULL); 1229 1230 seg_pinactive(seg, NULL, seg->s_base, sptd->spt_amp->size, 1231 sptd->spt_ppa, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim); 1232 1233 /* 1234 * If someone is blocked while unmapping, we purge 1235 * segment page cache and thus reclaim pplist synchronously 1236 * without waiting for seg_pasync_thread. This speeds up 1237 * unmapping in cases where munmap(2) is called, while 1238 * raw async i/o is still in progress or where a thread 1239 * exits on data fault in a multithreaded application. 1240 */ 1241 if (AS_ISUNMAPWAIT(seg->s_as) && (shmd->shm_softlockcnt > 0)) { 1242 segspt_purge(seg); 1243 } 1244 return (0); 1245 } 1246 1247 /* The L_PAGELOCK case... */ 1248 1249 /* 1250 * First try to find pages in segment page cache, without 1251 * holding the segment lock. 1252 */ 1253 pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size, 1254 S_WRITE, SEGP_FORCE_WIRED); 1255 if (pplist != NULL) { 1256 ASSERT(sptd->spt_ppa == pplist); 1257 ASSERT(sptd->spt_ppa[page_index]); 1258 /* 1259 * Since we cache the entire ISM segment, we want to 1260 * set ppp to point to the first slot that corresponds 1261 * to the requested addr, i.e. page_index. 1262 */ 1263 *ppp = &(sptd->spt_ppa[page_index]); 1264 return (0); 1265 } 1266 1267 mutex_enter(&sptd->spt_lock); 1268 1269 /* 1270 * try to find pages in segment page cache 1271 */ 1272 pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size, 1273 S_WRITE, SEGP_FORCE_WIRED); 1274 if (pplist != NULL) { 1275 ASSERT(sptd->spt_ppa == pplist); 1276 /* 1277 * Since we cache the entire segment, we want to 1278 * set ppp to point to the first slot that corresponds 1279 * to the requested addr, i.e. page_index. 1280 */ 1281 mutex_exit(&sptd->spt_lock); 1282 *ppp = &(sptd->spt_ppa[page_index]); 1283 return (0); 1284 } 1285 1286 if (seg_pinsert_check(seg, NULL, seg->s_base, sptd->spt_amp->size, 1287 SEGP_FORCE_WIRED) == SEGP_FAIL) { 1288 mutex_exit(&sptd->spt_lock); 1289 *ppp = NULL; 1290 return (ENOTSUP); 1291 } 1292 1293 /* 1294 * No need to worry about protections because ISM pages 1295 * are always rw. 1296 */ 1297 pl = pplist = NULL; 1298 1299 /* 1300 * Do we need to build the ppa array? 1301 */ 1302 if (sptd->spt_ppa == NULL) { 1303 ASSERT(sptd->spt_ppa == pplist); 1304 1305 spt_base = sptseg->s_base; 1306 pl_built = 1; 1307 1308 /* 1309 * availrmem is decremented once during anon_swap_adjust() 1310 * and is incremented during the anon_unresv(), which is 1311 * called from shm_rm_amp() when the segment is destroyed. 1312 */ 1313 amp = sptd->spt_amp; 1314 ASSERT(amp != NULL); 1315 1316 /* pcachecnt is protected by sptd->spt_lock */ 1317 ASSERT(sptd->spt_pcachecnt == 0); 1318 pplist = kmem_zalloc(sizeof (page_t *) 1319 * btopr(sptd->spt_amp->size), KM_SLEEP); 1320 pl = pplist; 1321 1322 anon_index = seg_page(sptseg, spt_base); 1323 1324 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 1325 for (a = spt_base; a < (spt_base + sptd->spt_amp->size); 1326 a += PAGESIZE, anon_index++, pplist++) { 1327 ap = anon_get_ptr(amp->ahp, anon_index); 1328 ASSERT(ap != NULL); 1329 swap_xlate(ap, &vp, &off); 1330 pp = page_lookup(vp, off, SE_SHARED); 1331 ASSERT(pp != NULL); 1332 *pplist = pp; 1333 } 1334 ANON_LOCK_EXIT(&->a_rwlock); 1335 1336 if (a < (spt_base + sptd->spt_amp->size)) { 1337 ret = ENOTSUP; 1338 goto insert_fail; 1339 } 1340 sptd->spt_ppa = pl; 1341 } else { 1342 /* 1343 * We already have a valid ppa[]. 1344 */ 1345 pl = sptd->spt_ppa; 1346 } 1347 1348 ASSERT(pl != NULL); 1349 1350 ret = seg_pinsert(seg, NULL, seg->s_base, sptd->spt_amp->size, 1351 sptd->spt_amp->size, pl, S_WRITE, SEGP_FORCE_WIRED, 1352 segspt_reclaim); 1353 if (ret == SEGP_FAIL) { 1354 /* 1355 * seg_pinsert failed. We return 1356 * ENOTSUP, so that the as_pagelock() code will 1357 * then try the slower F_SOFTLOCK path. 1358 */ 1359 if (pl_built) { 1360 /* 1361 * No one else has referenced the ppa[]. 1362 * We created it and we need to destroy it. 1363 */ 1364 sptd->spt_ppa = NULL; 1365 } 1366 ret = ENOTSUP; 1367 goto insert_fail; 1368 } 1369 1370 /* 1371 * In either case, we increment softlockcnt on the 'real' segment. 1372 */ 1373 sptd->spt_pcachecnt++; 1374 atomic_inc_ulong((ulong_t *)(&(shmd->shm_softlockcnt))); 1375 1376 /* 1377 * We can now drop the sptd->spt_lock since the ppa[] 1378 * exists and we have incremented pacachecnt. 1379 */ 1380 mutex_exit(&sptd->spt_lock); 1381 1382 /* 1383 * Since we cache the entire segment, we want to 1384 * set ppp to point to the first slot that corresponds 1385 * to the requested addr, i.e. page_index. 1386 */ 1387 *ppp = &(sptd->spt_ppa[page_index]); 1388 return (0); 1389 1390 insert_fail: 1391 /* 1392 * We will only reach this code if we tried and failed. 1393 * 1394 * And we can drop the lock on the dummy seg, once we've failed 1395 * to set up a new ppa[]. 1396 */ 1397 mutex_exit(&sptd->spt_lock); 1398 1399 if (pl_built) { 1400 /* 1401 * We created pl and we need to destroy it. 1402 */ 1403 pplist = pl; 1404 np = (((uintptr_t)(a - spt_base)) >> PAGESHIFT); 1405 while (np) { 1406 page_unlock(*pplist); 1407 np--; 1408 pplist++; 1409 } 1410 kmem_free(pl, sizeof (page_t *) * btopr(sptd->spt_amp->size)); 1411 } 1412 if (shmd->shm_softlockcnt <= 0) { 1413 if (AS_ISUNMAPWAIT(seg->s_as)) { 1414 mutex_enter(&seg->s_as->a_contents); 1415 if (AS_ISUNMAPWAIT(seg->s_as)) { 1416 AS_CLRUNMAPWAIT(seg->s_as); 1417 cv_broadcast(&seg->s_as->a_cv); 1418 } 1419 mutex_exit(&seg->s_as->a_contents); 1420 } 1421 } 1422 *ppp = NULL; 1423 return (ret); 1424 } 1425 1426 /* 1427 * purge any cached pages in the I/O page cache 1428 */ 1429 static void 1430 segspt_purge(struct seg *seg) 1431 { 1432 seg_ppurge(seg, NULL, SEGP_FORCE_WIRED); 1433 } 1434 1435 static int 1436 segspt_reclaim(void *ptag, caddr_t addr, size_t len, struct page **pplist, 1437 enum seg_rw rw, int async) 1438 { 1439 struct seg *seg = (struct seg *)ptag; 1440 struct shm_data *shmd = (struct shm_data *)seg->s_data; 1441 struct seg *sptseg; 1442 struct spt_data *sptd; 1443 pgcnt_t npages, i, free_availrmem = 0; 1444 int done = 0; 1445 1446 #ifdef lint 1447 addr = addr; 1448 #endif 1449 sptseg = shmd->shm_sptseg; 1450 sptd = sptseg->s_data; 1451 npages = (len >> PAGESHIFT); 1452 ASSERT(npages); 1453 ASSERT(sptd->spt_pcachecnt != 0); 1454 ASSERT(sptd->spt_ppa == pplist); 1455 ASSERT(npages == btopr(sptd->spt_amp->size)); 1456 ASSERT(async || AS_LOCK_HELD(seg->s_as)); 1457 1458 /* 1459 * Acquire the lock on the dummy seg and destroy the 1460 * ppa array IF this is the last pcachecnt. 1461 */ 1462 mutex_enter(&sptd->spt_lock); 1463 if (--sptd->spt_pcachecnt == 0) { 1464 for (i = 0; i < npages; i++) { 1465 if (pplist[i] == NULL) { 1466 continue; 1467 } 1468 if (rw == S_WRITE) { 1469 hat_setrefmod(pplist[i]); 1470 } else { 1471 hat_setref(pplist[i]); 1472 } 1473 if ((sptd->spt_flags & SHM_PAGEABLE) && 1474 (sptd->spt_ppa_lckcnt[i] == 0)) 1475 free_availrmem++; 1476 page_unlock(pplist[i]); 1477 } 1478 if ((sptd->spt_flags & SHM_PAGEABLE) && free_availrmem) { 1479 mutex_enter(&freemem_lock); 1480 availrmem += free_availrmem; 1481 mutex_exit(&freemem_lock); 1482 } 1483 /* 1484 * Since we want to cach/uncache the entire ISM segment, 1485 * we will track the pplist in a segspt specific field 1486 * ppa, that is initialized at the time we add an entry to 1487 * the cache. 1488 */ 1489 ASSERT(sptd->spt_pcachecnt == 0); 1490 kmem_free(pplist, sizeof (page_t *) * npages); 1491 sptd->spt_ppa = NULL; 1492 sptd->spt_flags &= ~DISM_PPA_CHANGED; 1493 sptd->spt_gen++; 1494 cv_broadcast(&sptd->spt_cv); 1495 done = 1; 1496 } 1497 mutex_exit(&sptd->spt_lock); 1498 1499 /* 1500 * If we are pcache async thread or called via seg_ppurge_wiredpp() we 1501 * may not hold AS lock (in this case async argument is not 0). This 1502 * means if softlockcnt drops to 0 after the decrement below address 1503 * space may get freed. We can't allow it since after softlock 1504 * derement to 0 we still need to access as structure for possible 1505 * wakeup of unmap waiters. To prevent the disappearance of as we take 1506 * this segment's shm_segfree_syncmtx. segspt_shmfree() also takes 1507 * this mutex as a barrier to make sure this routine completes before 1508 * segment is freed. 1509 * 1510 * The second complication we have to deal with in async case is a 1511 * possibility of missed wake up of unmap wait thread. When we don't 1512 * hold as lock here we may take a_contents lock before unmap wait 1513 * thread that was first to see softlockcnt was still not 0. As a 1514 * result we'll fail to wake up an unmap wait thread. To avoid this 1515 * race we set nounmapwait flag in as structure if we drop softlockcnt 1516 * to 0 if async is not 0. unmapwait thread 1517 * will not block if this flag is set. 1518 */ 1519 if (async) 1520 mutex_enter(&shmd->shm_segfree_syncmtx); 1521 1522 /* 1523 * Now decrement softlockcnt. 1524 */ 1525 ASSERT(shmd->shm_softlockcnt > 0); 1526 atomic_dec_ulong((ulong_t *)(&(shmd->shm_softlockcnt))); 1527 1528 if (shmd->shm_softlockcnt <= 0) { 1529 if (async || AS_ISUNMAPWAIT(seg->s_as)) { 1530 mutex_enter(&seg->s_as->a_contents); 1531 if (async) 1532 AS_SETNOUNMAPWAIT(seg->s_as); 1533 if (AS_ISUNMAPWAIT(seg->s_as)) { 1534 AS_CLRUNMAPWAIT(seg->s_as); 1535 cv_broadcast(&seg->s_as->a_cv); 1536 } 1537 mutex_exit(&seg->s_as->a_contents); 1538 } 1539 } 1540 1541 if (async) 1542 mutex_exit(&shmd->shm_segfree_syncmtx); 1543 1544 return (done); 1545 } 1546 1547 /* 1548 * Do a F_SOFTUNLOCK call over the range requested. 1549 * The range must have already been F_SOFTLOCK'ed. 1550 * 1551 * The calls to acquire and release the anon map lock mutex were 1552 * removed in order to avoid a deadly embrace during a DR 1553 * memory delete operation. (Eg. DR blocks while waiting for a 1554 * exclusive lock on a page that is being used for kaio; the 1555 * thread that will complete the kaio and call segspt_softunlock 1556 * blocks on the anon map lock; another thread holding the anon 1557 * map lock blocks on another page lock via the segspt_shmfault 1558 * -> page_lookup -> page_lookup_create -> page_lock_es code flow.) 1559 * 1560 * The appropriateness of the removal is based upon the following: 1561 * 1. If we are holding a segment's reader lock and the page is held 1562 * shared, then the corresponding element in anonmap which points to 1563 * anon struct cannot change and there is no need to acquire the 1564 * anonymous map lock. 1565 * 2. Threads in segspt_softunlock have a reader lock on the segment 1566 * and already have the shared page lock, so we are guaranteed that 1567 * the anon map slot cannot change and therefore can call anon_get_ptr() 1568 * without grabbing the anonymous map lock. 1569 * 3. Threads that softlock a shared page break copy-on-write, even if 1570 * its a read. Thus cow faults can be ignored with respect to soft 1571 * unlocking, since the breaking of cow means that the anon slot(s) will 1572 * not be shared. 1573 */ 1574 static void 1575 segspt_softunlock(struct seg *seg, caddr_t sptseg_addr, 1576 size_t len, enum seg_rw rw) 1577 { 1578 struct shm_data *shmd = (struct shm_data *)seg->s_data; 1579 struct seg *sptseg; 1580 struct spt_data *sptd; 1581 page_t *pp; 1582 caddr_t adr; 1583 struct vnode *vp; 1584 u_offset_t offset; 1585 ulong_t anon_index; 1586 struct anon_map *amp; /* XXX - for locknest */ 1587 struct anon *ap = NULL; 1588 pgcnt_t npages; 1589 1590 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 1591 1592 sptseg = shmd->shm_sptseg; 1593 sptd = sptseg->s_data; 1594 1595 /* 1596 * Some platforms assume that ISM mappings are HAT_LOAD_LOCK 1597 * and therefore their pages are SE_SHARED locked 1598 * for the entire life of the segment. 1599 */ 1600 if ((!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) && 1601 ((sptd->spt_flags & SHM_PAGEABLE) == 0)) { 1602 goto softlock_decrement; 1603 } 1604 1605 /* 1606 * Any thread is free to do a page_find and 1607 * page_unlock() on the pages within this seg. 1608 * 1609 * We are already holding the as->a_lock on the user's 1610 * real segment, but we need to hold the a_lock on the 1611 * underlying dummy as. This is mostly to satisfy the 1612 * underlying HAT layer. 1613 */ 1614 AS_LOCK_ENTER(sptseg->s_as, RW_READER); 1615 hat_unlock(sptseg->s_as->a_hat, sptseg_addr, len); 1616 AS_LOCK_EXIT(sptseg->s_as); 1617 1618 amp = sptd->spt_amp; 1619 ASSERT(amp != NULL); 1620 anon_index = seg_page(sptseg, sptseg_addr); 1621 1622 for (adr = sptseg_addr; adr < sptseg_addr + len; adr += PAGESIZE) { 1623 ap = anon_get_ptr(amp->ahp, anon_index++); 1624 ASSERT(ap != NULL); 1625 swap_xlate(ap, &vp, &offset); 1626 1627 /* 1628 * Use page_find() instead of page_lookup() to 1629 * find the page since we know that it has a 1630 * "shared" lock. 1631 */ 1632 pp = page_find(vp, offset); 1633 ASSERT(ap == anon_get_ptr(amp->ahp, anon_index - 1)); 1634 if (pp == NULL) { 1635 panic("segspt_softunlock: " 1636 "addr %p, ap %p, vp %p, off %llx", 1637 (void *)adr, (void *)ap, (void *)vp, offset); 1638 /*NOTREACHED*/ 1639 } 1640 1641 if (rw == S_WRITE) { 1642 hat_setrefmod(pp); 1643 } else if (rw != S_OTHER) { 1644 hat_setref(pp); 1645 } 1646 page_unlock(pp); 1647 } 1648 1649 softlock_decrement: 1650 npages = btopr(len); 1651 ASSERT(shmd->shm_softlockcnt >= npages); 1652 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), -npages); 1653 if (shmd->shm_softlockcnt == 0) { 1654 /* 1655 * All SOFTLOCKS are gone. Wakeup any waiting 1656 * unmappers so they can try again to unmap. 1657 * Check for waiters first without the mutex 1658 * held so we don't always grab the mutex on 1659 * softunlocks. 1660 */ 1661 if (AS_ISUNMAPWAIT(seg->s_as)) { 1662 mutex_enter(&seg->s_as->a_contents); 1663 if (AS_ISUNMAPWAIT(seg->s_as)) { 1664 AS_CLRUNMAPWAIT(seg->s_as); 1665 cv_broadcast(&seg->s_as->a_cv); 1666 } 1667 mutex_exit(&seg->s_as->a_contents); 1668 } 1669 } 1670 } 1671 1672 int 1673 segspt_shmattach(struct seg *seg, caddr_t *argsp) 1674 { 1675 struct shm_data *shmd_arg = (struct shm_data *)argsp; 1676 struct shm_data *shmd; 1677 struct anon_map *shm_amp = shmd_arg->shm_amp; 1678 struct spt_data *sptd; 1679 int error = 0; 1680 1681 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as)); 1682 1683 shmd = kmem_zalloc((sizeof (*shmd)), KM_NOSLEEP); 1684 if (shmd == NULL) 1685 return (ENOMEM); 1686 1687 shmd->shm_sptas = shmd_arg->shm_sptas; 1688 shmd->shm_amp = shm_amp; 1689 shmd->shm_sptseg = shmd_arg->shm_sptseg; 1690 1691 (void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, shm_amp, 0, 1692 NULL, 0, seg->s_size); 1693 1694 mutex_init(&shmd->shm_segfree_syncmtx, NULL, MUTEX_DEFAULT, NULL); 1695 1696 seg->s_data = (void *)shmd; 1697 seg->s_ops = &segspt_shmops; 1698 seg->s_szc = shmd->shm_sptseg->s_szc; 1699 sptd = shmd->shm_sptseg->s_data; 1700 1701 if (sptd->spt_flags & SHM_PAGEABLE) { 1702 if ((shmd->shm_vpage = kmem_zalloc(btopr(shm_amp->size), 1703 KM_NOSLEEP)) == NULL) { 1704 seg->s_data = (void *)NULL; 1705 kmem_free(shmd, (sizeof (*shmd))); 1706 return (ENOMEM); 1707 } 1708 shmd->shm_lckpgs = 0; 1709 if (hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) { 1710 if ((error = hat_share(seg->s_as->a_hat, seg->s_base, 1711 shmd_arg->shm_sptas->a_hat, SEGSPTADDR, 1712 seg->s_size, seg->s_szc)) != 0) { 1713 kmem_free(shmd->shm_vpage, 1714 btopr(shm_amp->size)); 1715 } 1716 } 1717 } else { 1718 error = hat_share(seg->s_as->a_hat, seg->s_base, 1719 shmd_arg->shm_sptas->a_hat, SEGSPTADDR, 1720 seg->s_size, seg->s_szc); 1721 } 1722 if (error) { 1723 seg->s_szc = 0; 1724 seg->s_data = (void *)NULL; 1725 kmem_free(shmd, (sizeof (*shmd))); 1726 } else { 1727 ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER); 1728 shm_amp->refcnt++; 1729 ANON_LOCK_EXIT(&shm_amp->a_rwlock); 1730 } 1731 return (error); 1732 } 1733 1734 int 1735 segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize) 1736 { 1737 struct shm_data *shmd = (struct shm_data *)seg->s_data; 1738 int reclaim = 1; 1739 1740 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as)); 1741 retry: 1742 if (shmd->shm_softlockcnt > 0) { 1743 if (reclaim == 1) { 1744 segspt_purge(seg); 1745 reclaim = 0; 1746 goto retry; 1747 } 1748 return (EAGAIN); 1749 } 1750 1751 if (ssize != seg->s_size) { 1752 #ifdef DEBUG 1753 cmn_err(CE_WARN, "Incompatible ssize %lx s_size %lx\n", 1754 ssize, seg->s_size); 1755 #endif 1756 return (EINVAL); 1757 } 1758 1759 (void) segspt_shmlockop(seg, raddr, shmd->shm_amp->size, 0, MC_UNLOCK, 1760 NULL, 0); 1761 hat_unshare(seg->s_as->a_hat, raddr, ssize, seg->s_szc); 1762 1763 seg_free(seg); 1764 1765 return (0); 1766 } 1767 1768 void 1769 segspt_shmfree(struct seg *seg) 1770 { 1771 struct shm_data *shmd = (struct shm_data *)seg->s_data; 1772 struct anon_map *shm_amp = shmd->shm_amp; 1773 1774 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as)); 1775 1776 (void) segspt_shmlockop(seg, seg->s_base, shm_amp->size, 0, 1777 MC_UNLOCK, NULL, 0); 1778 1779 /* 1780 * Need to increment refcnt when attaching 1781 * and decrement when detaching because of dup(). 1782 */ 1783 ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER); 1784 shm_amp->refcnt--; 1785 ANON_LOCK_EXIT(&shm_amp->a_rwlock); 1786 1787 if (shmd->shm_vpage) { /* only for DISM */ 1788 kmem_free(shmd->shm_vpage, btopr(shm_amp->size)); 1789 shmd->shm_vpage = NULL; 1790 } 1791 1792 /* 1793 * Take shm_segfree_syncmtx lock to let segspt_reclaim() finish if it's 1794 * still working with this segment without holding as lock. 1795 */ 1796 ASSERT(shmd->shm_softlockcnt == 0); 1797 mutex_enter(&shmd->shm_segfree_syncmtx); 1798 mutex_destroy(&shmd->shm_segfree_syncmtx); 1799 1800 kmem_free(shmd, sizeof (*shmd)); 1801 } 1802 1803 /*ARGSUSED*/ 1804 int 1805 segspt_shmsetprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot) 1806 { 1807 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 1808 1809 /* 1810 * Shared page table is more than shared mapping. 1811 * Individual process sharing page tables can't change prot 1812 * because there is only one set of page tables. 1813 * This will be allowed after private page table is 1814 * supported. 1815 */ 1816 /* need to return correct status error? */ 1817 return (0); 1818 } 1819 1820 1821 faultcode_t 1822 segspt_dismfault(struct hat *hat, struct seg *seg, caddr_t addr, 1823 size_t len, enum fault_type type, enum seg_rw rw) 1824 { 1825 struct shm_data *shmd = (struct shm_data *)seg->s_data; 1826 struct seg *sptseg = shmd->shm_sptseg; 1827 struct as *curspt = shmd->shm_sptas; 1828 struct spt_data *sptd = sptseg->s_data; 1829 pgcnt_t npages; 1830 size_t size; 1831 caddr_t segspt_addr, shm_addr; 1832 page_t **ppa; 1833 int i; 1834 ulong_t an_idx = 0; 1835 int err = 0; 1836 int dyn_ism_unmap = hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0); 1837 size_t pgsz; 1838 pgcnt_t pgcnt; 1839 caddr_t a; 1840 pgcnt_t pidx; 1841 1842 #ifdef lint 1843 hat = hat; 1844 #endif 1845 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 1846 1847 /* 1848 * Because of the way spt is implemented 1849 * the realsize of the segment does not have to be 1850 * equal to the segment size itself. The segment size is 1851 * often in multiples of a page size larger than PAGESIZE. 1852 * The realsize is rounded up to the nearest PAGESIZE 1853 * based on what the user requested. This is a bit of 1854 * ungliness that is historical but not easily fixed 1855 * without re-designing the higher levels of ISM. 1856 */ 1857 ASSERT(addr >= seg->s_base); 1858 if (((addr + len) - seg->s_base) > sptd->spt_realsize) 1859 return (FC_NOMAP); 1860 /* 1861 * For all of the following cases except F_PROT, we need to 1862 * make any necessary adjustments to addr and len 1863 * and get all of the necessary page_t's into an array called ppa[]. 1864 * 1865 * The code in shmat() forces base addr and len of ISM segment 1866 * to be aligned to largest page size supported. Therefore, 1867 * we are able to handle F_SOFTLOCK and F_INVAL calls in "large 1868 * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK 1869 * in large pagesize chunks, or else we will screw up the HAT 1870 * layer by calling hat_memload_array() with differing page sizes 1871 * over a given virtual range. 1872 */ 1873 pgsz = page_get_pagesize(sptseg->s_szc); 1874 pgcnt = page_get_pagecnt(sptseg->s_szc); 1875 shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), pgsz); 1876 size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), pgsz); 1877 npages = btopr(size); 1878 1879 /* 1880 * Now we need to convert from addr in segshm to addr in segspt. 1881 */ 1882 an_idx = seg_page(seg, shm_addr); 1883 segspt_addr = sptseg->s_base + ptob(an_idx); 1884 1885 ASSERT((segspt_addr + ptob(npages)) <= 1886 (sptseg->s_base + sptd->spt_realsize)); 1887 ASSERT(segspt_addr < (sptseg->s_base + sptseg->s_size)); 1888 1889 switch (type) { 1890 1891 case F_SOFTLOCK: 1892 1893 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages); 1894 /* 1895 * Fall through to the F_INVAL case to load up the hat layer 1896 * entries with the HAT_LOAD_LOCK flag. 1897 */ 1898 /* FALLTHRU */ 1899 case F_INVAL: 1900 1901 if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC)) 1902 return (FC_NOMAP); 1903 1904 ppa = kmem_zalloc(npages * sizeof (page_t *), KM_SLEEP); 1905 1906 err = spt_anon_getpages(sptseg, segspt_addr, size, ppa); 1907 if (err != 0) { 1908 if (type == F_SOFTLOCK) { 1909 atomic_add_long((ulong_t *)( 1910 &(shmd->shm_softlockcnt)), -npages); 1911 } 1912 goto dism_err; 1913 } 1914 AS_LOCK_ENTER(sptseg->s_as, RW_READER); 1915 a = segspt_addr; 1916 pidx = 0; 1917 if (type == F_SOFTLOCK) { 1918 1919 /* 1920 * Load up the translation keeping it 1921 * locked and don't unlock the page. 1922 */ 1923 for (; pidx < npages; a += pgsz, pidx += pgcnt) { 1924 hat_memload_array(sptseg->s_as->a_hat, 1925 a, pgsz, &ppa[pidx], sptd->spt_prot, 1926 HAT_LOAD_LOCK | HAT_LOAD_SHARE); 1927 } 1928 } else { 1929 /* 1930 * Migrate pages marked for migration 1931 */ 1932 if (lgrp_optimizations()) 1933 page_migrate(seg, shm_addr, ppa, npages); 1934 1935 for (; pidx < npages; a += pgsz, pidx += pgcnt) { 1936 hat_memload_array(sptseg->s_as->a_hat, 1937 a, pgsz, &ppa[pidx], 1938 sptd->spt_prot, 1939 HAT_LOAD_SHARE); 1940 } 1941 1942 /* 1943 * And now drop the SE_SHARED lock(s). 1944 */ 1945 if (dyn_ism_unmap) { 1946 for (i = 0; i < npages; i++) { 1947 page_unlock(ppa[i]); 1948 } 1949 } 1950 } 1951 1952 if (!dyn_ism_unmap) { 1953 if (hat_share(seg->s_as->a_hat, shm_addr, 1954 curspt->a_hat, segspt_addr, ptob(npages), 1955 seg->s_szc) != 0) { 1956 panic("hat_share err in DISM fault"); 1957 /* NOTREACHED */ 1958 } 1959 if (type == F_INVAL) { 1960 for (i = 0; i < npages; i++) { 1961 page_unlock(ppa[i]); 1962 } 1963 } 1964 } 1965 AS_LOCK_EXIT(sptseg->s_as); 1966 dism_err: 1967 kmem_free(ppa, npages * sizeof (page_t *)); 1968 return (err); 1969 1970 case F_SOFTUNLOCK: 1971 1972 /* 1973 * This is a bit ugly, we pass in the real seg pointer, 1974 * but the segspt_addr is the virtual address within the 1975 * dummy seg. 1976 */ 1977 segspt_softunlock(seg, segspt_addr, size, rw); 1978 return (0); 1979 1980 case F_PROT: 1981 1982 /* 1983 * This takes care of the unusual case where a user 1984 * allocates a stack in shared memory and a register 1985 * window overflow is written to that stack page before 1986 * it is otherwise modified. 1987 * 1988 * We can get away with this because ISM segments are 1989 * always rw. Other than this unusual case, there 1990 * should be no instances of protection violations. 1991 */ 1992 return (0); 1993 1994 default: 1995 #ifdef DEBUG 1996 panic("segspt_dismfault default type?"); 1997 #else 1998 return (FC_NOMAP); 1999 #endif 2000 } 2001 } 2002 2003 2004 faultcode_t 2005 segspt_shmfault(struct hat *hat, struct seg *seg, caddr_t addr, 2006 size_t len, enum fault_type type, enum seg_rw rw) 2007 { 2008 struct shm_data *shmd = (struct shm_data *)seg->s_data; 2009 struct seg *sptseg = shmd->shm_sptseg; 2010 struct as *curspt = shmd->shm_sptas; 2011 struct spt_data *sptd = sptseg->s_data; 2012 pgcnt_t npages; 2013 size_t size; 2014 caddr_t sptseg_addr, shm_addr; 2015 page_t *pp, **ppa; 2016 int i; 2017 u_offset_t offset; 2018 ulong_t anon_index = 0; 2019 struct vnode *vp; 2020 struct anon_map *amp; /* XXX - for locknest */ 2021 struct anon *ap = NULL; 2022 size_t pgsz; 2023 pgcnt_t pgcnt; 2024 caddr_t a; 2025 pgcnt_t pidx; 2026 size_t sz; 2027 2028 #ifdef lint 2029 hat = hat; 2030 #endif 2031 2032 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 2033 2034 if (sptd->spt_flags & SHM_PAGEABLE) { 2035 return (segspt_dismfault(hat, seg, addr, len, type, rw)); 2036 } 2037 2038 /* 2039 * Because of the way spt is implemented 2040 * the realsize of the segment does not have to be 2041 * equal to the segment size itself. The segment size is 2042 * often in multiples of a page size larger than PAGESIZE. 2043 * The realsize is rounded up to the nearest PAGESIZE 2044 * based on what the user requested. This is a bit of 2045 * ungliness that is historical but not easily fixed 2046 * without re-designing the higher levels of ISM. 2047 */ 2048 ASSERT(addr >= seg->s_base); 2049 if (((addr + len) - seg->s_base) > sptd->spt_realsize) 2050 return (FC_NOMAP); 2051 /* 2052 * For all of the following cases except F_PROT, we need to 2053 * make any necessary adjustments to addr and len 2054 * and get all of the necessary page_t's into an array called ppa[]. 2055 * 2056 * The code in shmat() forces base addr and len of ISM segment 2057 * to be aligned to largest page size supported. Therefore, 2058 * we are able to handle F_SOFTLOCK and F_INVAL calls in "large 2059 * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK 2060 * in large pagesize chunks, or else we will screw up the HAT 2061 * layer by calling hat_memload_array() with differing page sizes 2062 * over a given virtual range. 2063 */ 2064 pgsz = page_get_pagesize(sptseg->s_szc); 2065 pgcnt = page_get_pagecnt(sptseg->s_szc); 2066 shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), pgsz); 2067 size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), pgsz); 2068 npages = btopr(size); 2069 2070 /* 2071 * Now we need to convert from addr in segshm to addr in segspt. 2072 */ 2073 anon_index = seg_page(seg, shm_addr); 2074 sptseg_addr = sptseg->s_base + ptob(anon_index); 2075 2076 /* 2077 * And now we may have to adjust npages downward if we have 2078 * exceeded the realsize of the segment or initial anon 2079 * allocations. 2080 */ 2081 if ((sptseg_addr + ptob(npages)) > 2082 (sptseg->s_base + sptd->spt_realsize)) 2083 size = (sptseg->s_base + sptd->spt_realsize) - sptseg_addr; 2084 2085 npages = btopr(size); 2086 2087 ASSERT(sptseg_addr < (sptseg->s_base + sptseg->s_size)); 2088 ASSERT((sptd->spt_flags & SHM_PAGEABLE) == 0); 2089 2090 switch (type) { 2091 2092 case F_SOFTLOCK: 2093 2094 /* 2095 * availrmem is decremented once during anon_swap_adjust() 2096 * and is incremented during the anon_unresv(), which is 2097 * called from shm_rm_amp() when the segment is destroyed. 2098 */ 2099 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages); 2100 /* 2101 * Some platforms assume that ISM pages are SE_SHARED 2102 * locked for the entire life of the segment. 2103 */ 2104 if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) 2105 return (0); 2106 /* 2107 * Fall through to the F_INVAL case to load up the hat layer 2108 * entries with the HAT_LOAD_LOCK flag. 2109 */ 2110 2111 /* FALLTHRU */ 2112 case F_INVAL: 2113 2114 if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC)) 2115 return (FC_NOMAP); 2116 2117 /* 2118 * Some platforms that do NOT support DYNAMIC_ISM_UNMAP 2119 * may still rely on this call to hat_share(). That 2120 * would imply that those hat's can fault on a 2121 * HAT_LOAD_LOCK translation, which would seem 2122 * contradictory. 2123 */ 2124 if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) { 2125 if (hat_share(seg->s_as->a_hat, seg->s_base, 2126 curspt->a_hat, sptseg->s_base, 2127 sptseg->s_size, sptseg->s_szc) != 0) { 2128 panic("hat_share error in ISM fault"); 2129 /*NOTREACHED*/ 2130 } 2131 return (0); 2132 } 2133 ppa = kmem_zalloc(sizeof (page_t *) * npages, KM_SLEEP); 2134 2135 /* 2136 * I see no need to lock the real seg, 2137 * here, because all of our work will be on the underlying 2138 * dummy seg. 2139 * 2140 * sptseg_addr and npages now account for large pages. 2141 */ 2142 amp = sptd->spt_amp; 2143 ASSERT(amp != NULL); 2144 anon_index = seg_page(sptseg, sptseg_addr); 2145 2146 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 2147 for (i = 0; i < npages; i++) { 2148 ap = anon_get_ptr(amp->ahp, anon_index++); 2149 ASSERT(ap != NULL); 2150 swap_xlate(ap, &vp, &offset); 2151 pp = page_lookup(vp, offset, SE_SHARED); 2152 ASSERT(pp != NULL); 2153 ppa[i] = pp; 2154 } 2155 ANON_LOCK_EXIT(&->a_rwlock); 2156 ASSERT(i == npages); 2157 2158 /* 2159 * We are already holding the as->a_lock on the user's 2160 * real segment, but we need to hold the a_lock on the 2161 * underlying dummy as. This is mostly to satisfy the 2162 * underlying HAT layer. 2163 */ 2164 AS_LOCK_ENTER(sptseg->s_as, RW_READER); 2165 a = sptseg_addr; 2166 pidx = 0; 2167 if (type == F_SOFTLOCK) { 2168 /* 2169 * Load up the translation keeping it 2170 * locked and don't unlock the page. 2171 */ 2172 for (; pidx < npages; a += pgsz, pidx += pgcnt) { 2173 sz = MIN(pgsz, ptob(npages - pidx)); 2174 hat_memload_array(sptseg->s_as->a_hat, a, 2175 sz, &ppa[pidx], sptd->spt_prot, 2176 HAT_LOAD_LOCK | HAT_LOAD_SHARE); 2177 } 2178 } else { 2179 /* 2180 * Migrate pages marked for migration. 2181 */ 2182 if (lgrp_optimizations()) 2183 page_migrate(seg, shm_addr, ppa, npages); 2184 2185 for (; pidx < npages; a += pgsz, pidx += pgcnt) { 2186 sz = MIN(pgsz, ptob(npages - pidx)); 2187 hat_memload_array(sptseg->s_as->a_hat, 2188 a, sz, &ppa[pidx], 2189 sptd->spt_prot, HAT_LOAD_SHARE); 2190 } 2191 2192 /* 2193 * And now drop the SE_SHARED lock(s). 2194 */ 2195 for (i = 0; i < npages; i++) 2196 page_unlock(ppa[i]); 2197 } 2198 AS_LOCK_EXIT(sptseg->s_as); 2199 2200 kmem_free(ppa, sizeof (page_t *) * npages); 2201 return (0); 2202 case F_SOFTUNLOCK: 2203 2204 /* 2205 * This is a bit ugly, we pass in the real seg pointer, 2206 * but the sptseg_addr is the virtual address within the 2207 * dummy seg. 2208 */ 2209 segspt_softunlock(seg, sptseg_addr, ptob(npages), rw); 2210 return (0); 2211 2212 case F_PROT: 2213 2214 /* 2215 * This takes care of the unusual case where a user 2216 * allocates a stack in shared memory and a register 2217 * window overflow is written to that stack page before 2218 * it is otherwise modified. 2219 * 2220 * We can get away with this because ISM segments are 2221 * always rw. Other than this unusual case, there 2222 * should be no instances of protection violations. 2223 */ 2224 return (0); 2225 2226 default: 2227 #ifdef DEBUG 2228 cmn_err(CE_WARN, "segspt_shmfault default type?"); 2229 #endif 2230 return (FC_NOMAP); 2231 } 2232 } 2233 2234 /*ARGSUSED*/ 2235 static faultcode_t 2236 segspt_shmfaulta(struct seg *seg, caddr_t addr) 2237 { 2238 return (0); 2239 } 2240 2241 /*ARGSUSED*/ 2242 static int 2243 segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta) 2244 { 2245 return (0); 2246 } 2247 2248 /*ARGSUSED*/ 2249 static size_t 2250 segspt_shmswapout(struct seg *seg) 2251 { 2252 return (0); 2253 } 2254 2255 /* 2256 * duplicate the shared page tables 2257 */ 2258 int 2259 segspt_shmdup(struct seg *seg, struct seg *newseg) 2260 { 2261 struct shm_data *shmd = (struct shm_data *)seg->s_data; 2262 struct anon_map *amp = shmd->shm_amp; 2263 struct shm_data *shmd_new; 2264 struct seg *spt_seg = shmd->shm_sptseg; 2265 struct spt_data *sptd = spt_seg->s_data; 2266 int error = 0; 2267 2268 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as)); 2269 2270 shmd_new = kmem_zalloc((sizeof (*shmd_new)), KM_SLEEP); 2271 newseg->s_data = (void *)shmd_new; 2272 shmd_new->shm_sptas = shmd->shm_sptas; 2273 shmd_new->shm_amp = amp; 2274 shmd_new->shm_sptseg = shmd->shm_sptseg; 2275 newseg->s_ops = &segspt_shmops; 2276 newseg->s_szc = seg->s_szc; 2277 ASSERT(seg->s_szc == shmd->shm_sptseg->s_szc); 2278 2279 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 2280 amp->refcnt++; 2281 ANON_LOCK_EXIT(&->a_rwlock); 2282 2283 if (sptd->spt_flags & SHM_PAGEABLE) { 2284 shmd_new->shm_vpage = kmem_zalloc(btopr(amp->size), KM_SLEEP); 2285 shmd_new->shm_lckpgs = 0; 2286 if (hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) { 2287 if ((error = hat_share(newseg->s_as->a_hat, 2288 newseg->s_base, shmd->shm_sptas->a_hat, SEGSPTADDR, 2289 seg->s_size, seg->s_szc)) != 0) { 2290 kmem_free(shmd_new->shm_vpage, 2291 btopr(amp->size)); 2292 } 2293 } 2294 return (error); 2295 } else { 2296 return (hat_share(newseg->s_as->a_hat, newseg->s_base, 2297 shmd->shm_sptas->a_hat, SEGSPTADDR, seg->s_size, 2298 seg->s_szc)); 2299 2300 } 2301 } 2302 2303 /*ARGSUSED*/ 2304 int 2305 segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot) 2306 { 2307 struct shm_data *shmd = (struct shm_data *)seg->s_data; 2308 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 2309 2310 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 2311 2312 /* 2313 * ISM segment is always rw. 2314 */ 2315 return (((sptd->spt_prot & prot) != prot) ? EACCES : 0); 2316 } 2317 2318 /* 2319 * Return an array of locked large pages, for empty slots allocate 2320 * private zero-filled anon pages. 2321 */ 2322 static int 2323 spt_anon_getpages( 2324 struct seg *sptseg, 2325 caddr_t sptaddr, 2326 size_t len, 2327 page_t *ppa[]) 2328 { 2329 struct spt_data *sptd = sptseg->s_data; 2330 struct anon_map *amp = sptd->spt_amp; 2331 enum seg_rw rw = sptd->spt_prot; 2332 uint_t szc = sptseg->s_szc; 2333 size_t pg_sz, share_sz = page_get_pagesize(szc); 2334 pgcnt_t lp_npgs; 2335 caddr_t lp_addr, e_sptaddr; 2336 uint_t vpprot, ppa_szc = 0; 2337 struct vpage *vpage = NULL; 2338 ulong_t j, ppa_idx; 2339 int err, ierr = 0; 2340 pgcnt_t an_idx; 2341 anon_sync_obj_t cookie; 2342 int anon_locked = 0; 2343 pgcnt_t amp_pgs; 2344 2345 2346 ASSERT(IS_P2ALIGNED(sptaddr, share_sz) && IS_P2ALIGNED(len, share_sz)); 2347 ASSERT(len != 0); 2348 2349 pg_sz = share_sz; 2350 lp_npgs = btop(pg_sz); 2351 lp_addr = sptaddr; 2352 e_sptaddr = sptaddr + len; 2353 an_idx = seg_page(sptseg, sptaddr); 2354 ppa_idx = 0; 2355 2356 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 2357 2358 amp_pgs = page_get_pagecnt(amp->a_szc); 2359 2360 /*CONSTCOND*/ 2361 while (1) { 2362 for (; lp_addr < e_sptaddr; 2363 an_idx += lp_npgs, lp_addr += pg_sz, ppa_idx += lp_npgs) { 2364 2365 /* 2366 * If we're currently locked, and we get to a new 2367 * page, unlock our current anon chunk. 2368 */ 2369 if (anon_locked && P2PHASE(an_idx, amp_pgs) == 0) { 2370 anon_array_exit(&cookie); 2371 anon_locked = 0; 2372 } 2373 if (!anon_locked) { 2374 anon_array_enter(amp, an_idx, &cookie); 2375 anon_locked = 1; 2376 } 2377 ppa_szc = (uint_t)-1; 2378 ierr = anon_map_getpages(amp, an_idx, szc, sptseg, 2379 lp_addr, sptd->spt_prot, &vpprot, &ppa[ppa_idx], 2380 &ppa_szc, vpage, rw, 0, segvn_anypgsz, 0, kcred); 2381 2382 if (ierr != 0) { 2383 if (ierr > 0) { 2384 err = FC_MAKE_ERR(ierr); 2385 goto lpgs_err; 2386 } 2387 break; 2388 } 2389 } 2390 if (lp_addr == e_sptaddr) { 2391 break; 2392 } 2393 ASSERT(lp_addr < e_sptaddr); 2394 2395 /* 2396 * ierr == -1 means we failed to allocate a large page. 2397 * so do a size down operation. 2398 * 2399 * ierr == -2 means some other process that privately shares 2400 * pages with this process has allocated a larger page and we 2401 * need to retry with larger pages. So do a size up 2402 * operation. This relies on the fact that large pages are 2403 * never partially shared i.e. if we share any constituent 2404 * page of a large page with another process we must share the 2405 * entire large page. Note this cannot happen for SOFTLOCK 2406 * case, unless current address (lpaddr) is at the beginning 2407 * of the next page size boundary because the other process 2408 * couldn't have relocated locked pages. 2409 */ 2410 ASSERT(ierr == -1 || ierr == -2); 2411 if (segvn_anypgsz) { 2412 ASSERT(ierr == -2 || szc != 0); 2413 ASSERT(ierr == -1 || szc < sptseg->s_szc); 2414 szc = (ierr == -1) ? szc - 1 : szc + 1; 2415 } else { 2416 /* 2417 * For faults and segvn_anypgsz == 0 2418 * we need to be careful not to loop forever 2419 * if existing page is found with szc other 2420 * than 0 or seg->s_szc. This could be due 2421 * to page relocations on behalf of DR or 2422 * more likely large page creation. For this 2423 * case simply re-size to existing page's szc 2424 * if returned by anon_map_getpages(). 2425 */ 2426 if (ppa_szc == (uint_t)-1) { 2427 szc = (ierr == -1) ? 0 : sptseg->s_szc; 2428 } else { 2429 ASSERT(ppa_szc <= sptseg->s_szc); 2430 ASSERT(ierr == -2 || ppa_szc < szc); 2431 ASSERT(ierr == -1 || ppa_szc > szc); 2432 szc = ppa_szc; 2433 } 2434 } 2435 pg_sz = page_get_pagesize(szc); 2436 lp_npgs = btop(pg_sz); 2437 ASSERT(IS_P2ALIGNED(lp_addr, pg_sz)); 2438 } 2439 if (anon_locked) { 2440 anon_array_exit(&cookie); 2441 } 2442 ANON_LOCK_EXIT(&->a_rwlock); 2443 return (0); 2444 2445 lpgs_err: 2446 if (anon_locked) { 2447 anon_array_exit(&cookie); 2448 } 2449 ANON_LOCK_EXIT(&->a_rwlock); 2450 for (j = 0; j < ppa_idx; j++) 2451 page_unlock(ppa[j]); 2452 return (err); 2453 } 2454 2455 /* 2456 * count the number of bytes in a set of spt pages that are currently not 2457 * locked 2458 */ 2459 static rctl_qty_t 2460 spt_unlockedbytes(pgcnt_t npages, page_t **ppa) 2461 { 2462 ulong_t i; 2463 rctl_qty_t unlocked = 0; 2464 2465 for (i = 0; i < npages; i++) { 2466 if (ppa[i]->p_lckcnt == 0) 2467 unlocked += PAGESIZE; 2468 } 2469 return (unlocked); 2470 } 2471 2472 extern u_longlong_t randtick(void); 2473 /* number of locks to reserve/skip by spt_lockpages() and spt_unlockpages() */ 2474 #define NLCK (NCPU_P2) 2475 /* Random number with a range [0, n-1], n must be power of two */ 2476 #define RAND_P2(n) \ 2477 ((((long)curthread >> PTR24_LSB) ^ (long)randtick()) & ((n) - 1)) 2478 2479 int 2480 spt_lockpages(struct seg *seg, pgcnt_t anon_index, pgcnt_t npages, 2481 page_t **ppa, ulong_t *lockmap, size_t pos, 2482 rctl_qty_t *locked) 2483 { 2484 struct shm_data *shmd = seg->s_data; 2485 struct spt_data *sptd = shmd->shm_sptseg->s_data; 2486 ulong_t i; 2487 int kernel; 2488 pgcnt_t nlck = 0; 2489 int rv = 0; 2490 int use_reserved = 1; 2491 2492 /* return the number of bytes actually locked */ 2493 *locked = 0; 2494 2495 /* 2496 * To avoid contention on freemem_lock, availrmem and pages_locked 2497 * global counters are updated only every nlck locked pages instead of 2498 * every time. Reserve nlck locks up front and deduct from this 2499 * reservation for each page that requires a lock. When the reservation 2500 * is consumed, reserve again. nlck is randomized, so the competing 2501 * threads do not fall into a cyclic lock contention pattern. When 2502 * memory is low, the lock ahead is disabled, and instead page_pp_lock() 2503 * is used to lock pages. 2504 */ 2505 for (i = 0; i < npages; anon_index++, pos++, i++) { 2506 if (nlck == 0 && use_reserved == 1) { 2507 nlck = NLCK + RAND_P2(NLCK); 2508 /* if fewer loops left, decrease nlck */ 2509 nlck = MIN(nlck, npages - i); 2510 /* 2511 * Reserve nlck locks up front and deduct from this 2512 * reservation for each page that requires a lock. When 2513 * the reservation is consumed, reserve again. 2514 */ 2515 mutex_enter(&freemem_lock); 2516 if ((availrmem - nlck) < pages_pp_maximum) { 2517 /* Do not do advance memory reserves */ 2518 use_reserved = 0; 2519 } else { 2520 availrmem -= nlck; 2521 pages_locked += nlck; 2522 } 2523 mutex_exit(&freemem_lock); 2524 } 2525 if (!(shmd->shm_vpage[anon_index] & DISM_PG_LOCKED)) { 2526 if (sptd->spt_ppa_lckcnt[anon_index] < 2527 (ushort_t)DISM_LOCK_MAX) { 2528 if (++sptd->spt_ppa_lckcnt[anon_index] == 2529 (ushort_t)DISM_LOCK_MAX) { 2530 cmn_err(CE_WARN, 2531 "DISM page lock limit " 2532 "reached on DISM offset 0x%lx\n", 2533 anon_index << PAGESHIFT); 2534 } 2535 kernel = (sptd->spt_ppa && 2536 sptd->spt_ppa[anon_index]); 2537 if (!page_pp_lock(ppa[i], 0, kernel || 2538 use_reserved)) { 2539 sptd->spt_ppa_lckcnt[anon_index]--; 2540 rv = EAGAIN; 2541 break; 2542 } 2543 /* if this is a newly locked page, count it */ 2544 if (ppa[i]->p_lckcnt == 1) { 2545 if (kernel == 0 && use_reserved == 1) 2546 nlck--; 2547 *locked += PAGESIZE; 2548 } 2549 shmd->shm_lckpgs++; 2550 shmd->shm_vpage[anon_index] |= DISM_PG_LOCKED; 2551 if (lockmap != NULL) 2552 BT_SET(lockmap, pos); 2553 } 2554 } 2555 } 2556 /* Return unused lock reservation */ 2557 if (nlck != 0 && use_reserved == 1) { 2558 mutex_enter(&freemem_lock); 2559 availrmem += nlck; 2560 pages_locked -= nlck; 2561 mutex_exit(&freemem_lock); 2562 } 2563 2564 return (rv); 2565 } 2566 2567 int 2568 spt_unlockpages(struct seg *seg, pgcnt_t anon_index, pgcnt_t npages, 2569 rctl_qty_t *unlocked) 2570 { 2571 struct shm_data *shmd = seg->s_data; 2572 struct spt_data *sptd = shmd->shm_sptseg->s_data; 2573 struct anon_map *amp = sptd->spt_amp; 2574 struct anon *ap; 2575 struct vnode *vp; 2576 u_offset_t off; 2577 struct page *pp; 2578 int kernel; 2579 anon_sync_obj_t cookie; 2580 ulong_t i; 2581 pgcnt_t nlck = 0; 2582 pgcnt_t nlck_limit = NLCK; 2583 2584 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 2585 for (i = 0; i < npages; i++, anon_index++) { 2586 if (shmd->shm_vpage[anon_index] & DISM_PG_LOCKED) { 2587 anon_array_enter(amp, anon_index, &cookie); 2588 ap = anon_get_ptr(amp->ahp, anon_index); 2589 ASSERT(ap); 2590 2591 swap_xlate(ap, &vp, &off); 2592 anon_array_exit(&cookie); 2593 pp = page_lookup(vp, off, SE_SHARED); 2594 ASSERT(pp); 2595 /* 2596 * availrmem is decremented only for pages which are not 2597 * in seg pcache, for pages in seg pcache availrmem was 2598 * decremented in _dismpagelock() 2599 */ 2600 kernel = (sptd->spt_ppa && sptd->spt_ppa[anon_index]); 2601 ASSERT(pp->p_lckcnt > 0); 2602 2603 /* 2604 * lock page but do not change availrmem, we do it 2605 * ourselves every nlck loops. 2606 */ 2607 page_pp_unlock(pp, 0, 1); 2608 if (pp->p_lckcnt == 0) { 2609 if (kernel == 0) 2610 nlck++; 2611 *unlocked += PAGESIZE; 2612 } 2613 page_unlock(pp); 2614 shmd->shm_vpage[anon_index] &= ~DISM_PG_LOCKED; 2615 sptd->spt_ppa_lckcnt[anon_index]--; 2616 shmd->shm_lckpgs--; 2617 } 2618 2619 /* 2620 * To reduce freemem_lock contention, do not update availrmem 2621 * until at least NLCK pages have been unlocked. 2622 * 1. No need to update if nlck is zero 2623 * 2. Always update if the last iteration 2624 */ 2625 if (nlck > 0 && (nlck == nlck_limit || i == npages - 1)) { 2626 mutex_enter(&freemem_lock); 2627 availrmem += nlck; 2628 pages_locked -= nlck; 2629 mutex_exit(&freemem_lock); 2630 nlck = 0; 2631 nlck_limit = NLCK + RAND_P2(NLCK); 2632 } 2633 } 2634 ANON_LOCK_EXIT(&->a_rwlock); 2635 2636 return (0); 2637 } 2638 2639 /*ARGSUSED*/ 2640 static int 2641 segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len, 2642 int attr, int op, ulong_t *lockmap, size_t pos) 2643 { 2644 struct shm_data *shmd = seg->s_data; 2645 struct seg *sptseg = shmd->shm_sptseg; 2646 struct spt_data *sptd = sptseg->s_data; 2647 struct kshmid *sp = sptd->spt_amp->a_sp; 2648 pgcnt_t npages, a_npages; 2649 page_t **ppa; 2650 pgcnt_t an_idx, a_an_idx, ppa_idx; 2651 caddr_t spt_addr, a_addr; /* spt and aligned address */ 2652 size_t a_len; /* aligned len */ 2653 size_t share_sz; 2654 ulong_t i; 2655 int sts = 0; 2656 rctl_qty_t unlocked = 0; 2657 rctl_qty_t locked = 0; 2658 struct proc *p = curproc; 2659 kproject_t *proj; 2660 2661 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 2662 ASSERT(sp != NULL); 2663 2664 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) { 2665 return (0); 2666 } 2667 2668 addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 2669 an_idx = seg_page(seg, addr); 2670 npages = btopr(len); 2671 2672 if (an_idx + npages > btopr(shmd->shm_amp->size)) { 2673 return (ENOMEM); 2674 } 2675 2676 /* 2677 * A shm's project never changes, so no lock needed. 2678 * The shm has a hold on the project, so it will not go away. 2679 * Since we have a mapping to shm within this zone, we know 2680 * that the zone will not go away. 2681 */ 2682 proj = sp->shm_perm.ipc_proj; 2683 2684 if (op == MC_LOCK) { 2685 2686 /* 2687 * Need to align addr and size request if they are not 2688 * aligned so we can always allocate large page(s) however 2689 * we only lock what was requested in initial request. 2690 */ 2691 share_sz = page_get_pagesize(sptseg->s_szc); 2692 a_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_sz); 2693 a_len = P2ROUNDUP((uintptr_t)(((addr + len) - a_addr)), 2694 share_sz); 2695 a_npages = btop(a_len); 2696 a_an_idx = seg_page(seg, a_addr); 2697 spt_addr = sptseg->s_base + ptob(a_an_idx); 2698 ppa_idx = an_idx - a_an_idx; 2699 2700 if ((ppa = kmem_zalloc(((sizeof (page_t *)) * a_npages), 2701 KM_NOSLEEP)) == NULL) { 2702 return (ENOMEM); 2703 } 2704 2705 /* 2706 * Don't cache any new pages for IO and 2707 * flush any cached pages. 2708 */ 2709 mutex_enter(&sptd->spt_lock); 2710 if (sptd->spt_ppa != NULL) 2711 sptd->spt_flags |= DISM_PPA_CHANGED; 2712 2713 sts = spt_anon_getpages(sptseg, spt_addr, a_len, ppa); 2714 if (sts != 0) { 2715 mutex_exit(&sptd->spt_lock); 2716 kmem_free(ppa, ((sizeof (page_t *)) * a_npages)); 2717 return (sts); 2718 } 2719 2720 mutex_enter(&sp->shm_mlock); 2721 /* enforce locked memory rctl */ 2722 unlocked = spt_unlockedbytes(npages, &ppa[ppa_idx]); 2723 2724 mutex_enter(&p->p_lock); 2725 if (rctl_incr_locked_mem(p, proj, unlocked, 0)) { 2726 mutex_exit(&p->p_lock); 2727 sts = EAGAIN; 2728 } else { 2729 mutex_exit(&p->p_lock); 2730 sts = spt_lockpages(seg, an_idx, npages, 2731 &ppa[ppa_idx], lockmap, pos, &locked); 2732 2733 /* 2734 * correct locked count if not all pages could be 2735 * locked 2736 */ 2737 if ((unlocked - locked) > 0) { 2738 rctl_decr_locked_mem(NULL, proj, 2739 (unlocked - locked), 0); 2740 } 2741 } 2742 /* 2743 * unlock pages 2744 */ 2745 for (i = 0; i < a_npages; i++) 2746 page_unlock(ppa[i]); 2747 if (sptd->spt_ppa != NULL) 2748 sptd->spt_flags |= DISM_PPA_CHANGED; 2749 mutex_exit(&sp->shm_mlock); 2750 mutex_exit(&sptd->spt_lock); 2751 2752 kmem_free(ppa, ((sizeof (page_t *)) * a_npages)); 2753 2754 } else if (op == MC_UNLOCK) { /* unlock */ 2755 page_t **ppa; 2756 2757 mutex_enter(&sptd->spt_lock); 2758 if (shmd->shm_lckpgs == 0) { 2759 mutex_exit(&sptd->spt_lock); 2760 return (0); 2761 } 2762 /* 2763 * Don't cache new IO pages. 2764 */ 2765 if (sptd->spt_ppa != NULL) 2766 sptd->spt_flags |= DISM_PPA_CHANGED; 2767 2768 mutex_enter(&sp->shm_mlock); 2769 sts = spt_unlockpages(seg, an_idx, npages, &unlocked); 2770 if ((ppa = sptd->spt_ppa) != NULL) 2771 sptd->spt_flags |= DISM_PPA_CHANGED; 2772 mutex_exit(&sptd->spt_lock); 2773 2774 rctl_decr_locked_mem(NULL, proj, unlocked, 0); 2775 mutex_exit(&sp->shm_mlock); 2776 2777 if (ppa != NULL) 2778 seg_ppurge_wiredpp(ppa); 2779 } 2780 return (sts); 2781 } 2782 2783 /*ARGSUSED*/ 2784 int 2785 segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv) 2786 { 2787 struct shm_data *shmd = (struct shm_data *)seg->s_data; 2788 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 2789 spgcnt_t pgno = seg_page(seg, addr+len) - seg_page(seg, addr) + 1; 2790 2791 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 2792 2793 /* 2794 * ISM segment is always rw. 2795 */ 2796 while (--pgno >= 0) 2797 *protv++ = sptd->spt_prot; 2798 return (0); 2799 } 2800 2801 /*ARGSUSED*/ 2802 u_offset_t 2803 segspt_shmgetoffset(struct seg *seg, caddr_t addr) 2804 { 2805 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 2806 2807 /* Offset does not matter in ISM memory */ 2808 2809 return ((u_offset_t)0); 2810 } 2811 2812 /* ARGSUSED */ 2813 int 2814 segspt_shmgettype(struct seg *seg, caddr_t addr) 2815 { 2816 struct shm_data *shmd = (struct shm_data *)seg->s_data; 2817 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 2818 2819 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 2820 2821 /* 2822 * The shared memory mapping is always MAP_SHARED, SWAP is only 2823 * reserved for DISM 2824 */ 2825 return (MAP_SHARED | 2826 ((sptd->spt_flags & SHM_PAGEABLE) ? 0 : MAP_NORESERVE)); 2827 } 2828 2829 /*ARGSUSED*/ 2830 int 2831 segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp) 2832 { 2833 struct shm_data *shmd = (struct shm_data *)seg->s_data; 2834 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 2835 2836 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 2837 2838 *vpp = sptd->spt_vp; 2839 return (0); 2840 } 2841 2842 /* 2843 * We need to wait for pending IO to complete to a DISM segment in order for 2844 * pages to get kicked out of the seg_pcache. 120 seconds should be more 2845 * than enough time to wait. 2846 */ 2847 static clock_t spt_pcache_wait = 120; 2848 2849 /*ARGSUSED*/ 2850 static int 2851 segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len, uint_t behav) 2852 { 2853 struct shm_data *shmd = (struct shm_data *)seg->s_data; 2854 struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data; 2855 struct anon_map *amp; 2856 pgcnt_t pg_idx; 2857 ushort_t gen; 2858 clock_t end_lbolt; 2859 int writer; 2860 page_t **ppa; 2861 2862 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as)); 2863 2864 if (behav == MADV_FREE || behav == MADV_PURGE) { 2865 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) 2866 return (0); 2867 2868 amp = sptd->spt_amp; 2869 pg_idx = seg_page(seg, addr); 2870 2871 mutex_enter(&sptd->spt_lock); 2872 if ((ppa = sptd->spt_ppa) == NULL) { 2873 mutex_exit(&sptd->spt_lock); 2874 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 2875 (void) anon_disclaim(amp, pg_idx, len, behav, NULL); 2876 ANON_LOCK_EXIT(&->a_rwlock); 2877 return (0); 2878 } 2879 2880 sptd->spt_flags |= DISM_PPA_CHANGED; 2881 gen = sptd->spt_gen; 2882 2883 mutex_exit(&sptd->spt_lock); 2884 2885 /* 2886 * Purge all DISM cached pages 2887 */ 2888 seg_ppurge_wiredpp(ppa); 2889 2890 /* 2891 * Drop the AS_LOCK so that other threads can grab it 2892 * in the as_pageunlock path and hopefully get the segment 2893 * kicked out of the seg_pcache. We bump the shm_softlockcnt 2894 * to keep this segment resident. 2895 */ 2896 writer = AS_WRITE_HELD(seg->s_as); 2897 atomic_inc_ulong((ulong_t *)(&(shmd->shm_softlockcnt))); 2898 AS_LOCK_EXIT(seg->s_as); 2899 2900 mutex_enter(&sptd->spt_lock); 2901 2902 end_lbolt = ddi_get_lbolt() + (hz * spt_pcache_wait); 2903 2904 /* 2905 * Try to wait for pages to get kicked out of the seg_pcache. 2906 */ 2907 while (sptd->spt_gen == gen && 2908 (sptd->spt_flags & DISM_PPA_CHANGED) && 2909 ddi_get_lbolt() < end_lbolt) { 2910 if (!cv_timedwait_sig(&sptd->spt_cv, 2911 &sptd->spt_lock, end_lbolt)) { 2912 break; 2913 } 2914 } 2915 2916 mutex_exit(&sptd->spt_lock); 2917 2918 /* Regrab the AS_LOCK and release our hold on the segment */ 2919 AS_LOCK_ENTER(seg->s_as, writer ? RW_WRITER : RW_READER); 2920 atomic_dec_ulong((ulong_t *)(&(shmd->shm_softlockcnt))); 2921 if (shmd->shm_softlockcnt <= 0) { 2922 if (AS_ISUNMAPWAIT(seg->s_as)) { 2923 mutex_enter(&seg->s_as->a_contents); 2924 if (AS_ISUNMAPWAIT(seg->s_as)) { 2925 AS_CLRUNMAPWAIT(seg->s_as); 2926 cv_broadcast(&seg->s_as->a_cv); 2927 } 2928 mutex_exit(&seg->s_as->a_contents); 2929 } 2930 } 2931 2932 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 2933 (void) anon_disclaim(amp, pg_idx, len, behav, NULL); 2934 ANON_LOCK_EXIT(&->a_rwlock); 2935 } else if (lgrp_optimizations() && (behav == MADV_ACCESS_LWP || 2936 behav == MADV_ACCESS_MANY || behav == MADV_ACCESS_DEFAULT)) { 2937 int already_set; 2938 ulong_t anon_index; 2939 lgrp_mem_policy_t policy; 2940 caddr_t shm_addr; 2941 size_t share_size; 2942 size_t size; 2943 struct seg *sptseg = shmd->shm_sptseg; 2944 caddr_t sptseg_addr; 2945 2946 /* 2947 * Align address and length to page size of underlying segment 2948 */ 2949 share_size = page_get_pagesize(shmd->shm_sptseg->s_szc); 2950 shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_size); 2951 size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), 2952 share_size); 2953 2954 amp = shmd->shm_amp; 2955 anon_index = seg_page(seg, shm_addr); 2956 2957 /* 2958 * And now we may have to adjust size downward if we have 2959 * exceeded the realsize of the segment or initial anon 2960 * allocations. 2961 */ 2962 sptseg_addr = sptseg->s_base + ptob(anon_index); 2963 if ((sptseg_addr + size) > 2964 (sptseg->s_base + sptd->spt_realsize)) 2965 size = (sptseg->s_base + sptd->spt_realsize) - 2966 sptseg_addr; 2967 2968 /* 2969 * Set memory allocation policy for this segment 2970 */ 2971 policy = lgrp_madv_to_policy(behav, len, MAP_SHARED); 2972 already_set = lgrp_shm_policy_set(policy, amp, anon_index, 2973 NULL, 0, len); 2974 2975 /* 2976 * If random memory allocation policy set already, 2977 * don't bother reapplying it. 2978 */ 2979 if (already_set && !LGRP_MEM_POLICY_REAPPLICABLE(policy)) 2980 return (0); 2981 2982 /* 2983 * Mark any existing pages in the given range for 2984 * migration, flushing the I/O page cache, and using 2985 * underlying segment to calculate anon index and get 2986 * anonmap and vnode pointer from 2987 */ 2988 if (shmd->shm_softlockcnt > 0) 2989 segspt_purge(seg); 2990 2991 page_mark_migrate(seg, shm_addr, size, amp, 0, NULL, 0, 0); 2992 } 2993 2994 return (0); 2995 } 2996 2997 /*ARGSUSED*/ 2998 void 2999 segspt_shmdump(struct seg *seg) 3000 { 3001 /* no-op for ISM segment */ 3002 } 3003 3004 /*ARGSUSED*/ 3005 static faultcode_t 3006 segspt_shmsetpgsz(struct seg *seg, caddr_t addr, size_t len, uint_t szc) 3007 { 3008 return (ENOTSUP); 3009 } 3010 3011 /* 3012 * get a memory ID for an addr in a given segment 3013 */ 3014 static int 3015 segspt_shmgetmemid(struct seg *seg, caddr_t addr, memid_t *memidp) 3016 { 3017 struct shm_data *shmd = (struct shm_data *)seg->s_data; 3018 struct anon *ap; 3019 size_t anon_index; 3020 struct anon_map *amp = shmd->shm_amp; 3021 struct spt_data *sptd = shmd->shm_sptseg->s_data; 3022 struct seg *sptseg = shmd->shm_sptseg; 3023 anon_sync_obj_t cookie; 3024 3025 anon_index = seg_page(seg, addr); 3026 3027 if (addr > (seg->s_base + sptd->spt_realsize)) { 3028 return (EFAULT); 3029 } 3030 3031 ANON_LOCK_ENTER(&->a_rwlock, RW_READER); 3032 anon_array_enter(amp, anon_index, &cookie); 3033 ap = anon_get_ptr(amp->ahp, anon_index); 3034 if (ap == NULL) { 3035 struct page *pp; 3036 caddr_t spt_addr = sptseg->s_base + ptob(anon_index); 3037 3038 pp = anon_zero(sptseg, spt_addr, &ap, kcred); 3039 if (pp == NULL) { 3040 anon_array_exit(&cookie); 3041 ANON_LOCK_EXIT(&->a_rwlock); 3042 return (ENOMEM); 3043 } 3044 (void) anon_set_ptr(amp->ahp, anon_index, ap, ANON_SLEEP); 3045 page_unlock(pp); 3046 } 3047 anon_array_exit(&cookie); 3048 ANON_LOCK_EXIT(&->a_rwlock); 3049 memidp->val[0] = (uintptr_t)ap; 3050 memidp->val[1] = (uintptr_t)addr & PAGEOFFSET; 3051 return (0); 3052 } 3053 3054 /* 3055 * Get memory allocation policy info for specified address in given segment 3056 */ 3057 static lgrp_mem_policy_info_t * 3058 segspt_shmgetpolicy(struct seg *seg, caddr_t addr) 3059 { 3060 struct anon_map *amp; 3061 ulong_t anon_index; 3062 lgrp_mem_policy_info_t *policy_info; 3063 struct shm_data *shm_data; 3064 3065 ASSERT(seg != NULL); 3066 3067 /* 3068 * Get anon_map from segshm 3069 * 3070 * Assume that no lock needs to be held on anon_map, since 3071 * it should be protected by its reference count which must be 3072 * nonzero for an existing segment 3073 * Need to grab readers lock on policy tree though 3074 */ 3075 shm_data = (struct shm_data *)seg->s_data; 3076 if (shm_data == NULL) 3077 return (NULL); 3078 amp = shm_data->shm_amp; 3079 ASSERT(amp->refcnt != 0); 3080 3081 /* 3082 * Get policy info 3083 * 3084 * Assume starting anon index of 0 3085 */ 3086 anon_index = seg_page(seg, addr); 3087 policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0); 3088 3089 return (policy_info); 3090 } 3091 3092 /*ARGSUSED*/ 3093 static int 3094 segspt_shmcapable(struct seg *seg, segcapability_t capability) 3095 { 3096 return (0); 3097 } 3098