1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * Copyright (c) 2015, Joyent, Inc. All rights reserved. 28 */ 29 30 /* 31 * Memory special file 32 */ 33 34 #include <sys/types.h> 35 #include <sys/param.h> 36 #include <sys/user.h> 37 #include <sys/buf.h> 38 #include <sys/systm.h> 39 #include <sys/cred.h> 40 #include <sys/vm.h> 41 #include <sys/uio.h> 42 #include <sys/mman.h> 43 #include <sys/kmem.h> 44 #include <vm/seg.h> 45 #include <vm/page.h> 46 #include <sys/stat.h> 47 #include <sys/vmem.h> 48 #include <sys/memlist.h> 49 #include <sys/bootconf.h> 50 51 #include <vm/seg_vn.h> 52 #include <vm/seg_dev.h> 53 #include <vm/seg_kmem.h> 54 #include <vm/seg_kp.h> 55 #include <vm/seg_kpm.h> 56 #include <vm/hat.h> 57 58 #include <sys/conf.h> 59 #include <sys/mem.h> 60 #include <sys/types.h> 61 #include <sys/conf.h> 62 #include <sys/param.h> 63 #include <sys/systm.h> 64 #include <sys/errno.h> 65 #include <sys/modctl.h> 66 #include <sys/memlist.h> 67 #include <sys/ddi.h> 68 #include <sys/sunddi.h> 69 #include <sys/debug.h> 70 #include <sys/fm/protocol.h> 71 72 #if defined(__sparc) 73 extern int cpu_get_mem_name(uint64_t, uint64_t *, uint64_t, char *, int, int *); 74 extern int cpu_get_mem_info(uint64_t, uint64_t, uint64_t *, uint64_t *, 75 uint64_t *, int *, int *, int *); 76 extern size_t cpu_get_name_bufsize(void); 77 extern int cpu_get_mem_sid(char *, char *, int, int *); 78 extern int cpu_get_mem_addr(char *, char *, uint64_t, uint64_t *); 79 #elif defined(__x86) 80 #include <sys/cpu_module.h> 81 #endif /* __sparc */ 82 83 /* 84 * Turn a byte length into a pagecount. The DDI btop takes a 85 * 32-bit size on 32-bit machines, this handles 64-bit sizes for 86 * large physical-memory 32-bit machines. 87 */ 88 #define BTOP(x) ((pgcnt_t)((x) >> _pageshift)) 89 90 static kmutex_t mm_lock; 91 static caddr_t mm_map; 92 93 static dev_info_t *mm_dip; /* private copy of devinfo pointer */ 94 95 static int mm_kmem_io_access; 96 97 static int mm_kstat_update(kstat_t *ksp, int rw); 98 static int mm_kstat_snapshot(kstat_t *ksp, void *buf, int rw); 99 100 static int mm_read_mem_name(intptr_t data, mem_name_t *mem_name); 101 102 #define MM_KMEMLOG_NENTRIES 64 103 104 static int mm_kmemlogent; 105 static mm_logentry_t mm_kmemlog[MM_KMEMLOG_NENTRIES]; 106 107 /* 108 * On kmem/allmem writes, we log information that might be useful in the event 109 * that a write is errant (that is, due to operator error) and induces a later 110 * problem. Note that (in particular) in the event of such operator-induced 111 * corruption, a search over the kernel address space for the corrupted 112 * address will yield the ring buffer entry that recorded the write. And 113 * should it seem baroque or otherwise unnecessary, yes, we need this kind of 114 * auditing facility and yes, we learned that the hard way: disturbingly, 115 * there exist recommendations for "tuning" the system that involve writing to 116 * kernel memory addresses via the kernel debugger, and -- as we discovered -- 117 * these can easily be applied incorrectly or unsafely, yielding an entirely 118 * undebuggable "can't happen" kind of panic. 119 */ 120 static void 121 mm_logkmem(struct uio *uio) 122 { 123 mm_logentry_t *ent; 124 proc_t *p = curthread->t_procp; 125 126 mutex_enter(&mm_lock); 127 128 ent = &mm_kmemlog[mm_kmemlogent++]; 129 130 if (mm_kmemlogent == MM_KMEMLOG_NENTRIES) 131 mm_kmemlogent = 0; 132 133 ent->mle_vaddr = (uintptr_t)uio->uio_loffset; 134 ent->mle_len = uio->uio_resid; 135 gethrestime(&ent->mle_hrestime); 136 ent->mle_hrtime = gethrtime(); 137 ent->mle_pid = p->p_pidp->pid_id; 138 139 (void) strncpy(ent->mle_psargs, 140 p->p_user.u_psargs, sizeof (ent->mle_psargs)); 141 142 mutex_exit(&mm_lock); 143 } 144 145 /*ARGSUSED1*/ 146 static int 147 mm_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) 148 { 149 int i; 150 struct mem_minor { 151 char *name; 152 minor_t minor; 153 int privonly; 154 const char *rdpriv; 155 const char *wrpriv; 156 mode_t priv_mode; 157 } mm[] = { 158 { "mem", M_MEM, 0, NULL, "all", 0640 }, 159 { "kmem", M_KMEM, 0, NULL, "all", 0640 }, 160 { "allkmem", M_ALLKMEM, 0, "all", "all", 0600 }, 161 { "null", M_NULL, PRIVONLY_DEV, NULL, NULL, 0666 }, 162 { "zero", M_ZERO, PRIVONLY_DEV, NULL, NULL, 0666 }, 163 }; 164 kstat_t *ksp; 165 166 mutex_init(&mm_lock, NULL, MUTEX_DEFAULT, NULL); 167 mm_map = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP); 168 169 for (i = 0; i < (sizeof (mm) / sizeof (mm[0])); i++) { 170 if (ddi_create_priv_minor_node(devi, mm[i].name, S_IFCHR, 171 mm[i].minor, DDI_PSEUDO, mm[i].privonly, 172 mm[i].rdpriv, mm[i].wrpriv, mm[i].priv_mode) == 173 DDI_FAILURE) { 174 ddi_remove_minor_node(devi, NULL); 175 return (DDI_FAILURE); 176 } 177 } 178 179 mm_dip = devi; 180 181 ksp = kstat_create("mm", 0, "phys_installed", "misc", 182 KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VAR_SIZE | KSTAT_FLAG_VIRTUAL); 183 if (ksp != NULL) { 184 ksp->ks_update = mm_kstat_update; 185 ksp->ks_snapshot = mm_kstat_snapshot; 186 ksp->ks_lock = &mm_lock; /* XXX - not really needed */ 187 kstat_install(ksp); 188 } 189 190 mm_kmem_io_access = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, 191 "kmem_io_access", 0); 192 193 return (DDI_SUCCESS); 194 } 195 196 /*ARGSUSED*/ 197 static int 198 mm_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result) 199 { 200 register int error; 201 202 switch (infocmd) { 203 case DDI_INFO_DEVT2DEVINFO: 204 *result = (void *)mm_dip; 205 error = DDI_SUCCESS; 206 break; 207 case DDI_INFO_DEVT2INSTANCE: 208 *result = (void *)0; 209 error = DDI_SUCCESS; 210 break; 211 default: 212 error = DDI_FAILURE; 213 } 214 return (error); 215 } 216 217 /*ARGSUSED1*/ 218 static int 219 mmopen(dev_t *devp, int flag, int typ, struct cred *cred) 220 { 221 switch (getminor(*devp)) { 222 case M_NULL: 223 case M_ZERO: 224 case M_MEM: 225 case M_KMEM: 226 case M_ALLKMEM: 227 /* standard devices */ 228 break; 229 230 default: 231 /* Unsupported or unknown type */ 232 return (EINVAL); 233 } 234 /* must be character device */ 235 if (typ != OTYP_CHR) 236 return (EINVAL); 237 return (0); 238 } 239 240 struct pollhead mm_pollhd; 241 242 /*ARGSUSED*/ 243 static int 244 mmchpoll(dev_t dev, short events, int anyyet, short *reventsp, 245 struct pollhead **phpp) 246 { 247 switch (getminor(dev)) { 248 case M_NULL: 249 case M_ZERO: 250 case M_MEM: 251 case M_KMEM: 252 case M_ALLKMEM: 253 *reventsp = events & (POLLIN | POLLOUT | POLLPRI | POLLRDNORM | 254 POLLWRNORM | POLLRDBAND | POLLWRBAND); 255 /* 256 * A non NULL pollhead pointer should be returned in case 257 * user polls for 0 events. 258 */ 259 *phpp = !anyyet && !*reventsp ? 260 &mm_pollhd : (struct pollhead *)NULL; 261 return (0); 262 default: 263 /* no other devices currently support polling */ 264 return (ENXIO); 265 } 266 } 267 268 static int 269 mmpropop(dev_t dev, dev_info_t *dip, ddi_prop_op_t prop_op, int flags, 270 char *name, caddr_t valuep, int *lengthp) 271 { 272 /* 273 * implement zero size to reduce overhead (avoid two failing 274 * property lookups per stat). 275 */ 276 return (ddi_prop_op_size(dev, dip, prop_op, 277 flags, name, valuep, lengthp, 0)); 278 } 279 280 static int 281 mmio(struct uio *uio, enum uio_rw rw, pfn_t pfn, off_t pageoff, int allowio, 282 page_t *pp) 283 { 284 int error = 0; 285 int devload = 0; 286 int is_memory = pf_is_memory(pfn); 287 size_t nbytes = MIN((size_t)(PAGESIZE - pageoff), 288 (size_t)uio->uio_iov->iov_len); 289 caddr_t va = NULL; 290 291 mutex_enter(&mm_lock); 292 293 if (is_memory && kpm_enable) { 294 if (pp) 295 va = hat_kpm_mapin(pp, NULL); 296 else 297 va = hat_kpm_mapin_pfn(pfn); 298 } 299 300 if (va == NULL) { 301 hat_devload(kas.a_hat, mm_map, PAGESIZE, pfn, 302 (uint_t)(rw == UIO_READ ? PROT_READ : PROT_READ|PROT_WRITE), 303 HAT_LOAD_NOCONSIST|HAT_LOAD_LOCK); 304 va = mm_map; 305 devload = 1; 306 } 307 308 if (!is_memory) { 309 if (allowio) { 310 size_t c = uio->uio_iov->iov_len; 311 312 if (ddi_peekpokeio(NULL, uio, rw, 313 (caddr_t)(uintptr_t)uio->uio_loffset, c, 314 sizeof (int32_t)) != DDI_SUCCESS) 315 error = EFAULT; 316 } else 317 error = EIO; 318 } else 319 error = uiomove(va + pageoff, nbytes, rw, uio); 320 321 if (devload) 322 hat_unload(kas.a_hat, mm_map, PAGESIZE, HAT_UNLOAD_UNLOCK); 323 else if (pp) 324 hat_kpm_mapout(pp, NULL, va); 325 else 326 hat_kpm_mapout_pfn(pfn); 327 328 mutex_exit(&mm_lock); 329 return (error); 330 } 331 332 static int 333 mmpagelock(struct as *as, caddr_t va) 334 { 335 struct seg *seg; 336 int i; 337 338 AS_LOCK_ENTER(as, RW_READER); 339 seg = as_segat(as, va); 340 i = (seg != NULL)? SEGOP_CAPABLE(seg, S_CAPABILITY_NOMINFLT) : 0; 341 AS_LOCK_EXIT(as); 342 343 return (i); 344 } 345 346 #ifdef __sparc 347 348 #define NEED_LOCK_KVADDR(kva) mmpagelock(&kas, kva) 349 350 #else /* __i386, __amd64 */ 351 352 #define NEED_LOCK_KVADDR(va) 0 353 354 #endif /* __sparc */ 355 356 /*ARGSUSED3*/ 357 static int 358 mmrw(dev_t dev, struct uio *uio, enum uio_rw rw, cred_t *cred) 359 { 360 pfn_t v; 361 struct iovec *iov; 362 int error = 0; 363 size_t c; 364 ssize_t oresid = uio->uio_resid; 365 minor_t minor = getminor(dev); 366 367 while (uio->uio_resid > 0 && error == 0) { 368 iov = uio->uio_iov; 369 if (iov->iov_len == 0) { 370 uio->uio_iov++; 371 uio->uio_iovcnt--; 372 if (uio->uio_iovcnt < 0) 373 panic("mmrw"); 374 continue; 375 } 376 switch (minor) { 377 378 case M_MEM: 379 memlist_read_lock(); 380 if (!address_in_memlist(phys_install, 381 (uint64_t)uio->uio_loffset, 1)) { 382 memlist_read_unlock(); 383 error = EFAULT; 384 break; 385 } 386 memlist_read_unlock(); 387 388 v = BTOP((u_offset_t)uio->uio_loffset); 389 error = mmio(uio, rw, v, 390 uio->uio_loffset & PAGEOFFSET, 0, NULL); 391 break; 392 393 case M_KMEM: 394 case M_ALLKMEM: 395 { 396 page_t **ppp = NULL; 397 caddr_t vaddr = (caddr_t)uio->uio_offset; 398 int try_lock = NEED_LOCK_KVADDR(vaddr); 399 int locked = 0; 400 401 if ((error = plat_mem_do_mmio(uio, rw)) != ENOTSUP) 402 break; 403 404 if (rw == UIO_WRITE) 405 mm_logkmem(uio); 406 407 /* 408 * If vaddr does not map a valid page, as_pagelock() 409 * will return failure. Hence we can't check the 410 * return value and return EFAULT here as we'd like. 411 * seg_kp and seg_kpm do not properly support 412 * as_pagelock() for this context so we avoid it 413 * using the try_lock set check above. Some day when 414 * the kernel page locking gets redesigned all this 415 * muck can be cleaned up. 416 */ 417 if (try_lock) 418 locked = (as_pagelock(&kas, &ppp, vaddr, 419 PAGESIZE, S_WRITE) == 0); 420 421 v = hat_getpfnum(kas.a_hat, 422 (caddr_t)(uintptr_t)uio->uio_loffset); 423 if (v == PFN_INVALID) { 424 if (locked) 425 as_pageunlock(&kas, ppp, vaddr, 426 PAGESIZE, S_WRITE); 427 error = EFAULT; 428 break; 429 } 430 431 error = mmio(uio, rw, v, uio->uio_loffset & PAGEOFFSET, 432 minor == M_ALLKMEM || mm_kmem_io_access, 433 (locked && ppp) ? *ppp : NULL); 434 if (locked) 435 as_pageunlock(&kas, ppp, vaddr, PAGESIZE, 436 S_WRITE); 437 } 438 439 break; 440 441 case M_ZERO: 442 if (rw == UIO_READ) { 443 label_t ljb; 444 445 if (on_fault(&ljb)) { 446 no_fault(); 447 error = EFAULT; 448 break; 449 } 450 uzero(iov->iov_base, iov->iov_len); 451 no_fault(); 452 uio->uio_resid -= iov->iov_len; 453 uio->uio_loffset += iov->iov_len; 454 break; 455 } 456 /* else it's a write, fall through to NULL case */ 457 /*FALLTHROUGH*/ 458 459 case M_NULL: 460 if (rw == UIO_READ) 461 return (0); 462 c = iov->iov_len; 463 iov->iov_base += c; 464 iov->iov_len -= c; 465 uio->uio_loffset += c; 466 uio->uio_resid -= c; 467 break; 468 469 } 470 } 471 return (uio->uio_resid == oresid ? error : 0); 472 } 473 474 static int 475 mmread(dev_t dev, struct uio *uio, cred_t *cred) 476 { 477 return (mmrw(dev, uio, UIO_READ, cred)); 478 } 479 480 static int 481 mmwrite(dev_t dev, struct uio *uio, cred_t *cred) 482 { 483 return (mmrw(dev, uio, UIO_WRITE, cred)); 484 } 485 486 /* 487 * Private ioctl for libkvm to support kvm_physaddr(). 488 * Given an address space and a VA, compute the PA. 489 */ 490 static int 491 mmioctl_vtop(intptr_t data) 492 { 493 #ifdef _SYSCALL32 494 mem_vtop32_t vtop32; 495 #endif 496 mem_vtop_t mem_vtop; 497 proc_t *p; 498 pfn_t pfn = (pfn_t)PFN_INVALID; 499 pid_t pid = 0; 500 struct as *as; 501 struct seg *seg; 502 503 if (get_udatamodel() == DATAMODEL_NATIVE) { 504 if (copyin((void *)data, &mem_vtop, sizeof (mem_vtop_t))) 505 return (EFAULT); 506 } 507 #ifdef _SYSCALL32 508 else { 509 if (copyin((void *)data, &vtop32, sizeof (mem_vtop32_t))) 510 return (EFAULT); 511 mem_vtop.m_as = (struct as *)(uintptr_t)vtop32.m_as; 512 mem_vtop.m_va = (void *)(uintptr_t)vtop32.m_va; 513 514 if (mem_vtop.m_as != NULL) 515 return (EINVAL); 516 } 517 #endif 518 519 if (mem_vtop.m_as == &kas) { 520 pfn = hat_getpfnum(kas.a_hat, mem_vtop.m_va); 521 } else { 522 if (mem_vtop.m_as == NULL) { 523 /* 524 * Assume the calling process's address space if the 525 * caller didn't specify one. 526 */ 527 p = curthread->t_procp; 528 if (p == NULL) 529 return (EIO); 530 mem_vtop.m_as = p->p_as; 531 } 532 533 mutex_enter(&pidlock); 534 for (p = practive; p != NULL; p = p->p_next) { 535 if (p->p_as == mem_vtop.m_as) { 536 pid = p->p_pid; 537 break; 538 } 539 } 540 mutex_exit(&pidlock); 541 if (p == NULL) 542 return (EIO); 543 p = sprlock(pid); 544 if (p == NULL) 545 return (EIO); 546 as = p->p_as; 547 if (as == mem_vtop.m_as) { 548 mutex_exit(&p->p_lock); 549 AS_LOCK_ENTER(as, RW_READER); 550 for (seg = AS_SEGFIRST(as); seg != NULL; 551 seg = AS_SEGNEXT(as, seg)) 552 if ((uintptr_t)mem_vtop.m_va - 553 (uintptr_t)seg->s_base < seg->s_size) 554 break; 555 if (seg != NULL) 556 pfn = hat_getpfnum(as->a_hat, mem_vtop.m_va); 557 AS_LOCK_EXIT(as); 558 mutex_enter(&p->p_lock); 559 } 560 sprunlock(p); 561 } 562 mem_vtop.m_pfn = pfn; 563 if (pfn == PFN_INVALID) 564 return (EIO); 565 566 if (get_udatamodel() == DATAMODEL_NATIVE) { 567 if (copyout(&mem_vtop, (void *)data, sizeof (mem_vtop_t))) 568 return (EFAULT); 569 } 570 #ifdef _SYSCALL32 571 else { 572 vtop32.m_pfn = mem_vtop.m_pfn; 573 if (copyout(&vtop32, (void *)data, sizeof (mem_vtop32_t))) 574 return (EFAULT); 575 } 576 #endif 577 578 return (0); 579 } 580 581 /* 582 * Given a PA, execute the given page retire command on it. 583 */ 584 static int 585 mmioctl_page_retire(int cmd, intptr_t data) 586 { 587 extern int page_retire_test(void); 588 uint64_t pa; 589 590 if (copyin((void *)data, &pa, sizeof (uint64_t))) { 591 return (EFAULT); 592 } 593 594 switch (cmd) { 595 case MEM_PAGE_ISRETIRED: 596 return (page_retire_check(pa, NULL)); 597 598 case MEM_PAGE_UNRETIRE: 599 return (page_unretire(pa)); 600 601 case MEM_PAGE_RETIRE: 602 return (page_retire(pa, PR_FMA)); 603 604 case MEM_PAGE_RETIRE_MCE: 605 return (page_retire(pa, PR_MCE)); 606 607 case MEM_PAGE_RETIRE_UE: 608 return (page_retire(pa, PR_UE)); 609 610 case MEM_PAGE_GETERRORS: 611 { 612 uint64_t page_errors; 613 int rc = page_retire_check(pa, &page_errors); 614 if (copyout(&page_errors, (void *)data, 615 sizeof (uint64_t))) { 616 return (EFAULT); 617 } 618 return (rc); 619 } 620 621 case MEM_PAGE_RETIRE_TEST: 622 return (page_retire_test()); 623 624 } 625 626 return (EINVAL); 627 } 628 629 #ifdef __sparc 630 /* 631 * Given a syndrome, syndrome type, and address return the 632 * associated memory name in the provided data buffer. 633 */ 634 static int 635 mmioctl_get_mem_name(intptr_t data) 636 { 637 mem_name_t mem_name; 638 void *buf; 639 size_t bufsize; 640 int len, err; 641 642 if ((bufsize = cpu_get_name_bufsize()) == 0) 643 return (ENOTSUP); 644 645 if ((err = mm_read_mem_name(data, &mem_name)) < 0) 646 return (err); 647 648 buf = kmem_alloc(bufsize, KM_SLEEP); 649 650 /* 651 * Call into cpu specific code to do the lookup. 652 */ 653 if ((err = cpu_get_mem_name(mem_name.m_synd, mem_name.m_type, 654 mem_name.m_addr, buf, bufsize, &len)) != 0) { 655 kmem_free(buf, bufsize); 656 return (err); 657 } 658 659 if (len >= mem_name.m_namelen) { 660 kmem_free(buf, bufsize); 661 return (ENOSPC); 662 } 663 664 if (copyoutstr(buf, (char *)mem_name.m_name, 665 mem_name.m_namelen, NULL) != 0) { 666 kmem_free(buf, bufsize); 667 return (EFAULT); 668 } 669 670 kmem_free(buf, bufsize); 671 return (0); 672 } 673 674 /* 675 * Given a syndrome and address return information about the associated memory. 676 */ 677 static int 678 mmioctl_get_mem_info(intptr_t data) 679 { 680 mem_info_t mem_info; 681 int err; 682 683 if (copyin((void *)data, &mem_info, sizeof (mem_info_t))) 684 return (EFAULT); 685 686 if ((err = cpu_get_mem_info(mem_info.m_synd, mem_info.m_addr, 687 &mem_info.m_mem_size, &mem_info.m_seg_size, &mem_info.m_bank_size, 688 &mem_info.m_segments, &mem_info.m_banks, &mem_info.m_mcid)) != 0) 689 return (err); 690 691 if (copyout(&mem_info, (void *)data, sizeof (mem_info_t)) != 0) 692 return (EFAULT); 693 694 return (0); 695 } 696 697 /* 698 * Given a memory name, return its associated serial id 699 */ 700 static int 701 mmioctl_get_mem_sid(intptr_t data) 702 { 703 mem_name_t mem_name; 704 void *buf; 705 void *name; 706 size_t name_len; 707 size_t bufsize; 708 int len, err; 709 710 if ((bufsize = cpu_get_name_bufsize()) == 0) 711 return (ENOTSUP); 712 713 if ((err = mm_read_mem_name(data, &mem_name)) < 0) 714 return (err); 715 716 buf = kmem_alloc(bufsize, KM_SLEEP); 717 718 if (mem_name.m_namelen > 1024) 719 mem_name.m_namelen = 1024; /* cap at 1024 bytes */ 720 721 name = kmem_alloc(mem_name.m_namelen, KM_SLEEP); 722 723 if ((err = copyinstr((char *)mem_name.m_name, (char *)name, 724 mem_name.m_namelen, &name_len)) != 0) { 725 kmem_free(buf, bufsize); 726 kmem_free(name, mem_name.m_namelen); 727 return (err); 728 } 729 730 /* 731 * Call into cpu specific code to do the lookup. 732 */ 733 if ((err = cpu_get_mem_sid(name, buf, bufsize, &len)) != 0) { 734 kmem_free(buf, bufsize); 735 kmem_free(name, mem_name.m_namelen); 736 return (err); 737 } 738 739 if (len > mem_name.m_sidlen) { 740 kmem_free(buf, bufsize); 741 kmem_free(name, mem_name.m_namelen); 742 return (ENAMETOOLONG); 743 } 744 745 if (copyoutstr(buf, (char *)mem_name.m_sid, 746 mem_name.m_sidlen, NULL) != 0) { 747 kmem_free(buf, bufsize); 748 kmem_free(name, mem_name.m_namelen); 749 return (EFAULT); 750 } 751 752 kmem_free(buf, bufsize); 753 kmem_free(name, mem_name.m_namelen); 754 return (0); 755 } 756 #endif /* __sparc */ 757 758 /* 759 * Private ioctls for 760 * libkvm to support kvm_physaddr(). 761 * FMA support for page_retire() and memory attribute information. 762 */ 763 /*ARGSUSED*/ 764 static int 765 mmioctl(dev_t dev, int cmd, intptr_t data, int flag, cred_t *cred, int *rvalp) 766 { 767 if ((cmd == MEM_VTOP && getminor(dev) != M_KMEM) || 768 (cmd != MEM_VTOP && getminor(dev) != M_MEM)) 769 return (ENXIO); 770 771 switch (cmd) { 772 case MEM_VTOP: 773 return (mmioctl_vtop(data)); 774 775 case MEM_PAGE_RETIRE: 776 case MEM_PAGE_ISRETIRED: 777 case MEM_PAGE_UNRETIRE: 778 case MEM_PAGE_RETIRE_MCE: 779 case MEM_PAGE_RETIRE_UE: 780 case MEM_PAGE_GETERRORS: 781 case MEM_PAGE_RETIRE_TEST: 782 return (mmioctl_page_retire(cmd, data)); 783 784 #ifdef __sparc 785 case MEM_NAME: 786 return (mmioctl_get_mem_name(data)); 787 788 case MEM_INFO: 789 return (mmioctl_get_mem_info(data)); 790 791 case MEM_SID: 792 return (mmioctl_get_mem_sid(data)); 793 #else 794 case MEM_NAME: 795 case MEM_INFO: 796 case MEM_SID: 797 return (ENOTSUP); 798 #endif /* __sparc */ 799 } 800 return (ENXIO); 801 } 802 803 /*ARGSUSED2*/ 804 static int 805 mmmmap(dev_t dev, off_t off, int prot) 806 { 807 pfn_t pf; 808 struct memlist *pmem; 809 minor_t minor = getminor(dev); 810 811 switch (minor) { 812 case M_MEM: 813 pf = btop(off); 814 memlist_read_lock(); 815 for (pmem = phys_install; pmem != NULL; pmem = pmem->ml_next) { 816 if (pf >= BTOP(pmem->ml_address) && 817 pf < BTOP(pmem->ml_address + pmem->ml_size)) { 818 memlist_read_unlock(); 819 return (impl_obmem_pfnum(pf)); 820 } 821 } 822 memlist_read_unlock(); 823 break; 824 825 case M_KMEM: 826 case M_ALLKMEM: 827 /* no longer supported with KPR */ 828 return (-1); 829 830 case M_ZERO: 831 /* 832 * We shouldn't be mmap'ing to /dev/zero here as 833 * mmsegmap() should have already converted 834 * a mapping request for this device to a mapping 835 * using seg_vn for anonymous memory. 836 */ 837 break; 838 839 } 840 return (-1); 841 } 842 843 /* 844 * This function is called when a memory device is mmap'ed. 845 * Set up the mapping to the correct device driver. 846 */ 847 static int 848 mmsegmap(dev_t dev, off_t off, struct as *as, caddr_t *addrp, off_t len, 849 uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred) 850 { 851 struct segvn_crargs vn_a; 852 struct segdev_crargs dev_a; 853 int error; 854 minor_t minor; 855 off_t i; 856 857 minor = getminor(dev); 858 859 as_rangelock(as); 860 /* 861 * No need to worry about vac alignment on /dev/zero 862 * since this is a "clone" object that doesn't yet exist. 863 */ 864 error = choose_addr(as, addrp, len, off, 865 (minor == M_MEM) || (minor == M_KMEM), flags); 866 if (error != 0) { 867 as_rangeunlock(as); 868 return (error); 869 } 870 871 switch (minor) { 872 case M_MEM: 873 /* /dev/mem cannot be mmap'ed with MAP_PRIVATE */ 874 if ((flags & MAP_TYPE) != MAP_SHARED) { 875 as_rangeunlock(as); 876 return (EINVAL); 877 } 878 879 /* 880 * Check to ensure that the entire range is 881 * legal and we are not trying to map in 882 * more than the device will let us. 883 */ 884 for (i = 0; i < len; i += PAGESIZE) { 885 if (mmmmap(dev, off + i, maxprot) == -1) { 886 as_rangeunlock(as); 887 return (ENXIO); 888 } 889 } 890 891 /* 892 * Use seg_dev segment driver for /dev/mem mapping. 893 */ 894 dev_a.mapfunc = mmmmap; 895 dev_a.dev = dev; 896 dev_a.offset = off; 897 dev_a.type = (flags & MAP_TYPE); 898 dev_a.prot = (uchar_t)prot; 899 dev_a.maxprot = (uchar_t)maxprot; 900 dev_a.hat_attr = 0; 901 902 /* 903 * Make /dev/mem mappings non-consistent since we can't 904 * alias pages that don't have page structs behind them, 905 * such as kernel stack pages. If someone mmap()s a kernel 906 * stack page and if we give him a tte with cv, a line from 907 * that page can get into both pages of the spitfire d$. 908 * But snoop from another processor will only invalidate 909 * the first page. This later caused kernel (xc_attention) 910 * to go into an infinite loop at pil 13 and no interrupts 911 * could come in. See 1203630. 912 * 913 */ 914 dev_a.hat_flags = HAT_LOAD_NOCONSIST; 915 dev_a.devmap_data = NULL; 916 917 error = as_map(as, *addrp, len, segdev_create, &dev_a); 918 break; 919 920 case M_ZERO: 921 /* 922 * Use seg_vn segment driver for /dev/zero mapping. 923 * Passing in a NULL amp gives us the "cloning" effect. 924 */ 925 vn_a.vp = NULL; 926 vn_a.offset = 0; 927 vn_a.type = (flags & MAP_TYPE); 928 vn_a.prot = prot; 929 vn_a.maxprot = maxprot; 930 vn_a.flags = flags & ~MAP_TYPE; 931 vn_a.cred = cred; 932 vn_a.amp = NULL; 933 vn_a.szc = 0; 934 vn_a.lgrp_mem_policy_flags = 0; 935 error = as_map(as, *addrp, len, segvn_create, &vn_a); 936 break; 937 938 case M_KMEM: 939 case M_ALLKMEM: 940 /* No longer supported with KPR. */ 941 error = ENXIO; 942 break; 943 944 case M_NULL: 945 /* 946 * Use seg_dev segment driver for /dev/null mapping. 947 */ 948 dev_a.mapfunc = mmmmap; 949 dev_a.dev = dev; 950 dev_a.offset = off; 951 dev_a.type = 0; /* neither PRIVATE nor SHARED */ 952 dev_a.prot = dev_a.maxprot = (uchar_t)PROT_NONE; 953 dev_a.hat_attr = 0; 954 dev_a.hat_flags = 0; 955 error = as_map(as, *addrp, len, segdev_create, &dev_a); 956 break; 957 958 default: 959 error = ENXIO; 960 } 961 962 as_rangeunlock(as); 963 return (error); 964 } 965 966 static struct cb_ops mm_cb_ops = { 967 mmopen, /* open */ 968 nulldev, /* close */ 969 nodev, /* strategy */ 970 nodev, /* print */ 971 nodev, /* dump */ 972 mmread, /* read */ 973 mmwrite, /* write */ 974 mmioctl, /* ioctl */ 975 nodev, /* devmap */ 976 mmmmap, /* mmap */ 977 mmsegmap, /* segmap */ 978 mmchpoll, /* poll */ 979 mmpropop, /* prop_op */ 980 0, /* streamtab */ 981 D_NEW | D_MP | D_64BIT | D_U64BIT 982 }; 983 984 static struct dev_ops mm_ops = { 985 DEVO_REV, /* devo_rev, */ 986 0, /* refcnt */ 987 mm_info, /* get_dev_info */ 988 nulldev, /* identify */ 989 nulldev, /* probe */ 990 mm_attach, /* attach */ 991 nodev, /* detach */ 992 nodev, /* reset */ 993 &mm_cb_ops, /* driver operations */ 994 (struct bus_ops *)0, /* bus operations */ 995 NULL, /* power */ 996 ddi_quiesce_not_needed, /* quiesce */ 997 }; 998 999 static struct modldrv modldrv = { 1000 &mod_driverops, "memory driver", &mm_ops, 1001 }; 1002 1003 static struct modlinkage modlinkage = { 1004 MODREV_1, &modldrv, NULL 1005 }; 1006 1007 int 1008 _init(void) 1009 { 1010 return (mod_install(&modlinkage)); 1011 } 1012 1013 int 1014 _info(struct modinfo *modinfop) 1015 { 1016 return (mod_info(&modlinkage, modinfop)); 1017 } 1018 1019 int 1020 _fini(void) 1021 { 1022 return (mod_remove(&modlinkage)); 1023 } 1024 1025 static int 1026 mm_kstat_update(kstat_t *ksp, int rw) 1027 { 1028 struct memlist *pmem; 1029 uint_t count; 1030 1031 if (rw == KSTAT_WRITE) 1032 return (EACCES); 1033 1034 count = 0; 1035 memlist_read_lock(); 1036 for (pmem = phys_install; pmem != NULL; pmem = pmem->ml_next) { 1037 count++; 1038 } 1039 memlist_read_unlock(); 1040 1041 ksp->ks_ndata = count; 1042 ksp->ks_data_size = count * 2 * sizeof (uint64_t); 1043 1044 return (0); 1045 } 1046 1047 static int 1048 mm_kstat_snapshot(kstat_t *ksp, void *buf, int rw) 1049 { 1050 struct memlist *pmem; 1051 struct memunit { 1052 uint64_t address; 1053 uint64_t size; 1054 } *kspmem; 1055 1056 if (rw == KSTAT_WRITE) 1057 return (EACCES); 1058 1059 ksp->ks_snaptime = gethrtime(); 1060 1061 kspmem = (struct memunit *)buf; 1062 memlist_read_lock(); 1063 for (pmem = phys_install; pmem != NULL; 1064 pmem = pmem->ml_next, kspmem++) { 1065 if ((caddr_t)kspmem >= (caddr_t)buf + ksp->ks_data_size) 1066 break; 1067 kspmem->address = pmem->ml_address; 1068 kspmem->size = pmem->ml_size; 1069 } 1070 memlist_read_unlock(); 1071 1072 return (0); 1073 } 1074 1075 /* 1076 * Read a mem_name_t from user-space and store it in the mem_name_t 1077 * pointed to by the mem_name argument. 1078 */ 1079 static int 1080 mm_read_mem_name(intptr_t data, mem_name_t *mem_name) 1081 { 1082 if (get_udatamodel() == DATAMODEL_NATIVE) { 1083 if (copyin((void *)data, mem_name, sizeof (mem_name_t))) 1084 return (EFAULT); 1085 } 1086 #ifdef _SYSCALL32 1087 else { 1088 mem_name32_t mem_name32; 1089 1090 if (copyin((void *)data, &mem_name32, sizeof (mem_name32_t))) 1091 return (EFAULT); 1092 mem_name->m_addr = mem_name32.m_addr; 1093 mem_name->m_synd = mem_name32.m_synd; 1094 mem_name->m_type[0] = mem_name32.m_type[0]; 1095 mem_name->m_type[1] = mem_name32.m_type[1]; 1096 mem_name->m_name = (caddr_t)(uintptr_t)mem_name32.m_name; 1097 mem_name->m_namelen = (size_t)mem_name32.m_namelen; 1098 mem_name->m_sid = (caddr_t)(uintptr_t)mem_name32.m_sid; 1099 mem_name->m_sidlen = (size_t)mem_name32.m_sidlen; 1100 } 1101 #endif /* _SYSCALL32 */ 1102 1103 return (0); 1104 } 1105